From f40d8f47c2480feaa73e6b85d3401a38e8c38908 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 7 Mar 2022 15:12:01 +0100 Subject: [PATCH] chg: [v4.1] add Investigation with MISP Export + v4.1 update --- bin/export/MispExport.py | 52 +- bin/lib/Config_DB.py | 145 ++-- bin/lib/Investigations.py | 449 ++++++++++++ bin/lib/ail_objects.py | 406 ----------- bin/lib/ail_users.py | 30 + bin/lib/exceptions.py | 15 + bin/lib/objects/CryptoCurrencies.py | 76 ++ bin/lib/objects/Decodeds.py | 85 +++ bin/lib/objects/Domains.py | 77 ++ bin/lib/objects/Items.py | 690 ++++++++++++++++++ bin/lib/objects/Pgps.py | 72 ++ bin/lib/objects/Screenshots.py | 65 ++ bin/lib/objects/Usernames.py | 73 ++ bin/lib/objects/abstract_object.py | 135 ++++ bin/lib/objects/ail_objects.py | 87 +++ bin/packages/Item.py | 2 +- configs/core.cfg.sample | 23 +- update/v4.1/Update.py | 36 + update/v4.1/Update.sh | 25 + var/www/Flask_server.py | 2 + var/www/blueprints/crawler_splash.py | 27 +- var/www/blueprints/import_export.py | 14 + var/www/blueprints/investigations_b.py | 212 ++++++ var/www/modules/settings/Flask_settings.py | 6 + .../settings/templates/ail_configs.html | 93 +++ var/www/static/css/ail-project.css | 13 + .../metadata_card_cryptocurrency.html | 8 + .../correlation/metadata_card_decoded.html | 8 + .../correlation/metadata_card_domain.html | 8 + .../correlation/metadata_card_paste.html | 8 + .../correlation/metadata_card_pgp.html | 8 + .../correlation/metadata_card_screenshot.html | 8 + .../correlation/metadata_card_username.html | 8 + .../investigations/add_investigation.html | 214 ++++++ .../investigations/investigations.html | 90 +++ .../investigations/view_investigation.html | 178 +++++ .../modals/investigations_register_obj.html | 62 ++ var/www/templates/objects/item/show_item.html | 10 + var/www/templates/objects/obj_svg_block.html | 6 + var/www/templates/settings/menu_sidebar.html | 8 +- .../templates/sidebars/sidebar_objects.html | 94 +++ 41 files changed, 3140 insertions(+), 488 deletions(-) create mode 100755 bin/lib/Investigations.py delete mode 100755 bin/lib/ail_objects.py create mode 100755 bin/lib/ail_users.py create mode 100755 bin/lib/exceptions.py create mode 100755 bin/lib/objects/CryptoCurrencies.py create mode 100755 bin/lib/objects/Decodeds.py create mode 100755 bin/lib/objects/Domains.py create mode 100755 bin/lib/objects/Items.py create mode 100755 bin/lib/objects/Pgps.py create mode 100755 bin/lib/objects/Screenshots.py create mode 100755 bin/lib/objects/Usernames.py create mode 100755 bin/lib/objects/abstract_object.py create mode 100755 bin/lib/objects/ail_objects.py create mode 100755 update/v4.1/Update.py create mode 100755 update/v4.1/Update.sh create mode 100644 var/www/blueprints/investigations_b.py create mode 100644 var/www/modules/settings/templates/ail_configs.html create mode 100644 var/www/templates/investigations/add_investigation.html create mode 100644 var/www/templates/investigations/investigations.html create mode 100644 var/www/templates/investigations/view_investigation.html create mode 100644 var/www/templates/modals/investigations_register_obj.html create mode 100644 var/www/templates/objects/obj_svg_block.html create mode 100644 var/www/templates/sidebars/sidebar_objects.html diff --git a/bin/export/MispExport.py b/bin/export/MispExport.py index f1f1cf60..c42ed5cb 100755 --- a/bin/export/MispExport.py +++ b/bin/export/MispExport.py @@ -22,6 +22,10 @@ import Correlate_object import AILObjects import Export + +from Investigations import Investigation +import Tag + # # TODO: # FIXME: REFRACTOR ME => use UI/Global config sys.path.append('../../configs/keys') try: @@ -390,6 +394,7 @@ def create_misp_event(event, distribution=0, threat_level_id=4, publish=False, a # # TODO: handle multiple MISP instance misp = PyMISP(misp_url, misp_key, misp_verifycert) #print(event.to_json()) + misp_event = misp.add_event(event) #print(misp_event) # # TODO: handle error @@ -414,9 +419,50 @@ def extract_event_metadata(event): # LVL 1 => DETAILED Also add correlated_items correlation ###### -if __name__ == '__main__': +# # TODO: # create object relationships +def create_investigation_event(investigation_uuid): + investigation = Investigation(investigation_uuid) - l_obj = [{'id': 'bfd5f1d89e55b10a8b122a9d7ce31667ec1d086a', 'type': 'decoded', 'lvl': 2}] - create_list_of_objs_to_export(l_obj) + event = MISPEvent() + event.info = investigation.get_info() + event.uuid = investigation.get_uuid() + event.date = investigation.get_date() + event.analysis = investigation.get_analysis() + event.threat_level_id = investigation.get_threat_level() + + taxonomies_tags, galaxies_tags = Tag.sort_tags_taxonomies_galaxies(investigation.get_tags()) + event.Tag = taxonomies_tags + event.Galaxy = galaxies_tags + #event.add_galaxy(galaxies_tags) + + investigation_objs = investigation.get_objects() + for obj in investigation_objs: + # if subtype -> obj_id = 'subtype:type' + if obj['subtype']: + obj_id = f"{obj['subtype']}:{obj['id']}" + else: + obj_id = obj['id'] + misp_obj = create_misp_obj(obj['type'], obj_id) + if misp_obj: + event.add_object(misp_obj) + + # if publish: + # event.publish() + + # res = event.to_json() + # print(event.to_json()) + + misp = PyMISP(misp_url, misp_key, misp_verifycert) + misp_event = misp.add_event(event) + # print(misp_event) + + # # TODO: handle error + event_metadata = extract_event_metadata(misp_event) + return event_metadata + +# if __name__ == '__main__': + + # l_obj = [{'id': 'bfd5f1d89e55b10a8b122a9d7ce31667ec1d086a', 'type': 'decoded', 'lvl': 2}] + # create_list_of_objs_to_export(l_obj) #print(event.to_json()) diff --git a/bin/lib/Config_DB.py b/bin/lib/Config_DB.py index 67e106ab..40998dd6 100755 --- a/bin/lib/Config_DB.py +++ b/bin/lib/Config_DB.py @@ -24,44 +24,116 @@ config_loader = None ## data retention ######################### -default_config = { + +ail_config = { "crawler": { - "enable_har_by_default": False, - "enable_screenshot_by_default": True, - "default_depth_limit": 1, - "default_closespider_pagecount": 50, - "default_user_agent": "Mozilla/5.0 (Windows NT 10.0; rv:68.0) Gecko/20100101 Firefox/68.0", - "default_timeout": 30 + "enable_har_by_default": { + "default": False, + "type": bool, + "info": "Enable HAR by default" + }, + "enable_screenshot_by_default": { + "default": True, + "type": bool, + "info": "Enable screenshot by default" + }, + "depth_limit": { + "default": 1, + "type": int, + "info": "Maximum number of url depth" + }, + "closespider_pagecount": { + "default": 50, + "type": int, + "info": "Maximum number of pages" + }, + "user_agent": { + "default": 50, + "type": str, + "info": "User agent used by default" + }, + "timeout": { + "default": 30, + "type": int, + "info": "Crawler connection timeout" + }, + }, + "misp": { + "url": { + "default": "https://localhost:8443/", + "type": str, + "info": "Crawler connection timeout" + }, + "key": { + "default": "", + "type": str, + "info": "Crawler connection timeout" + }, + "verifycert": { + "default": True, + "type": bool, + "info": "Crawler connection timeout" + }, } } +# The MISP auth key can be found on the MISP web interface under the automation section + +def get_config_value(section, field, value): + return r_serv_db.hget(f'ail:config:global:{section}', field, value) + +def get_config_default_value(section, field, value): + return ail_config[section][field]['default'] + +def get_config_type(section, field, value): + return ail_config[section][field]['type'] + +def get_config_info(section, field, value): + return ail_config[section][field]['info'] + +def save_config(section, field, value): + if section in ail_config: + if is_valid_type(value, section, field, value_type=value_type): + # if value_type in ['list', 'set', 'dict']: + # pass + # else: + r_serv_db.hset(f'ail:config:global:{section}', field, value) + + +config_documentation = { + +} + +default_config = { + +} + def get_default_config(): return default_config def get_default_config_value(section, field): return default_config[section][field] -config_type = { - # crawler config - "crawler": { - "enable_har_by_default": bool, - "enable_screenshot_by_default": bool, - "default_depth_limit": int, - "default_closespider_pagecount": int, - "default_user_agent": str, - "default_timeout": int - } -} -def get_config_type(section, field): - return config_type[section][field] + + +#### DEFAULT CONFIG #### + +#### CONFIG TYPE #### +# CONFIG DOC +config_type = { + +} # # TODO: add set, dict, list and select_(multiple_)value def is_valid_type(obj, section, field, value_type=None): res = isinstance(obj, get_config_type(section, field)) return res +# # TODO: ########################################################### def reset_default_config(): + for section in config_type: + pass def set_default_config(section, field): @@ -92,45 +164,12 @@ def get_config_dict_by_section(section): config_dict[field] = get_config(section, field) return config_dict -def save_config(section, field, value, value_type=None): ########################################### - if section in default_config: - if is_valid_type(value, section, field, value_type=value_type): - if value_type in ['list', 'set', 'dict']: - pass - else: - r_serv_db.hset(f'config:global:{section}', field, value) - # used by check_integrity - r_serv_db.sadd('config:all_global_section', field, value) # check config value + type def check_integrity(): pass -config_documentation = { - "crawler": { - "enable_har_by_default": 'Enable HAR by default', - "enable_screenshot_by_default": 'Enable screenshot by default', - "default_depth_limit": 'Maximum number of url depth', - "default_closespider_pagecount": 'Maximum number of pages', - "default_user_agent": "User agent used by default", - "default_timeout": "Crawler connection timeout" - } -} - -def get_config_documentation(section, field): - return config_documentation[section][field] - -# def conf_view(): -# class F(MyBaseForm): -# pass -# -# F.username = TextField('username') -# for name in iterate_some_model_dynamically(): -# setattr(F, name, TextField(name.title())) -# -# form = F(request.POST, ...) - def get_field_full_config(section, field): dict_config = {} dict_config['value'] = get_config(section, field) diff --git a/bin/lib/Investigations.py b/bin/lib/Investigations.py new file mode 100755 index 00000000..733f938b --- /dev/null +++ b/bin/lib/Investigations.py @@ -0,0 +1,449 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +################################################################## +################################################################## + +# TODO: /!\ MISP ORG UUID + +################################################################## +################################################################## + +import os +import sys +import datetime +import redis +import time +import uuid + +from abc import ABC +from enum import Enum +from flask import escape + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) +import ConfigLoader +from exceptions import UpdateInvestigationError + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) +import Tag + +config_loader = ConfigLoader.ConfigLoader() +r_tracking = config_loader.get_redis_conn("DB_Tracking") +config_loader = None + + +#### UUID #### +def is_valid_uuid_v4(UUID): + if not UUID: + return False + UUID = UUID.replace('-', '') + try: + uuid_test = uuid.UUID(hex=UUID, version=4) + return uuid_test.hex == UUID + except: + return False + +def sanityze_uuid(UUID): + sanityzed_uuid = uuid.UUID(hex=UUID, version=4) + return str(sanityzed_uuid).replace('-', '') + +def exists_obj_type(obj_type): + return obj_type in ['domain', 'item', 'pgp', 'cryptocurrency', 'decoded', 'screenshot', 'username'] + +def generate_uuid(): + return str(uuid.uuid4()).replace('-', '') + +##-- UUID --## + +# status +# created +# last change +# tags +# comment/info +# level + +## threat_level: +# 1 = high +# 2 = medium +# 3 = low +# 4 = undefined + +## analysis: +# 0 = Initial +# 1 = Ongoing +# 2 = Complete + +# # TODO: Save correlation between investigations ? + +class ThreatLevel(Enum): + high = 1 + medium = 2 + low = 3 + undefined = 4 + +class Analysis(Enum): + initial = 0 + ongoing = 1 + completed = 2 + +class Investigation(object): + """Investigation.""" + + def __init__(self, investigation_uuid): + self.uuid = investigation_uuid + + def get_uuid(self): + return self.uuid + + # # TODO: Replace by title ?????? + def get_name(self): + return r_tracking.hget(f'investigations:data:{self.uuid}', 'name') + + def get_threat_level(self): + try: + return int(r_tracking.hget(f'investigations:data:{self.uuid}', 'threat_level')) + except: + return 1 + + def get_threat_level_str(self): + return ThreatLevel(self.get_threat_level()).name + + def get_analysis(self): + try: + return int(r_tracking.hget(f'investigations:data:{self.uuid}', 'analysis')) + except: + return 0 + + def get_analysis_str(self): + return Analysis(self.get_analysis()).name + + def get_tags(self): + return r_tracking.smembers(f'investigations:tags:{self.uuid}') + + # save all editor ?????? + def get_creator_user(self): + return r_tracking.hget(f'investigations:data:{self.uuid}', 'creator_user') + + def get_info(self): + return r_tracking.hget(f'investigations:data:{self.uuid}', 'info') + + def get_date(self): + return r_tracking.hget(f'investigations:data:{self.uuid}', 'date') + + def get_timestamp(self, r_str=False): + timestamp = r_tracking.hget(f'investigations:data:{self.uuid}', 'timestamp') + if r_str and timestamp: + timestamp = datetime.datetime.fromtimestamp(float(timestamp)).strftime('%Y-%m-%d %H:%M:%S') + return timestamp + + def get_last_change(self, r_str=False): + last_change = r_tracking.hget(f'investigations:data:{self.uuid}', 'last_change') + if r_str and last_change: + last_change = datetime.datetime.fromtimestamp(float(last_change)).strftime('%Y-%m-%d %H:%M:%S') + return last_change + + # # TODO: DATE FORMAT + def get_metadata(self, r_str=False): + if r_str: + analysis = self.get_analysis_str() + threat_level = self.get_threat_level_str() + else: + analysis = self.get_analysis() + threat_level = self.get_threat_level() + return {'uuid': self.uuid, + 'name': self.get_name(), + 'threat_level': threat_level, + 'analysis': analysis, + 'tags': self.get_tags(), + 'user_creator': self.get_creator_user(), + 'date': self.get_date(), + 'timestamp': self.get_timestamp(r_str=r_str), + 'last_change': self.get_last_change(r_str=r_str), + 'info': self.get_info(), + 'nb_objects': self.get_nb_objects()} + + def set_name(self, name): + r_tracking.hset(f'investigations:data:{self.uuid}', 'name', name) + + def set_info(self, info): + r_tracking.hset(f'investigations:data:{self.uuid}', 'info', info) + + def set_date(self, date): + r_tracking.hset(f'investigations:data:{self.uuid}', 'date', date) + + def set_last_change(self, last_change): + r_tracking.hset(f'investigations:data:{self.uuid}', 'last_change', last_change) + + def set_threat_level(self, threat_level): + try: + threat_level = int(threat_level) + except: + raise UpdateInvestigationError('threat_level Not an integer') + if threat_level >= 1 and threat_level <= 4: + r_tracking.hset(f'investigations:data:{self.uuid}', 'threat_level', threat_level) + else: + raise UpdateInvestigationError(f'Invalid threat_level: {threat_level}') + + def set_analysis(self, analysis): + try: + analysis = int(analysis) + except: + raise UpdateInvestigationError('analysis Not an integer') + if analysis >= 0 and analysis <= 2: + r_tracking.hset(f'investigations:data:{self.uuid}', 'analysis', analysis) + else: + raise UpdateInvestigationError(f'Invalid analysis: {analysis}') + + def set_tags(self, tags): + # delete previous tags + r_tracking.delete(f'investigations:tags:{self.uuid}') + for tag in tags: + r_tracking.sadd(f'investigations:tags:{self.uuid}', tag) + + def get_nb_objects(self): + return r_tracking.scard(f'investigations:objs:{self.uuid}') + + def _get_objects(self): + return r_tracking.smembers(f'investigations:objs:{self.uuid}') + + # # TODO: return Python object ???? ################################ + # TODO: PAGINATE + def get_objects(self): + # obj_dict = {} + # for str_obj in self._get_objects(): + # obj_type, subtype, id = str_obj.split(':', 2) + # if not obj_dict[obj_type]: + # obj_dict[obj_type] = [] + # obj_dict[obj_type].append({'subtype': subtype, 'id': id}) + objs = [] + for str_obj in self._get_objects(): + obj_type, subtype, obj_id = str_obj.split(':', 2) + dict_obj = {'type': obj_type, 'subtype': subtype, 'id': obj_id} + objs.append(dict_obj) + return objs + + # # TODO: def register_object(self, Object): in OBJECT CLASS + + def register_object(self, obj_id, obj_type, subtype): + r_tracking.sadd(f'investigations:objs:{self.uuid}', f'{obj_type}:{subtype}:{obj_id}') + r_tracking.sadd(f'obj:investigations:{obj_type}:{subtype}:{obj_id}', self.uuid) + timestamp = int(time.time()) + self.set_last_change(timestamp) + + + def unregister_object(self, obj_id, obj_type, subtype): + r_tracking.srem(f'investigations:objs:{self.uuid}', f'{obj_type}:{subtype}:{obj_id}') + r_tracking.srem(f'obj:investigations:{obj_type}:{subtype}:{obj_id}', self.uuid) + timestamp = int(time.time()) + self.set_last_change(timestamp) + + def delete(self): + for str_obj in self._get_objects(): + obj_type, subtype, obj_id = str_obj.split(':', 2) + self.unregister_object(obj_id, obj_type, subtype=subtype) + + r_tracking.srem('investigations:all', self.uuid) + # user map + r_tracking.srem(f'investigations:user:{self.get_creator_user()}', self.uuid) + # metadata + r_tracking.delete(f'investigations:data:{self.uuid}') + r_tracking.delete(f'investigations:tags:{self.uuid}') + +##-- Class --## + +def get_all_investigations(): + return r_tracking.smembers('investigations:all') + +def exists_investigation(investigation_uuid): + return r_tracking.sismember('investigations:all', investigation_uuid) + +# created by user +def get_user_all_investigations(user_id): + return r_tracking.smembers('investigations:user:{user_id}') + +def is_object_investigated(obj_id, obj_type, subtype=''): + return r_tracking.exists(f'obj:investigations:{obj_type}:{subtype}:{obj_id}') + +def get_obj_investigations(obj_id, obj_type, subtype=''): + return r_tracking.smembers(f'obj:investigations:{obj_type}:{subtype}:{obj_id}') + +# # TODO: fix default threat_level analysis +# # TODO: limit description + name +# # TODO: sanityze tags +# # TODO: sanityze date +def create_investigation(user_id, date, name, threat_level, analysis, info, tags=[]): + investigation_uuid = generate_uuid() + r_tracking.sadd('investigations:all', investigation_uuid) + # user map + r_tracking.sadd('investigations:user:{user_id}', investigation_uuid) + # metadata + r_tracking.hset(f'investigations:data:{investigation_uuid}', 'creator_user', user_id) + + # TODO: limit info + name + investigation = Investigation(investigation_uuid) + investigation.set_info(info) + #investigation.set_name(name) ############################################## + investigation.set_date(date) + investigation.set_threat_level(threat_level) + investigation.set_analysis(analysis) + + # # TODO: sanityze tags + if tags: + investigation.set_tags(tags) + + timestamp = int(time.time()) + r_tracking.hset(f'investigations:data:{investigation_uuid}', 'timestamp', timestamp) + investigation.set_last_change(timestamp) + + return investigation_uuid + +def get_all_investigations_meta(r_str=False): + investigations_meta = [] + for investigation_uuid in get_all_investigations(): + investigation = Investigation(investigation_uuid) + investigations_meta.append(investigation.get_metadata(r_str=r_str)) + return investigations_meta + +def get_investigations_selector(): + l_investigations = [] + for investigation_uuid in get_all_investigations(): + investigation = Investigation(investigation_uuid) + name = investigation.get_info() + l_investigations.append({"id":investigation_uuid, "name": name}) + return l_investigations + + #{id:'8dc4b81aeff94a9799bd70ba556fa345',name:"Paris"} + + +#### API #### + +# # TODO: CHECK Mandatory Fields +# # TODO: SANITYZE Fields +# # TODO: Name ????? +def api_add_investigation(json_dict): + user_id = json_dict.get('user_id') + name = json_dict.get('name') ##### mandatory ? + name = escape(name) + threat_level = json_dict.get('threat_level', 4) + analysis = json_dict.get('analysis', 0) + + # # TODO: sanityze date + date = json_dict.get('date') + + info = json_dict.get('info', '') + info = escape(info) + info = info[:1000] + tags = json_dict.get('tags', []) + if not Tag.are_enabled_tags(tags): + return {"status": "error", "reason": "Invalid/Disabled tags"}, 400 + + try: + res = create_investigation(user_id, date, name, threat_level, analysis, info, tags=tags) + except UpdateInvestigationError as e: + return e.message, 400 + return res, 200 + +# # TODO: edit threat level / status +def api_edit_investigation(json_dict): + investigation_uuid = json_dict.get('uuid', '').replace(' ', '') + if not is_valid_uuid_v4(investigation_uuid): + return {"status": "error", "reason": "Invalid Investigation uuid"}, 400 + investigation_uuid = sanityze_uuid(investigation_uuid) + if not exists_investigation(investigation_uuid): + return {"status": "error", "reason": "Investigation not found"}, 404 + investigation = Investigation(investigation_uuid) + + name = json_dict.get('name') ##### mandatory ? + name = escape(name) + threat_level = json_dict.get('threat_level', 4) + try: + investigation.set_threat_level(threat_level) + except UpdateInvestigationError: + return {"status": "error", "reason": "Invalid Investigation threat_level"}, 400 + + analysis = json_dict.get('analysis', 0) + try: + investigation.set_analysis(analysis) + except UpdateInvestigationError: + return {"status": "error", "reason": "Invalid Investigation analysis"}, 400 + + info = json_dict.get('info', '') + info = escape(info) + info = info[:1000] + tags = json_dict.get('tags', []) + if not Tag.are_enabled_tags(tags): + return {"status": "error", "reason": "Invalid/Disabled tags"}, 400 + + investigation.set_info(info) + investigation.set_name(name) + investigation.set_tags(tags) + + timestamp = int(time.time()) + investigation.set_last_change(timestamp) + + return investigation_uuid, 200 + +def api_delete_investigation(json_dict): + investigation_uuid = json_dict.get('uuid', '').replace(' ', '') + if not is_valid_uuid_v4(investigation_uuid): + return {"status": "error", "reason": "Invalid Investigation uuid"}, 400 + investigation_uuid = sanityze_uuid(investigation_uuid) + if not exists_investigation(investigation_uuid): + return {"status": "error", "reason": "Investigation not found"}, 404 + investigation = Investigation(investigation_uuid) + res = investigation.delete() + return res, 200 + +def api_register_object(json_dict): + investigation_uuid = json_dict.get('uuid', '').replace(' ', '') + if not is_valid_uuid_v4(investigation_uuid): + return {"status": "error", "reason": f"Invalid Investigation uuid: {investigation_uuid}"}, 400 + investigation_uuid = sanityze_uuid(investigation_uuid) + if not exists_investigation(investigation_uuid): + return {"status": "error", "reason": f"Investigation not found: {investigation_uuid}"}, 404 + investigation = Investigation(investigation_uuid) + + obj_type = json_dict.get('type', '').replace(' ', '') + if not exists_obj_type(obj_type): + return {"status": "error", "reason": f"Invalid Object Type: {obj_type}"}, 400 + + subtype = json_dict.get('subtype', '') + if subtype == 'None': + subtype = '' + obj_id = json_dict.get('id', '').replace(' ', '') + res = investigation.register_object(obj_id, obj_type, subtype) + return res, 200 + +def api_unregister_object(json_dict): + investigation_uuid = json_dict.get('uuid', '').replace(' ', '') + if not is_valid_uuid_v4(investigation_uuid): + return {"status": "error", "reason": f"Invalid Investigation uuid: {investigation_uuid}"}, 400 + investigation_uuid = sanityze_uuid(investigation_uuid) + if not exists_investigation(investigation_uuid): + return {"status": "error", "reason": f"Investigation not found: {investigation_uuid}"}, 404 + investigation = Investigation(investigation_uuid) + + obj_type = json_dict.get('type', '').replace(' ', '') + subtype = json_dict.get('subtype', '') + if subtype == 'None': + subtype = '' + obj_id = json_dict.get('id', '').replace(' ', '') + res = investigation.unregister_object(obj_id, obj_type, subtype) + return res, 200 + +##-- API --## + + +if __name__ == '__main__': + # res = create_star_list(user_id, name, description) + # print(res) + + # res = r_tracking.dbsize() + # print(res) + + investigation_uuid = 'e4e1c8e3b0a349bf81482f2f823efc0f' + + investigation = Investigation(investigation_uuid) + investigation.delete() + +# # TODO: PAGINATION diff --git a/bin/lib/ail_objects.py b/bin/lib/ail_objects.py deleted file mode 100755 index 97ec275d..00000000 --- a/bin/lib/ail_objects.py +++ /dev/null @@ -1,406 +0,0 @@ -#!/usr/bin/env python3 -# -*-coding:UTF-8 -* - -import os -import sys -import uuid -import redis - -from abc import ABC -from flask import url_for - -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) -import Tag - -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader - -config_loader = ConfigLoader.ConfigLoader() -r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") -config_loader = None - -class AbstractObject(ABC): - """ - Abstract Object - """ - - # first seen last/seen ?? - # # TODO: - tags - # - handle + refactor coorelations - # - creates others objects - - def __init__(self, obj_type, id, subtype=None): - """ Abstract for all the AIL object - - :param obj_type: object type (item, ...) - :param id: Object ID - """ - self.id = id - self.type = obj_type - self.subtype = None - - def get_id(self): - return self.id - - def get_type(self): - return self.type - - def get_subtype(self, r_str=False): - if not self.subtype: - if r_str: - return '' - return self.subtype - - def get_default_meta(self): - dict_meta = {'id': self.get_id(), - 'type': self.get_type()} - if self.subtype: - dict_meta['subtype'] = self.subtype - return dict_meta - - def get_tags(self, r_set=False): - tags = Tag.get_obj_tag(self.id) - if r_set: - tags = set(tags) - return tags - - ## ADD TAGS ???? - #def add_tags(self): - - def _delete(self): - # DELETE TAGS - Tag.delete_obj_all_tags(self.id, self.type) - if self.type == 'item': - # delete tracker - pass - - -def is_valid_object_type(object_type): - if object_type in ['domain', 'item', 'image', 'decoded']: - return True - else: - return False - -def get_all_objects(): - return ['domain', 'paste', 'pgp', 'cryptocurrency', 'decoded', 'screenshot'] - -def get_all_correlation_names(): - ''' - Return a list of all available correlations - ''' - return ['pgp', 'cryptocurrency', 'decoded', 'screenshot'] - -def get_all_correlation_objects(): - ''' - Return a list of all correllated objects - ''' - return ['domain', 'paste'] - -def exist_object(object_type, correlation_id, type_id=None): - if object_type == 'domain': - return Domain.verify_if_domain_exist(correlation_id) - elif object_type == 'paste' or object_type == 'item': - return Item.exist_item(correlation_id) - elif object_type == 'decoded': - return Decoded.exist_decoded(correlation_id) - elif object_type == 'pgp': - return Pgp.pgp._exist_corelation_field(type_id, correlation_id) - elif object_type == 'cryptocurrency': - return Cryptocurrency.cryptocurrency._exist_corelation_field(type_id, correlation_id) - elif object_type == 'screenshot' or object_type == 'image': - return Screenshot.exist_screenshot(correlation_id) - else: - return False - -def get_obj_date(object_type, object_id): - if object_type == "item": - return int(Item.get_item_date(object_id)) - else: - return None - -# request_type => api or ui -def get_object_metadata(object_type, correlation_id, type_id=None): - if object_type == 'domain': - return Domain.Domain(correlation_id).get_domain_metadata(tags=True) - elif object_type == 'paste' or object_type == 'item': - return Item.get_item({"id": correlation_id, "date": True, "date_separator": True, "tags": True})[0] - elif object_type == 'decoded': - return Decoded.get_decoded_metadata(correlation_id, nb_seen=True, size=True, file_type=True, tag=True) - elif object_type == 'pgp': - return Pgp.pgp.get_metadata(type_id, correlation_id) - elif object_type == 'cryptocurrency': - return Cryptocurrency.cryptocurrency.get_metadata(type_id, correlation_id) - elif object_type == 'screenshot' or object_type == 'image': - return Screenshot.get_metadata(correlation_id) - -def get_object_correlation(object_type, value, correlation_names=None, correlation_objects=None, requested_correl_type=None): - if object_type == 'domain': - return Domain.get_domain_all_correlation(value, correlation_names=correlation_names) - elif object_type == 'paste' or object_type == 'item': - return Item.get_item_all_correlation(value, correlation_names=correlation_names) - elif object_type == 'decoded': - return Decoded.get_decoded_correlated_object(value, correlation_objects=correlation_objects) - elif object_type == 'pgp': - return Pgp.pgp.get_correlation_all_object(requested_correl_type, value, correlation_objects=correlation_objects) - elif object_type == 'cryptocurrency': - return Cryptocurrency.cryptocurrency.get_correlation_all_object(requested_correl_type, value, correlation_objects=correlation_objects) - elif object_type == 'screenshot' or object_type == 'image': - return Screenshot.get_screenshot_correlated_object(value, correlation_objects=correlation_objects) - return {} - -def get_correlation_node_icon(correlation_name, correlation_type=None, value=None): - ''' - Used in UI Graph. - Return a font awesome icon for a given correlation_name. - - :param correlation_name: correlation name - :param correlation_name: str - :param correlation_type: correlation type - :type correlation_type: str, optional - - :return: a dictionnary {font awesome class, icon_code} - :rtype: dict - ''' - icon_class = 'fas' - icon_text = '' - node_color = "#332288" - node_radius = 6 - if correlation_name == "pgp": - node_color = '#44AA99' - if correlation_type == 'key': - icon_text = '\uf084' - elif correlation_type == 'name': - icon_text = '\uf507' - elif correlation_type == 'mail': - icon_text = '\uf1fa' - else: - icon_text = 'times' - - elif correlation_name == 'cryptocurrency': - node_color = '#DDCC77' - if correlation_type == 'bitcoin': - icon_class = 'fab' - icon_text = '\uf15a' - elif correlation_type == 'monero': - icon_class = 'fab' - icon_text = '\uf3d0' - elif correlation_type == 'ethereum': - icon_class = 'fab' - icon_text = '\uf42e' - else: - icon_text = '\uf51e' - - elif correlation_name == 'decoded': - node_color = '#88CCEE' - correlation_type = Decoded.get_decoded_item_type(value).split('/')[0] - if correlation_type == 'application': - icon_text = '\uf15b' - elif correlation_type == 'audio': - icon_text = '\uf1c7' - elif correlation_type == 'image': - icon_text = '\uf1c5' - elif correlation_type == 'text': - icon_text = '\uf15c' - else: - icon_text = '\uf249' - - elif correlation_name == 'screenshot' or correlation_name == 'image': - node_color = '#E1F5DF' - icon_text = '\uf03e' - - elif correlation_name == 'domain': - node_radius = 5 - node_color = '#3DA760' - if Domain.get_domain_type(value) == 'onion': - icon_text = '\uf06e' - else: - icon_class = 'fab' - icon_text = '\uf13b' - - elif correlation_name == 'paste': - node_radius = 5 - if Item.is_crawled(value): - node_color = 'red' - else: - node_color = '#332288' - - return {"icon_class": icon_class, "icon_text": icon_text, "node_color": node_color, "node_radius": node_radius} - -def get_item_url(correlation_name, value, correlation_type=None): - ''' - Warning: use only in flask - ''' - url = '#' - if correlation_name == "pgp": - endpoint = 'correlation.show_correlation' - url = url_for(endpoint, object_type="pgp", type_id=correlation_type, correlation_id=value) - elif correlation_name == 'cryptocurrency': - endpoint = 'correlation.show_correlation' - url = url_for(endpoint, object_type="cryptocurrency", type_id=correlation_type, correlation_id=value) - elif correlation_name == 'decoded': - endpoint = 'correlation.show_correlation' - url = url_for(endpoint, object_type="decoded", correlation_id=value) - elif correlation_name == 'screenshot' or correlation_name == 'image': ### # TODO: rename me - endpoint = 'correlation.show_correlation' - url = url_for(endpoint, object_type="screenshot", correlation_id=value) - elif correlation_name == 'domain': - endpoint = 'crawler_splash.showDomain' - url = url_for(endpoint, domain=value) - elif correlation_name == 'item': - endpoint = 'showsavedpastes.showsavedpaste' - url = url_for(endpoint, paste=value) - elif correlation_name == 'paste': ### # TODO: remove me - endpoint = 'showsavedpastes.showsavedpaste' - url = url_for(endpoint, paste=value) - return url - -def get_obj_tag_table_keys(object_type): - ''' - Warning: use only in flask (dynamic templates) - ''' - if object_type=="domain": - return ['id', 'first_seen', 'last_check', 'status'] # # TODO: add root screenshot - - -def create_graph_links(links_set): - graph_links_list = [] - for link in links_set: - graph_links_list.append({"source": link[0], "target": link[1]}) - return graph_links_list - -def create_graph_nodes(nodes_set, root_node_id): - graph_nodes_list = [] - for node_id in nodes_set: - correlation_name, correlation_type, value = node_id.split(';', 3) - dict_node = {"id": node_id} - dict_node['style'] = get_correlation_node_icon(correlation_name, correlation_type, value) - dict_node['text'] = value - if node_id == root_node_id: - dict_node["style"]["node_color"] = 'orange' - dict_node["style"]["node_radius"] = 7 - dict_node['url'] = get_item_url(correlation_name, value, correlation_type) - graph_nodes_list.append(dict_node) - return graph_nodes_list - -def create_node_id(correlation_name, value, correlation_type=''): - if correlation_type is None: - correlation_type = '' - return '{};{};{}'.format(correlation_name, correlation_type, value) - - - -# # TODO: filter by correlation type => bitcoin, mail, ... -def get_graph_node_object_correlation(object_type, root_value, mode, correlation_names, correlation_objects, max_nodes=300, requested_correl_type=None): - links = set() - nodes = set() - - root_node_id = create_node_id(object_type, root_value, requested_correl_type) - nodes.add(root_node_id) - - root_correlation = get_object_correlation(object_type, root_value, correlation_names, correlation_objects, requested_correl_type=requested_correl_type) - for correl in root_correlation: - if correl in ('pgp', 'cryptocurrency'): - for correl_type in root_correlation[correl]: - for correl_val in root_correlation[correl][correl_type]: - - # add correlation - correl_node_id = create_node_id(correl, correl_val, correl_type) - - if mode=="union": - if len(nodes) > max_nodes: - break - nodes.add(correl_node_id) - links.add((root_node_id, correl_node_id)) - - # get second correlation - res = get_object_correlation(correl, correl_val, correlation_names, correlation_objects, requested_correl_type=correl_type) - if res: - for corr_obj in res: - for correl_key_val in res[corr_obj]: - #filter root value - if correl_key_val == root_value: - continue - - if len(nodes) > max_nodes: - break - new_corel_1 = create_node_id(corr_obj, correl_key_val) - new_corel_2 = create_node_id(correl, correl_val, correl_type) - nodes.add(new_corel_1) - nodes.add(new_corel_2) - links.add((new_corel_1, new_corel_2)) - - if mode=="inter": - nodes.add(correl_node_id) - links.add((root_node_id, correl_node_id)) - if correl in ('decoded', 'screenshot', 'domain', 'paste'): - for correl_val in root_correlation[correl]: - - correl_node_id = create_node_id(correl, correl_val) - if mode=="union": - if len(nodes) > max_nodes: - break - nodes.add(correl_node_id) - links.add((root_node_id, correl_node_id)) - - res = get_object_correlation(correl, correl_val, correlation_names, correlation_objects) - if res: - for corr_obj in res: - if corr_obj in ('decoded', 'domain', 'paste', 'screenshot'): - for correl_key_val in res[corr_obj]: - #filter root value - if correl_key_val == root_value: - continue - - if len(nodes) > max_nodes: - break - new_corel_1 = create_node_id(corr_obj, correl_key_val) - new_corel_2 = create_node_id(correl, correl_val) - nodes.add(new_corel_1) - nodes.add(new_corel_2) - links.add((new_corel_1, new_corel_2)) - - if mode=="inter": - nodes.add(correl_node_id) - links.add((root_node_id, correl_node_id)) - - if corr_obj in ('pgp', 'cryptocurrency'): - for correl_key_type in res[corr_obj]: - for correl_key_val in res[corr_obj][correl_key_type]: - #filter root value - if correl_key_val == root_value: - continue - - if len(nodes) > max_nodes: - break - new_corel_1 = create_node_id(corr_obj, correl_key_val, correl_key_type) - new_corel_2 = create_node_id(correl, correl_val) - nodes.add(new_corel_1) - nodes.add(new_corel_2) - links.add((new_corel_1, new_corel_2)) - - if mode=="inter": - nodes.add(correl_node_id) - links.add((root_node_id, correl_node_id)) - - - return {"nodes": create_graph_nodes(nodes, root_node_id), "links": create_graph_links(links)} - - -def get_obj_global_id(obj_type, obj_id, obj_sub_type=None): - if obj_sub_type: - return '{}:{}:{}'.format(obj_type, obj_sub_type, obj_id) - else: - # # TODO: remove me - if obj_type=='paste': - obj_type='item' - # # TODO: remove me - if obj_type=='screenshot': - obj_type='image' - - return '{}:{}'.format(obj_type, obj_id) - -######## API EXPOSED ######## -def sanitize_object_type(object_type): - if not is_valid_object_type(object_type): - return ({'status': 'error', 'reason': 'Incorrect object_type'}, 400) -######## ######## diff --git a/bin/lib/ail_users.py b/bin/lib/ail_users.py new file mode 100755 index 00000000..e31fca6e --- /dev/null +++ b/bin/lib/ail_users.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys +import uuid +import redis + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) +import ConfigLoader + +config_loader = ConfigLoader.ConfigLoader() +r_serv_db = config_loader.get_redis_conn("ARDB_DB") +r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") +config_loader = None + +class User(object): + """AIL User.""" + + def __init__(self, id): + self.id = id + if self.id == '__anonymous__': + self.role = 'anonymous' + else: + self.role = None + + def get_role(self): + pass + + diff --git a/bin/lib/exceptions.py b/bin/lib/exceptions.py new file mode 100755 index 00000000..3b8ab98d --- /dev/null +++ b/bin/lib/exceptions.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +class AIL_ERROR(Exception): + """docstring for AIL_ERROR.""" + + def __init__(self, message): + super(AIL_ERROR, self).__init__(message) + self.message = message + +class UpdateInvestigationError(AIL_ERROR): + pass + +class NewTagError(AIL_ERROR): + pass diff --git a/bin/lib/objects/CryptoCurrencies.py b/bin/lib/objects/CryptoCurrencies.py new file mode 100755 index 00000000..bcffdf11 --- /dev/null +++ b/bin/lib/objects/CryptoCurrencies.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys +import redis + +# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) +import ConfigLoader + +from abstract_object import AbstractObject + +config_loader = ConfigLoader.ConfigLoader() + +config_loader = None + + +################################################################################ +################################################################################ +################################################################################ + +class CryptoCurrency(AbstractObject): + """ + AIL CryptoCurrency Object. (strings) + """ + + def __init__(self, id, subtype): + super(CryptoCurrency, self).__init__('cryptocurrency', id, subtype=subtype) + + # def get_ail_2_ail_payload(self): + # payload = {'raw': self.get_gzip_content(b64=True), + # 'compress': 'gzip'} + # return payload + + # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\ + def delete(self): + # # TODO: + pass + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('correlation.show_correlation', object_type=self.type, type_id=self.subtype, correlation_id=self.id) + else: + url = f'{baseurl}/correlation/show_correlation?object_type={self.type}&type_id={self.subtype}&correlation_id={self.id}' + return url + + def get_svg_icon(self): + if self.subtype == 'bitcoin': + style = 'fab' + icon = '\uf15a' + elif self.subtype == 'monero': + style = 'fab' + icon = '\uf3d0' + elif self.subtype == 'ethereum': + style = 'fab' + icon = '\uf42e' + else: + style = 'fas' + icon = '\uf51e' + return {'style': style, 'icon': icon, 'color': '#DDCC77', 'radius':5} + + ############################################################################ + ############################################################################ + ############################################################################ + + def exist_correlation(self): + pass + + ############################################################################ + ############################################################################ + + + +#if __name__ == '__main__': diff --git a/bin/lib/objects/Decodeds.py b/bin/lib/objects/Decodeds.py new file mode 100755 index 00000000..07ebfcd9 --- /dev/null +++ b/bin/lib/objects/Decodeds.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys +import redis + +# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) +import ConfigLoader + +from abstract_object import AbstractObject + +config_loader = ConfigLoader.ConfigLoader() +r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") +HASH_DIR = config_loader.get_config_str('Directories', 'hash') +baseurl = config_loader.get_config_str("Notifications", "ail_domain") +config_loader = None + + +################################################################################ +################################################################################ +################################################################################ + +# # TODO: COMPLETE CLASS + +class Decoded(AbstractObject): + """ + AIL Decoded Object. (strings) + """ + + def __init__(self, id): + super(Decoded, self).__init__('decoded', id) + + # def get_ail_2_ail_payload(self): + # payload = {'raw': self.get_gzip_content(b64=True), + # 'compress': 'gzip'} + # return payload + + def get_sha1(self): + return self.id.split('/')[0] + + def get_file_type(self): + return r_serv_metadata.hget(f'metadata_hash:{self.get_sha1()}', 'estimated_type') + + # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\ + def delete(self): + # # TODO: + pass + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('correlation.show_correlation', object_type="decoded", correlation_id=value) + else: + url = f'{baseurl}/correlation/show_correlation?object_type={self.type}&correlation_id={self.id}' + return url + + def get_svg_icon(self): + file_type = self.get_file_type() + if file_type == 'application': + icon = '\uf15b' + elif file_type == 'audio': + icon = '\uf1c7' + elif file_type == 'image': + icon = '\uf1c5' + elif file_type == 'text': + icon = '\uf15c' + else: + icon = '\uf249' + return {'style': 'fas', 'icon': icon, 'color': '#88CCEE', 'radius':5} + + ############################################################################ + ############################################################################ + ############################################################################ + + def exist_correlation(self): + pass + + ############################################################################ + ############################################################################ + + + +#if __name__ == '__main__': diff --git a/bin/lib/objects/Domains.py b/bin/lib/objects/Domains.py new file mode 100755 index 00000000..bab307e0 --- /dev/null +++ b/bin/lib/objects/Domains.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys +import redis + +# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) +import ConfigLoader + +from abstract_object import AbstractObject + +config_loader = ConfigLoader.ConfigLoader() + +config_loader = None + + +################################################################################ +################################################################################ +################################################################################ + +class Domain(AbstractObject): + """ + AIL Decoded Object. (strings) + """ + + def __init__(self, id): + super(Domain, self).__init__('domain', id) + + # def get_ail_2_ail_payload(self): + # payload = {'raw': self.get_gzip_content(b64=True), + # 'compress': 'gzip'} + # return payload + + def get_domain_type(self): + if str(self.id).endswith('.onion'): + return 'onion' + else: + return 'regular' + + # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\ + def delete(self): + # # TODO: + pass + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('crawler_splash.showDomain', domain=self.id) + else: + url = f'{baseurl}/crawlers/showDomain?domain={self.id}' + return url + + def get_svg_icon(self): + color = '#3DA760' + if self.get_domain_type() == 'onion': + style = 'fas' + icon = '\uf06e' + else: + style = 'fab' + icon = '\uf13b' + return {'style': style, 'icon': icon, 'color':color, 'radius':5} + + ############################################################################ + ############################################################################ + ############################################################################ + + def exist_correlation(self): + pass + + ############################################################################ + ############################################################################ + + + +#if __name__ == '__main__': diff --git a/bin/lib/objects/Items.py b/bin/lib/objects/Items.py new file mode 100755 index 00000000..02085c9b --- /dev/null +++ b/bin/lib/objects/Items.py @@ -0,0 +1,690 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import base64 +import os +import re +import sys +import redis +import cld3 +import html2text + +from io import BytesIO + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) +import Tag +import Cryptocurrency +import Pgp + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) +import item_basic +import domain_basic +import ConfigLoader +import Correlate_object +import Decoded +import Screenshot +import Username + +from abstract_object import AbstractObject +from item_basic import * + +config_loader = ConfigLoader.ConfigLoader() +# get and sanityze PASTE DIRECTORY +# # TODO: rename PASTES_FOLDER +PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/' +PASTES_FOLDER = os.path.join(os.path.realpath(PASTES_FOLDER), '') + +r_cache = config_loader.get_redis_conn("Redis_Cache") +r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") +screenshot_directory = config_loader.get_files_directory('screenshot') +har_directory = config_loader.get_files_directory('har') +baseurl = config_loader.get_config_str("Notifications", "ail_domain") +config_loader = None + + +################################################################################ +################################################################################ +################################################################################ + +class Item(AbstractObject): + """ + AIL Item Object. (strings) + """ + + def __init__(self, id): + super(Item, self).__init__('item', id) + + def get_date(self, separator=False): + """ + Returns Item date + """ + return item_basic.get_item_date(self.id, add_separator=separator) + + def get_source(self): + """ + Returns Item source/feeder name + """ + #return self.id.split('/')[-5] + l_source = self.id.split('/')[:-4] + return os.path.join(*l_source) + + def get_basename(self): + return os.path.basename(self.id) + + def get_filename(self): + # Creating the full filepath + filename = os.path.join(PASTES_FOLDER, self.id) + filename = os.path.realpath(filename) + + # incorrect filename + if not os.path.commonprefix([filename, PASTES_FOLDER]) == PASTES_FOLDER: + return None + else: + return filename + + def get_content(self): + """ + Returns Item content + """ + return item_basic.get_item_content(self.id) + + def get_gzip_content(self, b64=False): + with open(self.get_filename(), 'rb') as f: + content = f.read() + if b64: + content = base64.b64encode(content) + return content.decode() + + def get_ail_2_ail_payload(self): + payload = {'raw': self.get_gzip_content(b64=True), + 'compress': 'gzip'} + return payload + + # # TODO: + def create(self): + pass + + # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\ + # TODO: DELETE ITEM CORRELATION + TAGS + METADATA + ... + def delete(self): + try: + os.remove(self.get_filename()) + return True + except FileNotFoundError: + return False + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('objects_item.showItem', id=value) + else: + url = f'{baseurl}/object/item?id={self.id}' + return url + + def get_svg_icon(self): + if is_crawled(self.id): + color = 'red' + else: + color = '#332288' + return {'style': '', 'icon': '', 'color': color, 'radius':5} + + ############################################################################ + ############################################################################ + ############################################################################ + + def exist_correlation(self): + pass + + ############################################################################ + ############################################################################ + + +################################################################################ +################################################################################ +################################################################################ + +def exist_item(item_id): + return item_basic.exist_item(item_id) + +def get_basename(item_id): + return os.path.basename(item_id) + +def get_item_id(full_path): + return full_path.replace(PASTES_FOLDER, '', 1) + +def get_item_filepath(item_id): + return item_basic.get_item_filepath(item_id) + +def get_item_date(item_id, add_separator=False): + return item_basic.get_item_date(item_id, add_separator=add_separator) + +def get_source(item_id): + return item_basic.get_source(item_id) + +def get_all_sources(): + return item_basic.get_all_items_sources(r_list=True) + +def get_item_basename(item_id): + return os.path.basename(item_id) + +def get_item_size(item_id): + return round(os.path.getsize(os.path.join(PASTES_FOLDER, item_id))/1024.0, 2) + +def get_item_encoding(item_id): + return None + +def get_lines_info(item_id, item_content=None): + if not item_content: + item_content = get_item_content(item_id) + max_length = 0 + line_id = 0 + nb_line = 0 + for line in item_content.splitlines(): + length = len(line) + if length > max_length: + max_length = length + nb_line += 1 + return {'nb': nb_line, 'max_length': max_length} + + +def get_item_metadata(item_id, item_content=None): + ## TODO: FIXME ##performance + # encoding + # language + # lines info + item_metadata = {'date': get_item_date(item_id, add_separator=True), + 'source': get_source(item_id), + 'size': get_item_size(item_id), + 'encoding': get_item_encoding(item_id), + 'lines': get_lines_info(item_id, item_content=item_content) + } + return item_metadata + +def get_item_parent(item_id): + return item_basic.get_item_parent(item_id) + +def add_item_parent(item_parent, item_id): + return item_basic.add_item_parent(item_parent, item_id) + +def get_item_content(item_id): + return item_basic.get_item_content(item_id) + +def get_item_content_html2text(item_id, item_content=None, ignore_links=False): + if not item_content: + item_content = get_item_content(item_id) + h = html2text.HTML2Text() + h.ignore_links = ignore_links + h.ignore_images = ignore_links + return h.handle(item_content) + +def remove_all_urls_from_content(item_id, item_content=None): + if not item_content: + item_content = get_item_content(item_id) + regex = r'\b(?:http://|https://)?(?:[a-zA-Z\d-]{,63}(?:\.[a-zA-Z\d-]{,63})+)(?:\:[0-9]+)*(?:/(?:$|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*\b' + url_regex = re.compile(regex) + urls = url_regex.findall(item_content) + urls = sorted(urls, key=len, reverse=True) + for url in urls: + item_content = item_content.replace(url, '') + + regex_pgp_public_blocs = r'-----BEGIN PGP PUBLIC KEY BLOCK-----[\s\S]+?-----END PGP PUBLIC KEY BLOCK-----' + regex_pgp_signature = r'-----BEGIN PGP SIGNATURE-----[\s\S]+?-----END PGP SIGNATURE-----' + regex_pgp_message = r'-----BEGIN PGP MESSAGE-----[\s\S]+?-----END PGP MESSAGE-----' + re.compile(regex_pgp_public_blocs) + re.compile(regex_pgp_signature) + re.compile(regex_pgp_message) + + res = re.findall(regex_pgp_public_blocs, item_content) + for it in res: + item_content = item_content.replace(it, '') + res = re.findall(regex_pgp_signature, item_content) + for it in res: + item_content = item_content.replace(it, '') + res = re.findall(regex_pgp_message, item_content) + for it in res: + item_content = item_content.replace(it, '') + + return item_content + +def get_item_languages(item_id, min_len=600, num_langs=3, min_proportion=0.2, min_probability=0.7): + all_languages = [] + + ## CLEAN CONTENT ## + content = get_item_content_html2text(item_id, ignore_links=True) + content = remove_all_urls_from_content(item_id, item_content=content) + + # REMOVE USELESS SPACE + content = ' '.join(content.split()) + #- CLEAN CONTENT -# + + #print(content) + #print(len(content)) + if len(content) >= min_len: + for lang in cld3.get_frequent_languages(content, num_langs=num_langs): + if lang.proportion >= min_proportion and lang.probability >= min_probability and lang.is_reliable: + all_languages.append(lang) + return all_languages + +# API +def get_item(request_dict): + if not request_dict: + return {'status': 'error', 'reason': 'Malformed JSON'}, 400 + + item_id = request_dict.get('id', None) + if not item_id: + return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400 + if not exist_item(item_id): + return {'status': 'error', 'reason': 'Item not found'}, 404 + + dict_item = {} + dict_item['id'] = item_id + date = request_dict.get('date', True) + if date: + add_separator = False + if request_dict.get('date_separator', False): + add_separator = True + dict_item['date'] = get_item_date(item_id, add_separator=add_separator) + tags = request_dict.get('tags', True) + if tags: + dict_item['tags'] = Tag.get_obj_tag(item_id) + + size = request_dict.get('size', False) + if size: + dict_item['size'] = get_item_size(item_id) + + content = request_dict.get('content', False) + if content: + # UTF-8 outpout, # TODO: use base64 + dict_item['content'] = get_item_content(item_id) + + raw_content = request_dict.get('raw_content', False) + if raw_content: + dict_item['raw_content'] = get_raw_content(item_id) + + lines_info = request_dict.get('lines', False) + if lines_info: + dict_item['lines'] = get_lines_info(item_id, dict_item.get('content', 'None')) + + if request_dict.get('pgp'): + dict_item['pgp'] = {} + if request_dict['pgp'].get('key'): + dict_item['pgp']['key'] = get_item_pgp_key(item_id) + if request_dict['pgp'].get('mail'): + dict_item['pgp']['mail'] = get_item_pgp_mail(item_id) + if request_dict['pgp'].get('name'): + dict_item['pgp']['name'] = get_item_pgp_name(item_id) + + if request_dict.get('cryptocurrency'): + dict_item['cryptocurrency'] = {} + if request_dict['cryptocurrency'].get('bitcoin'): + dict_item['cryptocurrency']['bitcoin'] = get_item_bitcoin(item_id) + + return dict_item, 200 + + + +def api_get_item_content_base64_utf8(request_dict): + item_id = request_dict.get('id', None) + if not request_dict: + return {'status': 'error', 'reason': 'Malformed JSON'}, 400 + if not item_id: + return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400 + if not exist_item(item_id): + return {'status': 'error', 'reason': 'Item not found'}, 404 + + item_content = get_item_content(item_id) + item_content = base64.b64encode((item_content.encode('utf-8'))).decode('UTF-8') + return {'status': 'success', 'content': item_content}, 200 + + +def api_get_items_sources(): + item_content = {'sources': get_all_sources()} + return item_content, 200 + +# def check_item_source(request_dict): +# source = request_dict.get('source', None) +# if not request_dict: +# return {'status': 'error', 'reason': 'Malformed JSON'}, 400 +# if not source: +# return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400 +# +# all_sources = item_basic.get_all_items_sources() +# +# if source not in all_sources: +# return {'status': 'error', 'reason': 'Invalid source', 'provide': source}, 400 +# return {'status': 'success', 'reason': 'Valid source', 'provide': source}, 200 + +### +### correlation +### +def get_item_cryptocurrency(item_id, currencies_type=None, get_nb=False): + ''' + Return all cryptocurrencies of a given item. + + :param item_id: item id + :param currencies_type: list of cryptocurrencies type + :type currencies_type: list, optional + ''' + return Cryptocurrency.cryptocurrency.get_item_correlation_dict(item_id, correlation_type=currencies_type, get_nb=get_nb) + +def get_item_pgp(item_id, currencies_type=None, get_nb=False): + ''' + Return all pgp of a given item. + + :param item_id: item id + :param currencies_type: list of cryptocurrencies type + :type currencies_type: list, optional + ''' + return Pgp.pgp.get_item_correlation_dict(item_id, correlation_type=currencies_type, get_nb=get_nb) + +def get_item_username(item_id, sub_type=None, get_nb=False): + ''' + Return all pgp of a given item. + + :param item_id: item id + :param sub_type: list of username type + :type sub_type: list, optional + ''' + return Username.correlation.get_item_correlation_dict(item_id, correlation_type=sub_type, get_nb=get_nb) + +def get_item_decoded(item_id): + ''' + Return all pgp of a given item. + + :param item_id: item id + :param currencies_type: list of cryptocurrencies type + :type currencies_type: list, optional + ''' + return Decoded.get_item_decoded(item_id) + +def get_item_all_screenshot(item_id): + ''' + Return all screenshot of a given item. + + :param item_id: item id + ''' + return Screenshot.get_item_screenshot_list(item_id) + +def get_item_all_correlation(item_id, correlation_names=[], get_nb=False): + ''' + Retun all correlation of a given item id. + + :param item_id: item id + :type domain: str + + :return: a dict of all correlation for a item id + :rtype: dict + ''' + if not correlation_names: + correlation_names = Correlate_object.get_all_correlation_names() + item_correl = {} + for correlation_name in correlation_names: + if correlation_name=='cryptocurrency': + res = get_item_cryptocurrency(item_id, get_nb=get_nb) + elif correlation_name=='pgp': + res = get_item_pgp(item_id, get_nb=get_nb) + elif correlation_name=='username': + res = get_item_username(item_id, get_nb=get_nb) + elif correlation_name=='decoded': + res = get_item_decoded(item_id) + elif correlation_name=='screenshot': + res = get_item_all_screenshot(item_id) + else: + res = None + # add correllation to dict + if res: + item_correl[correlation_name] = res + return item_correl + + + +## TODO: REFRACTOR +def _get_item_correlation(correlation_name, correlation_type, item_id): + res = r_serv_metadata.smembers('item_{}_{}:{}'.format(correlation_name, correlation_type, item_id)) + if res: + return list(res) + else: + return [] + +## TODO: REFRACTOR +def get_item_bitcoin(item_id): + return _get_item_correlation('cryptocurrency', 'bitcoin', item_id) + +## TODO: REFRACTOR +def get_item_pgp_key(item_id): + return _get_item_correlation('pgpdump', 'key', item_id) + +## TODO: REFRACTOR +def get_item_pgp_name(item_id): + return _get_item_correlation('pgpdump', 'name', item_id) + +## TODO: REFRACTOR +def get_item_pgp_mail(item_id): + return _get_item_correlation('pgpdump', 'mail', item_id) + +## TODO: REFRACTOR +def get_item_pgp_correlation(item_id): + pass + +### +### GET Internal Module DESC +### +def get_item_list_desc(list_item_id): + desc_list = [] + for item_id in list_item_id: + desc_list.append( {'id': item_id, 'date': get_item_date(item_id), 'tags': Tag.get_obj_tag(item_id)} ) + return desc_list + +def is_crawled(item_id): + return item_basic.is_crawled(item_id) + +def get_crawler_matadata(item_id, ltags=None): + dict_crawler = {} + if is_crawled(item_id): + dict_crawler['domain'] = get_item_domain(item_id) + if not ltags: + ltags = Tag.get_obj_tag(item_id) + dict_crawler['is_tags_safe'] = Tag.is_tags_safe(ltags) + dict_crawler['url'] = get_item_link(item_id) + dict_crawler['screenshot'] = get_item_screenshot(item_id) + dict_crawler['har'] = get_item_har_name(item_id) + return dict_crawler + +def is_onion(item_id): + is_onion = False + if len(is_onion) > 62: + if is_crawled(item_id) and item_id[-42:-36] == '.onion': + is_onion = True + return is_onion + +def is_item_in_domain(domain, item_id): + is_in_domain = False + domain_lenght = len(domain) + if len(item_id) > (domain_lenght+48): + if item_id[-36-domain_lenght:-36] == domain: + is_in_domain = True + return is_in_domain + +def get_item_domain(item_id): + return item_basic.get_item_domain(item_id) + +def get_domain(item_id): + item_id = item_id.split('/') + item_id = item_id[-1] + return item_id[:-36] + +def get_item_domain_with_port(item_id): + return r_serv_metadata.hget('paste_metadata:{}'.format(item_id), 'domain') + +def get_item_link(item_id): + return r_serv_metadata.hget('paste_metadata:{}'.format(item_id), 'real_link') + +def get_item_screenshot(item_id): + screenshot = r_serv_metadata.hget('paste_metadata:{}'.format(item_id), 'screenshot') + if screenshot: + return os.path.join(screenshot[0:2], screenshot[2:4], screenshot[4:6], screenshot[6:8], screenshot[8:10], screenshot[10:12], screenshot[12:]) + return '' + +def get_item_har_name(item_id): + har_path = os.path.join(har_directory, item_id) + '.json' + if os.path.isfile(har_path): + return har_path + else: + return None + +def get_item_har(har_path): + pass + +def get_item_filename(item_id): + # Creating the full filepath + filename = os.path.join(PASTES_FOLDER, item_id) + filename = os.path.realpath(filename) + + # incorrect filename + if not os.path.commonprefix([filename, PASTES_FOLDER]) == PASTES_FOLDER: + return None + else: + return filename + +def get_item_duplicate(item_id, r_list=True): + res = r_serv_metadata.smembers('dup:{}'.format(item_id)) + if r_list: + if res: + return list(res) + else: + return [] + return res + +def get_item_nb_duplicates(item_id): + return r_serv_metadata.scard('dup:{}'.format(item_id)) + +def get_item_duplicates_dict(item_id): + dict_duplicates = {} + for duplicate in get_item_duplicate(item_id): + duplicate = duplicate[1:-1].replace('\'', '').replace(' ', '').split(',') + duplicate_id = duplicate[1] + if not duplicate_id in dict_duplicates: + dict_duplicates[duplicate_id] = {'date': get_item_date(duplicate_id, add_separator=True), 'algo': {}} + algo = duplicate[0] + if algo == 'tlsh': + similarity = 100 - int(duplicate[2]) + else: + similarity = int(duplicate[2]) + dict_duplicates[duplicate_id]['algo'][algo] = similarity + return dict_duplicates + +def add_item_duplicate(item_id, l_dup): + for item_dup in l_dup: + r_serv_metadata.sadd('dup:{}'.format(item_dup), item_id) + r_serv_metadata.sadd('dup:{}'.format(item_id), item_dup) + +def delete_item_duplicate(item_id): + item_dup = get_item_duplicate(item_id) + for item_dup in get_item_duplicate(item_id): + r_serv_metadata.srem('dup:{}'.format(item_dup), item_id) + r_serv_metadata.delete('dup:{}'.format(item_id)) + +def get_raw_content(item_id): + filepath = get_item_filepath(item_id) + with open(filepath, 'rb') as f: + file_content = BytesIO(f.read()) + return file_content + +def save_raw_content(item_id, io_content): + filepath = get_item_filename(item_id) + if os.path.isfile(filepath): + #print('File already exist') + return False + # create subdir + dirname = os.path.dirname(filepath) + if not os.path.exists(dirname): + os.makedirs(dirname) + # # TODO: check if is IO file + with open(filepath, 'wb') as f: + f.write(io_content.getvalue()) + return True + +# IDEA: send item to duplicate ? +def create_item(obj_id, obj_metadata, io_content): + ''' + Create a new Item (Import or Test only). + + :param obj_id: item id + :type obj_metadata: dict - 'first_seen', 'tags' + + :return: is item created + :rtype: boolean + ''' + # check if datetime match ?? + + + # # TODO: validate obj_id + + res = save_raw_content(obj_id, io_content) + # item saved + if res: + # creata tags + if 'tags' in obj_metadata: + # # TODO: handle mixed tags: taxonomies and Galaxies + Tag.api_add_obj_tags(tags=obj_metadata['tags'], object_id=obj_id, object_type="item") + return True + + # Item not created + return False + +def delete_item(obj_id): + # check if item exists + if not exist_item(obj_id): + return False + else: + delete_item_duplicate(obj_id) + # delete MISP event + r_serv_metadata.delete('misp_events:{}'.format(obj_id)) + r_serv_metadata.delete('hive_cases:{}'.format(obj_id)) + + os.remove(get_item_filename(obj_id)) + + # get all correlation + obj_correlations = get_item_all_correlation(obj_id) + for correlation in obj_correlations: + if correlation=='cryptocurrency' or correlation=='pgp': + for obj2_subtype in obj_correlations[correlation]: + for obj2_id in obj_correlations[correlation][obj2_subtype]: + Correlate_object.delete_obj_relationship(correlation, obj2_id, 'item', obj_id, + obj1_subtype=obj2_subtype) + else: + for obj2_id in obj_correlations[correlation]: + Correlate_object.delete_obj_relationship(correlation, obj2_id, 'item', obj_id) + + # delete father/child + delete_node(obj_id) + + # delete item metadata + r_serv_metadata.delete('paste_metadata:{}'.format(obj_id)) + + return True + + ### TODO in inport V2 + # delete from tracked items + + # # # TODO: # FIXME: LATER + # delete from queue + ### + return False + +#### #### +def delete_node(item_id): + if is_node(item_id): + if is_crawled(item_id): + delete_domain_node(item_id) + item_basic._delete_node(item_id) + +def delete_domain_node(item_id): + if is_domain_root(item_id): + # remove from domain history + domain, port = get_item_domain_with_port(item_id).split(':') + domain_basic.delete_domain_item_core(item_id, domain, port) + for child_id in get_all_domain_node_by_item_id(item_id): + delete_item(child_id) + + +#if __name__ == '__main__': diff --git a/bin/lib/objects/Pgps.py b/bin/lib/objects/Pgps.py new file mode 100755 index 00000000..68ab5b6b --- /dev/null +++ b/bin/lib/objects/Pgps.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys +import redis + +# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) +import ConfigLoader + +from abstract_object import AbstractObject + +config_loader = ConfigLoader.ConfigLoader() + +config_loader = None + + +################################################################################ +################################################################################ +################################################################################ + +class Pgp(AbstractObject): + """ + AIL Pgp Object. (strings) + """ + + def __init__(self, id, subtype): + super(Pgp, self).__init__('pgp', id, subtype=subtype) + + # def get_ail_2_ail_payload(self): + # payload = {'raw': self.get_gzip_content(b64=True), + # 'compress': 'gzip'} + # return payload + + # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\ + def delete(self): + # # TODO: + pass + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('correlation.show_correlation', object_type=self.type, type_id=self.subtype, correlation_id=self.id) + else: + url = f'{baseurl}/correlation/show_correlation?object_type={self.type}&type_id={self.subtype}&correlation_id={self.id}' + return url + + def get_svg_icon(self): + if self.subtype == 'key': + icon = '\uf084' + elif self.subtype == 'name': + icon = '\uf507' + elif self.subtype == 'mail': + icon = '\uf1fa' + else: + icon = 'times' + return {'style': 'fas', 'icon': icon, 'color': '#44AA99', 'radius':5} + + ############################################################################ + ############################################################################ + ############################################################################ + + def exist_correlation(self): + pass + + ############################################################################ + ############################################################################ + + + +#if __name__ == '__main__': diff --git a/bin/lib/objects/Screenshots.py b/bin/lib/objects/Screenshots.py new file mode 100755 index 00000000..41aff1be --- /dev/null +++ b/bin/lib/objects/Screenshots.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys +import redis + +# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) +import ConfigLoader + +from abstract_object import AbstractObject + +config_loader = ConfigLoader.ConfigLoader() +r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") +HASH_DIR = config_loader.get_config_str('Directories', 'hash') +config_loader = None + + +################################################################################ +################################################################################ +################################################################################ + +class Screenshot(AbstractObject): + """ + AIL Screenshot Object. (strings) + """ + + def __init__(self, id): + super(Screenshot, self).__init__('screenshot', id) + + # def get_ail_2_ail_payload(self): + # payload = {'raw': self.get_gzip_content(b64=True), + # 'compress': 'gzip'} + # return payload + + # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\ + def delete(self): + # # TODO: + pass + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('correlation.show_correlation', object_type=self.type, correlation_id=self.id) + else: + url = f'{baseurl}/correlation/show_correlation?object_type={self.type}&correlation_id={self.id}' + return url + + def get_svg_icon(self): + return {'style': 'fas', 'icon': '\uf03e', 'color': '#E1F5DF', 'radius':5} + + ############################################################################ + ############################################################################ + ############################################################################ + + def exist_correlation(self): + pass + + ############################################################################ + ############################################################################ + + + +#if __name__ == '__main__': diff --git a/bin/lib/objects/Usernames.py b/bin/lib/objects/Usernames.py new file mode 100755 index 00000000..17edf49f --- /dev/null +++ b/bin/lib/objects/Usernames.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys +import redis + +# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) +import ConfigLoader + +from abstract_object import AbstractObject + +config_loader = ConfigLoader.ConfigLoader() + +config_loader = None + + +################################################################################ +################################################################################ +################################################################################ + +class Username(AbstractObject): + """ + AIL Username Object. (strings) + """ + + def __init__(self, id, subtype): + super(Username, self).__init__('username', id, subtype=subtype) + + # def get_ail_2_ail_payload(self): + # payload = {'raw': self.get_gzip_content(b64=True), + # 'compress': 'gzip'} + # return payload + + # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\ + def delete(self): + # # TODO: + pass + + def get_link(self, flask_context=False): + if flask_context: + url = url_for('correlation.show_correlation', object_type=self.type, type_id=self.subtype, correlation_id=self.id) + else: + url = f'{baseurl}/correlation/show_correlation?object_type={self.type}&type_id={self.subtype}&correlation_id={self.id}' + return url + + def get_svg_icon(self): + if self.subtype == 'telegram': + style = 'fab' + icon = '\uf2c6' + elif self.subtype == 'twitter': + style = 'fab' + icon = '\uf099' + else: + style = 'fas' + icon = '\uf007' + return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius':5} + + ############################################################################ + ############################################################################ + ############################################################################ + + def exist_correlation(self): + pass + + ############################################################################ + ############################################################################ + + + +#if __name__ == '__main__': diff --git a/bin/lib/objects/abstract_object.py b/bin/lib/objects/abstract_object.py new file mode 100755 index 00000000..5b9cfce2 --- /dev/null +++ b/bin/lib/objects/abstract_object.py @@ -0,0 +1,135 @@ +# -*-coding:UTF-8 -* +""" +Base Class for AIL Objects +""" + +################################## +# Import External packages +################################## +import os +import sys +from abc import ABC, abstractmethod + +#from flask import url_for + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from packages import Tag +from lib.Investigations import is_object_investigated, get_obj_investigations + +# # TODO: ADD CORRELATION ENGINE + +class AbstractObject(ABC): + """ + Abstract Object + """ + + # first seen last/seen ?? + # # TODO: - tags + # - handle + refactor coorelations + # - creates others objects + + def __init__(self, obj_type, id, subtype=None): + """ Abstract for all the AIL object + + :param obj_type: object type (item, ...) + :param id: Object ID + """ + self.id = id + self.type = obj_type + self.subtype = subtype + + def get_id(self): + return self.id + + def get_type(self): + return self.type + + def get_subtype(self, r_str=False): + if not self.subtype: + if r_str: + return '' + return self.subtype + + def get_default_meta(self, tags=False): + dict_meta = {'id': self.get_id(), + 'type': self.get_type(), + 'subtype': self.get_subtype()} + if tags: + dict_meta['tags'] = self.get_tags() + return dict_meta + + ## Tags ## + def get_tags(self, r_set=False): + tags = Tag.get_obj_tag(self.id) + if r_set: + tags = set(tags) + return tags + + ## ADD TAGS ???? + #def add_tags(self): + + #- Tags -# + + ## Investigations ## + # # TODO: unregister ===== + + def is_investigated(self): + if not self.subtype: + is_investigated = is_object_investigated(self.id, self.type) + else: + is_investigated = is_object_investigated(self.id, self.type, self.subtype) + return is_investigated + + def get_investigations(self): + if not self.subtype: + investigations = get_obj_investigations(self.id, self.type) + else: + investigations = get_obj_investigations(self.id, self.type, self.subtype) + return investigations + #- Investigations -# + + def _delete(self): + # DELETE TAGS + Tag.delete_obj_all_tags(self.id, self.type) + # # TODO: remove from investigations + + @abstractmethod + def delete(self): + """ + Delete Object: used for the Data Retention + """ + pass + + # @abstractmethod + # def get_meta(self): + # """ + # get Object metadata + # """ + # pass + + @abstractmethod + def get_svg_icon(self): + """ + Get object svg icon + """ + pass + + @abstractmethod + def get_link(self, flask_context=False): + pass + + # # TODO: + # @abstractmethod + # def get_correlations(self, message): + # """ + # Get object correlations + # """ + # pass + + + # # TODO: get favicon + # # TODO: get url + # # TODO: get metadata diff --git a/bin/lib/objects/ail_objects.py b/bin/lib/objects/ail_objects.py new file mode 100755 index 00000000..d5346efb --- /dev/null +++ b/bin/lib/objects/ail_objects.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys +import uuid +import redis + +from abc import ABC +from flask import url_for + + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) +import ConfigLoader + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/objects')) +from Decodeds import Decoded +from Domains import Domain +from CryptoCurrencies import CryptoCurrency +from Items import Item +from Pgps import Pgp +from Screenshots import Screenshot +from Usernames import Username + +################################################################## +################################################################## +#sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) + +#sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) +################################################################## +################################################################## + +config_loader = ConfigLoader.ConfigLoader() +r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") +config_loader = None + +class AILObjects(object): + initial = 0 + ongoing = 1 + completed = 2 + + +# # TODO: check change paste => item +def get_all_objects(): + return ['domain', 'item', 'pgp', 'cryptocurrency', 'decoded', 'screenshot', 'username'] + +def get_object(obj_type, subtype, id): + if obj_type == 'item': + return Item(id) + elif obj_type == 'domain': + return Domain(id) + elif obj_type == 'decoded': + return Decoded(id) + elif obj_type == 'screenshot': + return Screenshot(id) + elif obj_type == 'cryptocurrency': + return CryptoCurrency(id, subtype) + elif obj_type == 'pgp': + return Pgp(id, subtype) + elif obj_type == 'username': + return Username(id, subtype) + +def get_object_svg(obj_type, subtype, id): + object = get_object(obj_type, subtype, id) + return object.get_svg_icon() + +def get_objects_meta(l_dict_objs, icon=False, url=False, flask_context=False): + l_meta = [] + for dict_obj in l_dict_objs: + object = get_object(dict_obj['type'], dict_obj['subtype'], dict_obj['id']) + dict_meta = object.get_default_meta(tags=True) + if icon: + dict_meta['icon'] = object.get_svg_icon() + if url: + dict_meta['link'] = object.get_link(flask_context=flask_context) + l_meta.append(dict_meta) + return l_meta + + +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ +################################################################################ diff --git a/bin/packages/Item.py b/bin/packages/Item.py index cce96a2e..f48f52d8 100755 --- a/bin/packages/Item.py +++ b/bin/packages/Item.py @@ -26,7 +26,7 @@ import Decoded import Screenshot import Username -from ail_objects import AbstractObject +from objects.abstract_object import AbstractObject from item_basic import * config_loader = ConfigLoader.ConfigLoader() diff --git a/configs/core.cfg.sample b/configs/core.cfg.sample index 4bc871a0..710b3b9b 100644 --- a/configs/core.cfg.sample +++ b/configs/core.cfg.sample @@ -73,9 +73,6 @@ criticalNumberToAlert=8 #Will be considered as false positive if less that X matches from the top password list minTopPassList=5 -[Curve] -max_execution_time = 90 - [Onion] save_i2p = False max_execution_time = 180 @@ -149,31 +146,27 @@ port = 6381 db = 1 ##### ARDB ##### -[ARDB_Curve] + +[ARDB_DB] host = localhost port = 6382 -db = 1 +db = 0 + +[DB_Tracking] +host = localhost +port = 6382 +db = 3 [ARDB_Sentiment] host = localhost port = 6382 db = 4 -[ARDB_TermFreq] -host = localhost -port = 6382 -db = 2 - [ARDB_TermCred] host = localhost port = 6382 db = 5 -[ARDB_DB] -host = localhost -port = 6382 -db = 0 - [ARDB_Trending] host = localhost port = 6382 diff --git a/update/v4.1/Update.py b/update/v4.1/Update.py new file mode 100755 index 00000000..630ee414 --- /dev/null +++ b/update/v4.1/Update.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import re +import sys +import time +import redis +import datetime + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) +import ConfigLoader + +sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin')) +from ail_updater import AIL_Updater + +class Updater(AIL_Updater): + """default Updater.""" + + def __init__(self, version): + super(Updater, self).__init__(version) + + def update(self): + config_loader = ConfigLoader.ConfigLoader() + r_tracking = config_loader.get_redis_conn("DB_Tracking") + config_loader = None + + # FLUSH OLD DB + r_tracking.flushdb() + +if __name__ == '__main__': + + updater = Updater('v4.1') + + + updater.run_update() diff --git a/update/v4.1/Update.sh b/update/v4.1/Update.sh new file mode 100755 index 00000000..2be5376a --- /dev/null +++ b/update/v4.1/Update.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; + +export PATH=$AIL_HOME:$PATH +export PATH=$AIL_REDIS:$PATH +export PATH=$AIL_ARDB:$PATH +export PATH=$AIL_BIN:$PATH +export PATH=$AIL_FLASK:$PATH + +GREEN="\\033[1;32m" +DEFAULT="\\033[0;39m" + +echo -e $GREEN"Shutting down AIL ..."$DEFAULT +bash ${AIL_BIN}/LAUNCH.sh -ks +wait + +# SUBMODULES # +git submodule update + +exit 0 diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index c8eea936..3ab8dddf 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -42,6 +42,7 @@ from blueprints.crawler_splash import crawler_splash from blueprints.correlation import correlation from blueprints.tags_ui import tags_ui from blueprints.import_export import import_export +from blueprints.investigations_b import investigations_b from blueprints.objects_item import objects_item from blueprints.hunters import hunters from blueprints.old_endpoints import old_endpoints @@ -101,6 +102,7 @@ app.register_blueprint(crawler_splash, url_prefix=baseUrl) app.register_blueprint(correlation, url_prefix=baseUrl) app.register_blueprint(tags_ui, url_prefix=baseUrl) app.register_blueprint(import_export, url_prefix=baseUrl) +app.register_blueprint(investigations_b, url_prefix=baseUrl) app.register_blueprint(objects_item, url_prefix=baseUrl) app.register_blueprint(hunters, url_prefix=baseUrl) app.register_blueprint(old_endpoints, url_prefix=baseUrl) diff --git a/var/www/blueprints/crawler_splash.py b/var/www/blueprints/crawler_splash.py index 8fe3ed13..7c79023e 100644 --- a/var/www/blueprints/crawler_splash.py +++ b/var/www/blueprints/crawler_splash.py @@ -28,7 +28,7 @@ import crawlers import Domain import Language -import Config_DB +#import Config_DB r_cache = Flask_config.r_cache r_serv_db = Flask_config.r_serv_db @@ -295,6 +295,15 @@ def domains_search_name(): l_dict_domains=l_dict_domains, bootstrap_label=bootstrap_label, domains_types=domains_types) +@crawler_splash.route('/domains/TODO', methods=['GET']) +@login_required +@login_analyst +def domains_todo(): + domain_type = request.args.get('type') + last_domains = Domain.get_last_crawled_domains(domain_type) + + + ##-- --## @@ -349,8 +358,8 @@ def crawler_cookiejar_add_post(): return redirect(url_for('crawler_splash.crawler_cookiejar_show', cookiejar_uuid=cookiejar_uuid)) @crawler_splash.route('/crawler/cookiejar/all', methods=['GET']) -#@login_required -#@login_read_only +@login_required +@login_read_only def crawler_cookiejar_all(): user_id = current_user.get_id() user_cookiejar = crawlers.get_cookiejar_metadata_by_iterator(crawlers.get_user_cookiejar(user_id)) @@ -358,8 +367,8 @@ def crawler_cookiejar_all(): return render_template("all_cookiejar.html", user_cookiejar=user_cookiejar, global_cookiejar=global_cookiejar) @crawler_splash.route('/crawler/cookiejar/show', methods=['GET']) -#@login_required -#@login_read_only +@login_required +@login_read_only def crawler_cookiejar_show(): user_id = current_user.get_id() cookiejar_uuid = request.args.get('cookiejar_uuid') @@ -379,8 +388,8 @@ def crawler_cookiejar_show(): l_cookies=l_cookies, l_cookie_uuid=l_cookie_uuid) @crawler_splash.route('/crawler/cookiejar/cookie/delete', methods=['GET']) -#@login_required -#@login_read_only +@login_required +@login_read_only def crawler_cookiejar_cookie_delete(): user_id = current_user.get_id() cookiejar_uuid = request.args.get('cookiejar_uuid') @@ -392,8 +401,8 @@ def crawler_cookiejar_cookie_delete(): return redirect(url_for('crawler_splash.crawler_cookiejar_show', cookiejar_uuid=cookiejar_uuid)) @crawler_splash.route('/crawler/cookiejar/delete', methods=['GET']) -#@login_required -#@login_read_only +@login_required +@login_read_only def crawler_cookiejar_delete(): user_id = current_user.get_id() cookiejar_uuid = request.args.get('cookiejar_uuid') diff --git a/var/www/blueprints/import_export.py b/var/www/blueprints/import_export.py index bd8879e1..ca424bb5 100644 --- a/var/www/blueprints/import_export.py +++ b/var/www/blueprints/import_export.py @@ -182,6 +182,20 @@ def add_object_id_to_export(): # redirect return redirect(url_for('import_export.export_object')) +@import_export.route("/import_export/investigation", methods=['GET']) +@login_required +@login_analyst +def export_investigation(): + investigation_uuid = request.args.get("uuid") + + if MispExport.ping_misp(): + event_metadata = MispExport.create_investigation_event(investigation_uuid) + else: + return Response(json.dumps({"error": "Can't reach MISP Instance"}, indent=2, sort_keys=True), mimetype='application/json'), 400 + + return redirect(url_for('investigations_b.show_investigation', uuid=investigation_uuid)) + + # @import_export.route("/import_export/delete_object_id_to_export", methods=['GET']) # @login_required # @login_analyst diff --git a/var/www/blueprints/investigations_b.py b/var/www/blueprints/investigations_b.py new file mode 100644 index 00000000..eaea2c53 --- /dev/null +++ b/var/www/blueprints/investigations_b.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +''' + Blueprint Flask: ail_investigations +''' + +import os +import sys +import json + +from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file +from flask_login import login_required, current_user + +# Import Role_Manager +from Role_Manager import login_admin, login_analyst, login_read_only + +sys.path.append('modules') +import Flask_config + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) +import Investigations +from lib.objects import ail_objects + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages')) +import Tag + +# ============ BLUEPRINT ============ +investigations_b = Blueprint('investigations_b', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/investigations')) + +# ============ VARIABLES ============ +bootstrap_label = Flask_config.bootstrap_label + +# ============ FUNCTIONS ============ +def create_json_response(data, status_code): + return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code + +# ============= ROUTES ============== + +@investigations_b.route("/investigations", methods=['GET']) +@login_required +@login_read_only +def investigations_dashboard(): + investigations = Investigations.get_all_investigations_meta(r_str=True) + return render_template("investigations.html", bootstrap_label=bootstrap_label, + investigations=investigations) + + +@investigations_b.route("/investigation", methods=['GET']) ## FIXME: add /view ???? +@login_required +@login_read_only +def show_investigation(): + investigation_uuid = request.args.get("uuid") + investigation = Investigations.Investigation(investigation_uuid) + metadata = investigation.get_metadata(r_str=True) + objs = ail_objects.get_objects_meta(investigation.get_objects(), icon=True) + return render_template("view_investigation.html", bootstrap_label=bootstrap_label, + metadata=metadata, investigation_objs=objs) + + +@investigations_b.route("/investigation/add", methods=['GET', 'POST']) +@login_required +@login_read_only +def add_investigation(): + if request.method == 'POST': + user_id = current_user.get_id() + name = request.form.get("investigation_name") + date = request.form.get("investigation_date") + threat_level = request.form.get("threat_level") + analysis = request.form.get("analysis") + info = request.form.get("investigation_info") + # tags + taxonomies_tags = request.form.get('taxonomies_tags') + if taxonomies_tags: + try: + taxonomies_tags = json.loads(taxonomies_tags) + except Exception: + taxonomies_tags = [] + else: + taxonomies_tags = [] + galaxies_tags = request.form.get('galaxies_tags') + if galaxies_tags: + try: + galaxies_tags = json.loads(galaxies_tags) + except Exception: + galaxies_tags = [] + tags = taxonomies_tags + galaxies_tags + + input_dict = {"user_id": user_id, "name": name, + "threat_level": threat_level, "date": date, + "analysis": analysis, "info": info, "tags": tags} + res = Investigations.api_add_investigation(input_dict) + if res[1] != 200: + return create_json_response(res[0], res[1]) + + return redirect(url_for('investigations_b.show_investigation', uuid=res[0])) + else: + return render_template("add_investigation.html", tags_selector_data=Tag.get_tags_selector_data()) + + +@investigations_b.route("/investigation/edit", methods=['GET', 'POST']) +@login_required +@login_read_only +def edit_investigation(): + if request.method == 'POST': + user_id = current_user.get_id() + investigation_uuid = request.form.get("investigation_uuid") + name = request.form.get("investigation_name") + date = request.form.get("investigation_date") + threat_level = request.form.get("threat_level") + analysis = request.form.get("analysis") + info = request.form.get("investigation_info") + + # tags + taxonomies_tags = request.form.get('taxonomies_tags') + if taxonomies_tags: + try: + taxonomies_tags = json.loads(taxonomies_tags) + except Exception: + taxonomies_tags = [] + else: + taxonomies_tags = [] + galaxies_tags = request.form.get('galaxies_tags') + if galaxies_tags: + try: + galaxies_tags = json.loads(galaxies_tags) + except Exception: + galaxies_tags = [] + tags = taxonomies_tags + galaxies_tags + + input_dict = {"user_id": user_id, "uuid": investigation_uuid, + "name": name, "threat_level": threat_level, + "analysis": analysis, "info": info, "tags": tags} + res = Investigations.api_edit_investigation(input_dict) + if res[1] != 200: + return create_json_response(res[0], res[1]) + + return redirect(url_for('investigations_b.show_investigation', uuid=res[0])) + else: + investigation_uuid = request.args.get('uuid') + investigation = Investigations.Investigation(investigation_uuid) + metadata = investigation.get_metadata(r_str=False) + taxonomies_tags, galaxies_tags = Tag.sort_tags_taxonomies_galaxies(metadata['tags']) + tags_selector_data = Tag.get_tags_selector_data() + tags_selector_data['taxonomies_tags'] = taxonomies_tags + tags_selector_data['galaxies_tags'] = galaxies_tags + return render_template("add_investigation.html", edit=True, + tags_selector_data=tags_selector_data, metadata=metadata) + +@investigations_b.route("/investigation/delete", methods=['GET']) +@login_required +@login_read_only +def delete_investigation(): + investigation_uuid = request.args.get('uuid') + input_dict = {"uuid": investigation_uuid} + res = Investigations.api_delete_investigation(input_dict) + if res[1] != 200: + return create_json_response(res[0], res[1]) + return redirect(url_for('investigations_b.investigations_dashboard')) + +@investigations_b.route("/investigation/object/register", methods=['GET']) +@login_required +@login_read_only +def register_investigation(): + investigations_uuid = request.args.get('uuids') + investigations_uuid = investigations_uuid.split(',') + + object_type = request.args.get('type') + object_subtype = request.args.get('subtype') + object_id = request.args.get('id') + + for investigation_uuid in investigations_uuid: + input_dict = {"uuid": investigation_uuid, "id": object_id, + "type": object_type, "subtype": object_subtype} + res = Investigations.api_register_object(input_dict) + if res[1] != 200: + return create_json_response(res[0], res[1]) + return redirect(url_for('investigations_b.investigations_dashboard', uuid=investigation_uuid)) + +@investigations_b.route("/investigation/object/unregister", methods=['GET']) +@login_required +@login_read_only +def unregister_investigation(): + investigation_uuid = request.args.get('uuid') + object_type = request.args.get('type') + object_subtype = request.args.get('subtype') + object_id = request.args.get('id') + input_dict = {"uuid": investigation_uuid, "id": object_id, + "type": object_type, "subtype": object_subtype} + res = Investigations.api_unregister_object(input_dict) + if res[1] != 200: + return create_json_response(res[0], res[1]) + return redirect(url_for('investigations_b.show_investigation', uuid=investigation_uuid)) + + +@investigations_b.route("/investigation/all/selector_json") +@login_required +@login_read_only +def get_investigations_selector_json(): + return jsonify(Investigations.get_investigations_selector()) + + +# +# @investigations_b.route("/object/item") #completely shows the paste in a new tab +# @login_required +# @login_analyst +# def showItem(): # # TODO: support post +# item_id = request.args.get('id') +# if not item_id or not Item.exist_item(item_id): +# abort(404) +# +# return render_template("show_item.html", bootstrap_label=bootstrap_label) diff --git a/var/www/modules/settings/Flask_settings.py b/var/www/modules/settings/Flask_settings.py index 68ccd517..ae97235d 100644 --- a/var/www/modules/settings/Flask_settings.py +++ b/var/www/modules/settings/Flask_settings.py @@ -288,5 +288,11 @@ def passive_dns_change_state(): passivedns_enabled = d4.change_passive_dns_state(new_state) return redirect(url_for('settings.passive_dns')) +@settings.route("/settings/ail", methods=['GET']) +@login_required +@login_admin +def ail_configs(): + return render_template("ail_configs.html", passivedns_enabled=None) + # ========= REGISTRATION ========= app.register_blueprint(settings, url_prefix=baseUrl) diff --git a/var/www/modules/settings/templates/ail_configs.html b/var/www/modules/settings/templates/ail_configs.html new file mode 100644 index 00000000..833d1011 --- /dev/null +++ b/var/www/modules/settings/templates/ail_configs.html @@ -0,0 +1,93 @@ + + + + + Passive DNS - AIL + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
+
+ + {% include 'settings/menu_sidebar.html' %} + +
+ +
+
+ +
+
+
+ + MISP project + +
+
+
+
+ + +
+
+ + +
+
+ + +
+ +
+
+
+
+ +
+
+ +
+
+
+ + + + + + diff --git a/var/www/static/css/ail-project.css b/var/www/static/css/ail-project.css index ba4d461b..653d4ea5 100644 --- a/var/www/static/css/ail-project.css +++ b/var/www/static/css/ail-project.css @@ -5,3 +5,16 @@ padding-left: 0.15em; background-color: #2e5; } + +.blue { + color: #0088cc !important; +} + +.bold { + font-weight: bold; +} + +.object_node_icon { + font-size: 16px; + pointer-events: none; +} diff --git a/var/www/templates/correlation/metadata_card_cryptocurrency.html b/var/www/templates/correlation/metadata_card_cryptocurrency.html index 6818a55e..967efa75 100644 --- a/var/www/templates/correlation/metadata_card_cryptocurrency.html +++ b/var/www/templates/correlation/metadata_card_cryptocurrency.html @@ -78,6 +78,14 @@ Expand Bitcoin address {% endif %} {% endif %} + + {% with obj_type='cryptocurrency', obj_id=dict_object['correlation_id'], obj_subtype=dict_object["metadata"]["type_id"] %} + {% include 'modals/investigations_register_obj.html' %} + {% endwith %} + + diff --git a/var/www/templates/correlation/metadata_card_decoded.html b/var/www/templates/correlation/metadata_card_decoded.html index f02d422c..a2554caf 100644 --- a/var/www/templates/correlation/metadata_card_decoded.html +++ b/var/www/templates/correlation/metadata_card_decoded.html @@ -88,6 +88,14 @@ + + {% with obj_type='decoded', obj_id=dict_object['correlation_id'], obj_subtype='' %} + {% include 'modals/investigations_register_obj.html' %} + {% endwith %} + + diff --git a/var/www/templates/correlation/metadata_card_domain.html b/var/www/templates/correlation/metadata_card_domain.html index b8d666b3..2566fb14 100644 --- a/var/www/templates/correlation/metadata_card_domain.html +++ b/var/www/templates/correlation/metadata_card_domain.html @@ -64,5 +64,13 @@ + + {% with obj_type='domain', obj_id=dict_object['correlation_id'], obj_subtype='' %} + {% include 'modals/investigations_register_obj.html' %} + {% endwith %} + + diff --git a/var/www/templates/correlation/metadata_card_paste.html b/var/www/templates/correlation/metadata_card_paste.html index 08744321..f929ca26 100644 --- a/var/www/templates/correlation/metadata_card_paste.html +++ b/var/www/templates/correlation/metadata_card_paste.html @@ -46,5 +46,13 @@ + + {% with obj_type='item', obj_id=dict_object['correlation_id'], obj_subtype='' %} + {% include 'modals/investigations_register_obj.html' %} + {% endwith %} + + diff --git a/var/www/templates/correlation/metadata_card_pgp.html b/var/www/templates/correlation/metadata_card_pgp.html index 25815058..819c4d0f 100644 --- a/var/www/templates/correlation/metadata_card_pgp.html +++ b/var/www/templates/correlation/metadata_card_pgp.html @@ -40,6 +40,14 @@ + + {% with obj_type='pgp', obj_id=dict_object['correlation_id'], obj_subtype=dict_object["metadata"]["type_id"] %} + {% include 'modals/investigations_register_obj.html' %} + {% endwith %} + + diff --git a/var/www/templates/correlation/metadata_card_screenshot.html b/var/www/templates/correlation/metadata_card_screenshot.html index daf988f5..804620e7 100644 --- a/var/www/templates/correlation/metadata_card_screenshot.html +++ b/var/www/templates/correlation/metadata_card_screenshot.html @@ -78,6 +78,14 @@ + + {% with obj_type='screenshot', obj_id=dict_object['correlation_id'], obj_subtype='' %} + {% include 'modals/investigations_register_obj.html' %} + {% endwith %} + + diff --git a/var/www/templates/correlation/metadata_card_username.html b/var/www/templates/correlation/metadata_card_username.html index 25815058..46053fa5 100644 --- a/var/www/templates/correlation/metadata_card_username.html +++ b/var/www/templates/correlation/metadata_card_username.html @@ -40,6 +40,14 @@ + + {% with obj_type='username', obj_id=dict_object['correlation_id'], obj_subtype=dict_object["metadata"]["type_id"] %} + {% include 'modals/investigations_register_obj.html' %} + {% endwith %} + + diff --git a/var/www/templates/investigations/add_investigation.html b/var/www/templates/investigations/add_investigation.html new file mode 100644 index 00000000..3184a0e4 --- /dev/null +++ b/var/www/templates/investigations/add_investigation.html @@ -0,0 +1,214 @@ + + + + + AIL-Framework + + + + + + + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
+
+ + {% include 'sidebars/sidebar_objects.html' %} + +
+ +
+
+
+ {% if edit %} + Edit Investigation + {% else %} + Create Investigation + {% endif %} +
+
+
+ +
+ + {% if edit %} + + {% endif %} + +
+
+
+
+
+
+ +
+ +
+
+
+ + +
+
+
+
+ + +
+
+
+ +
+
+ +
+ +
+
+ Tags +
+
+ {% include 'tags/block_tags_selector.html' %} +
+
+ +
+ +
+ {% if edit %} + Edit Investigation + {% else %} + Create a new Investigation + {% endif %} +
+
+ + +
+ + +
+ +
+
+ + +
+ +
+
+ + + + diff --git a/var/www/templates/investigations/investigations.html b/var/www/templates/investigations/investigations.html new file mode 100644 index 00000000..45a6d8de --- /dev/null +++ b/var/www/templates/investigations/investigations.html @@ -0,0 +1,90 @@ + + + + + Investigations + + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
+
+ + {% include 'sidebars/sidebar_objects.html' %} + +
+ +

+ Investigations: +

+ + + + + + + + + + + + + {% for dict_investigation in investigations %} + + + + + + + + {% endfor %} + +
NameDatelast modifiedInfoNb Objects
+ + {{ dict_investigation['info']}} +
+ {% for tag in dict_investigation['tags'] %} + {{ tag }} + {% endfor %} +
+ + 		{{ dict_investigation['date']}}{{ dict_investigation['last_change']}}{{ dict_investigation['info']}}{{ dict_investigation['nb_objects']}}
+ +
+ +
+
+ + + + diff --git a/var/www/templates/investigations/view_investigation.html b/var/www/templates/investigations/view_investigation.html new file mode 100644 index 00000000..be6e3567 --- /dev/null +++ b/var/www/templates/investigations/view_investigation.html @@ -0,0 +1,178 @@ + + + + + AIL-Framework + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
+
+ + {% include 'sidebars/sidebar_objects.html' %} + +
+ +
+
+

{{metadata['info']}}

+
+
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
UUID{{metadata['uuid']}}
Creator{{metadata['user_creator']}}
Tags + {% for tag in metadata['tags'] %} + {{ tag }} + {% endfor %} +
Date{{metadata['date']}}
Threat Level{{metadata['threat_level']}}
Analysis{{metadata['analysis']}}
Info{{metadata['info']}}
# Objects{{metadata['nb_objects']}}
Timestamp{{metadata['timestamp']}}
Last change{{metadata['last_change']}}
+ +
+
+ +
+ + + + + + + + + Export as Event + +
+ +
+
+
+ +
+
+ +

Objects

+ + + + + + + + + + + + + {% for object in investigation_objs %} + + + + + + + + {% endfor %} + +
TypeIdTags
+ {% with style=object['icon']['style'], icon=object['icon']['icon'] , color=object['icon']['color'] %} + {% include 'objects/obj_svg_block.html' %} + {% endwith %} + {{ object['type']}} + + {% if object['subtype'] %} + {{ object['subtype']}} + {% endif %} + {{ object['id']}} + {% for tag in object['tags'] %} + {{ tag }} + {% endfor %} + + + + +
+ + + +
+ +
+ + + + diff --git a/var/www/templates/modals/investigations_register_obj.html b/var/www/templates/modals/investigations_register_obj.html new file mode 100644 index 00000000..edac2853 --- /dev/null +++ b/var/www/templates/modals/investigations_register_obj.html @@ -0,0 +1,62 @@ + + + + + + diff --git a/var/www/templates/objects/item/show_item.html b/var/www/templates/objects/item/show_item.html index 084b3477..6442e0ae 100644 --- a/var/www/templates/objects/item/show_item.html +++ b/var/www/templates/objects/item/show_item.html @@ -97,6 +97,16 @@
+
+ {% with obj_type='item', obj_id=dict_item['id'], obj_subtype=''%} + {% include 'modals/investigations_register_obj.html' %} + {% endwith %} +
+ +
+
{% with obj_type='item', obj_id=dict_item['id'], obj_lvl=0%} {% include 'import_export/block_add_user_object_to_export.html' %} diff --git a/var/www/templates/objects/obj_svg_block.html b/var/www/templates/objects/obj_svg_block.html new file mode 100644 index 00000000..8bcacaa6 --- /dev/null +++ b/var/www/templates/objects/obj_svg_block.html @@ -0,0 +1,6 @@ + + + + {{ icon }} + + diff --git a/var/www/templates/settings/menu_sidebar.html b/var/www/templates/settings/menu_sidebar.html index 18f24242..209de341 100644 --- a/var/www/templates/settings/menu_sidebar.html +++ b/var/www/templates/settings/menu_sidebar.html @@ -44,7 +44,13 @@ Settings