chg: [v4.1] add Investigation with MISP Export + v4.1 update

This commit is contained in:
Terrtia 2022-03-07 15:12:01 +01:00
parent 303c8001e5
commit f40d8f47c2
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
41 changed files with 3140 additions and 488 deletions

View file

@ -22,6 +22,10 @@ import Correlate_object
import AILObjects import AILObjects
import Export import Export
from Investigations import Investigation
import Tag
# # TODO: # FIXME: REFRACTOR ME => use UI/Global config # # TODO: # FIXME: REFRACTOR ME => use UI/Global config
sys.path.append('../../configs/keys') sys.path.append('../../configs/keys')
try: try:
@ -390,6 +394,7 @@ def create_misp_event(event, distribution=0, threat_level_id=4, publish=False, a
# # TODO: handle multiple MISP instance # # TODO: handle multiple MISP instance
misp = PyMISP(misp_url, misp_key, misp_verifycert) misp = PyMISP(misp_url, misp_key, misp_verifycert)
#print(event.to_json()) #print(event.to_json())
misp_event = misp.add_event(event) misp_event = misp.add_event(event)
#print(misp_event) #print(misp_event)
# # TODO: handle error # # TODO: handle error
@ -414,9 +419,50 @@ def extract_event_metadata(event):
# LVL 1 => DETAILED Also add correlated_items correlation # LVL 1 => DETAILED Also add correlated_items correlation
###### ######
if __name__ == '__main__': # # TODO: # create object relationships
def create_investigation_event(investigation_uuid):
investigation = Investigation(investigation_uuid)
l_obj = [{'id': 'bfd5f1d89e55b10a8b122a9d7ce31667ec1d086a', 'type': 'decoded', 'lvl': 2}] event = MISPEvent()
create_list_of_objs_to_export(l_obj) event.info = investigation.get_info()
event.uuid = investigation.get_uuid()
event.date = investigation.get_date()
event.analysis = investigation.get_analysis()
event.threat_level_id = investigation.get_threat_level()
taxonomies_tags, galaxies_tags = Tag.sort_tags_taxonomies_galaxies(investigation.get_tags())
event.Tag = taxonomies_tags
event.Galaxy = galaxies_tags
#event.add_galaxy(galaxies_tags)
investigation_objs = investigation.get_objects()
for obj in investigation_objs:
# if subtype -> obj_id = 'subtype:type'
if obj['subtype']:
obj_id = f"{obj['subtype']}:{obj['id']}"
else:
obj_id = obj['id']
misp_obj = create_misp_obj(obj['type'], obj_id)
if misp_obj:
event.add_object(misp_obj)
# if publish:
# event.publish()
# res = event.to_json()
# print(event.to_json())
misp = PyMISP(misp_url, misp_key, misp_verifycert)
misp_event = misp.add_event(event)
# print(misp_event)
# # TODO: handle error
event_metadata = extract_event_metadata(misp_event)
return event_metadata
# if __name__ == '__main__':
# l_obj = [{'id': 'bfd5f1d89e55b10a8b122a9d7ce31667ec1d086a', 'type': 'decoded', 'lvl': 2}]
# create_list_of_objs_to_export(l_obj)
#print(event.to_json()) #print(event.to_json())

View file

@ -24,44 +24,116 @@ config_loader = None
## data retention ## data retention
######################### #########################
default_config = {
ail_config = {
"crawler": { "crawler": {
"enable_har_by_default": False, "enable_har_by_default": {
"enable_screenshot_by_default": True, "default": False,
"default_depth_limit": 1, "type": bool,
"default_closespider_pagecount": 50, "info": "Enable HAR by default"
"default_user_agent": "Mozilla/5.0 (Windows NT 10.0; rv:68.0) Gecko/20100101 Firefox/68.0", },
"default_timeout": 30 "enable_screenshot_by_default": {
"default": True,
"type": bool,
"info": "Enable screenshot by default"
},
"depth_limit": {
"default": 1,
"type": int,
"info": "Maximum number of url depth"
},
"closespider_pagecount": {
"default": 50,
"type": int,
"info": "Maximum number of pages"
},
"user_agent": {
"default": 50,
"type": str,
"info": "User agent used by default"
},
"timeout": {
"default": 30,
"type": int,
"info": "Crawler connection timeout"
},
},
"misp": {
"url": {
"default": "https://localhost:8443/",
"type": str,
"info": "Crawler connection timeout"
},
"key": {
"default": "",
"type": str,
"info": "Crawler connection timeout"
},
"verifycert": {
"default": True,
"type": bool,
"info": "Crawler connection timeout"
},
} }
} }
# The MISP auth key can be found on the MISP web interface under the automation section
def get_config_value(section, field, value):
return r_serv_db.hget(f'ail:config:global:{section}', field, value)
def get_config_default_value(section, field, value):
return ail_config[section][field]['default']
def get_config_type(section, field, value):
return ail_config[section][field]['type']
def get_config_info(section, field, value):
return ail_config[section][field]['info']
def save_config(section, field, value):
if section in ail_config:
if is_valid_type(value, section, field, value_type=value_type):
# if value_type in ['list', 'set', 'dict']:
# pass
# else:
r_serv_db.hset(f'ail:config:global:{section}', field, value)
config_documentation = {
}
default_config = {
}
def get_default_config(): def get_default_config():
return default_config return default_config
def get_default_config_value(section, field): def get_default_config_value(section, field):
return default_config[section][field] return default_config[section][field]
config_type = {
# crawler config
"crawler": {
"enable_har_by_default": bool,
"enable_screenshot_by_default": bool,
"default_depth_limit": int,
"default_closespider_pagecount": int,
"default_user_agent": str,
"default_timeout": int
}
}
def get_config_type(section, field):
return config_type[section][field]
#### DEFAULT CONFIG ####
#### CONFIG TYPE ####
# CONFIG DOC
config_type = {
}
# # TODO: add set, dict, list and select_(multiple_)value # # TODO: add set, dict, list and select_(multiple_)value
def is_valid_type(obj, section, field, value_type=None): def is_valid_type(obj, section, field, value_type=None):
res = isinstance(obj, get_config_type(section, field)) res = isinstance(obj, get_config_type(section, field))
return res return res
# # TODO: ###########################################################
def reset_default_config(): def reset_default_config():
for section in config_type:
pass pass
def set_default_config(section, field): def set_default_config(section, field):
@ -92,45 +164,12 @@ def get_config_dict_by_section(section):
config_dict[field] = get_config(section, field) config_dict[field] = get_config(section, field)
return config_dict return config_dict
def save_config(section, field, value, value_type=None): ###########################################
if section in default_config:
if is_valid_type(value, section, field, value_type=value_type):
if value_type in ['list', 'set', 'dict']:
pass
else:
r_serv_db.hset(f'config:global:{section}', field, value)
# used by check_integrity
r_serv_db.sadd('config:all_global_section', field, value)
# check config value + type # check config value + type
def check_integrity(): def check_integrity():
pass pass
config_documentation = {
"crawler": {
"enable_har_by_default": 'Enable HAR by default',
"enable_screenshot_by_default": 'Enable screenshot by default',
"default_depth_limit": 'Maximum number of url depth',
"default_closespider_pagecount": 'Maximum number of pages',
"default_user_agent": "User agent used by default",
"default_timeout": "Crawler connection timeout"
}
}
def get_config_documentation(section, field):
return config_documentation[section][field]
# def conf_view():
# class F(MyBaseForm):
# pass
#
# F.username = TextField('username')
# for name in iterate_some_model_dynamically():
# setattr(F, name, TextField(name.title()))
#
# form = F(request.POST, ...)
def get_field_full_config(section, field): def get_field_full_config(section, field):
dict_config = {} dict_config = {}
dict_config['value'] = get_config(section, field) dict_config['value'] = get_config(section, field)

449
bin/lib/Investigations.py Executable file
View file

@ -0,0 +1,449 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
##################################################################
##################################################################
# TODO: /!\ MISP ORG UUID
##################################################################
##################################################################
import os
import sys
import datetime
import redis
import time
import uuid
from abc import ABC
from enum import Enum
from flask import escape
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
from exceptions import UpdateInvestigationError
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
import Tag
config_loader = ConfigLoader.ConfigLoader()
r_tracking = config_loader.get_redis_conn("DB_Tracking")
config_loader = None
#### UUID ####
def is_valid_uuid_v4(UUID):
if not UUID:
return False
UUID = UUID.replace('-', '')
try:
uuid_test = uuid.UUID(hex=UUID, version=4)
return uuid_test.hex == UUID
except:
return False
def sanityze_uuid(UUID):
sanityzed_uuid = uuid.UUID(hex=UUID, version=4)
return str(sanityzed_uuid).replace('-', '')
def exists_obj_type(obj_type):
return obj_type in ['domain', 'item', 'pgp', 'cryptocurrency', 'decoded', 'screenshot', 'username']
def generate_uuid():
return str(uuid.uuid4()).replace('-', '')
##-- UUID --##
# status
# created
# last change
# tags
# comment/info
# level
## threat_level:
# 1 = high
# 2 = medium
# 3 = low
# 4 = undefined
## analysis:
# 0 = Initial
# 1 = Ongoing
# 2 = Complete
# # TODO: Save correlation between investigations ?
class ThreatLevel(Enum):
high = 1
medium = 2
low = 3
undefined = 4
class Analysis(Enum):
initial = 0
ongoing = 1
completed = 2
class Investigation(object):
"""Investigation."""
def __init__(self, investigation_uuid):
self.uuid = investigation_uuid
def get_uuid(self):
return self.uuid
# # TODO: Replace by title ??????
def get_name(self):
return r_tracking.hget(f'investigations:data:{self.uuid}', 'name')
def get_threat_level(self):
try:
return int(r_tracking.hget(f'investigations:data:{self.uuid}', 'threat_level'))
except:
return 1
def get_threat_level_str(self):
return ThreatLevel(self.get_threat_level()).name
def get_analysis(self):
try:
return int(r_tracking.hget(f'investigations:data:{self.uuid}', 'analysis'))
except:
return 0
def get_analysis_str(self):
return Analysis(self.get_analysis()).name
def get_tags(self):
return r_tracking.smembers(f'investigations:tags:{self.uuid}')
# save all editor ??????
def get_creator_user(self):
return r_tracking.hget(f'investigations:data:{self.uuid}', 'creator_user')
def get_info(self):
return r_tracking.hget(f'investigations:data:{self.uuid}', 'info')
def get_date(self):
return r_tracking.hget(f'investigations:data:{self.uuid}', 'date')
def get_timestamp(self, r_str=False):
timestamp = r_tracking.hget(f'investigations:data:{self.uuid}', 'timestamp')
if r_str and timestamp:
timestamp = datetime.datetime.fromtimestamp(float(timestamp)).strftime('%Y-%m-%d %H:%M:%S')
return timestamp
def get_last_change(self, r_str=False):
last_change = r_tracking.hget(f'investigations:data:{self.uuid}', 'last_change')
if r_str and last_change:
last_change = datetime.datetime.fromtimestamp(float(last_change)).strftime('%Y-%m-%d %H:%M:%S')
return last_change
# # TODO: DATE FORMAT
def get_metadata(self, r_str=False):
if r_str:
analysis = self.get_analysis_str()
threat_level = self.get_threat_level_str()
else:
analysis = self.get_analysis()
threat_level = self.get_threat_level()
return {'uuid': self.uuid,
'name': self.get_name(),
'threat_level': threat_level,
'analysis': analysis,
'tags': self.get_tags(),
'user_creator': self.get_creator_user(),
'date': self.get_date(),
'timestamp': self.get_timestamp(r_str=r_str),
'last_change': self.get_last_change(r_str=r_str),
'info': self.get_info(),
'nb_objects': self.get_nb_objects()}
def set_name(self, name):
r_tracking.hset(f'investigations:data:{self.uuid}', 'name', name)
def set_info(self, info):
r_tracking.hset(f'investigations:data:{self.uuid}', 'info', info)
def set_date(self, date):
r_tracking.hset(f'investigations:data:{self.uuid}', 'date', date)
def set_last_change(self, last_change):
r_tracking.hset(f'investigations:data:{self.uuid}', 'last_change', last_change)
def set_threat_level(self, threat_level):
try:
threat_level = int(threat_level)
except:
raise UpdateInvestigationError('threat_level Not an integer')
if threat_level >= 1 and threat_level <= 4:
r_tracking.hset(f'investigations:data:{self.uuid}', 'threat_level', threat_level)
else:
raise UpdateInvestigationError(f'Invalid threat_level: {threat_level}')
def set_analysis(self, analysis):
try:
analysis = int(analysis)
except:
raise UpdateInvestigationError('analysis Not an integer')
if analysis >= 0 and analysis <= 2:
r_tracking.hset(f'investigations:data:{self.uuid}', 'analysis', analysis)
else:
raise UpdateInvestigationError(f'Invalid analysis: {analysis}')
def set_tags(self, tags):
# delete previous tags
r_tracking.delete(f'investigations:tags:{self.uuid}')
for tag in tags:
r_tracking.sadd(f'investigations:tags:{self.uuid}', tag)
def get_nb_objects(self):
return r_tracking.scard(f'investigations:objs:{self.uuid}')
def _get_objects(self):
return r_tracking.smembers(f'investigations:objs:{self.uuid}')
# # TODO: return Python object ???? ################################
# TODO: PAGINATE
def get_objects(self):
# obj_dict = {}
# for str_obj in self._get_objects():
# obj_type, subtype, id = str_obj.split(':', 2)
# if not obj_dict[obj_type]:
# obj_dict[obj_type] = []
# obj_dict[obj_type].append({'subtype': subtype, 'id': id})
objs = []
for str_obj in self._get_objects():
obj_type, subtype, obj_id = str_obj.split(':', 2)
dict_obj = {'type': obj_type, 'subtype': subtype, 'id': obj_id}
objs.append(dict_obj)
return objs
# # TODO: def register_object(self, Object): in OBJECT CLASS
def register_object(self, obj_id, obj_type, subtype):
r_tracking.sadd(f'investigations:objs:{self.uuid}', f'{obj_type}:{subtype}:{obj_id}')
r_tracking.sadd(f'obj:investigations:{obj_type}:{subtype}:{obj_id}', self.uuid)
timestamp = int(time.time())
self.set_last_change(timestamp)
def unregister_object(self, obj_id, obj_type, subtype):
r_tracking.srem(f'investigations:objs:{self.uuid}', f'{obj_type}:{subtype}:{obj_id}')
r_tracking.srem(f'obj:investigations:{obj_type}:{subtype}:{obj_id}', self.uuid)
timestamp = int(time.time())
self.set_last_change(timestamp)
def delete(self):
for str_obj in self._get_objects():
obj_type, subtype, obj_id = str_obj.split(':', 2)
self.unregister_object(obj_id, obj_type, subtype=subtype)
r_tracking.srem('investigations:all', self.uuid)
# user map
r_tracking.srem(f'investigations:user:{self.get_creator_user()}', self.uuid)
# metadata
r_tracking.delete(f'investigations:data:{self.uuid}')
r_tracking.delete(f'investigations:tags:{self.uuid}')
##-- Class --##
def get_all_investigations():
return r_tracking.smembers('investigations:all')
def exists_investigation(investigation_uuid):
return r_tracking.sismember('investigations:all', investigation_uuid)
# created by user
def get_user_all_investigations(user_id):
return r_tracking.smembers('investigations:user:{user_id}')
def is_object_investigated(obj_id, obj_type, subtype=''):
return r_tracking.exists(f'obj:investigations:{obj_type}:{subtype}:{obj_id}')
def get_obj_investigations(obj_id, obj_type, subtype=''):
return r_tracking.smembers(f'obj:investigations:{obj_type}:{subtype}:{obj_id}')
# # TODO: fix default threat_level analysis
# # TODO: limit description + name
# # TODO: sanityze tags
# # TODO: sanityze date
def create_investigation(user_id, date, name, threat_level, analysis, info, tags=[]):
investigation_uuid = generate_uuid()
r_tracking.sadd('investigations:all', investigation_uuid)
# user map
r_tracking.sadd('investigations:user:{user_id}', investigation_uuid)
# metadata
r_tracking.hset(f'investigations:data:{investigation_uuid}', 'creator_user', user_id)
# TODO: limit info + name
investigation = Investigation(investigation_uuid)
investigation.set_info(info)
#investigation.set_name(name) ##############################################
investigation.set_date(date)
investigation.set_threat_level(threat_level)
investigation.set_analysis(analysis)
# # TODO: sanityze tags
if tags:
investigation.set_tags(tags)
timestamp = int(time.time())
r_tracking.hset(f'investigations:data:{investigation_uuid}', 'timestamp', timestamp)
investigation.set_last_change(timestamp)
return investigation_uuid
def get_all_investigations_meta(r_str=False):
investigations_meta = []
for investigation_uuid in get_all_investigations():
investigation = Investigation(investigation_uuid)
investigations_meta.append(investigation.get_metadata(r_str=r_str))
return investigations_meta
def get_investigations_selector():
l_investigations = []
for investigation_uuid in get_all_investigations():
investigation = Investigation(investigation_uuid)
name = investigation.get_info()
l_investigations.append({"id":investigation_uuid, "name": name})
return l_investigations
#{id:'8dc4b81aeff94a9799bd70ba556fa345',name:"Paris"}
#### API ####
# # TODO: CHECK Mandatory Fields
# # TODO: SANITYZE Fields
# # TODO: Name ?????
def api_add_investigation(json_dict):
user_id = json_dict.get('user_id')
name = json_dict.get('name') ##### mandatory ?
name = escape(name)
threat_level = json_dict.get('threat_level', 4)
analysis = json_dict.get('analysis', 0)
# # TODO: sanityze date
date = json_dict.get('date')
info = json_dict.get('info', '')
info = escape(info)
info = info[:1000]
tags = json_dict.get('tags', [])
if not Tag.are_enabled_tags(tags):
return {"status": "error", "reason": "Invalid/Disabled tags"}, 400
try:
res = create_investigation(user_id, date, name, threat_level, analysis, info, tags=tags)
except UpdateInvestigationError as e:
return e.message, 400
return res, 200
# # TODO: edit threat level / status
def api_edit_investigation(json_dict):
investigation_uuid = json_dict.get('uuid', '').replace(' ', '')
if not is_valid_uuid_v4(investigation_uuid):
return {"status": "error", "reason": "Invalid Investigation uuid"}, 400
investigation_uuid = sanityze_uuid(investigation_uuid)
if not exists_investigation(investigation_uuid):
return {"status": "error", "reason": "Investigation not found"}, 404
investigation = Investigation(investigation_uuid)
name = json_dict.get('name') ##### mandatory ?
name = escape(name)
threat_level = json_dict.get('threat_level', 4)
try:
investigation.set_threat_level(threat_level)
except UpdateInvestigationError:
return {"status": "error", "reason": "Invalid Investigation threat_level"}, 400
analysis = json_dict.get('analysis', 0)
try:
investigation.set_analysis(analysis)
except UpdateInvestigationError:
return {"status": "error", "reason": "Invalid Investigation analysis"}, 400
info = json_dict.get('info', '')
info = escape(info)
info = info[:1000]
tags = json_dict.get('tags', [])
if not Tag.are_enabled_tags(tags):
return {"status": "error", "reason": "Invalid/Disabled tags"}, 400
investigation.set_info(info)
investigation.set_name(name)
investigation.set_tags(tags)
timestamp = int(time.time())
investigation.set_last_change(timestamp)
return investigation_uuid, 200
def api_delete_investigation(json_dict):
investigation_uuid = json_dict.get('uuid', '').replace(' ', '')
if not is_valid_uuid_v4(investigation_uuid):
return {"status": "error", "reason": "Invalid Investigation uuid"}, 400
investigation_uuid = sanityze_uuid(investigation_uuid)
if not exists_investigation(investigation_uuid):
return {"status": "error", "reason": "Investigation not found"}, 404
investigation = Investigation(investigation_uuid)
res = investigation.delete()
return res, 200
def api_register_object(json_dict):
investigation_uuid = json_dict.get('uuid', '').replace(' ', '')
if not is_valid_uuid_v4(investigation_uuid):
return {"status": "error", "reason": f"Invalid Investigation uuid: {investigation_uuid}"}, 400
investigation_uuid = sanityze_uuid(investigation_uuid)
if not exists_investigation(investigation_uuid):
return {"status": "error", "reason": f"Investigation not found: {investigation_uuid}"}, 404
investigation = Investigation(investigation_uuid)
obj_type = json_dict.get('type', '').replace(' ', '')
if not exists_obj_type(obj_type):
return {"status": "error", "reason": f"Invalid Object Type: {obj_type}"}, 400
subtype = json_dict.get('subtype', '')
if subtype == 'None':
subtype = ''
obj_id = json_dict.get('id', '').replace(' ', '')
res = investigation.register_object(obj_id, obj_type, subtype)
return res, 200
def api_unregister_object(json_dict):
investigation_uuid = json_dict.get('uuid', '').replace(' ', '')
if not is_valid_uuid_v4(investigation_uuid):
return {"status": "error", "reason": f"Invalid Investigation uuid: {investigation_uuid}"}, 400
investigation_uuid = sanityze_uuid(investigation_uuid)
if not exists_investigation(investigation_uuid):
return {"status": "error", "reason": f"Investigation not found: {investigation_uuid}"}, 404
investigation = Investigation(investigation_uuid)
obj_type = json_dict.get('type', '').replace(' ', '')
subtype = json_dict.get('subtype', '')
if subtype == 'None':
subtype = ''
obj_id = json_dict.get('id', '').replace(' ', '')
res = investigation.unregister_object(obj_id, obj_type, subtype)
return res, 200
##-- API --##
if __name__ == '__main__':
# res = create_star_list(user_id, name, description)
# print(res)
# res = r_tracking.dbsize()
# print(res)
investigation_uuid = 'e4e1c8e3b0a349bf81482f2f823efc0f'
investigation = Investigation(investigation_uuid)
investigation.delete()
# # TODO: PAGINATION

View file

@ -1,406 +0,0 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import sys
import uuid
import redis
from abc import ABC
from flask import url_for
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
import Tag
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
config_loader = ConfigLoader.ConfigLoader()
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
config_loader = None
class AbstractObject(ABC):
"""
Abstract Object
"""
# first seen last/seen ??
# # TODO: - tags
# - handle + refactor coorelations
# - creates others objects
def __init__(self, obj_type, id, subtype=None):
""" Abstract for all the AIL object
:param obj_type: object type (item, ...)
:param id: Object ID
"""
self.id = id
self.type = obj_type
self.subtype = None
def get_id(self):
return self.id
def get_type(self):
return self.type
def get_subtype(self, r_str=False):
if not self.subtype:
if r_str:
return ''
return self.subtype
def get_default_meta(self):
dict_meta = {'id': self.get_id(),
'type': self.get_type()}
if self.subtype:
dict_meta['subtype'] = self.subtype
return dict_meta
def get_tags(self, r_set=False):
tags = Tag.get_obj_tag(self.id)
if r_set:
tags = set(tags)
return tags
## ADD TAGS ????
#def add_tags(self):
def _delete(self):
# DELETE TAGS
Tag.delete_obj_all_tags(self.id, self.type)
if self.type == 'item':
# delete tracker
pass
def is_valid_object_type(object_type):
if object_type in ['domain', 'item', 'image', 'decoded']:
return True
else:
return False
def get_all_objects():
return ['domain', 'paste', 'pgp', 'cryptocurrency', 'decoded', 'screenshot']
def get_all_correlation_names():
'''
Return a list of all available correlations
'''
return ['pgp', 'cryptocurrency', 'decoded', 'screenshot']
def get_all_correlation_objects():
'''
Return a list of all correllated objects
'''
return ['domain', 'paste']
def exist_object(object_type, correlation_id, type_id=None):
if object_type == 'domain':
return Domain.verify_if_domain_exist(correlation_id)
elif object_type == 'paste' or object_type == 'item':
return Item.exist_item(correlation_id)
elif object_type == 'decoded':
return Decoded.exist_decoded(correlation_id)
elif object_type == 'pgp':
return Pgp.pgp._exist_corelation_field(type_id, correlation_id)
elif object_type == 'cryptocurrency':
return Cryptocurrency.cryptocurrency._exist_corelation_field(type_id, correlation_id)
elif object_type == 'screenshot' or object_type == 'image':
return Screenshot.exist_screenshot(correlation_id)
else:
return False
def get_obj_date(object_type, object_id):
if object_type == "item":
return int(Item.get_item_date(object_id))
else:
return None
# request_type => api or ui
def get_object_metadata(object_type, correlation_id, type_id=None):
if object_type == 'domain':
return Domain.Domain(correlation_id).get_domain_metadata(tags=True)
elif object_type == 'paste' or object_type == 'item':
return Item.get_item({"id": correlation_id, "date": True, "date_separator": True, "tags": True})[0]
elif object_type == 'decoded':
return Decoded.get_decoded_metadata(correlation_id, nb_seen=True, size=True, file_type=True, tag=True)
elif object_type == 'pgp':
return Pgp.pgp.get_metadata(type_id, correlation_id)
elif object_type == 'cryptocurrency':
return Cryptocurrency.cryptocurrency.get_metadata(type_id, correlation_id)
elif object_type == 'screenshot' or object_type == 'image':
return Screenshot.get_metadata(correlation_id)
def get_object_correlation(object_type, value, correlation_names=None, correlation_objects=None, requested_correl_type=None):
if object_type == 'domain':
return Domain.get_domain_all_correlation(value, correlation_names=correlation_names)
elif object_type == 'paste' or object_type == 'item':
return Item.get_item_all_correlation(value, correlation_names=correlation_names)
elif object_type == 'decoded':
return Decoded.get_decoded_correlated_object(value, correlation_objects=correlation_objects)
elif object_type == 'pgp':
return Pgp.pgp.get_correlation_all_object(requested_correl_type, value, correlation_objects=correlation_objects)
elif object_type == 'cryptocurrency':
return Cryptocurrency.cryptocurrency.get_correlation_all_object(requested_correl_type, value, correlation_objects=correlation_objects)
elif object_type == 'screenshot' or object_type == 'image':
return Screenshot.get_screenshot_correlated_object(value, correlation_objects=correlation_objects)
return {}
def get_correlation_node_icon(correlation_name, correlation_type=None, value=None):
'''
Used in UI Graph.
Return a font awesome icon for a given correlation_name.
:param correlation_name: correlation name
:param correlation_name: str
:param correlation_type: correlation type
:type correlation_type: str, optional
:return: a dictionnary {font awesome class, icon_code}
:rtype: dict
'''
icon_class = 'fas'
icon_text = ''
node_color = "#332288"
node_radius = 6
if correlation_name == "pgp":
node_color = '#44AA99'
if correlation_type == 'key':
icon_text = '\uf084'
elif correlation_type == 'name':
icon_text = '\uf507'
elif correlation_type == 'mail':
icon_text = '\uf1fa'
else:
icon_text = 'times'
elif correlation_name == 'cryptocurrency':
node_color = '#DDCC77'
if correlation_type == 'bitcoin':
icon_class = 'fab'
icon_text = '\uf15a'
elif correlation_type == 'monero':
icon_class = 'fab'
icon_text = '\uf3d0'
elif correlation_type == 'ethereum':
icon_class = 'fab'
icon_text = '\uf42e'
else:
icon_text = '\uf51e'
elif correlation_name == 'decoded':
node_color = '#88CCEE'
correlation_type = Decoded.get_decoded_item_type(value).split('/')[0]
if correlation_type == 'application':
icon_text = '\uf15b'
elif correlation_type == 'audio':
icon_text = '\uf1c7'
elif correlation_type == 'image':
icon_text = '\uf1c5'
elif correlation_type == 'text':
icon_text = '\uf15c'
else:
icon_text = '\uf249'
elif correlation_name == 'screenshot' or correlation_name == 'image':
node_color = '#E1F5DF'
icon_text = '\uf03e'
elif correlation_name == 'domain':
node_radius = 5
node_color = '#3DA760'
if Domain.get_domain_type(value) == 'onion':
icon_text = '\uf06e'
else:
icon_class = 'fab'
icon_text = '\uf13b'
elif correlation_name == 'paste':
node_radius = 5
if Item.is_crawled(value):
node_color = 'red'
else:
node_color = '#332288'
return {"icon_class": icon_class, "icon_text": icon_text, "node_color": node_color, "node_radius": node_radius}
def get_item_url(correlation_name, value, correlation_type=None):
'''
Warning: use only in flask
'''
url = '#'
if correlation_name == "pgp":
endpoint = 'correlation.show_correlation'
url = url_for(endpoint, object_type="pgp", type_id=correlation_type, correlation_id=value)
elif correlation_name == 'cryptocurrency':
endpoint = 'correlation.show_correlation'
url = url_for(endpoint, object_type="cryptocurrency", type_id=correlation_type, correlation_id=value)
elif correlation_name == 'decoded':
endpoint = 'correlation.show_correlation'
url = url_for(endpoint, object_type="decoded", correlation_id=value)
elif correlation_name == 'screenshot' or correlation_name == 'image': ### # TODO: rename me
endpoint = 'correlation.show_correlation'
url = url_for(endpoint, object_type="screenshot", correlation_id=value)
elif correlation_name == 'domain':
endpoint = 'crawler_splash.showDomain'
url = url_for(endpoint, domain=value)
elif correlation_name == 'item':
endpoint = 'showsavedpastes.showsavedpaste'
url = url_for(endpoint, paste=value)
elif correlation_name == 'paste': ### # TODO: remove me
endpoint = 'showsavedpastes.showsavedpaste'
url = url_for(endpoint, paste=value)
return url
def get_obj_tag_table_keys(object_type):
'''
Warning: use only in flask (dynamic templates)
'''
if object_type=="domain":
return ['id', 'first_seen', 'last_check', 'status'] # # TODO: add root screenshot
def create_graph_links(links_set):
graph_links_list = []
for link in links_set:
graph_links_list.append({"source": link[0], "target": link[1]})
return graph_links_list
def create_graph_nodes(nodes_set, root_node_id):
graph_nodes_list = []
for node_id in nodes_set:
correlation_name, correlation_type, value = node_id.split(';', 3)
dict_node = {"id": node_id}
dict_node['style'] = get_correlation_node_icon(correlation_name, correlation_type, value)
dict_node['text'] = value
if node_id == root_node_id:
dict_node["style"]["node_color"] = 'orange'
dict_node["style"]["node_radius"] = 7
dict_node['url'] = get_item_url(correlation_name, value, correlation_type)
graph_nodes_list.append(dict_node)
return graph_nodes_list
def create_node_id(correlation_name, value, correlation_type=''):
if correlation_type is None:
correlation_type = ''
return '{};{};{}'.format(correlation_name, correlation_type, value)
# # TODO: filter by correlation type => bitcoin, mail, ...
def get_graph_node_object_correlation(object_type, root_value, mode, correlation_names, correlation_objects, max_nodes=300, requested_correl_type=None):
links = set()
nodes = set()
root_node_id = create_node_id(object_type, root_value, requested_correl_type)
nodes.add(root_node_id)
root_correlation = get_object_correlation(object_type, root_value, correlation_names, correlation_objects, requested_correl_type=requested_correl_type)
for correl in root_correlation:
if correl in ('pgp', 'cryptocurrency'):
for correl_type in root_correlation[correl]:
for correl_val in root_correlation[correl][correl_type]:
# add correlation
correl_node_id = create_node_id(correl, correl_val, correl_type)
if mode=="union":
if len(nodes) > max_nodes:
break
nodes.add(correl_node_id)
links.add((root_node_id, correl_node_id))
# get second correlation
res = get_object_correlation(correl, correl_val, correlation_names, correlation_objects, requested_correl_type=correl_type)
if res:
for corr_obj in res:
for correl_key_val in res[corr_obj]:
#filter root value
if correl_key_val == root_value:
continue
if len(nodes) > max_nodes:
break
new_corel_1 = create_node_id(corr_obj, correl_key_val)
new_corel_2 = create_node_id(correl, correl_val, correl_type)
nodes.add(new_corel_1)
nodes.add(new_corel_2)
links.add((new_corel_1, new_corel_2))
if mode=="inter":
nodes.add(correl_node_id)
links.add((root_node_id, correl_node_id))
if correl in ('decoded', 'screenshot', 'domain', 'paste'):
for correl_val in root_correlation[correl]:
correl_node_id = create_node_id(correl, correl_val)
if mode=="union":
if len(nodes) > max_nodes:
break
nodes.add(correl_node_id)
links.add((root_node_id, correl_node_id))
res = get_object_correlation(correl, correl_val, correlation_names, correlation_objects)
if res:
for corr_obj in res:
if corr_obj in ('decoded', 'domain', 'paste', 'screenshot'):
for correl_key_val in res[corr_obj]:
#filter root value
if correl_key_val == root_value:
continue
if len(nodes) > max_nodes:
break
new_corel_1 = create_node_id(corr_obj, correl_key_val)
new_corel_2 = create_node_id(correl, correl_val)
nodes.add(new_corel_1)
nodes.add(new_corel_2)
links.add((new_corel_1, new_corel_2))
if mode=="inter":
nodes.add(correl_node_id)
links.add((root_node_id, correl_node_id))
if corr_obj in ('pgp', 'cryptocurrency'):
for correl_key_type in res[corr_obj]:
for correl_key_val in res[corr_obj][correl_key_type]:
#filter root value
if correl_key_val == root_value:
continue
if len(nodes) > max_nodes:
break
new_corel_1 = create_node_id(corr_obj, correl_key_val, correl_key_type)
new_corel_2 = create_node_id(correl, correl_val)
nodes.add(new_corel_1)
nodes.add(new_corel_2)
links.add((new_corel_1, new_corel_2))
if mode=="inter":
nodes.add(correl_node_id)
links.add((root_node_id, correl_node_id))
return {"nodes": create_graph_nodes(nodes, root_node_id), "links": create_graph_links(links)}
def get_obj_global_id(obj_type, obj_id, obj_sub_type=None):
if obj_sub_type:
return '{}:{}:{}'.format(obj_type, obj_sub_type, obj_id)
else:
# # TODO: remove me
if obj_type=='paste':
obj_type='item'
# # TODO: remove me
if obj_type=='screenshot':
obj_type='image'
return '{}:{}'.format(obj_type, obj_id)
######## API EXPOSED ########
def sanitize_object_type(object_type):
if not is_valid_object_type(object_type):
return ({'status': 'error', 'reason': 'Incorrect object_type'}, 400)
######## ########

30
bin/lib/ail_users.py Executable file
View file

@ -0,0 +1,30 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import sys
import uuid
import redis
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
config_loader = ConfigLoader.ConfigLoader()
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
config_loader = None
class User(object):
"""AIL User."""
def __init__(self, id):
self.id = id
if self.id == '__anonymous__':
self.role = 'anonymous'
else:
self.role = None
def get_role(self):
pass

15
bin/lib/exceptions.py Executable file
View file

@ -0,0 +1,15 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
class AIL_ERROR(Exception):
"""docstring for AIL_ERROR."""
def __init__(self, message):
super(AIL_ERROR, self).__init__(message)
self.message = message
class UpdateInvestigationError(AIL_ERROR):
pass
class NewTagError(AIL_ERROR):
pass

View file

@ -0,0 +1,76 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import sys
import redis
# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
from abstract_object import AbstractObject
config_loader = ConfigLoader.ConfigLoader()
config_loader = None
################################################################################
################################################################################
################################################################################
class CryptoCurrency(AbstractObject):
"""
AIL CryptoCurrency Object. (strings)
"""
def __init__(self, id, subtype):
super(CryptoCurrency, self).__init__('cryptocurrency', id, subtype=subtype)
# def get_ail_2_ail_payload(self):
# payload = {'raw': self.get_gzip_content(b64=True),
# 'compress': 'gzip'}
# return payload
# # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
def delete(self):
# # TODO:
pass
def get_link(self, flask_context=False):
if flask_context:
url = url_for('correlation.show_correlation', object_type=self.type, type_id=self.subtype, correlation_id=self.id)
else:
url = f'{baseurl}/correlation/show_correlation?object_type={self.type}&type_id={self.subtype}&correlation_id={self.id}'
return url
def get_svg_icon(self):
if self.subtype == 'bitcoin':
style = 'fab'
icon = '\uf15a'
elif self.subtype == 'monero':
style = 'fab'
icon = '\uf3d0'
elif self.subtype == 'ethereum':
style = 'fab'
icon = '\uf42e'
else:
style = 'fas'
icon = '\uf51e'
return {'style': style, 'icon': icon, 'color': '#DDCC77', 'radius':5}
############################################################################
############################################################################
############################################################################
def exist_correlation(self):
pass
############################################################################
############################################################################
#if __name__ == '__main__':

85
bin/lib/objects/Decodeds.py Executable file
View file

@ -0,0 +1,85 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import sys
import redis
# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
from abstract_object import AbstractObject
config_loader = ConfigLoader.ConfigLoader()
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
HASH_DIR = config_loader.get_config_str('Directories', 'hash')
baseurl = config_loader.get_config_str("Notifications", "ail_domain")
config_loader = None
################################################################################
################################################################################
################################################################################
# # TODO: COMPLETE CLASS
class Decoded(AbstractObject):
"""
AIL Decoded Object. (strings)
"""
def __init__(self, id):
super(Decoded, self).__init__('decoded', id)
# def get_ail_2_ail_payload(self):
# payload = {'raw': self.get_gzip_content(b64=True),
# 'compress': 'gzip'}
# return payload
def get_sha1(self):
return self.id.split('/')[0]
def get_file_type(self):
return r_serv_metadata.hget(f'metadata_hash:{self.get_sha1()}', 'estimated_type')
# # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
def delete(self):
# # TODO:
pass
def get_link(self, flask_context=False):
if flask_context:
url = url_for('correlation.show_correlation', object_type="decoded", correlation_id=value)
else:
url = f'{baseurl}/correlation/show_correlation?object_type={self.type}&correlation_id={self.id}'
return url
def get_svg_icon(self):
file_type = self.get_file_type()
if file_type == 'application':
icon = '\uf15b'
elif file_type == 'audio':
icon = '\uf1c7'
elif file_type == 'image':
icon = '\uf1c5'
elif file_type == 'text':
icon = '\uf15c'
else:
icon = '\uf249'
return {'style': 'fas', 'icon': icon, 'color': '#88CCEE', 'radius':5}
############################################################################
############################################################################
############################################################################
def exist_correlation(self):
pass
############################################################################
############################################################################
#if __name__ == '__main__':

77
bin/lib/objects/Domains.py Executable file
View file

@ -0,0 +1,77 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import sys
import redis
# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
from abstract_object import AbstractObject
config_loader = ConfigLoader.ConfigLoader()
config_loader = None
################################################################################
################################################################################
################################################################################
class Domain(AbstractObject):
"""
AIL Decoded Object. (strings)
"""
def __init__(self, id):
super(Domain, self).__init__('domain', id)
# def get_ail_2_ail_payload(self):
# payload = {'raw': self.get_gzip_content(b64=True),
# 'compress': 'gzip'}
# return payload
def get_domain_type(self):
if str(self.id).endswith('.onion'):
return 'onion'
else:
return 'regular'
# # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
def delete(self):
# # TODO:
pass
def get_link(self, flask_context=False):
if flask_context:
url = url_for('crawler_splash.showDomain', domain=self.id)
else:
url = f'{baseurl}/crawlers/showDomain?domain={self.id}'
return url
def get_svg_icon(self):
color = '#3DA760'
if self.get_domain_type() == 'onion':
style = 'fas'
icon = '\uf06e'
else:
style = 'fab'
icon = '\uf13b'
return {'style': style, 'icon': icon, 'color':color, 'radius':5}
############################################################################
############################################################################
############################################################################
def exist_correlation(self):
pass
############################################################################
############################################################################
#if __name__ == '__main__':

690
bin/lib/objects/Items.py Executable file
View file

@ -0,0 +1,690 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import base64
import os
import re
import sys
import redis
import cld3
import html2text
from io import BytesIO
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
import Tag
import Cryptocurrency
import Pgp
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import item_basic
import domain_basic
import ConfigLoader
import Correlate_object
import Decoded
import Screenshot
import Username
from abstract_object import AbstractObject
from item_basic import *
config_loader = ConfigLoader.ConfigLoader()
# get and sanityze PASTE DIRECTORY
# # TODO: rename PASTES_FOLDER
PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/'
PASTES_FOLDER = os.path.join(os.path.realpath(PASTES_FOLDER), '')
r_cache = config_loader.get_redis_conn("Redis_Cache")
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
screenshot_directory = config_loader.get_files_directory('screenshot')
har_directory = config_loader.get_files_directory('har')
baseurl = config_loader.get_config_str("Notifications", "ail_domain")
config_loader = None
################################################################################
################################################################################
################################################################################
class Item(AbstractObject):
"""
AIL Item Object. (strings)
"""
def __init__(self, id):
super(Item, self).__init__('item', id)
def get_date(self, separator=False):
"""
Returns Item date
"""
return item_basic.get_item_date(self.id, add_separator=separator)
def get_source(self):
"""
Returns Item source/feeder name
"""
#return self.id.split('/')[-5]
l_source = self.id.split('/')[:-4]
return os.path.join(*l_source)
def get_basename(self):
return os.path.basename(self.id)
def get_filename(self):
# Creating the full filepath
filename = os.path.join(PASTES_FOLDER, self.id)
filename = os.path.realpath(filename)
# incorrect filename
if not os.path.commonprefix([filename, PASTES_FOLDER]) == PASTES_FOLDER:
return None
else:
return filename
def get_content(self):
"""
Returns Item content
"""
return item_basic.get_item_content(self.id)
def get_gzip_content(self, b64=False):
with open(self.get_filename(), 'rb') as f:
content = f.read()
if b64:
content = base64.b64encode(content)
return content.decode()
def get_ail_2_ail_payload(self):
payload = {'raw': self.get_gzip_content(b64=True),
'compress': 'gzip'}
return payload
# # TODO:
def create(self):
pass
# # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
# TODO: DELETE ITEM CORRELATION + TAGS + METADATA + ...
def delete(self):
try:
os.remove(self.get_filename())
return True
except FileNotFoundError:
return False
def get_link(self, flask_context=False):
if flask_context:
url = url_for('objects_item.showItem', id=value)
else:
url = f'{baseurl}/object/item?id={self.id}'
return url
def get_svg_icon(self):
if is_crawled(self.id):
color = 'red'
else:
color = '#332288'
return {'style': '', 'icon': '', 'color': color, 'radius':5}
############################################################################
############################################################################
############################################################################
def exist_correlation(self):
pass
############################################################################
############################################################################
################################################################################
################################################################################
################################################################################
def exist_item(item_id):
return item_basic.exist_item(item_id)
def get_basename(item_id):
return os.path.basename(item_id)
def get_item_id(full_path):
return full_path.replace(PASTES_FOLDER, '', 1)
def get_item_filepath(item_id):
return item_basic.get_item_filepath(item_id)
def get_item_date(item_id, add_separator=False):
return item_basic.get_item_date(item_id, add_separator=add_separator)
def get_source(item_id):
return item_basic.get_source(item_id)
def get_all_sources():
return item_basic.get_all_items_sources(r_list=True)
def get_item_basename(item_id):
return os.path.basename(item_id)
def get_item_size(item_id):
return round(os.path.getsize(os.path.join(PASTES_FOLDER, item_id))/1024.0, 2)
def get_item_encoding(item_id):
return None
def get_lines_info(item_id, item_content=None):
if not item_content:
item_content = get_item_content(item_id)
max_length = 0
line_id = 0
nb_line = 0
for line in item_content.splitlines():
length = len(line)
if length > max_length:
max_length = length
nb_line += 1
return {'nb': nb_line, 'max_length': max_length}
def get_item_metadata(item_id, item_content=None):
## TODO: FIXME ##performance
# encoding
# language
# lines info
item_metadata = {'date': get_item_date(item_id, add_separator=True),
'source': get_source(item_id),
'size': get_item_size(item_id),
'encoding': get_item_encoding(item_id),
'lines': get_lines_info(item_id, item_content=item_content)
}
return item_metadata
def get_item_parent(item_id):
return item_basic.get_item_parent(item_id)
def add_item_parent(item_parent, item_id):
return item_basic.add_item_parent(item_parent, item_id)
def get_item_content(item_id):
return item_basic.get_item_content(item_id)
def get_item_content_html2text(item_id, item_content=None, ignore_links=False):
if not item_content:
item_content = get_item_content(item_id)
h = html2text.HTML2Text()
h.ignore_links = ignore_links
h.ignore_images = ignore_links
return h.handle(item_content)
def remove_all_urls_from_content(item_id, item_content=None):
if not item_content:
item_content = get_item_content(item_id)
regex = r'\b(?:http://|https://)?(?:[a-zA-Z\d-]{,63}(?:\.[a-zA-Z\d-]{,63})+)(?:\:[0-9]+)*(?:/(?:$|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*\b'
url_regex = re.compile(regex)
urls = url_regex.findall(item_content)
urls = sorted(urls, key=len, reverse=True)
for url in urls:
item_content = item_content.replace(url, '')
regex_pgp_public_blocs = r'-----BEGIN PGP PUBLIC KEY BLOCK-----[\s\S]+?-----END PGP PUBLIC KEY BLOCK-----'
regex_pgp_signature = r'-----BEGIN PGP SIGNATURE-----[\s\S]+?-----END PGP SIGNATURE-----'
regex_pgp_message = r'-----BEGIN PGP MESSAGE-----[\s\S]+?-----END PGP MESSAGE-----'
re.compile(regex_pgp_public_blocs)
re.compile(regex_pgp_signature)
re.compile(regex_pgp_message)
res = re.findall(regex_pgp_public_blocs, item_content)
for it in res:
item_content = item_content.replace(it, '')
res = re.findall(regex_pgp_signature, item_content)
for it in res:
item_content = item_content.replace(it, '')
res = re.findall(regex_pgp_message, item_content)
for it in res:
item_content = item_content.replace(it, '')
return item_content
def get_item_languages(item_id, min_len=600, num_langs=3, min_proportion=0.2, min_probability=0.7):
all_languages = []
## CLEAN CONTENT ##
content = get_item_content_html2text(item_id, ignore_links=True)
content = remove_all_urls_from_content(item_id, item_content=content)
# REMOVE USELESS SPACE
content = ' '.join(content.split())
#- CLEAN CONTENT -#
#print(content)
#print(len(content))
if len(content) >= min_len:
for lang in cld3.get_frequent_languages(content, num_langs=num_langs):
if lang.proportion >= min_proportion and lang.probability >= min_probability and lang.is_reliable:
all_languages.append(lang)
return all_languages
# API
def get_item(request_dict):
if not request_dict:
return {'status': 'error', 'reason': 'Malformed JSON'}, 400
item_id = request_dict.get('id', None)
if not item_id:
return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400
if not exist_item(item_id):
return {'status': 'error', 'reason': 'Item not found'}, 404
dict_item = {}
dict_item['id'] = item_id
date = request_dict.get('date', True)
if date:
add_separator = False
if request_dict.get('date_separator', False):
add_separator = True
dict_item['date'] = get_item_date(item_id, add_separator=add_separator)
tags = request_dict.get('tags', True)
if tags:
dict_item['tags'] = Tag.get_obj_tag(item_id)
size = request_dict.get('size', False)
if size:
dict_item['size'] = get_item_size(item_id)
content = request_dict.get('content', False)
if content:
# UTF-8 outpout, # TODO: use base64
dict_item['content'] = get_item_content(item_id)
raw_content = request_dict.get('raw_content', False)
if raw_content:
dict_item['raw_content'] = get_raw_content(item_id)
lines_info = request_dict.get('lines', False)
if lines_info:
dict_item['lines'] = get_lines_info(item_id, dict_item.get('content', 'None'))
if request_dict.get('pgp'):
dict_item['pgp'] = {}
if request_dict['pgp'].get('key'):
dict_item['pgp']['key'] = get_item_pgp_key(item_id)
if request_dict['pgp'].get('mail'):
dict_item['pgp']['mail'] = get_item_pgp_mail(item_id)
if request_dict['pgp'].get('name'):
dict_item['pgp']['name'] = get_item_pgp_name(item_id)
if request_dict.get('cryptocurrency'):
dict_item['cryptocurrency'] = {}
if request_dict['cryptocurrency'].get('bitcoin'):
dict_item['cryptocurrency']['bitcoin'] = get_item_bitcoin(item_id)
return dict_item, 200
def api_get_item_content_base64_utf8(request_dict):
item_id = request_dict.get('id', None)
if not request_dict:
return {'status': 'error', 'reason': 'Malformed JSON'}, 400
if not item_id:
return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400
if not exist_item(item_id):
return {'status': 'error', 'reason': 'Item not found'}, 404
item_content = get_item_content(item_id)
item_content = base64.b64encode((item_content.encode('utf-8'))).decode('UTF-8')
return {'status': 'success', 'content': item_content}, 200
def api_get_items_sources():
item_content = {'sources': get_all_sources()}
return item_content, 200
# def check_item_source(request_dict):
# source = request_dict.get('source', None)
# if not request_dict:
# return {'status': 'error', 'reason': 'Malformed JSON'}, 400
# if not source:
# return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400
#
# all_sources = item_basic.get_all_items_sources()
#
# if source not in all_sources:
# return {'status': 'error', 'reason': 'Invalid source', 'provide': source}, 400
# return {'status': 'success', 'reason': 'Valid source', 'provide': source}, 200
###
### correlation
###
def get_item_cryptocurrency(item_id, currencies_type=None, get_nb=False):
'''
Return all cryptocurrencies of a given item.
:param item_id: item id
:param currencies_type: list of cryptocurrencies type
:type currencies_type: list, optional
'''
return Cryptocurrency.cryptocurrency.get_item_correlation_dict(item_id, correlation_type=currencies_type, get_nb=get_nb)
def get_item_pgp(item_id, currencies_type=None, get_nb=False):
'''
Return all pgp of a given item.
:param item_id: item id
:param currencies_type: list of cryptocurrencies type
:type currencies_type: list, optional
'''
return Pgp.pgp.get_item_correlation_dict(item_id, correlation_type=currencies_type, get_nb=get_nb)
def get_item_username(item_id, sub_type=None, get_nb=False):
'''
Return all pgp of a given item.
:param item_id: item id
:param sub_type: list of username type
:type sub_type: list, optional
'''
return Username.correlation.get_item_correlation_dict(item_id, correlation_type=sub_type, get_nb=get_nb)
def get_item_decoded(item_id):
'''
Return all pgp of a given item.
:param item_id: item id
:param currencies_type: list of cryptocurrencies type
:type currencies_type: list, optional
'''
return Decoded.get_item_decoded(item_id)
def get_item_all_screenshot(item_id):
'''
Return all screenshot of a given item.
:param item_id: item id
'''
return Screenshot.get_item_screenshot_list(item_id)
def get_item_all_correlation(item_id, correlation_names=[], get_nb=False):
'''
Retun all correlation of a given item id.
:param item_id: item id
:type domain: str
:return: a dict of all correlation for a item id
:rtype: dict
'''
if not correlation_names:
correlation_names = Correlate_object.get_all_correlation_names()
item_correl = {}
for correlation_name in correlation_names:
if correlation_name=='cryptocurrency':
res = get_item_cryptocurrency(item_id, get_nb=get_nb)
elif correlation_name=='pgp':
res = get_item_pgp(item_id, get_nb=get_nb)
elif correlation_name=='username':
res = get_item_username(item_id, get_nb=get_nb)
elif correlation_name=='decoded':
res = get_item_decoded(item_id)
elif correlation_name=='screenshot':
res = get_item_all_screenshot(item_id)
else:
res = None
# add correllation to dict
if res:
item_correl[correlation_name] = res
return item_correl
## TODO: REFRACTOR
def _get_item_correlation(correlation_name, correlation_type, item_id):
res = r_serv_metadata.smembers('item_{}_{}:{}'.format(correlation_name, correlation_type, item_id))
if res:
return list(res)
else:
return []
## TODO: REFRACTOR
def get_item_bitcoin(item_id):
return _get_item_correlation('cryptocurrency', 'bitcoin', item_id)
## TODO: REFRACTOR
def get_item_pgp_key(item_id):
return _get_item_correlation('pgpdump', 'key', item_id)
## TODO: REFRACTOR
def get_item_pgp_name(item_id):
return _get_item_correlation('pgpdump', 'name', item_id)
## TODO: REFRACTOR
def get_item_pgp_mail(item_id):
return _get_item_correlation('pgpdump', 'mail', item_id)
## TODO: REFRACTOR
def get_item_pgp_correlation(item_id):
pass
###
### GET Internal Module DESC
###
def get_item_list_desc(list_item_id):
desc_list = []
for item_id in list_item_id:
desc_list.append( {'id': item_id, 'date': get_item_date(item_id), 'tags': Tag.get_obj_tag(item_id)} )
return desc_list
def is_crawled(item_id):
return item_basic.is_crawled(item_id)
def get_crawler_matadata(item_id, ltags=None):
dict_crawler = {}
if is_crawled(item_id):
dict_crawler['domain'] = get_item_domain(item_id)
if not ltags:
ltags = Tag.get_obj_tag(item_id)
dict_crawler['is_tags_safe'] = Tag.is_tags_safe(ltags)
dict_crawler['url'] = get_item_link(item_id)
dict_crawler['screenshot'] = get_item_screenshot(item_id)
dict_crawler['har'] = get_item_har_name(item_id)
return dict_crawler
def is_onion(item_id):
is_onion = False
if len(is_onion) > 62:
if is_crawled(item_id) and item_id[-42:-36] == '.onion':
is_onion = True
return is_onion
def is_item_in_domain(domain, item_id):
is_in_domain = False
domain_lenght = len(domain)
if len(item_id) > (domain_lenght+48):
if item_id[-36-domain_lenght:-36] == domain:
is_in_domain = True
return is_in_domain
def get_item_domain(item_id):
return item_basic.get_item_domain(item_id)
def get_domain(item_id):
item_id = item_id.split('/')
item_id = item_id[-1]
return item_id[:-36]
def get_item_domain_with_port(item_id):
return r_serv_metadata.hget('paste_metadata:{}'.format(item_id), 'domain')
def get_item_link(item_id):
return r_serv_metadata.hget('paste_metadata:{}'.format(item_id), 'real_link')
def get_item_screenshot(item_id):
screenshot = r_serv_metadata.hget('paste_metadata:{}'.format(item_id), 'screenshot')
if screenshot:
return os.path.join(screenshot[0:2], screenshot[2:4], screenshot[4:6], screenshot[6:8], screenshot[8:10], screenshot[10:12], screenshot[12:])
return ''
def get_item_har_name(item_id):
har_path = os.path.join(har_directory, item_id) + '.json'
if os.path.isfile(har_path):
return har_path
else:
return None
def get_item_har(har_path):
pass
def get_item_filename(item_id):
# Creating the full filepath
filename = os.path.join(PASTES_FOLDER, item_id)
filename = os.path.realpath(filename)
# incorrect filename
if not os.path.commonprefix([filename, PASTES_FOLDER]) == PASTES_FOLDER:
return None
else:
return filename
def get_item_duplicate(item_id, r_list=True):
res = r_serv_metadata.smembers('dup:{}'.format(item_id))
if r_list:
if res:
return list(res)
else:
return []
return res
def get_item_nb_duplicates(item_id):
return r_serv_metadata.scard('dup:{}'.format(item_id))
def get_item_duplicates_dict(item_id):
dict_duplicates = {}
for duplicate in get_item_duplicate(item_id):
duplicate = duplicate[1:-1].replace('\'', '').replace(' ', '').split(',')
duplicate_id = duplicate[1]
if not duplicate_id in dict_duplicates:
dict_duplicates[duplicate_id] = {'date': get_item_date(duplicate_id, add_separator=True), 'algo': {}}
algo = duplicate[0]
if algo == 'tlsh':
similarity = 100 - int(duplicate[2])
else:
similarity = int(duplicate[2])
dict_duplicates[duplicate_id]['algo'][algo] = similarity
return dict_duplicates
def add_item_duplicate(item_id, l_dup):
for item_dup in l_dup:
r_serv_metadata.sadd('dup:{}'.format(item_dup), item_id)
r_serv_metadata.sadd('dup:{}'.format(item_id), item_dup)
def delete_item_duplicate(item_id):
item_dup = get_item_duplicate(item_id)
for item_dup in get_item_duplicate(item_id):
r_serv_metadata.srem('dup:{}'.format(item_dup), item_id)
r_serv_metadata.delete('dup:{}'.format(item_id))
def get_raw_content(item_id):
filepath = get_item_filepath(item_id)
with open(filepath, 'rb') as f:
file_content = BytesIO(f.read())
return file_content
def save_raw_content(item_id, io_content):
filepath = get_item_filename(item_id)
if os.path.isfile(filepath):
#print('File already exist')
return False
# create subdir
dirname = os.path.dirname(filepath)
if not os.path.exists(dirname):
os.makedirs(dirname)
# # TODO: check if is IO file
with open(filepath, 'wb') as f:
f.write(io_content.getvalue())
return True
# IDEA: send item to duplicate ?
def create_item(obj_id, obj_metadata, io_content):
'''
Create a new Item (Import or Test only).
:param obj_id: item id
:type obj_metadata: dict - 'first_seen', 'tags'
:return: is item created
:rtype: boolean
'''
# check if datetime match ??
# # TODO: validate obj_id
res = save_raw_content(obj_id, io_content)
# item saved
if res:
# creata tags
if 'tags' in obj_metadata:
# # TODO: handle mixed tags: taxonomies and Galaxies
Tag.api_add_obj_tags(tags=obj_metadata['tags'], object_id=obj_id, object_type="item")
return True
# Item not created
return False
def delete_item(obj_id):
# check if item exists
if not exist_item(obj_id):
return False
else:
delete_item_duplicate(obj_id)
# delete MISP event
r_serv_metadata.delete('misp_events:{}'.format(obj_id))
r_serv_metadata.delete('hive_cases:{}'.format(obj_id))
os.remove(get_item_filename(obj_id))
# get all correlation
obj_correlations = get_item_all_correlation(obj_id)
for correlation in obj_correlations:
if correlation=='cryptocurrency' or correlation=='pgp':
for obj2_subtype in obj_correlations[correlation]:
for obj2_id in obj_correlations[correlation][obj2_subtype]:
Correlate_object.delete_obj_relationship(correlation, obj2_id, 'item', obj_id,
obj1_subtype=obj2_subtype)
else:
for obj2_id in obj_correlations[correlation]:
Correlate_object.delete_obj_relationship(correlation, obj2_id, 'item', obj_id)
# delete father/child
delete_node(obj_id)
# delete item metadata
r_serv_metadata.delete('paste_metadata:{}'.format(obj_id))
return True
### TODO in inport V2
# delete from tracked items
# # # TODO: # FIXME: LATER
# delete from queue
###
return False
#### ####
def delete_node(item_id):
if is_node(item_id):
if is_crawled(item_id):
delete_domain_node(item_id)
item_basic._delete_node(item_id)
def delete_domain_node(item_id):
if is_domain_root(item_id):
# remove from domain history
domain, port = get_item_domain_with_port(item_id).split(':')
domain_basic.delete_domain_item_core(item_id, domain, port)
for child_id in get_all_domain_node_by_item_id(item_id):
delete_item(child_id)
#if __name__ == '__main__':

72
bin/lib/objects/Pgps.py Executable file
View file

@ -0,0 +1,72 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import sys
import redis
# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
from abstract_object import AbstractObject
config_loader = ConfigLoader.ConfigLoader()
config_loader = None
################################################################################
################################################################################
################################################################################
class Pgp(AbstractObject):
"""
AIL Pgp Object. (strings)
"""
def __init__(self, id, subtype):
super(Pgp, self).__init__('pgp', id, subtype=subtype)
# def get_ail_2_ail_payload(self):
# payload = {'raw': self.get_gzip_content(b64=True),
# 'compress': 'gzip'}
# return payload
# # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
def delete(self):
# # TODO:
pass
def get_link(self, flask_context=False):
if flask_context:
url = url_for('correlation.show_correlation', object_type=self.type, type_id=self.subtype, correlation_id=self.id)
else:
url = f'{baseurl}/correlation/show_correlation?object_type={self.type}&type_id={self.subtype}&correlation_id={self.id}'
return url
def get_svg_icon(self):
if self.subtype == 'key':
icon = '\uf084'
elif self.subtype == 'name':
icon = '\uf507'
elif self.subtype == 'mail':
icon = '\uf1fa'
else:
icon = 'times'
return {'style': 'fas', 'icon': icon, 'color': '#44AA99', 'radius':5}
############################################################################
############################################################################
############################################################################
def exist_correlation(self):
pass
############################################################################
############################################################################
#if __name__ == '__main__':

65
bin/lib/objects/Screenshots.py Executable file
View file

@ -0,0 +1,65 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import sys
import redis
# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
from abstract_object import AbstractObject
config_loader = ConfigLoader.ConfigLoader()
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
HASH_DIR = config_loader.get_config_str('Directories', 'hash')
config_loader = None
################################################################################
################################################################################
################################################################################
class Screenshot(AbstractObject):
"""
AIL Screenshot Object. (strings)
"""
def __init__(self, id):
super(Screenshot, self).__init__('screenshot', id)
# def get_ail_2_ail_payload(self):
# payload = {'raw': self.get_gzip_content(b64=True),
# 'compress': 'gzip'}
# return payload
# # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
def delete(self):
# # TODO:
pass
def get_link(self, flask_context=False):
if flask_context:
url = url_for('correlation.show_correlation', object_type=self.type, correlation_id=self.id)
else:
url = f'{baseurl}/correlation/show_correlation?object_type={self.type}&correlation_id={self.id}'
return url
def get_svg_icon(self):
return {'style': 'fas', 'icon': '\uf03e', 'color': '#E1F5DF', 'radius':5}
############################################################################
############################################################################
############################################################################
def exist_correlation(self):
pass
############################################################################
############################################################################
#if __name__ == '__main__':

73
bin/lib/objects/Usernames.py Executable file
View file

@ -0,0 +1,73 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import sys
import redis
# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
from abstract_object import AbstractObject
config_loader = ConfigLoader.ConfigLoader()
config_loader = None
################################################################################
################################################################################
################################################################################
class Username(AbstractObject):
"""
AIL Username Object. (strings)
"""
def __init__(self, id, subtype):
super(Username, self).__init__('username', id, subtype=subtype)
# def get_ail_2_ail_payload(self):
# payload = {'raw': self.get_gzip_content(b64=True),
# 'compress': 'gzip'}
# return payload
# # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
def delete(self):
# # TODO:
pass
def get_link(self, flask_context=False):
if flask_context:
url = url_for('correlation.show_correlation', object_type=self.type, type_id=self.subtype, correlation_id=self.id)
else:
url = f'{baseurl}/correlation/show_correlation?object_type={self.type}&type_id={self.subtype}&correlation_id={self.id}'
return url
def get_svg_icon(self):
if self.subtype == 'telegram':
style = 'fab'
icon = '\uf2c6'
elif self.subtype == 'twitter':
style = 'fab'
icon = '\uf099'
else:
style = 'fas'
icon = '\uf007'
return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius':5}
############################################################################
############################################################################
############################################################################
def exist_correlation(self):
pass
############################################################################
############################################################################
#if __name__ == '__main__':

View file

@ -0,0 +1,135 @@
# -*-coding:UTF-8 -*
"""
Base Class for AIL Objects
"""
##################################
# Import External packages
##################################
import os
import sys
from abc import ABC, abstractmethod
#from flask import url_for
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from packages import Tag
from lib.Investigations import is_object_investigated, get_obj_investigations
# # TODO: ADD CORRELATION ENGINE
class AbstractObject(ABC):
"""
Abstract Object
"""
# first seen last/seen ??
# # TODO: - tags
# - handle + refactor coorelations
# - creates others objects
def __init__(self, obj_type, id, subtype=None):
""" Abstract for all the AIL object
:param obj_type: object type (item, ...)
:param id: Object ID
"""
self.id = id
self.type = obj_type
self.subtype = subtype
def get_id(self):
return self.id
def get_type(self):
return self.type
def get_subtype(self, r_str=False):
if not self.subtype:
if r_str:
return ''
return self.subtype
def get_default_meta(self, tags=False):
dict_meta = {'id': self.get_id(),
'type': self.get_type(),
'subtype': self.get_subtype()}
if tags:
dict_meta['tags'] = self.get_tags()
return dict_meta
## Tags ##
def get_tags(self, r_set=False):
tags = Tag.get_obj_tag(self.id)
if r_set:
tags = set(tags)
return tags
## ADD TAGS ????
#def add_tags(self):
#- Tags -#
## Investigations ##
# # TODO: unregister =====
def is_investigated(self):
if not self.subtype:
is_investigated = is_object_investigated(self.id, self.type)
else:
is_investigated = is_object_investigated(self.id, self.type, self.subtype)
return is_investigated
def get_investigations(self):
if not self.subtype:
investigations = get_obj_investigations(self.id, self.type)
else:
investigations = get_obj_investigations(self.id, self.type, self.subtype)
return investigations
#- Investigations -#
def _delete(self):
# DELETE TAGS
Tag.delete_obj_all_tags(self.id, self.type)
# # TODO: remove from investigations
@abstractmethod
def delete(self):
"""
Delete Object: used for the Data Retention
"""
pass
# @abstractmethod
# def get_meta(self):
# """
# get Object metadata
# """
# pass
@abstractmethod
def get_svg_icon(self):
"""
Get object svg icon
"""
pass
@abstractmethod
def get_link(self, flask_context=False):
pass
# # TODO:
# @abstractmethod
# def get_correlations(self, message):
# """
# Get object correlations
# """
# pass
# # TODO: get favicon
# # TODO: get url
# # TODO: get metadata

87
bin/lib/objects/ail_objects.py Executable file
View file

@ -0,0 +1,87 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import sys
import uuid
import redis
from abc import ABC
from flask import url_for
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/objects'))
from Decodeds import Decoded
from Domains import Domain
from CryptoCurrencies import CryptoCurrency
from Items import Item
from Pgps import Pgp
from Screenshots import Screenshot
from Usernames import Username
##################################################################
##################################################################
#sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
#sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
##################################################################
##################################################################
config_loader = ConfigLoader.ConfigLoader()
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
config_loader = None
class AILObjects(object):
initial = 0
ongoing = 1
completed = 2
# # TODO: check change paste => item
def get_all_objects():
return ['domain', 'item', 'pgp', 'cryptocurrency', 'decoded', 'screenshot', 'username']
def get_object(obj_type, subtype, id):
if obj_type == 'item':
return Item(id)
elif obj_type == 'domain':
return Domain(id)
elif obj_type == 'decoded':
return Decoded(id)
elif obj_type == 'screenshot':
return Screenshot(id)
elif obj_type == 'cryptocurrency':
return CryptoCurrency(id, subtype)
elif obj_type == 'pgp':
return Pgp(id, subtype)
elif obj_type == 'username':
return Username(id, subtype)
def get_object_svg(obj_type, subtype, id):
object = get_object(obj_type, subtype, id)
return object.get_svg_icon()
def get_objects_meta(l_dict_objs, icon=False, url=False, flask_context=False):
l_meta = []
for dict_obj in l_dict_objs:
object = get_object(dict_obj['type'], dict_obj['subtype'], dict_obj['id'])
dict_meta = object.get_default_meta(tags=True)
if icon:
dict_meta['icon'] = object.get_svg_icon()
if url:
dict_meta['link'] = object.get_link(flask_context=flask_context)
l_meta.append(dict_meta)
return l_meta
################################################################################
################################################################################
################################################################################
################################################################################
################################################################################
################################################################################
################################################################################
################################################################################

View file

@ -26,7 +26,7 @@ import Decoded
import Screenshot import Screenshot
import Username import Username
from ail_objects import AbstractObject from objects.abstract_object import AbstractObject
from item_basic import * from item_basic import *
config_loader = ConfigLoader.ConfigLoader() config_loader = ConfigLoader.ConfigLoader()

View file

@ -73,9 +73,6 @@ criticalNumberToAlert=8
#Will be considered as false positive if less that X matches from the top password list #Will be considered as false positive if less that X matches from the top password list
minTopPassList=5 minTopPassList=5
[Curve]
max_execution_time = 90
[Onion] [Onion]
save_i2p = False save_i2p = False
max_execution_time = 180 max_execution_time = 180
@ -149,31 +146,27 @@ port = 6381
db = 1 db = 1
##### ARDB ##### ##### ARDB #####
[ARDB_Curve]
[ARDB_DB]
host = localhost host = localhost
port = 6382 port = 6382
db = 1 db = 0
[DB_Tracking]
host = localhost
port = 6382
db = 3
[ARDB_Sentiment] [ARDB_Sentiment]
host = localhost host = localhost
port = 6382 port = 6382
db = 4 db = 4
[ARDB_TermFreq]
host = localhost
port = 6382
db = 2
[ARDB_TermCred] [ARDB_TermCred]
host = localhost host = localhost
port = 6382 port = 6382
db = 5 db = 5
[ARDB_DB]
host = localhost
port = 6382
db = 0
[ARDB_Trending] [ARDB_Trending]
host = localhost host = localhost
port = 6382 port = 6382

36
update/v4.1/Update.py Executable file
View file

@ -0,0 +1,36 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import re
import sys
import time
import redis
import datetime
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin'))
from ail_updater import AIL_Updater
class Updater(AIL_Updater):
"""default Updater."""
def __init__(self, version):
super(Updater, self).__init__(version)
def update(self):
config_loader = ConfigLoader.ConfigLoader()
r_tracking = config_loader.get_redis_conn("DB_Tracking")
config_loader = None
# FLUSH OLD DB
r_tracking.flushdb()
if __name__ == '__main__':
updater = Updater('v4.1')
updater.run_update()

25
update/v4.1/Update.sh Executable file
View file

@ -0,0 +1,25 @@
#!/bin/bash
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
export PATH=$AIL_HOME:$PATH
export PATH=$AIL_REDIS:$PATH
export PATH=$AIL_ARDB:$PATH
export PATH=$AIL_BIN:$PATH
export PATH=$AIL_FLASK:$PATH
GREEN="\\033[1;32m"
DEFAULT="\\033[0;39m"
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
bash ${AIL_BIN}/LAUNCH.sh -ks
wait
# SUBMODULES #
git submodule update
exit 0

View file

@ -42,6 +42,7 @@ from blueprints.crawler_splash import crawler_splash
from blueprints.correlation import correlation from blueprints.correlation import correlation
from blueprints.tags_ui import tags_ui from blueprints.tags_ui import tags_ui
from blueprints.import_export import import_export from blueprints.import_export import import_export
from blueprints.investigations_b import investigations_b
from blueprints.objects_item import objects_item from blueprints.objects_item import objects_item
from blueprints.hunters import hunters from blueprints.hunters import hunters
from blueprints.old_endpoints import old_endpoints from blueprints.old_endpoints import old_endpoints
@ -101,6 +102,7 @@ app.register_blueprint(crawler_splash, url_prefix=baseUrl)
app.register_blueprint(correlation, url_prefix=baseUrl) app.register_blueprint(correlation, url_prefix=baseUrl)
app.register_blueprint(tags_ui, url_prefix=baseUrl) app.register_blueprint(tags_ui, url_prefix=baseUrl)
app.register_blueprint(import_export, url_prefix=baseUrl) app.register_blueprint(import_export, url_prefix=baseUrl)
app.register_blueprint(investigations_b, url_prefix=baseUrl)
app.register_blueprint(objects_item, url_prefix=baseUrl) app.register_blueprint(objects_item, url_prefix=baseUrl)
app.register_blueprint(hunters, url_prefix=baseUrl) app.register_blueprint(hunters, url_prefix=baseUrl)
app.register_blueprint(old_endpoints, url_prefix=baseUrl) app.register_blueprint(old_endpoints, url_prefix=baseUrl)

View file

@ -28,7 +28,7 @@ import crawlers
import Domain import Domain
import Language import Language
import Config_DB #import Config_DB
r_cache = Flask_config.r_cache r_cache = Flask_config.r_cache
r_serv_db = Flask_config.r_serv_db r_serv_db = Flask_config.r_serv_db
@ -295,6 +295,15 @@ def domains_search_name():
l_dict_domains=l_dict_domains, bootstrap_label=bootstrap_label, l_dict_domains=l_dict_domains, bootstrap_label=bootstrap_label,
domains_types=domains_types) domains_types=domains_types)
@crawler_splash.route('/domains/TODO', methods=['GET'])
@login_required
@login_analyst
def domains_todo():
domain_type = request.args.get('type')
last_domains = Domain.get_last_crawled_domains(domain_type)
##-- --## ##-- --##
@ -349,8 +358,8 @@ def crawler_cookiejar_add_post():
return redirect(url_for('crawler_splash.crawler_cookiejar_show', cookiejar_uuid=cookiejar_uuid)) return redirect(url_for('crawler_splash.crawler_cookiejar_show', cookiejar_uuid=cookiejar_uuid))
@crawler_splash.route('/crawler/cookiejar/all', methods=['GET']) @crawler_splash.route('/crawler/cookiejar/all', methods=['GET'])
#@login_required @login_required
#@login_read_only @login_read_only
def crawler_cookiejar_all(): def crawler_cookiejar_all():
user_id = current_user.get_id() user_id = current_user.get_id()
user_cookiejar = crawlers.get_cookiejar_metadata_by_iterator(crawlers.get_user_cookiejar(user_id)) user_cookiejar = crawlers.get_cookiejar_metadata_by_iterator(crawlers.get_user_cookiejar(user_id))
@ -358,8 +367,8 @@ def crawler_cookiejar_all():
return render_template("all_cookiejar.html", user_cookiejar=user_cookiejar, global_cookiejar=global_cookiejar) return render_template("all_cookiejar.html", user_cookiejar=user_cookiejar, global_cookiejar=global_cookiejar)
@crawler_splash.route('/crawler/cookiejar/show', methods=['GET']) @crawler_splash.route('/crawler/cookiejar/show', methods=['GET'])
#@login_required @login_required
#@login_read_only @login_read_only
def crawler_cookiejar_show(): def crawler_cookiejar_show():
user_id = current_user.get_id() user_id = current_user.get_id()
cookiejar_uuid = request.args.get('cookiejar_uuid') cookiejar_uuid = request.args.get('cookiejar_uuid')
@ -379,8 +388,8 @@ def crawler_cookiejar_show():
l_cookies=l_cookies, l_cookie_uuid=l_cookie_uuid) l_cookies=l_cookies, l_cookie_uuid=l_cookie_uuid)
@crawler_splash.route('/crawler/cookiejar/cookie/delete', methods=['GET']) @crawler_splash.route('/crawler/cookiejar/cookie/delete', methods=['GET'])
#@login_required @login_required
#@login_read_only @login_read_only
def crawler_cookiejar_cookie_delete(): def crawler_cookiejar_cookie_delete():
user_id = current_user.get_id() user_id = current_user.get_id()
cookiejar_uuid = request.args.get('cookiejar_uuid') cookiejar_uuid = request.args.get('cookiejar_uuid')
@ -392,8 +401,8 @@ def crawler_cookiejar_cookie_delete():
return redirect(url_for('crawler_splash.crawler_cookiejar_show', cookiejar_uuid=cookiejar_uuid)) return redirect(url_for('crawler_splash.crawler_cookiejar_show', cookiejar_uuid=cookiejar_uuid))
@crawler_splash.route('/crawler/cookiejar/delete', methods=['GET']) @crawler_splash.route('/crawler/cookiejar/delete', methods=['GET'])
#@login_required @login_required
#@login_read_only @login_read_only
def crawler_cookiejar_delete(): def crawler_cookiejar_delete():
user_id = current_user.get_id() user_id = current_user.get_id()
cookiejar_uuid = request.args.get('cookiejar_uuid') cookiejar_uuid = request.args.get('cookiejar_uuid')

View file

@ -182,6 +182,20 @@ def add_object_id_to_export():
# redirect # redirect
return redirect(url_for('import_export.export_object')) return redirect(url_for('import_export.export_object'))
@import_export.route("/import_export/investigation", methods=['GET'])
@login_required
@login_analyst
def export_investigation():
investigation_uuid = request.args.get("uuid")
if MispExport.ping_misp():
event_metadata = MispExport.create_investigation_event(investigation_uuid)
else:
return Response(json.dumps({"error": "Can't reach MISP Instance"}, indent=2, sort_keys=True), mimetype='application/json'), 400
return redirect(url_for('investigations_b.show_investigation', uuid=investigation_uuid))
# @import_export.route("/import_export/delete_object_id_to_export", methods=['GET']) # @import_export.route("/import_export/delete_object_id_to_export", methods=['GET'])
# @login_required # @login_required
# @login_analyst # @login_analyst

View file

@ -0,0 +1,212 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
'''
Blueprint Flask: ail_investigations
'''
import os
import sys
import json
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file
from flask_login import login_required, current_user
# Import Role_Manager
from Role_Manager import login_admin, login_analyst, login_read_only
sys.path.append('modules')
import Flask_config
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
import Investigations
from lib.objects import ail_objects
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
import Tag
# ============ BLUEPRINT ============
investigations_b = Blueprint('investigations_b', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/investigations'))
# ============ VARIABLES ============
bootstrap_label = Flask_config.bootstrap_label
# ============ FUNCTIONS ============
def create_json_response(data, status_code):
return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code
# ============= ROUTES ==============
@investigations_b.route("/investigations", methods=['GET'])
@login_required
@login_read_only
def investigations_dashboard():
investigations = Investigations.get_all_investigations_meta(r_str=True)
return render_template("investigations.html", bootstrap_label=bootstrap_label,
investigations=investigations)
@investigations_b.route("/investigation", methods=['GET']) ## FIXME: add /view ????
@login_required
@login_read_only
def show_investigation():
investigation_uuid = request.args.get("uuid")
investigation = Investigations.Investigation(investigation_uuid)
metadata = investigation.get_metadata(r_str=True)
objs = ail_objects.get_objects_meta(investigation.get_objects(), icon=True)
return render_template("view_investigation.html", bootstrap_label=bootstrap_label,
metadata=metadata, investigation_objs=objs)
@investigations_b.route("/investigation/add", methods=['GET', 'POST'])
@login_required
@login_read_only
def add_investigation():
if request.method == 'POST':
user_id = current_user.get_id()
name = request.form.get("investigation_name")
date = request.form.get("investigation_date")
threat_level = request.form.get("threat_level")
analysis = request.form.get("analysis")
info = request.form.get("investigation_info")
# tags
taxonomies_tags = request.form.get('taxonomies_tags')
if taxonomies_tags:
try:
taxonomies_tags = json.loads(taxonomies_tags)
except Exception:
taxonomies_tags = []
else:
taxonomies_tags = []
galaxies_tags = request.form.get('galaxies_tags')
if galaxies_tags:
try:
galaxies_tags = json.loads(galaxies_tags)
except Exception:
galaxies_tags = []
tags = taxonomies_tags + galaxies_tags
input_dict = {"user_id": user_id, "name": name,
"threat_level": threat_level, "date": date,
"analysis": analysis, "info": info, "tags": tags}
res = Investigations.api_add_investigation(input_dict)
if res[1] != 200:
return create_json_response(res[0], res[1])
return redirect(url_for('investigations_b.show_investigation', uuid=res[0]))
else:
return render_template("add_investigation.html", tags_selector_data=Tag.get_tags_selector_data())
@investigations_b.route("/investigation/edit", methods=['GET', 'POST'])
@login_required
@login_read_only
def edit_investigation():
if request.method == 'POST':
user_id = current_user.get_id()
investigation_uuid = request.form.get("investigation_uuid")
name = request.form.get("investigation_name")
date = request.form.get("investigation_date")
threat_level = request.form.get("threat_level")
analysis = request.form.get("analysis")
info = request.form.get("investigation_info")
# tags
taxonomies_tags = request.form.get('taxonomies_tags')
if taxonomies_tags:
try:
taxonomies_tags = json.loads(taxonomies_tags)
except Exception:
taxonomies_tags = []
else:
taxonomies_tags = []
galaxies_tags = request.form.get('galaxies_tags')
if galaxies_tags:
try:
galaxies_tags = json.loads(galaxies_tags)
except Exception:
galaxies_tags = []
tags = taxonomies_tags + galaxies_tags
input_dict = {"user_id": user_id, "uuid": investigation_uuid,
"name": name, "threat_level": threat_level,
"analysis": analysis, "info": info, "tags": tags}
res = Investigations.api_edit_investigation(input_dict)
if res[1] != 200:
return create_json_response(res[0], res[1])
return redirect(url_for('investigations_b.show_investigation', uuid=res[0]))
else:
investigation_uuid = request.args.get('uuid')
investigation = Investigations.Investigation(investigation_uuid)
metadata = investigation.get_metadata(r_str=False)
taxonomies_tags, galaxies_tags = Tag.sort_tags_taxonomies_galaxies(metadata['tags'])
tags_selector_data = Tag.get_tags_selector_data()
tags_selector_data['taxonomies_tags'] = taxonomies_tags
tags_selector_data['galaxies_tags'] = galaxies_tags
return render_template("add_investigation.html", edit=True,
tags_selector_data=tags_selector_data, metadata=metadata)
@investigations_b.route("/investigation/delete", methods=['GET'])
@login_required
@login_read_only
def delete_investigation():
investigation_uuid = request.args.get('uuid')
input_dict = {"uuid": investigation_uuid}
res = Investigations.api_delete_investigation(input_dict)
if res[1] != 200:
return create_json_response(res[0], res[1])
return redirect(url_for('investigations_b.investigations_dashboard'))
@investigations_b.route("/investigation/object/register", methods=['GET'])
@login_required
@login_read_only
def register_investigation():
investigations_uuid = request.args.get('uuids')
investigations_uuid = investigations_uuid.split(',')
object_type = request.args.get('type')
object_subtype = request.args.get('subtype')
object_id = request.args.get('id')
for investigation_uuid in investigations_uuid:
input_dict = {"uuid": investigation_uuid, "id": object_id,
"type": object_type, "subtype": object_subtype}
res = Investigations.api_register_object(input_dict)
if res[1] != 200:
return create_json_response(res[0], res[1])
return redirect(url_for('investigations_b.investigations_dashboard', uuid=investigation_uuid))
@investigations_b.route("/investigation/object/unregister", methods=['GET'])
@login_required
@login_read_only
def unregister_investigation():
investigation_uuid = request.args.get('uuid')
object_type = request.args.get('type')
object_subtype = request.args.get('subtype')
object_id = request.args.get('id')
input_dict = {"uuid": investigation_uuid, "id": object_id,
"type": object_type, "subtype": object_subtype}
res = Investigations.api_unregister_object(input_dict)
if res[1] != 200:
return create_json_response(res[0], res[1])
return redirect(url_for('investigations_b.show_investigation', uuid=investigation_uuid))
@investigations_b.route("/investigation/all/selector_json")
@login_required
@login_read_only
def get_investigations_selector_json():
return jsonify(Investigations.get_investigations_selector())
#
# @investigations_b.route("/object/item") #completely shows the paste in a new tab
# @login_required
# @login_analyst
# def showItem(): # # TODO: support post
# item_id = request.args.get('id')
# if not item_id or not Item.exist_item(item_id):
# abort(404)
#
# return render_template("show_item.html", bootstrap_label=bootstrap_label)

View file

@ -288,5 +288,11 @@ def passive_dns_change_state():
passivedns_enabled = d4.change_passive_dns_state(new_state) passivedns_enabled = d4.change_passive_dns_state(new_state)
return redirect(url_for('settings.passive_dns')) return redirect(url_for('settings.passive_dns'))
@settings.route("/settings/ail", methods=['GET'])
@login_required
@login_admin
def ail_configs():
return render_template("ail_configs.html", passivedns_enabled=None)
# ========= REGISTRATION ========= # ========= REGISTRATION =========
app.register_blueprint(settings, url_prefix=baseUrl) app.register_blueprint(settings, url_prefix=baseUrl)

View file

@ -0,0 +1,93 @@
<!DOCTYPE html>
<html>
<head>
<title>Passive DNS - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap4.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'settings/menu_sidebar.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div class="card my-3">
<div class="card-body">
<div class="container-fluid">
<div class="row">
<div class="col-12 col-lg-4">
<a href="https://www.misp-project.org/">
<img src="{{ url_for('static', filename='image/misp-logo.png')}}" alt="MISP project">
</a>
</div>
<div class="col-12 col-lg-8">
<form>
<div class="form-group">
<label for="misp_url">Url</label>
<input class="form-control" type="text" id="misp_url" name="misp_url" placeholder="URL of the MISP instance">
</div>
<div class="form-group">
<label for="misp_key">Key</label>
<input class="form-control" type="text" id="misp_key" name="misp_key" placeholder="API key of the user you want to use">
</div>
<div class="form-group">
<label for="misp_key">Org UUID</label>
<input class="form-control" type="text" id="org_uuid" name="org_uuid" placeholder="Org UUID">
</div>
<button type="submit" class="btn btn-primary">Update</button>
</form>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</body>
<script>
$(document).ready(function(){
$("#nav_settings").removeClass("text-muted");
$("#nav_ail_configs").addClass("active");
} );
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>
</html>

View file

@ -5,3 +5,16 @@
padding-left: 0.15em; padding-left: 0.15em;
background-color: #2e5; background-color: #2e5;
} }
.blue {
color: #0088cc !important;
}
.bold {
font-weight: bold;
}
.object_node_icon {
font-size: 16px;
pointer-events: none;
}

View file

@ -78,6 +78,14 @@
<a class="btn btn-secondary" href="{{ url_for('correlation.show_correlation') }}?object_type=cryptocurrency&type_id=bitcoin&correlation_id={{ dict_object['correlation_id'] }}&expand_card=True&correlation_objects=paste">Expand Bitcoin address</a> <a class="btn btn-secondary" href="{{ url_for('correlation.show_correlation') }}?object_type=cryptocurrency&type_id=bitcoin&correlation_id={{ dict_object['correlation_id'] }}&expand_card=True&correlation_objects=paste">Expand Bitcoin address</a>
{% endif %} {% endif %}
{% endif %} {% endif %}
{% with obj_type='cryptocurrency', obj_id=dict_object['correlation_id'], obj_subtype=dict_object["metadata"]["type_id"] %}
{% include 'modals/investigations_register_obj.html' %}
{% endwith %}
<button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
<i class="fas fa-microscope"></i> Investigations
</button>
</div> </div>
</div> </div>

View file

@ -88,6 +88,14 @@
<button class='btn btn-info'><i class="fas fa-download"></i> Download Decoded file <button class='btn btn-info'><i class="fas fa-download"></i> Download Decoded file
</button> </button>
</a> </a>
{% with obj_type='decoded', obj_id=dict_object['correlation_id'], obj_subtype='' %}
{% include 'modals/investigations_register_obj.html' %}
{% endwith %}
<button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
<i class="fas fa-microscope"></i> Investigations
</button>
</div> </div>
</div> </div>

View file

@ -64,5 +64,13 @@
</li> </li>
</ul> </ul>
{% with obj_type='domain', obj_id=dict_object['correlation_id'], obj_subtype='' %}
{% include 'modals/investigations_register_obj.html' %}
{% endwith %}
<button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
<i class="fas fa-microscope"></i> Investigations
</button>
</div> </div>
</div> </div>

View file

@ -46,5 +46,13 @@
</li> </li>
</ul> </ul>
{% with obj_type='item', obj_id=dict_object['correlation_id'], obj_subtype='' %}
{% include 'modals/investigations_register_obj.html' %}
{% endwith %}
<button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
<i class="fas fa-microscope"></i> Investigations
</button>
</div> </div>
</div> </div>

View file

@ -40,6 +40,14 @@
</div> </div>
</li> </li>
</ul> </ul>
{% with obj_type='pgp', obj_id=dict_object['correlation_id'], obj_subtype=dict_object["metadata"]["type_id"] %}
{% include 'modals/investigations_register_obj.html' %}
{% endwith %}
<button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
<i class="fas fa-microscope"></i> Investigations
</button>
</div> </div>
</div> </div>

View file

@ -78,6 +78,14 @@
</div> </div>
</li> </li>
</ul> </ul>
{% with obj_type='screenshot', obj_id=dict_object['correlation_id'], obj_subtype='' %}
{% include 'modals/investigations_register_obj.html' %}
{% endwith %}
<button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
<i class="fas fa-microscope"></i> Investigations
</button>
</div> </div>
</div> </div>

View file

@ -40,6 +40,14 @@
</div> </div>
</li> </li>
</ul> </ul>
{% with obj_type='username', obj_id=dict_object['correlation_id'], obj_subtype=dict_object["metadata"]["type_id"] %}
{% include 'modals/investigations_register_obj.html' %}
{% endwith %}
<button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
<i class="fas fa-microscope"></i> Investigations
</button>
</div> </div>
</div> </div>

View file

@ -0,0 +1,214 @@
<!DOCTYPE html>
<html>
<head>
<title>AIL-Framework</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
<!-- Core CSS -->
<link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='css/ail-project.css') }}" />
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet" type="text/css" />
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'sidebars/sidebar_objects.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div class="card my-3">
<div class="card-header bg-dark text-white">
<h5 class="card-title">
{% if edit %}
Edit Investigation
{% else %}
Create Investigation
{% endif %}
</h5>
</div>
<div class="card-body">
<form action="{% if edit %}{{ url_for('investigations_b.edit_investigation') }}{% else %}{{ url_for('investigations_b.add_investigation') }}{% endif %}" method='post' onsubmit="SubmitCreateInvestigation();">
{% if edit %}
<input id="investigation_uuid" name="investigation_uuid" type="text" value="{{ metadata['uuid'] }}" hidden>
{% endif %}
<div class="row">
<div class="col-12 col-xl-9">
<div class="input-group mb-2 mr-sm-2">
<div class="input-group-prepend">
<div class="input-group-text bg-dark text-white"><i class="fas fa-quote-right"></i></div>
</div>
<input id="investigation_info" name="investigation_info" class="form-control" placeholder="Quick Investigation Info" type="text" {% if edit %}value="{{metadata['info']}}"{% endif %} required>
</div>
<div class="row">
<div class="col-12 col-xl-6">
<div class="form-group">
<label for="threat_level">Threat Level:
<span id="threat_level_idInfoPopover" class="fas fa-info-circle" data-toggle="popover" data-trigger="hover"></span>
<script type="text/javascript">
$(function() {
$('#threat_level_idInfoPopover').popover({
html: true,
content: function() {
var tempSelector = '#threat_level';
return $('<div>').append(
$('<span>').attr('class', 'blue bold').text($(tempSelector +" option:selected").text())
).append(
$('<span>').text(': ' + fieldDesc[$(tempSelector).val()])
);
}
});
var fieldDesc = {"1":"*high* means sophisticated APT malware or 0-day attack","2":"*medium* means APT malware","3":"*low* means mass-malware","4":"*undefined* no risk"};
});
</script>
</label>
<select class="form-control" id="threat_level" name="threat_level" required>
<option value="1">High</option>
<option value="2">Medium</option>
<option value="3">Low</option>
<option value="4">Undefined</option>
</select>
</div>
</div>
<div class="col-12 col-xl-6">
<div class="form-group">
<label for="analysis">Analysis:
<span id="analysis_idInfoPopover" class="fas fa-info-circle" data-toggle="popover" data-trigger="hover"></span>
<script type="text/javascript">
$(function() {
$('#analysis_idInfoPopover').popover({
html: true,
content: function() {
var tempSelector = '#analysis';
return $('<div>').append(
$('<span>').attr('class', 'blue bold').text($(tempSelector +" option:selected").text())
).append(
$('<span>').text(': ' + fieldDesc[$(tempSelector).val()])
);
}
});
var fieldDesc = {"0":"Investigation has just been created and is in an initial state","1":"The analysis is still ongoing","2":"The Investigation creator considers the analysis complete"};
});
</script>
</label>
<select class="form-control" id="analysis" name="analysis" required>
<option value="0">Initial</option>
<option value="1">Ongoing</option>
<option value="2">Completed</option>
</select>
</div>
</div>
</div>
<div class="input-group" id="ivestigation-date">
<div class="input-group-prepend"><span class="input-group-text bg-secondary text-white"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
<input class="form-control" id="ivestigation-date-input" placeholder="yyyy-mm-dd" name="investigation_date" autocomplete="off" required>
</div>
<div class="card my-4">
<div class="card-header bg-secondary text-white">
<b>Tags</b>
</div>
<div class="card-body">
{% include 'tags/block_tags_selector.html' %}
</div>
</div>
</div>
<div class="col-12 col-xl-3">
{% if edit %}
Edit Investigation
{% else %}
Create a new Investigation
{% endif %}
</div>
</div>
<br>
<button class="btn btn-primary mt-2">
{% if edit %}
<i class="fas fa-pencil-alt"></i> Edit Investigation
{% else %}
<i class="fas fa-plus"></i> Create Investigation
{% endif %}
</button>
</form>
</div>
</div>
</div>
</div>
</div>
</body>
<script>
$(document).ready(function(){
$('#nav_investigation').removeClass("text-muted");
$("#nav_add_investigation").addClass("active");
{% if edit %}
var investigation_date = "{{metadata['date']}}";
{% else %}
var investigation_date = getTodayDate();
{% endif %}
$('#ivestigation-date-input').val(investigation_date);
$('#ivestigation-date-input').dateRangePicker({
autoClose: true,
singleDate : true,
showShortcuts: false,
startDate: false,
});
{% if edit %}
$('#threat_level').val({{metadata['threat_level']}});
$('#analysis').val({{metadata['analysis']}});
{% endif %}
});
function SubmitCreateInvestigation() {
var tags = ltags.getValue();
var tagsgalaxy = ltagsgalaxies.getValue();
$('#ltags').val(tags);
$('#ltagsgalaxies').val(tagsgalaxy);
return true;
}
function getTodayDate() {
var date = new Date();
var month = date.getMonth()+1;
var day = date.getDate();
return date.getFullYear() +'-'+ (month<10 ? '0' : '') + month +'-'+ (day<10 ? '0' : '') + day;
}
</script>

View file

@ -0,0 +1,90 @@
<!DOCTYPE html>
<html>
<head>
<title>Investigations</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'sidebars/sidebar_objects.html' %}
<div class="col-12 col-lg-10" id="core_content">
<h3 class="mt-2 text-secondary">
<i class="fas fa-microscope"></i> Investigations:
</h3>
<table id="table_investigation" class="table table-striped border-primary">
<thead class="bg-dark text-white">
<tr>
<th>Name</th>
<th>Date</th>
<th>last modified</th>
<td>Info</td>
<th>Nb Objects</th>
</tr>
</thead>
<tbody style="font-size: 15px;">
{% for dict_investigation in investigations %}
<tr class="border-color: blue;">
<td>
<a href="{{ url_for('investigations_b.show_investigation') }}?uuid={{ dict_investigation['uuid'] }}">
{{ dict_investigation['info']}}
<div>
{% for tag in dict_investigation['tags'] %}
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
{% endfor %}
</div>
</a>
</td>
<td>{{ dict_investigation['date']}}</td>
<td>{{ dict_investigation['last_change']}}</td>
<td>{{ dict_investigation['info']}}</td>
<td>{{ dict_investigation['nb_objects']}}</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
</div>
</body>
<script>
$(document).ready(function(){
$('#nav_sync').removeClass("text-muted");
$("#navsync_queues").addClass("active");
$('#table_investigation').DataTable({
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
"iDisplayLength": 10,
"order": [[ 2, "desc" ]]
});
});
</script>

View file

@ -0,0 +1,178 @@
<!DOCTYPE html>
<html>
<head>
<title>AIL-Framework</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/ail-project.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'sidebars/sidebar_objects.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div class="card my-1">
<div class="card-header bg-dark text-white">
<h4 class="card-title">{{metadata['info']}}</h4>
</div>
<div class="card-body">
<div class="container-fluid">
<div class="row">
<div class="col-12 col-lg-6">
<table class="table table-hover">
<tr>
<th style="width:30%">UUID</th>
<td>{{metadata['uuid']}}</td>
</tr>
<tr>
<th>Creator</th>
<td>{{metadata['user_creator']}}</td>
</tr>
<tr>
<th>Tags</th>
<td>
{% for tag in metadata['tags'] %}
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
{% endfor %}
</td>
</tr>
<tr>
<th>Date</th>
<td>{{metadata['date']}}</td>
</tr>
<tr>
<th>Threat Level</th>
<td>{{metadata['threat_level']}}</td>
</tr>
<tr>
<th>Analysis</th>
<td>{{metadata['analysis']}}</td>
</tr>
<tr>
<th>Info</th>
<td>{{metadata['info']}}</td>
</tr>
<tr>
<th># Objects</th>
<td>{{metadata['nb_objects']}}</td>
</tr>
<tr>
<th>Timestamp</th>
<td>{{metadata['timestamp']}}</td>
</tr>
<tr>
<th>Last change</th>
<td>{{metadata['last_change']}}</td>
</tr>
</table>
</div>
<div class="col-12 col-lg-6">
<div class="my-4">
<a href="{{ url_for('investigations_b.delete_investigation') }}?uuid={{metadata['uuid']}}">
<button type="button" class="btn btn-danger">
<i class="fas fa-trash-alt"></i> <b>Delete</b>
</button>
</a>
<a href="{{ url_for('investigations_b.edit_investigation') }}?uuid={{metadata['uuid']}}">
<button type="button" class="btn btn-info">
<i class="fas fa-pencil-alt"></i> <b>Edit</b>
</button>
</a>
<a class="btn btn-outline-dark" target="_blank" href="{{ url_for('import_export.export_investigation')}}?uuid={{metadata['uuid']}}">
<img id="misp-logo" src="{{ url_for('static', filename='image/misp-logo.png')}}" height="25">
Export as Event
</a>
</div>
</div>
</div>
</div>
</div>
</div>
<h3>Objects</h3>
<table id="table_sync_queues" class="table table-striped border-primary">
<thead class="bg-dark text-white">
<tr>
<th>Type</th>
<th></th>
<th>Id</th>
<th>Tags</th>
<th></th>
</tr>
</thead>
<tbody style="font-size: 15px;">
{% for object in investigation_objs %}
<tr class="border-color: blue;">
<td>
{% with style=object['icon']['style'], icon=object['icon']['icon'] , color=object['icon']['color'] %}
{% include 'objects/obj_svg_block.html' %}
{% endwith %}
{{ object['type']}}
</td>
<td>
{% if object['subtype'] %}
{{ object['subtype']}}
{% endif %}
</td>
<td>{{ object['id']}}</td>
<td>
{% for tag in object['tags'] %}
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
{% endfor %}
</td>
<td class="text-right">
<a href="{{ url_for('investigations_b.unregister_investigation') }}?uuid={{ metadata['uuid']}}&type={{ object['type'] }}&subtype={{ object['subtype']}}&id={{ object['id']}}">
<button type="button" class="btn btn-danger"><i class="fas fa-trash-alt"></i></button>
</a>
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
</body>
<script>
$(document).ready(function(){
$('#nav_sync').removeClass("text-muted");
$('#table_sync_queues').DataTable({
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
"iDisplayLength": 10,
"order": [[ 0, "asc" ]]
});
});
</script>

View file

@ -0,0 +1,62 @@
<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet" type="text/css" />
<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
<div id="investigations_register_obj_modal" class="modal fade" role="dialog">
<div class="modal-dialog modal-lg">
<div id="investigations_register_obj_content" class="modal-content">
<div class="modal-header" style="border-bottom: 4px solid #cccccc; background-color: #cccccc; color: #ffffff;">
<h4>Add to Investigations</h4>
</div>
<div class="modal-body">
<div class="input-group" >
<input id="linvestigations" type="text" class="form-control" autocomplete="off" style="width: 760px">
</div>
</div>
<div class="modal-footer">
<a class="btn btn-info" href="{{ url_for('investigations_b.add_investigation') }}" target="_blank">
<i class="fas fa-microscope"></i>
<span class="label-icon">Create Investigation </span>
</a>
<button class="btn btn-primary" onclick="Register_Obj()">
<i class="fas fa-plus"></i>
Add to Investigations
</button>
<button type="button" class="btn btn-outline-dark" data-dismiss="modal" >Close</button>
</div>
</div>
</div>
</div>
<script>
var linvestigations;
$('#investigations_register_obj_modal').on('shown.bs.modal', function () {
$.getJSON("{{ url_for('investigations_b.get_investigations_selector_json') }}",
function(data) {
console.log(data);
linvestigations = $('#linvestigations').tagSuggest({
data: data,
maxDropHeight: 200,
name: 'linvestigations',
emptyText: 'Select Investigations'
});
}
);
});
function Register_Obj() {
var uuids = linvestigations.getValue();
// TODO: REQUEST
window.location.replace("{{ url_for('investigations_b.register_investigation') }}?uuids=" + uuids + "&type={{ obj_type }}&subtype={{ obj_subtype }}&id={{ obj_id }}");
}
</script>

View file

@ -97,6 +97,16 @@
</button> </button>
</a> </a>
</div> </div>
<div>
{% with obj_type='item', obj_id=dict_item['id'], obj_subtype=''%}
{% include 'modals/investigations_register_obj.html' %}
{% endwith %}
<div class="mr-2">
<button type="button" class="btn btn-lg btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
<i class="fas fa-microscope"></i> Investigations
</button>
</div>
</div>
<div class="mx-2"> <div class="mx-2">
{% with obj_type='item', obj_id=dict_item['id'], obj_lvl=0%} {% with obj_type='item', obj_id=dict_item['id'], obj_lvl=0%}
{% include 'import_export/block_add_user_object_to_export.html' %} {% include 'import_export/block_add_user_object_to_export.html' %}

View file

@ -0,0 +1,6 @@
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="{{ color }}"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="object_node_icon {{ style }}">{{ icon }}</text>
</g>
</svg>

After

Width:  |  Height:  |  Size: 249 B

View file

@ -44,7 +44,13 @@
<span>Settings</span> <span>Settings</span>
</h5> </h5>
<ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills--> <ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills-->
<li class="nav-item"> <!-- <li class="nav-item">
<a class="nav-link" href="{{url_for('settings.ail_configs')}}" id="nav_ail_configs">
<img src="{{ url_for('static', filename='image/ail.png')}}" alt="AIL servers" style="width:25px;">
<span>AIL Configs</span>
</a>
</li> -->
<li class="nav-item">
<a class="nav-link" href="{{url_for('settings.passive_dns')}}" id="passive_dns"> <a class="nav-link" href="{{url_for('settings.passive_dns')}}" id="passive_dns">
<img src="{{ url_for('static', filename='image/d4-logo.png')}}" alt="D4 project" style="width:25px;"> <img src="{{ url_for('static', filename='image/d4-logo.png')}}" alt="D4 project" style="width:25px;">
<span>Passive DNS</span> <span>Passive DNS</span>

View file

@ -0,0 +1,94 @@
<div class="col-12 col-lg-2 p-0 bg-light border-right" id="side_menu">
<button type="button" class="btn btn-outline-secondary mt-1 ml-3" onclick="toggle_sidebar()">
<i class="fas fa-align-left"></i>
<span>Toggle Sidebar</span>
</button>
<nav class="navbar navbar-expand navbar-light bg-light flex-md-column flex-row align-items-start py-2" id="nav_investigation">
<h5 class="d-flex text-muted w-100">
<span>Investigations</span>
</h5>
<ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100 mb-4">
<li class="nav-item">
<a class="nav-link" href="{{url_for('investigations_b.investigations_dashboard')}}" id="nav_investigation_dashboard">
<i class="fas fa-microscope"></i>
<span>Investigations</span>
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="{{url_for('investigations_b.add_investigation')}}" id="nav_add_investigation">
<i class="fas fa-plus"></i>
<span>Add Investigation</span>
</a>
</li>
</ul>
<h5 class="d-flex text-muted w-100">
<span>Objects</span>
</h5>
<ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100 mb-4">
<li class="nav-item">
<a class="nav-link" href="{{url_for('hashDecoded.hashDecoded_page')}}" id="nav_dashboard">
<i class="fas fa-lock-open"></i>
<span>Decoded</span>
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="{{url_for('hashDecoded.pgpdump_page')}}" id="nav_dashboard_pgpdump">
<i class="fas fa-key"></i>
<span>PGP Dumps</span>
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="{{url_for('hashDecoded.cryptocurrency_page')}}" id="nav_dashboard_cryptocurrency">
<i class="fas fa-coins"></i>
<span>Cryptocurrency</span>
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="{{url_for('hashDecoded.username_page')}}" id="nav_dashboard_username">
<i class="fas fa-user"></i>
<span>Username</span>
</a>
</li>
</ul>
<h5 class="d-flex text-muted w-100">
<span>
<img src="{{ url_for('static', filename='image/misp-logo.png')}}" alt="MISP" style="width:80px;">
Format
</span>
</h5>
<ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100">
<li class="nav-item">
<a class="nav-link" href="{{url_for('import_export.import_object')}}" id="nav_misp_import">
<b>Import</b>
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="{{url_for('import_export.export_object')}}" id="nav_misp_export">
<b>Export</b>
</a>
</li>
</ul>
</nav>
</div>
<script>
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>