AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-10 00:28:22 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

chg: [MISP export + redis] refactor MISP, export all relationships between objects, relationships between investigated objects + bump redis-py version + theHive export

Browse source
This commit is contained in:
Terrtia 2023-01-16 16:27:49 +01:00
parent 7fa19efc1e
commit 0e41c95b5c
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
28 changed files with 775 additions and 1013 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
bin/DB_KVROCKS_MIGRATION.py
View file

@ -564,7 +564,7 @@ def domain_migration():
for history in get_domain_history_by_port(domain_type, dom, port):
epoch = history['epoch']
# DOMAIN DOWN
if not history.get('status'): # domain DOWN
if not history.get('status'): # domain DOWN
domain.add_history(epoch)
print(f'DOWN {epoch}')
# DOMAIN UP
@ -585,6 +585,7 @@ def domain_migration():
#print()
# TODO REMOVE INVALID DOMAINS
for domain_type in ['onion', 'regular']:
for date in Date.get_date_range_today('20190101'):
for dom in get_domain_down_by_date(domain_type, date):
@ -601,7 +602,7 @@ def domain_migration():
domain.update_daterange(last_check)
if last_origin:
domain.set_last_origin(last_origin)
domain.add_history(None, None, date=date)
domain.add_history(0, None, date=date)
###############################
@ -871,12 +872,12 @@ if __name__ == '__main__':
#core_migration()
#user_migration()
#tags_migration()
items_migration()
#crawler_migration()
# domain_migration() # TO TEST ###########################
# items_migration()
# crawler_migration()
domain_migration() # TO TEST ###########################
# decodeds_migration()
# screenshots_migration()
# subtypes_obj_migration()
subtypes_obj_migration()
# ail_2_ail_migration()
# trackers_migration()
# investigations_migration()

53
bin/export/AILObjects.py
View file

@ -1,53 +0,0 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import sys
import uuid
sys.path.append(os.environ['AIL_BIN'])
from lib.objects import ail_objects
from lib import ConfigLoader
config_loader = ConfigLoader.ConfigLoader()
r_serv_objects = config_loader.get_redis_conn("ARDB_Objects")
config_loader = None
def create_map_obj_uuid_golbal_id(obj_uuid, global_id):
r_serv_objects.sadd('all_object:uuid', obj_uuid)
r_serv_objects.sadd('all_object:global_id', global_id)
r_serv_objects.sadd('object:map:uuid_id:{}'.format(obj_uuid), global_id)
r_serv_objects.sadd('object:map:id_uuid:{}'.format(global_id), obj_uuid)
def create_map_obj_event_uuid(event_uuid, global_id):
r_serv_objects.sadd('export:all_object:event_uuid', event_uuid)
r_serv_objects.sadd('export:all_object:global_id', global_id)
r_serv_objects.sadd('object:map:event_id:{}'.format(event_uuid), global_id)
r_serv_objects.sadd('object:map:id_event:{}'.format(global_id), event_uuid)
# def get_user_list_of_obj_to_export(user_id, add_uuid=False):
# objs_to_export = []
# res = r_serv_objects.hgetall('user:all_objs_to_export:{}'.format(user_id))
# for global_id in res:
# dict_obj = Correlate_object.get_global_id_from_id(global_id)
# dict_obj['lvl'] = int(res[global_id])
# if add_uuid:
# obj_dict['uuid'] = str(uuid.uuid4())
# objs_to_export.append(dict_obj)
# return objs_to_export
def add_user_object_to_export(user_id, obj_type, obj_id, lvl, obj_subtype=None):
## TODO: check if user exist
# # TODO: check if valid object
# # TODO: check lvl
global_id = ail_objects.get_obj_global_id(obj_type, obj_subtype, obj_id)
return r_serv_objects.hset('user:all_objs_to_export:{}'.format(user_id), global_id, lvl)
def delete_user_object_to_export(user_id, obj_type, obj_id, obj_subtype=None):
## TODO: check if user exist
global_id = ail_objects.get_obj_global_id(obj_type, obj_subtype, obj_id)
r_serv_objects.hdel('user:all_objs_to_export:{}'.format(user_id), global_id)
def delete_all_user_object_to_export(user_id):
## TODO: check if user exist
r_serv_objects.delete('user:all_objs_to_export:{}'.format(user_id))

345
bin/export/Export.py
View file

@ -3,53 +3,172 @@
import os
import sys
from uuid import uuid4
import uuid
sys.path.append(os.environ['AIL_BIN'])
from lib import ConfigLoader
sys.path.append('../../configs/keys')
try:
from thehive4py.api import TheHiveApi
import thehive4py.exceptions
from theHiveKEYS import the_hive_url, the_hive_key, the_hive_verifycert
if the_hive_url == '':
is_hive_connected = False
else:
is_hive_connected = TheHiveApi(the_hive_url, the_hive_key, cert=the_hive_verifycert)
except:
is_hive_connected = False
if is_hive_connected != False:
try:
is_hive_connected.get_alert(0)
is_hive_connected = True
except thehive4py.exceptions.AlertException:
is_hive_connected = False
##################################
# Import Project packages
##################################
from lib.ConfigLoader import ConfigLoader
from lib.objects.Items import Item
from lib.ail_core import get_ail_uuid
from lib.Investigations import Investigation
from lib.objects import ail_objects
## LOAD CONFIG ##
config_loader = ConfigLoader.ConfigLoader()
r_serv_cache = config_loader.get_redis_conn("Redis_Cache")
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
config_loader = ConfigLoader()
r_cache = config_loader.get_redis_conn("Redis_Cache")
r_db = config_loader.get_db_conn("Kvrocks_DB")
r_serv_db = config_loader.get_redis_conn("ARDB_DB") ######################################
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") ######################################
config_loader = None
## -- ##
def load_tags_to_export_in_cache():
all_exports = ['misp', 'thehive']
for export_target in all_exports:
# save solo tags in cache
all_tags_to_export = Tag.get_list_of_solo_tags_to_export_by_type()
if len(all_tags_to_export) > 1:
r_serv_cache.sadd('to_export:solo_tags:{}'.format(export_target), *all_tags_to_export)
elif all_tags_to_export:
r_serv_cache.sadd('to_export:solo_tags:{}'.format(export_target), all_tags_to_export[0])
sys.path.append('../../configs/keys')
##################################
# Import Keys
##################################
from thehive4py.api import TheHiveApi
from thehive4py.models import Alert, AlertArtifact, Case, CaseObservable
import thehive4py.exceptions
# save combinaison of tags in cache
pass
from pymisp import MISPEvent, MISPObject, PyMISP
def is_hive_connected(): # # TODO: REFRACTOR, put in cache (with retry)
return is_hive_connected
##################################
# THE HIVE
##################################
HIVE_CLIENT = None
try:
from theHiveKEYS import the_hive_url, the_hive_key, the_hive_verifycert
HIVE_URL = the_hive_url
HIVE_KEY = the_hive_key
HIVE_VERIFY_CERT = the_hive_verifycert
except:
HIVE_URL = None
HIVE_KEY = None
HIVE_VERIFY_CERT = None
def get_hive_client():
global HIVE_CLIENT
try:
HIVE_CLIENT = TheHiveApi(HIVE_URL, HIVE_KEY, cert=HIVE_VERIFY_CERT)
except:
HIVE_CLIENT = None
return HIVE_CLIENT
def is_hive_connected():
try:
# print(hive_client.health())
HIVE_CLIENT.get_alert(0)
return True
except thehive4py.exceptions.AlertException:
return False
HIVE_CLIENT = get_hive_client()
def create_thehive_alert(item_id, tag_trigger):
item = Item(item_id)
meta = item.get_meta()
# TheHive expects a file
content = item.get_raw_content(decompress=True)
# remove .gz from submitted path to TheHive because we've decompressed it
if item_id.endswith(".gz"):
item_id = item_id[:-3]
# add .txt it's easier to open when downloaded from TheHive
item_id = f'{item_id}.txt'
artifacts = [
AlertArtifact(dataType='other', message='uuid-ail', data=(get_ail_uuid())),
AlertArtifact(dataType='file', data=(content, item_id), tags=meta['tags'])
]
# Prepare the sample Alert
sourceRef = str(uuid.uuid4())[0:6]
alert = Alert(title='AIL Leak',
tlp=3,
tags=meta['tags'],
description='AIL Leak, triggered by {}'.format(tag_trigger),
type='ail',
source=meta['source'], # Use item ID ?
sourceRef=sourceRef,
artifacts=artifacts)
# Create the Alert
alert_id = None
try:
response = HIVE_CLIENT.create_alert(alert)
if response.status_code == 201:
# print(json.dumps(response.json(), indent=4, sort_keys=True))
print('Alert Created')
print(response.json())
alert_id = response.json()['id']
else:
print(f'ko: {response.status_code}/{response.text}')
return 0
except:
print('hive connection error')
print(alert_id)
# TODO SAVE CASE URL ????????????????????????
def create_thehive_case(item_id, title=None, tlp=2, threat_level=2, description=None):
item = Item(item_id)
ail_uuid = get_ail_uuid()
if not title:
title = f'AIL Case {item.id}'
if not description:
description = f'AIL {ail_uuid} Case'
date = item.get_date()
date = f'{date[0:4]}-{date[4:6]}-{date[6:8]}'
tags = item.get_tags(r_list=True)
case = Case(title=title,
tlp=tlp,
severity=threat_level,
flag=False,
tags=tags,
description=description)
# Create Case
response = get_hive_client().create_case(case)
if response.status_code == 201:
case_id = response.json()['id']
observables = [
CaseObservable(dataType="other", data=[ail_uuid], message="uuid-ail"),
CaseObservable(dataType="file", data=item.get_filename(), tags=tags),
CaseObservable(dataType="other", data=[item.get_source()], message="source"),
CaseObservable(dataType="other", data=[date], message="last-seen")
]
for observable in observables:
resp = HIVE_CLIENT.create_case_observable(case_id, observable)
if resp.status_code != 201:
print(f'error observable creation: {resp.status_code}/{resp.text}')
# print(case_id)
# return HIVE_URL /thehive/cases/~37040/details
return case_id
# r_serv_metadata.set('hive_cases:'+path, id)
else:
print(f'ko: {response.status_code}/{response.text}')
return None
def get_case_url(case_id):
return f'{HIVE_URL}/cases/{case_id}/details'
# TODO
def get_item_hive_cases(item_id):
hive_case = r_serv_metadata.get('hive_cases:{}'.format(item_id))
if hive_case:
@ -57,6 +176,150 @@ def get_item_hive_cases(item_id):
return hive_case
##################################
# MISP
##################################
try:
from mispKEYS import misp_url, misp_key, misp_verifycert
MISP_URL = misp_url
MISP_KEY = misp_key
MISP_VERIFYCERT = misp_verifycert
if MISP_URL.endswith('/'):
MISP_URL = MISP_URL[:-1]
except:
MISP_URL = None
MISP_KEY = None
MISP_VERIFYCERT = None
def get_misp_client():
return PyMISP(misp_url, misp_key, misp_verifycert)
# # TODO: return error
def ping_misp():
try:
PyMISP(misp_url, misp_key, misp_verifycert)
return True
except Exception as e:
print(e)
return False
def sanitize_misp_event_distribution(distribution):
try:
int(distribution)
if 0 <= distribution <= 3:
return distribution
else:
return 0
except:
return 0
def sanitize_misp_event_threat_level(threat_level):
try:
int(threat_level)
if 1 <= threat_level <= 4:
return threat_level
else:
return 4
except:
return 4
def sanitize_misp_event_analysis(analysis):
try:
int(analysis)
if 0 <= analysis <= 2:
return analysis
else:
return 0
except:
return 0
def create_misp_event(objs, event_uuid=None, date=None, distribution=0, threat_level_id=4, publish=False, analysis=0,
info=None, tags=None, export=False):
if tags is None:
tags = []
event = MISPEvent()
if not event_uuid:
event_uuid = str(uuid.uuid4())
event.uuid = event_uuid
if date:
event.date = date
if not info:
info = 'AIL framework export'
event.info = info
if publish:
event.publish()
for tag in tags:
event.add_tag(tag)
event.distribution = sanitize_misp_event_distribution(distribution)
event.threat_level_id = sanitize_misp_event_threat_level(threat_level_id)
event.analysis = sanitize_misp_event_analysis(analysis)
misp_objects = ail_objects.get_misp_objects(objs)
for obj in misp_objects:
event.add_object(obj)
# print(event.to_json())
if export:
misp = get_misp_client()
misp_event = misp.add_event(event)
# TODO: handle error
misp_event['url'] = f'{MISP_URL}/events/view/{misp_event["Event"]["uuid"]}'
return misp_event
else:
return {'uuid': event['uuid'], 'event': event.to_json()}
def create_investigation_misp_event(investigation_uuid):
investigation = Investigation(investigation_uuid)
objs = ail_objects.get_objects(investigation.get_objects())
event = create_misp_event(objs,
# event_uuid=investigation.get_uuid(separator=True), # TODO EDIT EXISTING EVENT ????
date=investigation.get_date(),
distribution=0,
threat_level_id=investigation.get_threat_level(),
analysis=investigation.get_analysis(),
info=investigation.get_info(),
tags=investigation.get_tags(),
export=True)
url = event['url']
if url:
investigation.add_misp_events(url)
return url
def get_user_misp_objects_to_export(user_id):
objs = []
objects = r_db.hgetall(f'user:obj:misp:export:{user_id}')
for obj in objects:
obj_type, obj_subtype, obj_id = obj.split(':', 2)
lvl = objects[obj]
try:
lvl = int(lvl)
except:
lvl = 0
objs.append({'type': obj_type, 'subtype': obj_subtype, 'id': obj_id, 'lvl': lvl})
return objs
def add_user_misp_object_to_export(user_id, obj_type, obj_subtype, obj_id, lvl=0):
if not obj_subtype:
obj_subtype = ''
r_db.hset(f'user:obj:misp:export:{user_id}', f'{obj_type}:{obj_subtype}:{obj_id}', lvl)
def delete_user_misp_object_to_export(user_id, obj_type, obj_subtype, obj_id):
r_db.hdel(f'user:obj:misp:export:{user_id}', f'{obj_type}:{obj_subtype}:{obj_id}')
def delete_user_misp_objects_to_export(user_id):
r_db.delete(f'user:obj:misp:export:{user_id}')
###########################################################
# # set default
# if r_serv_db.get('hive:auto-alerts') is None:
@ -64,3 +327,11 @@ def get_item_hive_cases(item_id):
#
# if r_serv_db.get('misp:auto-events') is None:
# r_serv_db.set('misp:auto-events', 0)
# if __name__ == '__main__':
# from lib.objects.Cves import Cve
# create_misp_event([Item('crawled/2020/09/14/circl.lu0f4976a4-dda4-4189-ba11-6618c4a8c951'),
# Cve('CVE-2020-16856'), Cve('CVE-2014-6585'), Cve('CVE-2015-0383'),
# Cve('CVE-2015-0410')])
# create_investigation_misp_event('c6bbf8fa9ead4cc698eaeb07835cca5d)

329
bin/export/MispExport.py
View file

@ -1,329 +0,0 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import io
import sys
import uuid
sys.path.append(os.environ['AIL_BIN'])
from lib.objects import ail_objects
from export import AILObjects
from lib.Investigations import Investigation
# # TODO: # FIXME: REFACTOR ME => use UI/Global config
sys.path.append('../../configs/keys')
try:
from mispKEYS import misp_url, misp_key, misp_verifycert
except:
misp_url = ''
misp_key = ''
misp_verifycert = False
# MISP
from pymisp import MISPEvent, MISPObject, PyMISP
def is_valid_obj_to_export(obj_type, obj_subtype, obj_id):
if not ail_objects.is_valid_object_type(obj_type):
return False
if not ail_objects.is_valid_object_subtype(obj_type, obj_subtype):
return False
if not ail_objects.exists_obj(obj_type, obj_subtype, obj_id):
return False
return True
def sanitize_obj_export_lvl(lvl):
try:
lvl = int(lvl)
except:
lvl = 0
return lvl
def get_export_filename(json_content):
return 'ail_export_{}.json'.format(json_content.uuid)
def create_in_memory_file(json_content):
return io.BytesIO(json_content.encode())
def add_relation_ship_to_create(set_relationship, dict_obj, dict_new_obj):
global_id = ail_objects.get_obj_global_id(dict_obj['type'], dict_obj.get('subtype', ''), dict_obj['id'])
global_id_new = ail_objects.get_obj_global_id(dict_new_obj['type'], dict_new_obj.get('subtype', ''), dict_new_obj['id'])
if global_id > global_id_new:
res = (global_id, global_id_new)
else:
res = (global_id_new, global_id)
set_relationship.add( res )
# # TODO: add action by obj type
# ex => Domain
def add_obj_to_create(all_obj_to_export, set_relationship, dict_obj):
all_obj_to_export.add(ail_objects.get_obj_global_id(dict_obj['type'], dict_obj.get('subtype', ''), dict_obj['id']))
def add_obj_to_create_by_lvl(all_obj_to_export, set_relationship, dict_obj, lvl):
# # TODO: filter by export mode or filter on all global ?
if lvl >= 0:
add_obj_to_create(all_obj_to_export, add_obj_to_create, dict_obj)
if lvl > 0:
lvl = lvl - 1
# # TODO: filter by correlation types
obj_correlations = ail_objects.get_obj_correlations(dict_obj['type'], dict_obj.get('subtype', ''), dict_obj['id'])
for obj_type in obj_correlations:
dict_new_obj = {'type': obj_type}
if obj_type=='pgp' or obj_type=='cryptocurrency' or obj_type=='username':
for subtype in obj_correlations[obj_type]:
dict_new_obj['subtype'] = subtype
for obj_id in obj_correlations[obj_type][subtype]:
dict_new_obj['id'] = obj_id
add_obj_to_create_by_lvl(all_obj_to_export, set_relationship, dict_new_obj, lvl)
add_relation_ship_to_create(set_relationship, dict_obj, dict_new_obj)
else:
for obj_id in obj_correlations[obj_type]:
dict_new_obj['id'] = obj_id
add_obj_to_create_by_lvl(all_obj_to_export, set_relationship, dict_new_obj, lvl)
add_relation_ship_to_create(set_relationship, dict_obj, dict_new_obj)
add_obj_to_create_by_lvl(all_obj_to_export, set_relationship, dict_obj, lvl)
def create_list_of_objs_to_export(l_obj, r_type='json'):
all_obj_to_export = set()
set_relationship = set()
for obj in l_obj:
add_obj_to_create_by_lvl(all_obj_to_export, set_relationship, obj, obj.get('lvl', 1))
# create MISP objects
dict_misp_obj = create_all_misp_obj(all_obj_to_export, set_relationship)
# create object relationships
for obj_global_id_1, obj_global_id_2 in set_relationship:
dict_relationship = get_relationship_between_global_obj(obj_global_id_1, obj_global_id_2)
if dict_relationship:
obj_src = dict_misp_obj[dict_relationship['src']]
obj_dest = dict_misp_obj[dict_relationship['dest']]
obj_src.add_reference(obj_dest.uuid, dict_relationship['relation'], 'add a comment')
event = MISPEvent()
event.info = 'AIL framework export'
event.uuid = str(uuid.uuid4())
for obj_global_id in dict_misp_obj:
misp_obj = dict_misp_obj[obj_global_id]
AILObjects.create_map_obj_event_uuid(event.uuid, obj_global_id)
AILObjects.create_map_obj_uuid_golbal_id(event.uuid, obj_global_id)
if misp_obj:
# add object to event
event.add_object(dict_misp_obj[obj_global_id])
return event
# TODO REFACTOR ME
def create_all_misp_obj(all_obj_to_export, set_relationship):
dict_misp_obj = {}
for obj_global_id in all_obj_to_export:
obj_type, obj_id = obj_global_id.split(':', 1)
dict_misp_obj[obj_global_id] = create_misp_obj(obj_type, obj_id)
return dict_misp_obj
# TODO REFACTOR ME
def create_misp_obj(obj_type, obj_id):
if obj_type in ['cryptocurrency', 'pgp', 'username']:
obj_subtype, obj_id = obj_id.split(':', 1)
else:
obj_subtype = ''
misp_obj = ail_objects.get_misp_object(obj_type, obj_subtype, obj_id)
return misp_obj
def get_relationship_between_global_obj(obj_global_id_1, obj_global_id_2):
obj_type_1 = obj_global_id_1.split(':', 1)[0]
obj_type_2 = obj_global_id_2.split(':', 1)[0]
type_tuple = [obj_type_1, obj_type_2]
if 'image' in type_tuple: # or screenshot ## TODO:
if obj_type_1 == 'image':
src = obj_global_id_1
dest = obj_global_id_2
else:
src = obj_global_id_2
dest = obj_global_id_1
return {'relation': 'screenshot-of', 'src': src, 'dest': dest}
elif 'decoded' in type_tuple:
if obj_type_1 == 'decoded':
src = obj_global_id_1
dest = obj_global_id_2
else:
src = obj_global_id_2
dest = obj_global_id_1
return {'relation': 'included-in', 'src': src, 'dest': dest}
elif 'pgp' in type_tuple:
if obj_type_1 == 'pgp':
src = obj_global_id_1
dest = obj_global_id_2
else:
src = obj_global_id_2
dest = obj_global_id_1
return {'relation': 'extracted-from', 'src': src, 'dest': dest}
elif 'cryptocurrency':
if obj_type_1 == 'cryptocurrency':
src = obj_global_id_1
dest = obj_global_id_2
else:
src = obj_global_id_2
dest = obj_global_id_1
return {'relation': 'extracted-from', 'src': src, 'dest': dest}
elif 'domain' in type_tuple:
if 'item' in type_tuple:
if obj_type_1 == 'item':
src = obj_global_id_1
dest = obj_global_id_2
else:
src = obj_global_id_2
dest = obj_global_id_1
return {'relation': 'extracted-from', 'src': src, 'dest': dest} # replave by crawled-from
elif 'item' in type_tuple:
if 'domain' in type_tuple:
if obj_type_1 == 'item':
src = obj_global_id_1
dest = obj_global_id_2
else:
src = obj_global_id_2
dest = obj_global_id_1
return {'relation': 'extracted-from', 'src': src, 'dest': dest} # replave by crawled-from
return None
def sanitize_event_distribution(distribution):
try:
int(distribution)
if (0 <= distribution <= 3):
return distribution
else:
return 0
except:
return 0
def sanitize_event_threat_level_id(threat_level_id):
try:
int(threat_level_id)
if (1 <= threat_level_id <= 4):
return threat_level_id
else:
return 4
except:
return 4
def sanitize_event_analysis(analysis):
try:
int(analysis)
if (0 <= analysis <= 2):
return analysis
else:
return 0
except:
return 0
# # TODO: return error
def ping_misp():
try:
PyMISP(misp_url, misp_key, misp_verifycert)
return True
except Exception as e:
print(e)
return False
def create_misp_event(event, distribution=0, threat_level_id=4, publish=False, analysis=0, event_info=None):
if event_info:
event.info = event_info
event.distribution = sanitize_event_distribution(distribution)
event.threat_level_id = sanitize_event_threat_level_id(threat_level_id)
event.analysis = sanitize_event_analysis(analysis)
if publish:
event.publish()
# # TODO: handle multiple MISP instance
misp = PyMISP(misp_url, misp_key, misp_verifycert)
#print(event.to_json())
misp_event = misp.add_event(event)
#print(misp_event)
# # TODO: handle error
event_metadata = extract_event_metadata(misp_event)
return event_metadata
def extract_event_metadata(event):
event_metadata = {}
event_metadata['uuid'] = event['Event']['uuid']
event_metadata['id'] = event['Event']['id']
if misp_url[-1] == '/':
event_metadata['url'] = misp_url + 'events/view/' + str(event_metadata['id'])
else:
event_metadata['url'] = misp_url + '/events/view/' + str(event_metadata['id'])
return event_metadata
######
#
# EXPORT LVL DEFINITION: (== Correl<tion DEPTH)
#
# LVL 0 => PARTIAL Only add core item Correlation
# LVL 1 => DETAILED Also add correlated_items correlation
######
# # TODO: # create object relationships
def create_investigation_event(investigation_uuid):
investigation = Investigation(investigation_uuid)
event = MISPEvent()
event.info = investigation.get_info()
event.uuid = investigation.get_uuid(separator=True)
event.date = investigation.get_date()
event.analysis = investigation.get_analysis()
event.threat_level_id = investigation.get_threat_level()
event.distribution = 0
# tags
for tag in investigation.get_tags():
event.add_tag(tag)
# objects
investigation_objs = investigation.get_objects()
for obj in investigation_objs:
# if subtype -> obj_id = 'subtype:type'
if obj['subtype']:
obj_id = f"{obj['subtype']}:{obj['id']}"
else:
obj_id = obj['id']
misp_obj = create_misp_obj(obj['type'], obj_id)
if misp_obj:
event.add_object(misp_obj)
# if publish:
# event.publish()
# print(event.to_json())
misp = PyMISP(misp_url, misp_key, misp_verifycert)
if misp.event_exists(event.uuid):
misp_event = misp.update_event(event)
else:
misp_event = misp.add_event(event)
# # TODO: handle error
event_metadata = extract_event_metadata(misp_event)
if event_metadata.get('uuid'):
if misp_url[-1] == '/':
url = misp_url[:-1]
else:
url = misp_url
investigation.add_misp_events(url)
return event_metadata
# if __name__ == '__main__':
# l_obj = [{'id': 'bfd5f1d89e55b10a8b122a9d7ce31667ec1d086a', 'type': 'decoded', 'lvl': 2}]
# create_list_of_objs_to_export(l_obj)
#print(event.to_json())

7
bin/export/MispImport.py
View file

@ -13,8 +13,6 @@ from lib.objects import ail_objects
from lib.objects import Items
from export import AILObjects
# MISP
from pymisp import MISPEvent, MISPObject, PyMISP
@ -218,10 +216,6 @@ def create_obj_relationships(map_uuid_global_id, misp_obj):
# TODO CREATE OBJ RELATIONSHIP
def create_map_all_obj_uuid_golbal_id(map_uuid_global_id):
for obj_uuid in map_uuid_global_id:
AILObjects.create_map_obj_uuid_golbal_id(obj_uuid, map_uuid_global_id[obj_uuid])
def import_objs_from_file(filepath):
map_uuid_global_id = {}
@ -237,7 +231,6 @@ def import_objs_from_file(filepath):
for misp_obj in event_to_import.objects:
create_obj_relationships(map_uuid_global_id, misp_obj)
create_map_all_obj_uuid_golbal_id(map_uuid_global_id)
return map_uuid_global_id

23
bin/import/ail_json_importer/Ail_feeder_urlextract.py
View file

@ -12,12 +12,16 @@ import sys
import datetime
import uuid
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
import item_basic
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'import', 'ail_json_importer'))
from Default_json import Default_json
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from lib.objects.Items import Item
class Ail_feeder_urlextract(Default_json):
"""urlextract Feeder functions"""
@ -46,13 +50,14 @@ class Ail_feeder_urlextract(Default_json):
# # TODO:
def process_json_meta(self, process, item_id):
'''
"""
Process JSON meta filed.
'''
"""
json_meta = self.get_json_meta()
parent_id = str(json_meta['parent:twitter:tweet_id']) # TODO SEARCH IN CACHE !!!
item = Item(item_id)
item.set_parent(parent_id)
# # TODO: change me
parent_type = 'twitter_id'
parent_id = str(json_meta['parent:twitter:tweet_id'])
item_basic.add_item_parent_by_parent_id(parent_type, parent_id, item_id)
# parent_type = 'twitter_id'
# item_basic.add_item_parent_by_parent_id(parent_type, parent_id, item_id)

4
bin/lib/Investigations.py
View file

@ -200,8 +200,8 @@ class Investigation(object):
else:
raise UpdateInvestigationError(f'Invalid analysis: {analysis}')
def add_misp_events(self, misp_url):
r_tracking.sadd(f'investigations:misp:{self.uuid}', misp_url)
def add_misp_events(self, event_uuid):
r_tracking.sadd(f'investigations:misp:{self.uuid}', event_uuid)
def set_tags(self, tags):
# delete previous tags

20
bin/lib/ail_core.py
View file

@ -12,13 +12,13 @@ sys.path.append(os.environ['AIL_BIN'])
from lib.ConfigLoader import ConfigLoader
config_loader = ConfigLoader()
r_serv_db = config_loader.get_db_conn("Kvrocks_DB")
config_loader = None
AIL_OBJECTS = {'cve', 'cryptocurrency', 'decoded', 'domain', 'item', 'pgp', 'screenshot', 'username'}
AIL_OBJECTS = sorted({'cve', 'cryptocurrency', 'decoded', 'domain', 'item', 'pgp', 'screenshot', 'username'})
def get_ail_uuid():
pass
return r_serv_db.get('ail:uuid')
#### AIL OBJECTS ####
@ -33,11 +33,21 @@ def get_object_all_subtypes(obj_type):
if obj_type == 'username':
return ['telegram', 'twitter', 'jabber']
def get_all_objects_with_subtypes_tuple():
str_objs = []
for obj_type in get_all_objects():
subtypes = get_object_all_subtypes(obj_type)
if subtypes:
for subtype in subtypes:
str_objs.append((obj_type, subtype))
else:
str_objs.append((obj_type, ''))
return str_objs
##-- AIL OBJECTS --##
def paginate_iterator(iter_elems, nb_obj=50, page=1):
dict_page = {}
dict_page['nb_all_elem'] = len(iter_elems)
dict_page = {'nb_all_elem': len(iter_elems)}
nb_pages = dict_page['nb_all_elem'] / nb_obj
if not nb_pages.is_integer():
nb_pages = int(nb_pages)+1

1
bin/lib/data_retention_engine.py
View file

@ -96,4 +96,3 @@ def update_obj_date(date, obj_type, subtype=''):
if __name__ == '__main__':
print(r_obj.hgetall(f'date:first'))
print(r_obj.hgetall(f'date:last'))

12
bin/lib/objects/Domains.py
View file

@ -420,11 +420,15 @@ class Domain(AbstractObject):
def add_history(self, epoch, root_item=None, date=None):
if not date:
date = time.strftime('%Y%m%d', time.gmtime(epoch))
try:
root_item = int(root_item)
if not root_item:
root_item = int(epoch)
status = False
except (ValueError, TypeError):
status = True
else:
try:
root_item = int(root_item)
status = False
except (ValueError, TypeError):
status = True
update_obj_date(date, 'domain', self.domain_type)
# UP

9
bin/lib/objects/Items.py
View file

@ -88,10 +88,13 @@ class Item(AbstractObject):
else:
return item_basic.get_item_content(self.id)
def get_raw_content(self):
def get_raw_content(self, decompress=False):
filepath = self.get_filename()
with open(filepath, 'rb') as f:
raw_content = BytesIO(f.read())
if decompress:
raw_content = BytesIO(self.get_content(binary=True))
else:
with open(filepath, 'rb') as f:
raw_content = BytesIO(f.read())
return raw_content
def get_gzip_content(self, b64=False):

2
bin/lib/objects/Usernames.py
View file

@ -18,7 +18,7 @@ from lib.ConfigLoader import ConfigLoader
from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id
config_loader = ConfigLoader()
baseurl = config_loader.get_config_str("Notifications", "ail_domain")
config_loader = None

8
bin/lib/objects/abstract_object.py
View file

@ -243,6 +243,14 @@ class AbstractObject(ABC):
"""
return is_obj_correlated(self.type, self.subtype, self.id, type2, subtype2, id2)
def are_correlated(self, object2):
"""
Check if an object is correlated with another Object
:type object2 AbstractObject
"""
return is_obj_correlated(self.type, self.subtype, self.id,
object2.get_type(), object2.get_subtype(r_str=True), object2.get_id())
def delete_correlation(self, type2, subtype2, id2):
"""
Get object correlations

218
bin/lib/objects/ail_objects.py
View file

@ -4,15 +4,12 @@
import os
import sys
from abc import ABC
from flask import url_for
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from lib.ConfigLoader import ConfigLoader
from lib.ail_core import get_all_objects
from lib.ail_core import get_all_objects, get_object_all_subtypes, get_all_objects_with_subtypes_tuple
from lib import correlations_engine
from lib import btc_ail
from lib import Tag
@ -26,26 +23,18 @@ from lib.objects import Pgps
from lib.objects.Screenshots import Screenshot
from lib.objects import Usernames
config_loader = ConfigLoader()
config_loader = None
class AILObjects(object): ## ??????????????????????
initial = 0
ongoing = 1
completed = 2
def is_valid_object_type(obj_type):
return obj_type in get_all_objects()
def is_valid_object_subtype(obj_type, subtype):
if obj_type == 'cryptocurrency':
return subtype in CryptoCurrencies.get_all_subtypes()
elif obj_type == 'pgp':
return subtype in Pgps.get_all_subtypes()
elif obj_type == 'username':
return subtype in CryptoCurrencies.get_all_subtypes()
return subtype in get_object_all_subtypes(obj_type)
def sanitize_objs_types(objs):
l_types = []
@ -56,6 +45,7 @@ def sanitize_objs_types(objs):
l_types = get_all_objects()
return l_types
def get_object(obj_type, subtype, id):
if obj_type == 'item':
return Item(id)
@ -74,6 +64,25 @@ def get_object(obj_type, subtype, id):
elif obj_type == 'username':
return Usernames.Username(id, subtype)
def get_objects(objects):
objs = set()
for obj in objects:
if isinstance(obj, dict):
obj_type = obj['type']
obj_subtype = obj['subtype']
obj_id = obj['id']
if 'lvl' in obj:
correl_objs = get_obj_correlations_objs(obj_type, obj_subtype, obj_id, lvl=obj['lvl'])
objs = objs.union(correl_objs)
else:
obj_type, obj_subtype, obj_id = obj
objs.add((obj_type, obj_subtype, obj_id))
ail_objects = []
for obj in objs:
ail_objects.append(get_object(obj[0], obj[1], obj[2]))
return ail_objects
def exists_obj(obj_type, subtype, obj_id):
obj = get_object(obj_type, subtype, obj_id)
if obj:
@ -81,35 +90,44 @@ def exists_obj(obj_type, subtype, obj_id):
else:
return False
def get_obj_global_id(obj_type, subtype, obj_id):
obj = get_object(obj_type, subtype, obj_id)
return obj.get_global_id()
def get_obj_from_global_id(global_id):
obj = global_id.split(':', 3)
return get_object(obj[0], obj[1], obj[2])
def get_object_link(obj_type, subtype, id, flask_context=False):
obj = get_object(obj_type, subtype, id)
return obj.get_link(flask_context=flask_context)
def get_object_svg(obj_type, subtype, id):
obj = get_object(obj_type, subtype, id)
return obj.get_svg_icon()
## TAGS ##
def get_obj_tags(obj_type, subtype, id):
obj = get_object(obj_type, subtype, id)
return obj.get_tags()
def add_obj_tag(obj_type, subtype, id, tag):
obj = get_object(obj_type, subtype, id)
obj.add_tag(tag)
def add_obj_tags(obj_type, subtype, id, tags):
obj = get_object(obj_type, subtype, id)
for tag in tags:
obj.add_tag(tag)
# -TAGS- #
def get_object_meta(obj_type, subtype, id, options=[], flask_context=False):
@ -119,6 +137,7 @@ def get_object_meta(obj_type, subtype, id, options=[], flask_context=False):
meta['link'] = obj.get_link(flask_context=flask_context)
return meta
def get_objects_meta(objs, options=[], flask_context=False):
metas = []
for obj_dict in objs:
@ -126,13 +145,15 @@ def get_objects_meta(objs, options=[], flask_context=False):
flask_context=flask_context))
return metas
def get_object_card_meta(obj_type, subtype, id, related_btc=False):
obj = get_object(obj_type, subtype, id)
meta = obj.get_meta()
meta['icon'] = obj.get_svg_icon()
if subtype or obj_type == 'cve':
meta['sparkline'] = obj.get_sparkline()
meta['cve_search'] = obj.get_cve_search()
if obj_type == 'cve':
meta['cve_search'] = obj.get_cve_search()
if subtype == 'bitcoin' and related_btc:
meta["related_btc"] = btc_ail.get_bitcoin_info(obj.id)
if obj.get_type() == 'decoded':
@ -143,6 +164,7 @@ def get_object_card_meta(obj_type, subtype, id, related_btc=False):
meta["add_tags_modal"] = Tag.get_modal_add_tags(obj.id, obj.get_type(), obj.get_subtype(r_str=True))
return meta
def get_ui_obj_tag_table_keys(obj_type):
'''
Warning: use only in flask (dynamic templates)
@ -150,15 +172,58 @@ def get_ui_obj_tag_table_keys(obj_type):
if obj_type == "domain":
return ['id', 'first_seen', 'last_check', 'status'] # # TODO: add root screenshot
# # # # MISP OBJECTS # # # #
# # TODO: CHECK IF object already have an UUID
def get_misp_object(obj_type, subtype, id):
obj = get_object(obj_type, subtype, id)
return obj.get_misp_object()
def get_misp_objects(objs):
misp_objects = {}
for obj in objs:
misp_objects[obj] = obj.get_misp_object()
for relation in get_objects_relationships(objs):
obj_src = misp_objects[relation['src']]
obj_dest = misp_objects[relation['dest']]
# print(relation['src'].get_id(), relation['dest'].get_id())
obj_src.add_reference(obj_dest.uuid, relation['relationship'], 'ail correlation')
return misp_objects.values()
# get misp relationship
def get_objects_relationship(obj_1, obj2):
relationship = {}
obj_types = (obj_1.get_type(), obj2.get_type())
def get_objects_relationships(objs):
relation = []
if len(objs) == 2:
if objs[0].are_correlated(objs[1]):
relationship = get_objects_relationship(objs[0], objs[1])
if relationship:
relation.append(relationship)
else:
iterator = objs.copy() # [obj1, obj2, obj3, obj4]
for obj in objs[:-1]: # [obj1, obj2, obj3]
iterator.pop(0) # [obj2, obj3, obj4] obj1 correlation already checked
for obj2 in iterator:
# CHECK CORRELATION obj - > obj2
if obj.are_correlated(obj2):
relationship = get_objects_relationship(obj, obj2)
if relationship:
relation.append(relationship)
return relation
def get_relationship_src_dest(src_type, obj1, obj2):
if obj1.get_type() == src_type:
src = obj1
dest = obj2
else:
src = obj2
dest = obj1
return src, dest
# get misp relationship
def get_objects_relationship(obj1, obj2):
obj_types = (obj1.get_type(), obj2.get_type())
##############################################################
# if ['cryptocurrency', 'pgp', 'username', 'decoded', 'screenshot']:
@ -166,63 +231,79 @@ def get_objects_relationship(obj_1, obj2):
# relationship[relation] =
##############################################################
if 'cryptocurrency' in obj_types:
relationship['relation'] = 'extracted-from'
if obj_1.get_type() == 'cryptocurrency':
relationship['src'] = obj_1.get_id()
relationship['dest'] = obj2.get_id()
else:
relationship['src'] = obj2.get_id()
relationship['dest'] = obj_1.get_id()
relationship = 'extracted-from'
src, dest = get_relationship_src_dest('cryptocurrency', obj1, obj2)
elif 'cve' in obj_types:
relationship = 'extracted-from'
src, dest = get_relationship_src_dest('cve', obj1, obj2)
elif 'pgp' in obj_types:
relationship['relation'] = 'extracted-from'
relationship = 'extracted-from'
src, dest = get_relationship_src_dest('pgp', obj1, obj2)
elif 'username' in obj_types:
relationship['relation'] = 'extracted-from'
relationship = 'extracted-from'
src, dest = get_relationship_src_dest('username', obj1, obj2)
elif 'decoded' in obj_types:
relationship['relation'] = 'included-in'
relationship = 'included-in'
src, dest = get_relationship_src_dest('decoded', obj1, obj2)
elif 'screenshot' in obj_types:
relationship['relation'] = 'screenshot-of'
relationship = 'screenshot-of'
src, dest = get_relationship_src_dest('screenshot', obj1, obj2)
elif 'domain' in obj_types:
relationship['relation'] = 'extracted-from'
relationship = 'extracted-from'
src, dest = get_relationship_src_dest('domain', obj1, obj2)
# default
else:
pass
relationship = None
src = None
dest = None
if not relationship:
return {}
else:
return {'src': src, 'dest': dest, 'relationship': relationship}
# - - - MISP OBJECTS - - - #
return relationship
def api_sanitize_object_type(obj_type):
if not is_valid_object_type(obj_type):
return {'status': 'error', 'reason': 'Incorrect object type'}, 400
def get_obj_correlations(obj_type, subtype, id):
obj = get_object(obj_type, subtype, id)
def get_obj_correlations(obj_type, subtype, obj_id):
obj = get_object(obj_type, subtype, obj_id)
return obj.get_correlations()
################################################################################
# DATA RETENTION
# # TODO: TO ADD ??????????????????????
# def get_first_objects_date():
# return r_object.zrange('objs:first_date', 0, -1)
#
# def get_first_object_date(obj_type, subtype):
# return r_object.zscore('objs:first_date', f'{obj_type}:{subtype}')
#
# def set_first_object_date(obj_type, subtype, date):
# return r_object.zadd('objs:first_date', f'{obj_type}:{subtype}', date)
def _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, lvl=0):
if lvl > 0 and (obj_type, subtype, obj_id) not in objs: # Avoid looking for the same correlation
objs.add((obj_type, subtype, obj_id))
lvl = lvl - 1
obj = get_object(obj_type, subtype, obj_id)
correlations = obj.get_correlations()
# print('--------------------------')
# print( obj_id, correlations)
# print(lvl)
# print('--------------------------')
for obj2_type in correlations:
for str_obj in correlations[obj2_type]:
obj2_subtype, obj2_id = str_obj.split(':', 1)
_get_obj_correlations_objs(objs, obj2_type, obj2_subtype, obj2_id, lvl=lvl)
# print(len(objs))
objs.add((obj_type, subtype, obj_id))
return objs
def get_obj_correlations_objs(obj_type, subtype, obj_id, lvl=0):
objs = set()
_get_obj_correlations_objs(objs, obj_type, subtype, obj_id, lvl=lvl)
return objs
################################################################################
################################################################################
################################################################################
def delete_obj(obj_type, subtype, id):
object = get_object(obj_type, subtype, id)
return object.delete()
def delete_obj(obj_type, subtype, obj_id):
obj = get_object(obj_type, subtype, obj_id)
return obj.delete()
################################################################################
################################################################################
@ -236,11 +317,12 @@ def create_correlation_graph_links(links_set):
links.append({"source": link[0], "target": link[1]})
return links
def create_correlation_graph_nodes(nodes_set, obj_str_id, flask_context=True):
graph_nodes_list = []
for node_id in nodes_set:
obj_type, subtype, obj_id = node_id.split(';', 2)
dict_node = {"id": node_id}
dict_node = {'id': node_id}
dict_node['style'] = get_object_svg(obj_type, subtype, obj_id)
# # TODO: # FIXME: in UI
@ -258,11 +340,27 @@ def create_correlation_graph_nodes(nodes_set, obj_str_id, flask_context=True):
graph_nodes_list.append(dict_node)
return graph_nodes_list
def get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=[], max_nodes=300, level=1, flask_context=False):
obj_str_id, nodes, links = correlations_engine.get_correlations_graph_nodes_links(obj_type, subtype, obj_id, filter_types=filter_types, max_nodes=max_nodes, level=level, flask_context=flask_context)
return {"nodes": create_correlation_graph_nodes(nodes, obj_str_id, flask_context=flask_context), "links": create_correlation_graph_links(links)}
def get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=[], max_nodes=300, level=1,
flask_context=False):
obj_str_id, nodes, links = correlations_engine.get_correlations_graph_nodes_links(obj_type, subtype, obj_id,
filter_types=filter_types,
max_nodes=max_nodes, level=level,
flask_context=flask_context)
return {"nodes": create_correlation_graph_nodes(nodes, obj_str_id, flask_context=flask_context),
"links": create_correlation_graph_links(links)}
###############
# if __name__ == '__main__':
# r = get_objects([{'lvl': 1, 'type': 'item', 'subtype': '', 'id': 'crawled/2020/09/14/circl.lu0f4976a4-dda4-4189-ba11-6618c4a8c951'}])
# r = get_misp_objects([Item('crawled/2020/09/14/circl.lu0f4976a4-dda4-4189-ba11-6618c4a8c951'),
# Cve('CVE-2020-16856'), Cve('CVE-2014-6585'), Cve('CVE-2015-0383'),
# Cve('CVE-2015-0410')])
# print()
# print(r)
# res = get_obj_correlations_objs('username', 'telegram', 'corona', lvl=100)
# print(res)

2
requirements.txt
View file

@ -6,7 +6,7 @@ d4-pyclient>=0.1.6
thehive4py
# Core
redis==3.0.0
redis>4.4.0
python-magic>0.4.15
yara-python>4.0.2

2
var/www/blueprints/crawler_splash.py
View file

@ -34,7 +34,7 @@ from lib import Tag
from packages import Date
#import Config_DB
# import Config_DB
bootstrap_label = Flask_config.bootstrap_label
# ============ BLUEPRINT ============

217
var/www/blueprints/import_export.py
View file

@ -4,17 +4,16 @@
'''
Blueprint Flask: MISP format import export
'''
import io
import os
import sys
import uuid
import json
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, send_file
from flask import render_template, jsonify, request, Blueprint, redirect, url_for, Response, send_file, abort
from flask_login import login_required, current_user, login_user, logout_user
sys.path.append('modules')
import Flask_config
# Import Role_Manager
from Role_Manager import login_admin, login_analyst, login_read_only
@ -23,20 +22,21 @@ sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from export import MispExport
from export import MispImport
from export import AILObjects ####################### # # # # ########################## ## # ####################################
from export import Export
from lib.objects import ail_objects
from export import MispImport # TODO REMOVE ME
# TODO REMOVE ME
# ============ BLUEPRINT ============
import_export = Blueprint('import_export', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/import_export'))
import_export = Blueprint('import_export', __name__,
template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/import_export'))
# ============ VARIABLES ============
# ============ FUNCTIONS ============
@ -47,6 +47,7 @@ import_export = Blueprint('import_export', __name__, template_folder=os.path.joi
def import_object():
return render_template("import_object.html")
# TODO
@import_export.route("/import_export/import_file", methods=['POST'])
@login_required
@login_analyst
@ -79,137 +80,129 @@ def import_object_file():
return render_template("import_object.html", all_imported_obj=all_imported_obj, error=error)
@import_export.route('/import_export/export')
@import_export.route("/objects/misp/export", methods=['GET'])
@login_required
@login_analyst
def export_object():
def objects_misp_export():
user_id = current_user.get_id()
object_types = ail_objects.get_all_objects_with_subtypes_tuple()
to_export = Export.get_user_misp_objects_to_export(user_id)
return render_template("export_object.html", object_types=object_types, to_export=to_export)
# get user saved obj to export
l_obj_to_export = AILObjects.get_user_list_of_obj_to_export(user_id)
return render_template("export_object.html", l_obj_to_export=l_obj_to_export)
@import_export.route("/import_export/export_file", methods=['POST'])
@import_export.route("/objects/misp/export/post", methods=['POST'])
@login_required
@login_analyst
def export_object_file():
def objects_misp_export_post():
user_id = current_user.get_id()
l_obj_to_export = []
l_obj_invalid = []
export_to_misp = False
dict_misp_event_export = {}
# Get new added Object
new_export = []
user_lvl_export = {}
for obj_tuple in list(request.form):
# GET NEW LEVEL
if obj_tuple[0] == '{':
obj_j = obj_tuple.replace("'", "\"")
obj_dict = json.loads(obj_j) # TODO sanitize
lvl = int(request.form.getlist(obj_tuple)[0])
user_lvl_export[f"{obj_dict['type']}:{obj_dict['subtype']}:{obj_dict['id']}"] = lvl
l_input = request.form.getlist(obj_tuple)
if len(l_input) == 3:
obj_type = l_input[0]
obj_id = l_input[1]
lvl = l_input[2]
lvl = MispExport.sanitize_obj_export_lvl(lvl)
if l_input[0] != 'Object type...':
new_type, new_subtype = l_input[0].split(':', 1)
if not new_subtype:
new_subtype = ''
new_export.append({'type': new_type, 'subtype': new_subtype, 'id': l_input[1], 'lvl': l_input[2]})
obj_subtype = obj_type.split(';')
if len(obj_subtype) == 2:
obj_type = obj_subtype[0]
obj_subtype = obj_subtype[1]
else:
obj_subtype = None
obj_dict = {'id': obj_id, 'type': obj_type, 'lvl': lvl}
if obj_subtype:
obj_dict['subtype'] = obj_subtype
if MispExport.is_valid_obj_to_export(obj_type, obj_subtype, obj_id):
l_obj_to_export.append(obj_dict)
AILObjects.add_user_object_to_export(user_id, obj_dict['type'], obj_dict['id'], obj_dict['lvl'], obj_subtype=obj_dict.get('subtype', None))
else:
if obj_id:
l_obj_invalid.append(obj_dict)
objects = []
invalid_obj = []
for obj in new_export:
if not ail_objects.exists_obj(obj['type'], obj['subtype'], obj['id']):
invalid_obj.append(obj)
else:
dict_misp_event_export[str(obj_tuple)] = request.form.get(obj_tuple)
# print(dict_misp_event_export)
if dict_misp_event_export.get('export_to_misp', None):
export_to_misp = True
else:
dict_misp_event_export = None
if l_obj_invalid:
# get user saved obj to export # # TODO: # performance
l_obj_to_export = AILObjects.get_user_list_of_obj_to_export(user_id)
for obj_dict in l_obj_invalid: # set uuid input
obj_dict['uuid'] = str(uuid.uuid4())
subtype = obj_dict.get('subtype', None)
if subtype:
obj_dict['type'] = f'{obj_dict["type"]};{subtype}'
return render_template("export_object.html", l_obj_to_export=l_obj_to_export,
l_obj_invalid=l_obj_invalid, dict_misp_event_export=dict_misp_event_export)
else:
if export_to_misp and MispExport.ping_misp():
event = MispExport.create_list_of_objs_to_export(l_obj_to_export, r_type='event')
event_metadata = MispExport.create_misp_event(event, distribution=dict_misp_event_export.get('export_to_misp', None),
threat_level_id=dict_misp_event_export.get('misp_threat_level_id', None),
publish=dict_misp_event_export.get('misp_publish', None),
analysis=dict_misp_event_export.get('misp_event_analysis', None),
event_info=dict_misp_event_export.get('misp_event_info', None))
AILObjects.delete_all_user_object_to_export(user_id)
return render_template("export_object.html", l_obj_to_export=l_obj_to_export,
event_metadata=event_metadata,
l_obj_invalid=[], dict_misp_event_export=[])
objects.append(obj)
for obj in Export.get_user_misp_objects_to_export(user_id):
if not ail_objects.exists_obj(obj['type'], obj['subtype'], obj['id']):
invalid_obj.append(obj)
else:
# get user saved obj to export # # TODO: # performance
json_export = MispExport.create_list_of_objs_to_export(l_obj_to_export)
export_filename = MispExport.get_export_filename(json_export)
json_export = MispExport.create_in_memory_file(json_export.to_json())
AILObjects.delete_all_user_object_to_export(user_id)
return send_file(json_export, as_attachment=True, attachment_filename=export_filename)
str_id = f"{obj['type']}:{obj['subtype']}:{obj['id']}"
if str_id in user_lvl_export:
obj['lvl'] = user_lvl_export[str_id]
objects.append(obj)
if invalid_obj:
object_types = ail_objects.get_all_objects_with_subtypes_tuple()
return render_template("export_object.html", object_types=object_types,
to_export=objects, l_obj_invalid=invalid_obj)
export = request.form.get('export_to_misp', False)
distribution = request.form.get('misp_event_distribution')
threat_level_id = request.form.get('threat_level_id')
analysis = request.form.get('misp_event_analysis')
info = request.form.get('misp_event_info')
publish = request.form.get('misp_event_info', False)
objs = ail_objects.get_objects(objects)
event = Export.create_misp_event(objs, distribution=distribution, threat_level_id=threat_level_id,
analysis=analysis, info=info, export=export, publish=publish)
Export.delete_user_misp_objects_to_export(user_id)
if not export:
return send_file(io.BytesIO(event['event'].encode()), as_attachment=True,
download_name=f'ail_export_{event["uuid"]}.json')
else:
object_types = ail_objects.get_all_objects_with_subtypes_tuple()
return render_template("export_object.html", object_types=object_types,
misp_url=event['url'])
@import_export.route("/import_export/add_object_id_to_export", methods=['GET'])
@import_export.route("/objects/misp/export/add", methods=['GET'])
@login_required
@login_analyst
def add_object_id_to_export():
user_id = current_user.get_id()
user_id = current_user.get_id()
obj_type = request.args.get('obj_type')
obj_id = request.args.get('obj_id')
obj_subtype = request.args.get('obj_subtype')
obj_lvl = request.args.get('obj_lvl')
AILObjects.add_user_object_to_export(user_id, obj_type, obj_id, obj_lvl, obj_subtype=obj_subtype)
obj_type = request.args.get('type')
obj_id = request.args.get('id')
obj_subtype = request.args.get('subtype')
obj_lvl = request.args.get('lvl')
try:
obj_lvl = int(obj_lvl)
except:
obj_lvl = 0
if not ail_objects.exists_obj(obj_type, obj_subtype, obj_id):
abort(404)
Export.add_user_misp_object_to_export(user_id, obj_type, obj_subtype, obj_id, lvl=obj_lvl)
# redirect
return redirect(url_for('import_export.export_object'))
return redirect(url_for('import_export.objects_misp_export'))
@import_export.route("/objects/misp/export/delete", methods=['GET'])
@login_required
@login_analyst
def delete_object_id_to_export():
user_id = current_user.get_id()
obj_type = request.args.get('type')
obj_id = request.args.get('id')
obj_subtype = request.args.get('subtype')
Export.delete_user_misp_object_to_export(user_id, obj_type, obj_subtype, obj_id)
return jsonify(success=True)
@import_export.route("/import_export/investigation", methods=['GET'])
@login_required
@login_analyst
def export_investigation():
investigation_uuid = request.args.get("uuid")
if MispExport.ping_misp():
event_metadata = MispExport.create_investigation_event(investigation_uuid)
if Export.ping_misp():
event = Export.create_investigation_misp_event(investigation_uuid)
print(event)
else:
return Response(json.dumps({"error": "Can't reach MISP Instance"}, indent=2, sort_keys=True), mimetype='application/json'), 400
return Response(json.dumps({"error": "Can't reach MISP Instance"}, indent=2, sort_keys=True),
mimetype='application/json'), 400
return redirect(url_for('investigations_b.show_investigation', uuid=investigation_uuid))
# @import_export.route("/import_export/delete_object_id_to_export", methods=['GET'])
# @login_required
# @login_analyst
# def delete_object_id_to_export():
# user_id = current_user.get_id()
# obj_type = request.args.get('obj_type')
# obj_id = request.args.get('obj_id')
# obj_subtype = request.args.get('obj_subtype')
# obj_lvl = request.args.get('obj_lvl')
# AILObjects.delete_user_object_to_export(user_id, object_type, object_id, obj_lvl, obj_subtype=obj_subtype)
# # redirect
# return 'ok'

226
var/www/modules/PasteSubmit/Flask_PasteSubmit.py
View file

@ -1,16 +1,15 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
'''
Flask functions and routes for the trending modules page
'''
"""
Flask functions and routes for the Item Submit modules page
"""
##################################
# Import External packages
##################################
import re
import os
import sys
import json
import string
import datetime
import unicodedata
@ -20,7 +19,7 @@ from io import BytesIO
from functools import wraps
# Flask
from flask import Flask, render_template, jsonify, request, Blueprint, url_for, redirect, abort
from flask import render_template, jsonify, request, Blueprint, url_for, redirect, abort
from Role_Manager import login_admin, login_analyst
from flask_login import login_required
@ -28,56 +27,39 @@ from flask_login import login_required
##################################
# Import Project packages
##################################
from export import Export
from lib import Tag
from lib.objects.Items import Item
from packages import Import_helper
from pytaxonomies import Taxonomies
from pytaxonomies import Taxonomies # TODO REMOVE ME
try:
from pymisp.mispevent import MISPObject
flag_misp = True
except:
flag_misp = False
try:
from thehive4py.models import Case, CaseTask, CustomFieldHelper, CaseObservable
flag_hive = True
except:
flag_hive = False
# ============ VARIABLES ============
import Flask_config
app = Flask_config.app
baseUrl = Flask_config.baseUrl
r_serv_metadata = Flask_config.r_serv_metadata
r_serv_db = Flask_config.r_serv_db
r_serv_log_submit = Flask_config.r_serv_log_submit
r_serv_metadata = Flask_config.r_serv_metadata # TODO REMOVE ME
r_serv_db = Flask_config.r_serv_db # TODO REMOVE ME
r_serv_log_submit = Flask_config.r_serv_log_submit # TODO REMOVE ME
logger = Flask_config.redis_logger
pymisp = Flask_config.pymisp
if pymisp is False:
flag_misp = False
HiveApi = Flask_config.HiveApi
if HiveApi is False:
flag_hive = False
PasteSubmit = Blueprint('PasteSubmit', __name__, template_folder='templates')
valid_filename_chars = "-_ %s%s" % (string.ascii_letters, string.digits)
UPLOAD_FOLDER = Flask_config.UPLOAD_FOLDER
misp_event_url = Flask_config.misp_event_url
hive_case_url = Flask_config.hive_case_url
text_max_size = int(Flask_config.SUBMIT_PASTE_TEXT_MAX_SIZE) / (1000*1000)
file_max_size = int(Flask_config.SUBMIT_PASTE_FILE_MAX_SIZE) / (1000*1000*1000)
allowed_extensions = ", ". join(Flask_config.SUBMIT_PASTE_FILE_ALLOWED_EXTENSIONS)
PasteSubmit = Blueprint('PasteSubmit', __name__, template_folder='templates')
# ============ Validators ============
def limit_content_length():
@ -117,133 +99,6 @@ def clean_filename(filename, whitelist=valid_filename_chars, replace=' '):
# keep only whitelisted chars
return ''.join(c for c in cleaned_filename if c in whitelist)
def date_to_str(date):
return "{0}-{1}-{2}".format(date.year, date.month, date.day)
def misp_create_event(distribution, threat_level_id, analysis, info, l_tags, publish, path):
item = Item(path)
source = item.get_source()
ail_uuid = r_serv_db.get('ail:uuid')
pseudofile = BytesIO(item.get_content(binary=True))
temp = item.get_duplicates()
# beautifier
if not temp:
temp = ''
p_duplicate_number = len(temp) if len(temp) >= 0 else 0
to_ret = ""
for dup in temp[:10]:
dup = dup.replace('\'','\"').replace('(','[').replace(')',']')
dup = json.loads(dup)
algo = dup[0]
path = dup[1].split('/')[-6:]
path = '/'.join(path)[:-3] # -3 removes .gz
if algo == 'tlsh':
perc = 100 - int(dup[2])
else:
perc = dup[2]
to_ret += "{}: {} [{}%]\n".format(path, algo, perc)
p_duplicate = to_ret
today = datetime.date.today()
# [0-3]
if publish == 'True':
published = True
else:
published = False
org_id = None
orgc_id = None
sharing_group_id = None
date = today
event = pymisp.new_event(distribution, threat_level_id,
analysis, info, date,
published, orgc_id, org_id, sharing_group_id)
eventUuid = event['Event']['uuid']
eventid = event['Event']['id']
r_serv_metadata.set('misp_events:'+path, eventid)
# add tags
for tag in l_tags:
pymisp.tag(eventUuid, tag)
# create attributes
obj_name = 'ail-leak'
leak_obj = MISPObject(obj_name)
leak_obj.add_attribute('sensor', value=ail_uuid, type="text")
leak_obj.add_attribute('origin', value=source, type='text')
leak_obj.add_attribute('last-seen', value=date_to_str(item.get_date()), type='datetime')
leak_obj.add_attribute('raw-data', value=source, data=pseudofile, type="attachment")
if p_duplicate_number > 0:
leak_obj.add_attribute('duplicate', value=p_duplicate, type='text')
leak_obj.add_attribute('duplicate_number', value=p_duplicate_number, type='counter')
try:
templateID = [x['ObjectTemplate']['id'] for x in pymisp.get_object_templates_list()['response'] if x['ObjectTemplate']['name'] == obj_name][0]
except IndexError:
valid_types = ", ".join([x['ObjectTemplate']['name'] for x in pymisp.get_object_templates_list()])
print (f"Template for type {obj_name} not found! Valid types are: {valid_types}")
return False
r = pymisp.add_object(eventid, templateID, leak_obj)
if 'errors' in r:
print(r)
return False
else:
event_url = misp_event_url + eventid
return event_url
def hive_create_case(hive_tlp, threat_level, hive_description, hive_case_title, l_tags, path):
ail_uuid = r_serv_db.get('ail:uuid')
source = path.split('/')[-6:]
source = '/'.join(source)[:-3]
# get item date
var = path.split('/')
last_seen = "{0}-{1}-{2}".format(var[-4], var[-3], var[-2])
case = Case(title=hive_case_title,
tlp=hive_tlp,
severity=threat_level,
flag=False,
tags=l_tags,
description='hive_description')
# Create the case
id = None
response = HiveApi.create_case(case)
if response.status_code == 201:
id = response.json()['id']
observ_sensor = CaseObservable(dataType="other", data=[ail_uuid], message="sensor")
observ_file = CaseObservable(dataType="file", data=[path], tags=l_tags)
observ_source = CaseObservable(dataType="other", data=[source], message="source")
observ_last_seen = CaseObservable(dataType="other", data=[last_seen], message="last-seen")
res = HiveApi.create_case_observable(id,observ_sensor)
if res.status_code != 201:
logger.info(f'ko sensor: {res.status_code}/{res.text}')
res = HiveApi.create_case_observable(id, observ_source)
if res.status_code != 201:
logger.info(f'ko source: {res.status_code}/{res.text}')
res = HiveApi.create_case_observable(id, observ_file)
if res.status_code != 201:
logger.info(f'ko file: {res.status_code}/{res.text}')
res = HiveApi.create_case_observable(id, observ_last_seen)
if res.status_code != 201:
logger.info(f'ko last_seen: {res.status_code}/{res.text}')
r_serv_metadata.set('hive_cases:'+path, id)
return hive_case_url.replace('id_here', id)
else:
logger.info(f'ko: {response.status_code}/{response.text}')
return False
# ============= ROUTES ==============
@PasteSubmit.route("/PasteSubmit/", methods=['GET'])
@ -462,50 +317,31 @@ def submit_status():
else:
return 'INVALID UUID'
@PasteSubmit.route("/PasteSubmit/create_misp_event", methods=['POST'])
@login_required
@login_analyst
def create_misp_event():
distribution = int(request.form['misp_data[Event][distribution]'])
threat_level_id = int(request.form['misp_data[Event][threat_level_id]'])
analysis = int(request.form['misp_data[Event][analysis]'])
info = request.form['misp_data[Event][info]']
path = request.form['paste']
publish = request.form.get('misp_publish')
#verify input
if (0 <= distribution <= 3) and (1 <= threat_level_id <= 4) and (0 <= analysis <= 2):
l_tags = list(r_serv_metadata.smembers('tag:'+path))
event = misp_create_event(distribution, threat_level_id, analysis, info, l_tags, publish, path)
if event != False:
return redirect(event)
else:
return 'error, event creation'
return 'error0'
# TODO MOVE ME IN import_export blueprint
@PasteSubmit.route("/PasteSubmit/create_hive_case", methods=['POST'])
@login_required
@login_analyst
def create_hive_case():
hive_tlp = int(request.form['hive_tlp'])
threat_level = int(request.form['threat_level_hive'])
hive_description = request.form['hive_description']
hive_case_title = request.form['hive_case_title']
path = os.environ['AIL_HOME'] + "/PASTES/"+ request.form['paste']
tlp = request.form['hive_tlp']
if tlp:
tlp = int(tlp)
else:
tlp = 2
threat_level = request.form['threat_level_hive']
if threat_level:
threat_level = int(threat_level)
else:
threat_level = 2
description = request.form['hive_description']
title = request.form['hive_case_title']
item_id = request.form['obj_id']
#verify input
if (0 <= hive_tlp <= 3) and (1 <= threat_level <= 4):
if (0 <= tlp <= 3) and (1 <= threat_level <= 4):
l_tags = list(r_serv_metadata.smembers('tag:'+path))
case = hive_create_case(hive_tlp, threat_level, hive_description, hive_case_title, l_tags, path)
if case != False:
return redirect(case)
case_id = Export.create_thehive_case(item_id, title=title, tlp=tlp, threat_level=threat_level, description=description)
if case_id:
return redirect(get_case_url(case_id))
else:
return 'error'

2
var/www/modules/hashDecoded/Flask_hashDecoded.py
View file

@ -131,7 +131,7 @@ def get_all_types_id(correlation_type):
def get_key_id_metadata(obj_type, subtype, obj_id):
obj = ail_objects.get_object_meta(obj_type, subtype, obj_id)
return obj._get_meta()
return obj
def list_sparkline_type_id_values(date_range_sparkline, correlation_type, type_id, key_id):
sparklines_value = []

2
var/www/modules/hashDecoded/templates/DaysCorrelation.html
View file

@ -171,7 +171,7 @@
{% for key_id in all_metadata %}
<tr>
<td><i class="{{ all_metadata[key_id]['type_icon'] }}"></i>&nbsp;&nbsp;{{ all_metadata[key_id]['type_id'] }}</td>
<td><a target="_blank" href="{{ url_for(show_key_id_endpoint) }}?object_type={{correlation_type_n}}&type_id={{ all_metadata[key_id]['type_id'] }}&correlation_id={{ key_id }}&correlation_objects=paste">{{ key_id }}</a></td>
<td><a target="_blank" href="{{ url_for(show_key_id_endpoint) }}?type={{correlation_type_n}}&subtype={{ all_metadata[key_id]['type_id'] }}&id={{ key_id }}">{{ key_id }}</a></td>
<td>{{ all_metadata[key_id]['first_seen'] }}</td>
<td>{{ all_metadata[key_id]['last_seen'] }}</td>
<td>{{ all_metadata[key_id]['nb_seen'] }}</td>

2
var/www/modules/hashDecoded/templates/hashDecoded.html
View file

@ -158,7 +158,7 @@
{% for b64 in l_64 %}
<tr>
<td><i class="fas {{ b64[0] }}"></i>&nbsp;&nbsp;{{ b64[1] }}</td>
<td><a target="_blank" href="{{ url_for('correlation.show_correlation') }}?object_type=decoded&correlation_id={{ b64[2] }}&correlation_objects=paste">{{ b64[2] }}</a></td>
<td><a target="_blank" href="{{ url_for('correlation.show_correlation') }}?type=decoded&id={{ b64[2] }}">{{ b64[2] }}</a></td>
<td>{{ b64[5] }}</td>
<td>{{ b64[6] }}</td>
<td>{{ b64[3] }}</td>

2
var/www/templates/import_export/block_add_user_object_to_export.html
View file

@ -1,4 +1,4 @@
<a class="btn btn-lg btn-outline-dark" target="_blank" href="{{ url_for('import_export.add_object_id_to_export')}}?obj_type={{obj_type}}&obj_id={{obj_id}}&obj_lvl={{obj_lvl}}{%if obj_subtype%}&obj_subtype={{obj_subtype}}{%endif%}">
<a class="btn btn-lg btn-outline-dark" target="_blank" href="{{ url_for('import_export.add_object_id_to_export')}}?type={{obj_type}}{%if obj_subtype%}&subtype={{obj_subtype}}{%endif%}&id={{obj_id}}&lvl={{obj_lvl}}">
Add to
<img id="misp-logo" src="{{ url_for('static', filename='image/misp-logo.png')}}" height="32">
Export

2
var/www/templates/import_export/block_add_user_object_to_export_small.html
View file

@ -1,6 +1,6 @@
<div class="d-flex flex-row-reverse bd-highlight">
<div>
<a class="btn" target="_blank" href="{{ url_for('import_export.add_object_id_to_export')}}?obj_type={{obj_type}}&obj_id={{obj_id}}&obj_lvl={{obj_lvl}}{%if obj_subtype%}&obj_subtype={{obj_subtype}}{%endif%}">
<a class="btn" target="_blank" href="{{ url_for('import_export.add_object_id_to_export')}}?type={{obj_type}}{%if obj_subtype%}&subtype={{obj_subtype}}{%endif%}&id={{obj_id}}&lvl={{obj_lvl}}">
<img id="misp-logo" src="{{ url_for('static', filename='image/misp-logo.png')}}" height="25">
</a>
</div>

39
var/www/templates/import_export/block_to_export_input.html
View file

@ -1,38 +1,21 @@
<div class="input-group mb-1">
<select class="custom-select col-2" name="{{input_uuid}}" value="{{obj_type}}">
{% if not obj_type %}
<option selected="">Object type...</option>
{% else %}
<option>Object type...</option>
{% endif %}
<option value="item" {%if obj_type=="item"%}selected{%endif%}>Item</option>
<option value="domain" {%if obj_type=="domain"%}selected{%endif%}>Domain</option>
<option value="image" {%if obj_type=="image"%}selected{%endif%}>Image</option>
<option value="decoded" {%if obj_type=="decoded"%}selected{%endif%}>Decoded</option>
<option value="pgp;key" {%if obj_type=="pgp" and obj_subtype=="key"%}selected{%endif%}>PGP - Key</option>
<option value="pgp;name" {%if obj_type=="pgp" and obj_subtype=="name"%}selected{%endif%}>PGP - Name</option>
<option value="pgp;mail" {%if obj_type=="pgp" and obj_subtype=="mail"%}selected{%endif%}>PGP - Mail</option>
<option value="cryptocurrency;bitcoin" {%if obj_type=="cryptocurrency" and obj_subtype=="bitcoin"%}selected{%endif%}>Cryptocurrency - Bitcoin</option>
<option value="cryptocurrency;bitcoin-cash" {%if obj_type=="cryptocurrency" and obj_subtype=="bitcoin-cash"%}selected{%endif%}>Cryptocurrency - Bitcoin Cash</option>
<option value="cryptocurrency;dash" {%if obj_type=="cryptocurrency" and obj_subtype=="dash"%}selected{%endif%}>Cryptocurrency - Dash</option>
<option value="cryptocurrency;etherum" {%if obj_type=="cryptocurrency" and obj_subtype=="etherum"%}selected{%endif%}>Cryptocurrency - Etherum</option>
<option value="cryptocurrency;litecoin" {%if obj_type=="cryptocurrency" and obj_subtype=="litecoin"%}selected{%endif%}>Cryptocurrency - Litecoin</option>
<option value="cryptocurrency;monero" {%if obj_type=="cryptocurrency" and obj_subtype=="monero"%}selected{%endif%}>Cryptocurrency - Monero</option>
<option value="cryptocurrency;zcash" {%if obj_type=="cryptocurrency" and obj_subtype=="zcash"%}selected{%endif%}>Cryptocurrency - Zcash</option>
<option value="username;telegram" {%if obj_type=="username" and obj_subtype=="telegram"%}selected{%endif%}>Username - telegram</option>
<option value="username;twitter" {%if obj_type=="username" and obj_subtype=="twitter"%}selected{%endif%}>Username - twitter</option>
<option value="username;jabber" {%if obj_type=="username" and obj_subtype=="jabber"%}selected{%endif%}>Username - jabber</option>
<select class="custom-select col-2" name="{{input_uuid}}" disabled>
<option selected="" value="{{ obj_type }}:{{ obj_subtype }}">
{% if obj_subtype %}
{{ obj_type }} - {{ obj_subtype }}
{% else %}
{{ obj_type }}
{% endif %}
</option>
</select>
<input type="text" class="form-control col-8 {%if obj_error%}is-invalid{%else%}is-valid{%endif%}" name="{{input_uuid}}" value="{{obj_id}}">
<input type="text" class="form-control col-8 {%if obj_error%}is-invalid{%else%}is-valid{%endif%}" name="{{input_uuid}}" value="{{obj_id}}" disabled>
<input class="form-control col-1" type="number" min="0" name="{{input_uuid}}" {%if obj_lvl%}value="{{obj_lvl}}"{%else%}value="0"{%endif%}>
<span class="btn btn-danger input-group-addon delete-field col-1"><i class="fas fa-trash-alt"></i></span>
<span class="btn btn-danger input-group-addon delete-field col-1" id="{{ obj_type }}:{{ obj_subtype }}:{{ obj_id }}"><i class="fas fa-trash-alt"></i></span>
<div class="invalid-feedback">
{%if obj_type=="Object type..."%}
&nbsp;&nbsp;&nbsp; Please select an object type.
{%else%}
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Unknow or Invalid ID.
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; Unknown or Invalid ID.
{%endif%}
</div>
</div>

84
var/www/templates/import_export/export_object.html
View file

@ -34,7 +34,7 @@
</div>
<div class="card-body">
<form action="{{ url_for('import_export.export_object_file') }}" method="post" enctype=multipart/form-data onsubmit="submitPaste()">
<form action="{{ url_for('import_export.objects_misp_export_post') }}" method="post" enctype=multipart/form-data onsubmit="submitPaste()">
<div id="container-id-to-import">
@ -47,36 +47,30 @@
</div>
<div class="form-horizontal">
<div class="form-body">
<div class="form-group">
<div class="fields">
<div class="input-group mb-1">
<select class="custom-select col-2" name="first_obj_to_export" id="obj_input_type">
<option selected>Object type...</option>
<option value="item">Item</option>
<option value="domain">Domain</option>
<option value="image">Image</option>
<option value="decoded">Decoded</option>
<option value="pgp;key">PGP - Key</option>
<option value="pgp;name">PGP - Name</option>
<option value="pgp;mail">PGP - Mail</option>
<option value="cryptocurrency;bitcoin">Cryptocurrency - Bitcoin</option>
<option value="cryptocurrency;bitcoin-cash">Cryptocurrency - Bitcoin Cash</option>
<option value="cryptocurrency;dash">Cryptocurrency - Dash</option>
<option value="cryptocurrency;etherum">Cryptocurrency - Etherum</option>
<option value="cryptocurrency;litecoin">Cryptocurrency - Litecoin</option>
<option value="cryptocurrency;monero">Cryptocurrency - Monero</option>
<option value="cryptocurrency;zcash">Cryptocurrency - Zcash</option>
<option value="username;telegram">Username - telegram</option>
<option value="username;twitter">Username - twitter</option>
<option value="username;jabber">Username - jabber</option>
</select>
<input type="text" class="form-control col-8" name="first_obj_to_export" id="obj_input_id">
<div class="form-body">
<div class="form-group">
<div class="fields">
<div class="input-group mb-1">
<select class="custom-select col-2" name="first_obj_to_export" id="obj_input_type">
<option selected>Object type...</option>
{% for object_type in object_types %}
<option value="{{ object_type[0] }}:{{ object_type[1] }}">
{% if object_type[1] %}
{{ object_type[0] }} - {{ object_type[1] }}
{% else %}
{{ object_type[0] }}
{% endif %}
</option>
{% endfor %}
</select>
<input type="text" class="form-control col-8" name="first_obj_to_export" id="obj_input_id">
<input class="form-control col-1" type="number" min="0" value="0" name="first_obj_to_export" id="obj_input_lvl">
<span class="btn btn-info input-group-addon add-field col-1"><i class="fas fa-plus"></i></span>
</div>
{% for obj_dict in l_obj_to_export %}
{% with obj_type=obj_dict['type'], obj_subtype=obj_dict.get('subtype', None), obj_id=obj_dict['id'], obj_lvl=obj_dict['lvl'], input_uuid=obj_dict, obj_error=False%}
<span class="btn btn-info input-group-addon add-field col-1"><i class="fas fa-plus"></i></span>
</div>
{% for obj_dict in to_export %}
{% with obj_type=obj_dict['type'], obj_subtype=obj_dict['subtype'], obj_id=obj_dict['id'], obj_lvl=obj_dict['lvl'], input_uuid=obj_dict, obj_error=False%}
{% include 'import_export/block_to_export_input.html' %}
{% endwith %}
{% endfor %}
@ -84,7 +78,7 @@
<br>
{% for obj_dict in l_obj_invalid %}
{% with obj_type=obj_dict['type'], obj_subtype=obj_dict.get('subtype', None), obj_id=obj_dict['id'], obj_lvl=obj_dict['lvl'], input_uuid=obj_dict['uuid'], obj_error=True%}
{% with obj_type=obj_dict['type'], obj_subtype=obj_dict.get('subtype', None), obj_id=obj_dict['id'], obj_lvl=obj_dict['lvl'], input_uuid=obj_dict, obj_error=True%}
{% include 'import_export/block_to_export_input.html' %}
{% endwith %}
{% endfor %}
@ -119,10 +113,10 @@
</div>
</div>
{% if event_metadata %}
{% if misp_url %}
MISP Event Created:
<a target="_blank" href="{{ event_metadata['url'] }}">
{{ event_metadata['url'] }}
<a target="_blank" href="{{ misp_url }}">
{{ misp_url }}
</a>
{% endif %}
@ -155,7 +149,17 @@ function toggle_sidebar(){
}
var input_1 = '<div class="input-group mb-1"><select class="custom-select col-2" name="'
var input_2 = '"><option selected>Object type...</option><option value="item">Item</option><option value="domain">Domain</option><option value="image">Image</option><option value="decoded">Decoded</option><option value="pgp;key">PGP - Key</option><option value="pgp;name">PGP - Name</option><option value="pgp;mail">PGP - Mail</option><option value="cryptocurrency;bitcoin">Cryptocurrency - Bitcoin</option><option value="cryptocurrency;bitcoin-cash">Cryptocurrency - Bitcoin Cash</option><option value="cryptocurrency;dash">Cryptocurrency - Dash</option><option value="cryptocurrency;etherum">Cryptocurrency - Etherum</option><option value="cryptocurrency;litecoin">Cryptocurrency - Litecoin</option><option value="cryptocurrency;monero">Cryptocurrency - Monero</option><option value="cryptocurrency;zcash">Cryptocurrency - Zcash</option></select><input type="text" class="form-control col-8" name="'
var input_2 = '"><option selected>Object type...</option>' +
{% for object_type in object_types %}
'<option value="{{ object_type[0] }}:{{ object_type[1] }}">' +
{% if object_type[1] %}
'{{ object_type[0] }} - {{ object_type[1] }}' +
{% else %}
'{{ object_type[0] }}' +
{% endif %}
'</option>' +
{% endfor %}
'</select><input type="text" class="form-control col-8" name="'
var input_3 = '"><input class="form-control col-1" type="number" min="0" value="0" name="'
var input_4 = '"></div>';
var minusButton = '<span class="btn btn-danger input-group-addon delete-field col-1"><i class="fas fa-trash-alt"></i></span>';
@ -163,13 +167,15 @@ var minusButton = '<span class="btn btn-danger input-group-addon delete-field co
$('.add-field').click(function() {
var new_uuid = uuidv4();
var template = input_1 + new_uuid + input_2 + new_uuid + input_3+ new_uuid + input_4;
var temp = $(template).insertBefore('.help-block');
temp.append(minusButton);
var temp = $(template).insertBefore('.help-block');
temp.append(minusButton);
});
$('.fields').on('click', '.delete-field', function(){
$(this).parent().remove();
//$.get( "#")
var obj = $(this).attr('id').split(":", 3);
var params = "?type=" + obj[0] + "&subtype=" + obj[1] +"&id=" + obj[2];
$.ajax({url: "{{ url_for('import_export.delete_object_id_to_export') }}" + params })
$(this).parent().remove();
});

4
var/www/templates/investigations/view_investigation.html
View file

@ -107,8 +107,8 @@
{% if metadata['misp_events'] %}
{% for misp_url in metadata['misp_events'] %}
<a target="_blank" href="{{misp_url}}/events/view/{{metadata['uuid'][:8]}}-{{metadata['uuid'][8:12]}}-{{metadata['uuid'][12:16]}}-{{metadata['uuid'][16:20]}}-{{metadata['uuid'][20:]}}">
{{misp_url}}/events/view/{{metadata['uuid'][:8]}}-{{metadata['uuid'][8:12]}}-{{metadata['uuid'][12:16]}}-{{metadata['uuid'][16:20]}}-{{metadata['uuid'][20:]}}</br>
<a target="_blank" href="{{ misp_url }}">
{{ misp_url }}</br>
</a>
{% endfor %}
{% endif %}

158
var/www/templates/modals/create_hive_case.html
View file

@ -1,120 +1,54 @@
<div id="create_hive_case_modal" class="modal fade" role="dialog">
<div class="modal-dialog modal-lg">
<div class="modal-dialog modal-lg">
<div id="create_hive_case_modal_content" class="modal-content">
<div class="modal-header justify-content-center">
<img id="hive-logo" src="{{url_for('static', filename='image/thehive-logo.png') }}" width="500" >
</div>
<div id="create_hive_case_modal_content" class="modal-content">
<div class="modal-header justify-content-center">
<img id="hive-logo" src="{{url_for('static', filename='image/thehive-logo.png') }}" width="500" >
</div>
<div class="modal-body">
<form method="post" action="{{ url_for('PasteSubmit.create_hive_case') }}" target="_blank">
<form method="post" action="{{ url_for('PasteSubmit.create_hive_case') }}" target="_blank">
<div class="modal-body">
<div class="input clear required">
<label for="EventThreatLevelId">Threat Level</label>
<select name="threat_level_hive" id="EventThreatLevelId" required="required">
<option value="1">High</option>
<option value="2" selected="selected">Medium</option>
<option value="3">Low</option>
</select>
</div>
<div class="input select required">
<label for="hive_tlp">TLP</label>
<select name="hive_tlp" id="hive_tlp" required="required" class="selectpicker">
<option value="0">White</option>
<option value="1">Green</option>
<option value="2" selected="selected">Amber</option>
<option value="3">Red</option>
</select>
</div>
<div class="clear">
<label for="hive_case_title">Title</label>
<input name="hive_case_title" class="form-control span6" placeholder="Title" type="text" id="hive_case_title"/>
</div>
<div class="clear">
<label for="hive_description">Description</label>
<input name="hive_description" class="form-control span6" placeholder="Quick Case Description" type="text" id="hive_description"/>
</div>
<input type="hidden" id="obj_id" name="obj_id" value="{{ dict_item['id'] }}">
</div>
<div class="modal-footer">
<button class="btn btn-primary" type="submit">
<i class="fas fa-plus"></i>
Create Case
</button>
<button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
</div>
</form>
<div class="input clear required">
<label for="EventThreatLevelId">Threat Level</label>
<select name="threat_level_hive" id="EventThreatLevelId" required="required">
<option value="1">High</option>
<option value="2" selected="selected">Medium</option>
<option value="3">Low</option>
</select>
</div>
<div class="input select required">
<label for="TLP">TLP</label>
<select name="hive_tlp" id="hive_tlp" required="required" class="selectpicker">
<option value="0">White</option>
<option value="1">Green</option>
<option value="2" selected="selected">Amber</option>
<option value="3">Red</option>
</select>
</div>
<div class="clear required">
<label for="hive_case_title">Title</label>
<input name="hive_case_title" class="form-control span6" placeholder="Title" type="text" id="hive_case_title" required="required"/>
</div>
<div class="clear required">
<label for="hive_description">Description</label>
<input name="hive_description" class="form-control span6" placeholder="Quick Case Description" type="text" id="hive_description" required="required"/>
</div>
<input type="hidden" id="paste" name="paste" value="{{ dict_item['id'] }}">
</div>
<div class="modal-footer">
<button class="btn btn-primary" type="submit">
<i class="fas fa-plus"></i>
Create Case
</button>
</form>
<button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
</div>
</div>
</div>
</div>
<script> // TODO: add tags to PASTE or DOMAIN
var ltags;
var ltagsgalaxies;
$.getJSON("{{ url_for('Tags.get_all_tags_taxonomies') }}",
function(data) {
ltags = $('#ltags').tagSuggest({
data: data,
maxDropHeight: 200,
name: 'ltags'
});
});
$.getJSON("{{ url_for('Tags.get_all_tags_galaxy') }}",
function(data) {
ltagsgalaxies = $('#ltagsgalaxies').tagSuggest({
data: data,
maxDropHeight: 200,
name: 'ltagsgalaxies'
});
});
jQuery("#all-tags-taxonomies").click(function(e){
//change input tags list
$.getJSON("{{ url_for('Tags.get_all_tags_taxonomies') }}",
function(data) {
ltags.setData(data)
});
});
jQuery("#all-tags-galaxies").click(function(e){
$.getJSON("{{ url_for('Tags.get_all_tags_galaxy') }}",
function(data) {
ltagsgalaxies.setData(data)
});
});
{% for taxo in modal_add_tags['active_taxonomies'] %}
jQuery("#{{ taxo }}-id{{ loop.index0 }}").click(function(e){
$.getJSON("{{ url_for('Tags.get_tags_taxonomie') }}?taxonomie={{ taxo }}",
function(data) {
ltags.setData(data)
});
});
{% endfor %}
{% for galaxy in modal_add_tags['active_galaxies'] %}
jQuery("#{{ galaxy }}-idgalax{{ loop.index0 }}").click(function(e){
$.getJSON("{{ url_for('Tags.get_tags_galaxy') }}?galaxy={{ galaxy }}",
function(data) {
ltagsgalaxies.setData(data)
});
});
{% endfor %}
function addTags() {
var tags = ltags.getValue()
var tagsgalaxy = ltagsgalaxies.getValue()
window.location.replace("{{ url_for('tags_ui.add_tags') }}?tags=" + tags + "&tagsgalaxies=" + tagsgalaxy + "&object_id={{ modal_add_tags['object_id'] }}&object_type={{ modal_add_tags['object_type'] }}");
}
</script>

2
var/www/templates/sidebars/sidebar_objects.html
View file

@ -75,7 +75,7 @@
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="{{url_for('import_export.export_object')}}" id="nav_misp_export">
<a class="nav-link" href="{{url_for('import_export.objects_misp_export')}}" id="nav_misp_export">
<b>Export</b>
</a>
</li>