mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 00:28:22 +00:00
chg: [MISP export + redis] refactor MISP, export all relationships between objects, relationships between investigated objects + bump redis-py version + theHive export
This commit is contained in:
Terrtia
parent
7fa19efc1e
commit
0e41c95b5c
28 changed files with 775 additions and 1013 deletions
|
|
@ -585,6 +585,7 @@ def domain_migration():
|
|||
|
||||
#print()
|
||||
|
||||
# TODO REMOVE INVALID DOMAINS
|
||||
for domain_type in ['onion', 'regular']:
|
||||
for date in Date.get_date_range_today('20190101'):
|
||||
for dom in get_domain_down_by_date(domain_type, date):
|
||||
|
|
@ -601,7 +602,7 @@ def domain_migration():
|
|||
domain.update_daterange(last_check)
|
||||
if last_origin:
|
||||
domain.set_last_origin(last_origin)
|
||||
domain.add_history(None, None, date=date)
|
||||
domain.add_history(0, None, date=date)
|
||||
|
||||
|
||||
###############################
|
||||
|
|
@ -871,12 +872,12 @@ if __name__ == '__main__':
|
|||
#core_migration()
|
||||
#user_migration()
|
||||
#tags_migration()
|
||||
items_migration()
|
||||
#crawler_migration()
|
||||
# domain_migration() # TO TEST ###########################
|
||||
# items_migration()
|
||||
# crawler_migration()
|
||||
domain_migration() # TO TEST ###########################
|
||||
# decodeds_migration()
|
||||
# screenshots_migration()
|
||||
# subtypes_obj_migration()
|
||||
subtypes_obj_migration()
|
||||
# ail_2_ail_migration()
|
||||
# trackers_migration()
|
||||
# investigations_migration()
|
||||
|
|
|
|||
|
|
@ -1,53 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
import uuid
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
from lib.objects import ail_objects
|
||||
from lib import ConfigLoader
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
r_serv_objects = config_loader.get_redis_conn("ARDB_Objects")
|
||||
config_loader = None
|
||||
|
||||
def create_map_obj_uuid_golbal_id(obj_uuid, global_id):
|
||||
r_serv_objects.sadd('all_object:uuid', obj_uuid)
|
||||
r_serv_objects.sadd('all_object:global_id', global_id)
|
||||
r_serv_objects.sadd('object:map:uuid_id:{}'.format(obj_uuid), global_id)
|
||||
r_serv_objects.sadd('object:map:id_uuid:{}'.format(global_id), obj_uuid)
|
||||
|
||||
def create_map_obj_event_uuid(event_uuid, global_id):
|
||||
r_serv_objects.sadd('export:all_object:event_uuid', event_uuid)
|
||||
r_serv_objects.sadd('export:all_object:global_id', global_id)
|
||||
r_serv_objects.sadd('object:map:event_id:{}'.format(event_uuid), global_id)
|
||||
r_serv_objects.sadd('object:map:id_event:{}'.format(global_id), event_uuid)
|
||||
|
||||
# def get_user_list_of_obj_to_export(user_id, add_uuid=False):
|
||||
# objs_to_export = []
|
||||
# res = r_serv_objects.hgetall('user:all_objs_to_export:{}'.format(user_id))
|
||||
# for global_id in res:
|
||||
# dict_obj = Correlate_object.get_global_id_from_id(global_id)
|
||||
# dict_obj['lvl'] = int(res[global_id])
|
||||
# if add_uuid:
|
||||
# obj_dict['uuid'] = str(uuid.uuid4())
|
||||
# objs_to_export.append(dict_obj)
|
||||
# return objs_to_export
|
||||
|
||||
def add_user_object_to_export(user_id, obj_type, obj_id, lvl, obj_subtype=None):
|
||||
## TODO: check if user exist
|
||||
# # TODO: check if valid object
|
||||
# # TODO: check lvl
|
||||
global_id = ail_objects.get_obj_global_id(obj_type, obj_subtype, obj_id)
|
||||
return r_serv_objects.hset('user:all_objs_to_export:{}'.format(user_id), global_id, lvl)
|
||||
|
||||
def delete_user_object_to_export(user_id, obj_type, obj_id, obj_subtype=None):
|
||||
## TODO: check if user exist
|
||||
global_id = ail_objects.get_obj_global_id(obj_type, obj_subtype, obj_id)
|
||||
r_serv_objects.hdel('user:all_objs_to_export:{}'.format(user_id), global_id)
|
||||
|
||||
def delete_all_user_object_to_export(user_id):
|
||||
## TODO: check if user exist
|
||||
r_serv_objects.delete('user:all_objs_to_export:{}'.format(user_id))
|
||||
|
|
@ -3,53 +3,172 @@
|
|||
|
||||
import os
|
||||
import sys
|
||||
from uuid import uuid4
|
||||
import uuid
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
from lib import ConfigLoader
|
||||
|
||||
sys.path.append('../../configs/keys')
|
||||
try:
|
||||
from thehive4py.api import TheHiveApi
|
||||
import thehive4py.exceptions
|
||||
from theHiveKEYS import the_hive_url, the_hive_key, the_hive_verifycert
|
||||
if the_hive_url == '':
|
||||
is_hive_connected = False
|
||||
else:
|
||||
is_hive_connected = TheHiveApi(the_hive_url, the_hive_key, cert=the_hive_verifycert)
|
||||
except:
|
||||
is_hive_connected = False
|
||||
if is_hive_connected != False:
|
||||
try:
|
||||
is_hive_connected.get_alert(0)
|
||||
is_hive_connected = True
|
||||
except thehive4py.exceptions.AlertException:
|
||||
is_hive_connected = False
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from lib.ConfigLoader import ConfigLoader
|
||||
from lib.objects.Items import Item
|
||||
from lib.ail_core import get_ail_uuid
|
||||
from lib.Investigations import Investigation
|
||||
from lib.objects import ail_objects
|
||||
|
||||
## LOAD CONFIG ##
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
r_serv_cache = config_loader.get_redis_conn("Redis_Cache")
|
||||
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
|
||||
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
|
||||
config_loader = ConfigLoader()
|
||||
r_cache = config_loader.get_redis_conn("Redis_Cache")
|
||||
r_db = config_loader.get_db_conn("Kvrocks_DB")
|
||||
|
||||
r_serv_db = config_loader.get_redis_conn("ARDB_DB") ######################################
|
||||
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") ######################################
|
||||
config_loader = None
|
||||
## -- ##
|
||||
|
||||
def load_tags_to_export_in_cache():
|
||||
all_exports = ['misp', 'thehive']
|
||||
for export_target in all_exports:
|
||||
# save solo tags in cache
|
||||
all_tags_to_export = Tag.get_list_of_solo_tags_to_export_by_type()
|
||||
if len(all_tags_to_export) > 1:
|
||||
r_serv_cache.sadd('to_export:solo_tags:{}'.format(export_target), *all_tags_to_export)
|
||||
elif all_tags_to_export:
|
||||
r_serv_cache.sadd('to_export:solo_tags:{}'.format(export_target), all_tags_to_export[0])
|
||||
sys.path.append('../../configs/keys')
|
||||
##################################
|
||||
# Import Keys
|
||||
##################################
|
||||
from thehive4py.api import TheHiveApi
|
||||
from thehive4py.models import Alert, AlertArtifact, Case, CaseObservable
|
||||
import thehive4py.exceptions
|
||||
|
||||
# save combinaison of tags in cache
|
||||
pass
|
||||
from pymisp import MISPEvent, MISPObject, PyMISP
|
||||
|
||||
def is_hive_connected(): # # TODO: REFRACTOR, put in cache (with retry)
|
||||
return is_hive_connected
|
||||
##################################
|
||||
# THE HIVE
|
||||
##################################
|
||||
|
||||
HIVE_CLIENT = None
|
||||
try:
|
||||
from theHiveKEYS import the_hive_url, the_hive_key, the_hive_verifycert
|
||||
|
||||
HIVE_URL = the_hive_url
|
||||
HIVE_KEY = the_hive_key
|
||||
HIVE_VERIFY_CERT = the_hive_verifycert
|
||||
except:
|
||||
HIVE_URL = None
|
||||
HIVE_KEY = None
|
||||
HIVE_VERIFY_CERT = None
|
||||
|
||||
|
||||
def get_hive_client():
|
||||
global HIVE_CLIENT
|
||||
try:
|
||||
HIVE_CLIENT = TheHiveApi(HIVE_URL, HIVE_KEY, cert=HIVE_VERIFY_CERT)
|
||||
except:
|
||||
HIVE_CLIENT = None
|
||||
return HIVE_CLIENT
|
||||
|
||||
|
||||
def is_hive_connected():
|
||||
try:
|
||||
# print(hive_client.health())
|
||||
HIVE_CLIENT.get_alert(0)
|
||||
return True
|
||||
except thehive4py.exceptions.AlertException:
|
||||
return False
|
||||
|
||||
|
||||
HIVE_CLIENT = get_hive_client()
|
||||
|
||||
|
||||
def create_thehive_alert(item_id, tag_trigger):
|
||||
item = Item(item_id)
|
||||
meta = item.get_meta()
|
||||
# TheHive expects a file
|
||||
content = item.get_raw_content(decompress=True)
|
||||
|
||||
# remove .gz from submitted path to TheHive because we've decompressed it
|
||||
if item_id.endswith(".gz"):
|
||||
item_id = item_id[:-3]
|
||||
# add .txt it's easier to open when downloaded from TheHive
|
||||
item_id = f'{item_id}.txt'
|
||||
|
||||
artifacts = [
|
||||
AlertArtifact(dataType='other', message='uuid-ail', data=(get_ail_uuid())),
|
||||
AlertArtifact(dataType='file', data=(content, item_id), tags=meta['tags'])
|
||||
]
|
||||
|
||||
# Prepare the sample Alert
|
||||
sourceRef = str(uuid.uuid4())[0:6]
|
||||
alert = Alert(title='AIL Leak',
|
||||
tlp=3,
|
||||
tags=meta['tags'],
|
||||
description='AIL Leak, triggered by {}'.format(tag_trigger),
|
||||
type='ail',
|
||||
source=meta['source'], # Use item ID ?
|
||||
sourceRef=sourceRef,
|
||||
artifacts=artifacts)
|
||||
|
||||
# Create the Alert
|
||||
alert_id = None
|
||||
try:
|
||||
response = HIVE_CLIENT.create_alert(alert)
|
||||
if response.status_code == 201:
|
||||
# print(json.dumps(response.json(), indent=4, sort_keys=True))
|
||||
print('Alert Created')
|
||||
print(response.json())
|
||||
alert_id = response.json()['id']
|
||||
else:
|
||||
print(f'ko: {response.status_code}/{response.text}')
|
||||
return 0
|
||||
except:
|
||||
print('hive connection error')
|
||||
print(alert_id)
|
||||
|
||||
|
||||
# TODO SAVE CASE URL ????????????????????????
|
||||
def create_thehive_case(item_id, title=None, tlp=2, threat_level=2, description=None):
|
||||
item = Item(item_id)
|
||||
ail_uuid = get_ail_uuid()
|
||||
|
||||
if not title:
|
||||
title = f'AIL Case {item.id}'
|
||||
if not description:
|
||||
description = f'AIL {ail_uuid} Case'
|
||||
date = item.get_date()
|
||||
date = f'{date[0:4]}-{date[4:6]}-{date[6:8]}'
|
||||
tags = item.get_tags(r_list=True)
|
||||
|
||||
case = Case(title=title,
|
||||
tlp=tlp,
|
||||
severity=threat_level,
|
||||
flag=False,
|
||||
tags=tags,
|
||||
description=description)
|
||||
|
||||
# Create Case
|
||||
response = get_hive_client().create_case(case)
|
||||
if response.status_code == 201:
|
||||
case_id = response.json()['id']
|
||||
|
||||
observables = [
|
||||
CaseObservable(dataType="other", data=[ail_uuid], message="uuid-ail"),
|
||||
CaseObservable(dataType="file", data=item.get_filename(), tags=tags),
|
||||
CaseObservable(dataType="other", data=[item.get_source()], message="source"),
|
||||
CaseObservable(dataType="other", data=[date], message="last-seen")
|
||||
]
|
||||
|
||||
for observable in observables:
|
||||
resp = HIVE_CLIENT.create_case_observable(case_id, observable)
|
||||
if resp.status_code != 201:
|
||||
print(f'error observable creation: {resp.status_code}/{resp.text}')
|
||||
# print(case_id)
|
||||
# return HIVE_URL /thehive/cases/~37040/details
|
||||
return case_id
|
||||
|
||||
# r_serv_metadata.set('hive_cases:'+path, id)
|
||||
else:
|
||||
print(f'ko: {response.status_code}/{response.text}')
|
||||
return None
|
||||
|
||||
|
||||
def get_case_url(case_id):
|
||||
return f'{HIVE_URL}/cases/{case_id}/details'
|
||||
|
||||
|
||||
# TODO
|
||||
def get_item_hive_cases(item_id):
|
||||
hive_case = r_serv_metadata.get('hive_cases:{}'.format(item_id))
|
||||
if hive_case:
|
||||
|
|
@ -57,6 +176,150 @@ def get_item_hive_cases(item_id):
|
|||
return hive_case
|
||||
|
||||
|
||||
##################################
|
||||
# MISP
|
||||
##################################
|
||||
|
||||
try:
|
||||
from mispKEYS import misp_url, misp_key, misp_verifycert
|
||||
|
||||
MISP_URL = misp_url
|
||||
MISP_KEY = misp_key
|
||||
MISP_VERIFYCERT = misp_verifycert
|
||||
if MISP_URL.endswith('/'):
|
||||
MISP_URL = MISP_URL[:-1]
|
||||
except:
|
||||
MISP_URL = None
|
||||
MISP_KEY = None
|
||||
MISP_VERIFYCERT = None
|
||||
|
||||
|
||||
def get_misp_client():
|
||||
return PyMISP(misp_url, misp_key, misp_verifycert)
|
||||
|
||||
|
||||
# # TODO: return error
|
||||
def ping_misp():
|
||||
try:
|
||||
PyMISP(misp_url, misp_key, misp_verifycert)
|
||||
return True
|
||||
except Exception as e:
|
||||
print(e)
|
||||
return False
|
||||
|
||||
|
||||
def sanitize_misp_event_distribution(distribution):
|
||||
try:
|
||||
int(distribution)
|
||||
if 0 <= distribution <= 3:
|
||||
return distribution
|
||||
else:
|
||||
return 0
|
||||
except:
|
||||
return 0
|
||||
|
||||
|
||||
def sanitize_misp_event_threat_level(threat_level):
|
||||
try:
|
||||
int(threat_level)
|
||||
if 1 <= threat_level <= 4:
|
||||
return threat_level
|
||||
else:
|
||||
return 4
|
||||
except:
|
||||
return 4
|
||||
|
||||
|
||||
def sanitize_misp_event_analysis(analysis):
|
||||
try:
|
||||
int(analysis)
|
||||
if 0 <= analysis <= 2:
|
||||
return analysis
|
||||
else:
|
||||
return 0
|
||||
except:
|
||||
return 0
|
||||
|
||||
def create_misp_event(objs, event_uuid=None, date=None, distribution=0, threat_level_id=4, publish=False, analysis=0,
|
||||
info=None, tags=None, export=False):
|
||||
if tags is None:
|
||||
tags = []
|
||||
event = MISPEvent()
|
||||
if not event_uuid:
|
||||
event_uuid = str(uuid.uuid4())
|
||||
event.uuid = event_uuid
|
||||
if date:
|
||||
event.date = date
|
||||
if not info:
|
||||
info = 'AIL framework export'
|
||||
event.info = info
|
||||
if publish:
|
||||
event.publish()
|
||||
for tag in tags:
|
||||
event.add_tag(tag)
|
||||
event.distribution = sanitize_misp_event_distribution(distribution)
|
||||
event.threat_level_id = sanitize_misp_event_threat_level(threat_level_id)
|
||||
event.analysis = sanitize_misp_event_analysis(analysis)
|
||||
|
||||
misp_objects = ail_objects.get_misp_objects(objs)
|
||||
for obj in misp_objects:
|
||||
event.add_object(obj)
|
||||
|
||||
# print(event.to_json())
|
||||
|
||||
if export:
|
||||
misp = get_misp_client()
|
||||
misp_event = misp.add_event(event)
|
||||
# TODO: handle error
|
||||
|
||||
misp_event['url'] = f'{MISP_URL}/events/view/{misp_event["Event"]["uuid"]}'
|
||||
return misp_event
|
||||
else:
|
||||
return {'uuid': event['uuid'], 'event': event.to_json()}
|
||||
|
||||
def create_investigation_misp_event(investigation_uuid):
|
||||
investigation = Investigation(investigation_uuid)
|
||||
objs = ail_objects.get_objects(investigation.get_objects())
|
||||
|
||||
event = create_misp_event(objs,
|
||||
# event_uuid=investigation.get_uuid(separator=True), # TODO EDIT EXISTING EVENT ????
|
||||
date=investigation.get_date(),
|
||||
distribution=0,
|
||||
threat_level_id=investigation.get_threat_level(),
|
||||
analysis=investigation.get_analysis(),
|
||||
info=investigation.get_info(),
|
||||
tags=investigation.get_tags(),
|
||||
export=True)
|
||||
url = event['url']
|
||||
if url:
|
||||
investigation.add_misp_events(url)
|
||||
return url
|
||||
|
||||
def get_user_misp_objects_to_export(user_id):
|
||||
objs = []
|
||||
objects = r_db.hgetall(f'user:obj:misp:export:{user_id}')
|
||||
for obj in objects:
|
||||
obj_type, obj_subtype, obj_id = obj.split(':', 2)
|
||||
lvl = objects[obj]
|
||||
try:
|
||||
lvl = int(lvl)
|
||||
except:
|
||||
lvl = 0
|
||||
objs.append({'type': obj_type, 'subtype': obj_subtype, 'id': obj_id, 'lvl': lvl})
|
||||
return objs
|
||||
|
||||
def add_user_misp_object_to_export(user_id, obj_type, obj_subtype, obj_id, lvl=0):
|
||||
if not obj_subtype:
|
||||
obj_subtype = ''
|
||||
r_db.hset(f'user:obj:misp:export:{user_id}', f'{obj_type}:{obj_subtype}:{obj_id}', lvl)
|
||||
|
||||
def delete_user_misp_object_to_export(user_id, obj_type, obj_subtype, obj_id):
|
||||
r_db.hdel(f'user:obj:misp:export:{user_id}', f'{obj_type}:{obj_subtype}:{obj_id}')
|
||||
|
||||
def delete_user_misp_objects_to_export(user_id):
|
||||
r_db.delete(f'user:obj:misp:export:{user_id}')
|
||||
|
||||
|
||||
###########################################################
|
||||
# # set default
|
||||
# if r_serv_db.get('hive:auto-alerts') is None:
|
||||
|
|
@ -64,3 +327,11 @@ def get_item_hive_cases(item_id):
|
|||
#
|
||||
# if r_serv_db.get('misp:auto-events') is None:
|
||||
# r_serv_db.set('misp:auto-events', 0)
|
||||
|
||||
# if __name__ == '__main__':
|
||||
# from lib.objects.Cves import Cve
|
||||
# create_misp_event([Item('crawled/2020/09/14/circl.lu0f4976a4-dda4-4189-ba11-6618c4a8c951'),
|
||||
# Cve('CVE-2020-16856'), Cve('CVE-2014-6585'), Cve('CVE-2015-0383'),
|
||||
# Cve('CVE-2015-0410')])
|
||||
|
||||
# create_investigation_misp_event('c6bbf8fa9ead4cc698eaeb07835cca5d)
|
||||
|
|
|
|||
|
|
@ -1,329 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import io
|
||||
import sys
|
||||
import uuid
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
from lib.objects import ail_objects
|
||||
|
||||
from export import AILObjects
|
||||
|
||||
|
||||
from lib.Investigations import Investigation
|
||||
|
||||
# # TODO: # FIXME: REFACTOR ME => use UI/Global config
|
||||
sys.path.append('../../configs/keys')
|
||||
try:
|
||||
from mispKEYS import misp_url, misp_key, misp_verifycert
|
||||
except:
|
||||
misp_url = ''
|
||||
misp_key = ''
|
||||
misp_verifycert = False
|
||||
|
||||
# MISP
|
||||
from pymisp import MISPEvent, MISPObject, PyMISP
|
||||
|
||||
def is_valid_obj_to_export(obj_type, obj_subtype, obj_id):
|
||||
if not ail_objects.is_valid_object_type(obj_type):
|
||||
return False
|
||||
if not ail_objects.is_valid_object_subtype(obj_type, obj_subtype):
|
||||
return False
|
||||
if not ail_objects.exists_obj(obj_type, obj_subtype, obj_id):
|
||||
return False
|
||||
return True
|
||||
|
||||
def sanitize_obj_export_lvl(lvl):
|
||||
try:
|
||||
lvl = int(lvl)
|
||||
except:
|
||||
lvl = 0
|
||||
return lvl
|
||||
|
||||
def get_export_filename(json_content):
|
||||
return 'ail_export_{}.json'.format(json_content.uuid)
|
||||
|
||||
def create_in_memory_file(json_content):
|
||||
return io.BytesIO(json_content.encode())
|
||||
|
||||
def add_relation_ship_to_create(set_relationship, dict_obj, dict_new_obj):
|
||||
global_id = ail_objects.get_obj_global_id(dict_obj['type'], dict_obj.get('subtype', ''), dict_obj['id'])
|
||||
global_id_new = ail_objects.get_obj_global_id(dict_new_obj['type'], dict_new_obj.get('subtype', ''), dict_new_obj['id'])
|
||||
if global_id > global_id_new:
|
||||
res = (global_id, global_id_new)
|
||||
else:
|
||||
res = (global_id_new, global_id)
|
||||
set_relationship.add( res )
|
||||
|
||||
# # TODO: add action by obj type
|
||||
# ex => Domain
|
||||
def add_obj_to_create(all_obj_to_export, set_relationship, dict_obj):
|
||||
all_obj_to_export.add(ail_objects.get_obj_global_id(dict_obj['type'], dict_obj.get('subtype', ''), dict_obj['id']))
|
||||
|
||||
def add_obj_to_create_by_lvl(all_obj_to_export, set_relationship, dict_obj, lvl):
|
||||
# # TODO: filter by export mode or filter on all global ?
|
||||
if lvl >= 0:
|
||||
add_obj_to_create(all_obj_to_export, add_obj_to_create, dict_obj)
|
||||
|
||||
if lvl > 0:
|
||||
lvl = lvl - 1
|
||||
|
||||
# # TODO: filter by correlation types
|
||||
obj_correlations = ail_objects.get_obj_correlations(dict_obj['type'], dict_obj.get('subtype', ''), dict_obj['id'])
|
||||
for obj_type in obj_correlations:
|
||||
dict_new_obj = {'type': obj_type}
|
||||
if obj_type=='pgp' or obj_type=='cryptocurrency' or obj_type=='username':
|
||||
for subtype in obj_correlations[obj_type]:
|
||||
dict_new_obj['subtype'] = subtype
|
||||
for obj_id in obj_correlations[obj_type][subtype]:
|
||||
dict_new_obj['id'] = obj_id
|
||||
add_obj_to_create_by_lvl(all_obj_to_export, set_relationship, dict_new_obj, lvl)
|
||||
add_relation_ship_to_create(set_relationship, dict_obj, dict_new_obj)
|
||||
|
||||
else:
|
||||
for obj_id in obj_correlations[obj_type]:
|
||||
dict_new_obj['id'] = obj_id
|
||||
add_obj_to_create_by_lvl(all_obj_to_export, set_relationship, dict_new_obj, lvl)
|
||||
add_relation_ship_to_create(set_relationship, dict_obj, dict_new_obj)
|
||||
|
||||
|
||||
add_obj_to_create_by_lvl(all_obj_to_export, set_relationship, dict_obj, lvl)
|
||||
|
||||
|
||||
def create_list_of_objs_to_export(l_obj, r_type='json'):
|
||||
all_obj_to_export = set()
|
||||
set_relationship = set()
|
||||
for obj in l_obj:
|
||||
add_obj_to_create_by_lvl(all_obj_to_export, set_relationship, obj, obj.get('lvl', 1))
|
||||
|
||||
# create MISP objects
|
||||
dict_misp_obj = create_all_misp_obj(all_obj_to_export, set_relationship)
|
||||
|
||||
# create object relationships
|
||||
for obj_global_id_1, obj_global_id_2 in set_relationship:
|
||||
dict_relationship = get_relationship_between_global_obj(obj_global_id_1, obj_global_id_2)
|
||||
if dict_relationship:
|
||||
obj_src = dict_misp_obj[dict_relationship['src']]
|
||||
obj_dest = dict_misp_obj[dict_relationship['dest']]
|
||||
obj_src.add_reference(obj_dest.uuid, dict_relationship['relation'], 'add a comment')
|
||||
|
||||
event = MISPEvent()
|
||||
event.info = 'AIL framework export'
|
||||
event.uuid = str(uuid.uuid4())
|
||||
for obj_global_id in dict_misp_obj:
|
||||
misp_obj = dict_misp_obj[obj_global_id]
|
||||
AILObjects.create_map_obj_event_uuid(event.uuid, obj_global_id)
|
||||
AILObjects.create_map_obj_uuid_golbal_id(event.uuid, obj_global_id)
|
||||
if misp_obj:
|
||||
# add object to event
|
||||
event.add_object(dict_misp_obj[obj_global_id])
|
||||
|
||||
return event
|
||||
|
||||
# TODO REFACTOR ME
|
||||
def create_all_misp_obj(all_obj_to_export, set_relationship):
|
||||
dict_misp_obj = {}
|
||||
for obj_global_id in all_obj_to_export:
|
||||
obj_type, obj_id = obj_global_id.split(':', 1)
|
||||
dict_misp_obj[obj_global_id] = create_misp_obj(obj_type, obj_id)
|
||||
return dict_misp_obj
|
||||
|
||||
# TODO REFACTOR ME
|
||||
def create_misp_obj(obj_type, obj_id):
|
||||
if obj_type in ['cryptocurrency', 'pgp', 'username']:
|
||||
obj_subtype, obj_id = obj_id.split(':', 1)
|
||||
else:
|
||||
obj_subtype = ''
|
||||
misp_obj = ail_objects.get_misp_object(obj_type, obj_subtype, obj_id)
|
||||
return misp_obj
|
||||
|
||||
|
||||
def get_relationship_between_global_obj(obj_global_id_1, obj_global_id_2):
|
||||
obj_type_1 = obj_global_id_1.split(':', 1)[0]
|
||||
obj_type_2 = obj_global_id_2.split(':', 1)[0]
|
||||
type_tuple = [obj_type_1, obj_type_2]
|
||||
|
||||
if 'image' in type_tuple: # or screenshot ## TODO:
|
||||
if obj_type_1 == 'image':
|
||||
src = obj_global_id_1
|
||||
dest = obj_global_id_2
|
||||
else:
|
||||
src = obj_global_id_2
|
||||
dest = obj_global_id_1
|
||||
return {'relation': 'screenshot-of', 'src': src, 'dest': dest}
|
||||
elif 'decoded' in type_tuple:
|
||||
if obj_type_1 == 'decoded':
|
||||
src = obj_global_id_1
|
||||
dest = obj_global_id_2
|
||||
else:
|
||||
src = obj_global_id_2
|
||||
dest = obj_global_id_1
|
||||
return {'relation': 'included-in', 'src': src, 'dest': dest}
|
||||
elif 'pgp' in type_tuple:
|
||||
if obj_type_1 == 'pgp':
|
||||
src = obj_global_id_1
|
||||
dest = obj_global_id_2
|
||||
else:
|
||||
src = obj_global_id_2
|
||||
dest = obj_global_id_1
|
||||
return {'relation': 'extracted-from', 'src': src, 'dest': dest}
|
||||
elif 'cryptocurrency':
|
||||
if obj_type_1 == 'cryptocurrency':
|
||||
src = obj_global_id_1
|
||||
dest = obj_global_id_2
|
||||
else:
|
||||
src = obj_global_id_2
|
||||
dest = obj_global_id_1
|
||||
return {'relation': 'extracted-from', 'src': src, 'dest': dest}
|
||||
elif 'domain' in type_tuple:
|
||||
if 'item' in type_tuple:
|
||||
if obj_type_1 == 'item':
|
||||
src = obj_global_id_1
|
||||
dest = obj_global_id_2
|
||||
else:
|
||||
src = obj_global_id_2
|
||||
dest = obj_global_id_1
|
||||
return {'relation': 'extracted-from', 'src': src, 'dest': dest} # replave by crawled-from
|
||||
elif 'item' in type_tuple:
|
||||
if 'domain' in type_tuple:
|
||||
if obj_type_1 == 'item':
|
||||
src = obj_global_id_1
|
||||
dest = obj_global_id_2
|
||||
else:
|
||||
src = obj_global_id_2
|
||||
dest = obj_global_id_1
|
||||
return {'relation': 'extracted-from', 'src': src, 'dest': dest} # replave by crawled-from
|
||||
return None
|
||||
|
||||
def sanitize_event_distribution(distribution):
|
||||
try:
|
||||
int(distribution)
|
||||
if (0 <= distribution <= 3):
|
||||
return distribution
|
||||
else:
|
||||
return 0
|
||||
except:
|
||||
return 0
|
||||
|
||||
def sanitize_event_threat_level_id(threat_level_id):
|
||||
try:
|
||||
int(threat_level_id)
|
||||
if (1 <= threat_level_id <= 4):
|
||||
return threat_level_id
|
||||
else:
|
||||
return 4
|
||||
except:
|
||||
return 4
|
||||
|
||||
def sanitize_event_analysis(analysis):
|
||||
try:
|
||||
int(analysis)
|
||||
if (0 <= analysis <= 2):
|
||||
return analysis
|
||||
else:
|
||||
return 0
|
||||
except:
|
||||
return 0
|
||||
|
||||
# # TODO: return error
|
||||
def ping_misp():
|
||||
try:
|
||||
PyMISP(misp_url, misp_key, misp_verifycert)
|
||||
return True
|
||||
except Exception as e:
|
||||
print(e)
|
||||
return False
|
||||
|
||||
def create_misp_event(event, distribution=0, threat_level_id=4, publish=False, analysis=0, event_info=None):
|
||||
if event_info:
|
||||
event.info = event_info
|
||||
event.distribution = sanitize_event_distribution(distribution)
|
||||
event.threat_level_id = sanitize_event_threat_level_id(threat_level_id)
|
||||
event.analysis = sanitize_event_analysis(analysis)
|
||||
if publish:
|
||||
event.publish()
|
||||
|
||||
# # TODO: handle multiple MISP instance
|
||||
misp = PyMISP(misp_url, misp_key, misp_verifycert)
|
||||
#print(event.to_json())
|
||||
|
||||
misp_event = misp.add_event(event)
|
||||
#print(misp_event)
|
||||
# # TODO: handle error
|
||||
event_metadata = extract_event_metadata(misp_event)
|
||||
return event_metadata
|
||||
|
||||
def extract_event_metadata(event):
|
||||
event_metadata = {}
|
||||
event_metadata['uuid'] = event['Event']['uuid']
|
||||
event_metadata['id'] = event['Event']['id']
|
||||
if misp_url[-1] == '/':
|
||||
event_metadata['url'] = misp_url + 'events/view/' + str(event_metadata['id'])
|
||||
else:
|
||||
event_metadata['url'] = misp_url + '/events/view/' + str(event_metadata['id'])
|
||||
return event_metadata
|
||||
|
||||
######
|
||||
#
|
||||
# EXPORT LVL DEFINITION: (== Correl<tion DEPTH)
|
||||
#
|
||||
# LVL 0 => PARTIAL Only add core item Correlation
|
||||
# LVL 1 => DETAILED Also add correlated_items correlation
|
||||
######
|
||||
|
||||
# # TODO: # create object relationships
|
||||
def create_investigation_event(investigation_uuid):
|
||||
investigation = Investigation(investigation_uuid)
|
||||
|
||||
event = MISPEvent()
|
||||
event.info = investigation.get_info()
|
||||
event.uuid = investigation.get_uuid(separator=True)
|
||||
event.date = investigation.get_date()
|
||||
event.analysis = investigation.get_analysis()
|
||||
event.threat_level_id = investigation.get_threat_level()
|
||||
|
||||
event.distribution = 0
|
||||
|
||||
# tags
|
||||
for tag in investigation.get_tags():
|
||||
event.add_tag(tag)
|
||||
# objects
|
||||
investigation_objs = investigation.get_objects()
|
||||
for obj in investigation_objs:
|
||||
# if subtype -> obj_id = 'subtype:type'
|
||||
if obj['subtype']:
|
||||
obj_id = f"{obj['subtype']}:{obj['id']}"
|
||||
else:
|
||||
obj_id = obj['id']
|
||||
misp_obj = create_misp_obj(obj['type'], obj_id)
|
||||
if misp_obj:
|
||||
event.add_object(misp_obj)
|
||||
|
||||
# if publish:
|
||||
# event.publish()
|
||||
|
||||
# print(event.to_json())
|
||||
misp = PyMISP(misp_url, misp_key, misp_verifycert)
|
||||
if misp.event_exists(event.uuid):
|
||||
misp_event = misp.update_event(event)
|
||||
else:
|
||||
misp_event = misp.add_event(event)
|
||||
|
||||
# # TODO: handle error
|
||||
event_metadata = extract_event_metadata(misp_event)
|
||||
if event_metadata.get('uuid'):
|
||||
if misp_url[-1] == '/':
|
||||
url = misp_url[:-1]
|
||||
else:
|
||||
url = misp_url
|
||||
investigation.add_misp_events(url)
|
||||
return event_metadata
|
||||
|
||||
# if __name__ == '__main__':
|
||||
|
||||
# l_obj = [{'id': 'bfd5f1d89e55b10a8b122a9d7ce31667ec1d086a', 'type': 'decoded', 'lvl': 2}]
|
||||
# create_list_of_objs_to_export(l_obj)
|
||||
|
||||
#print(event.to_json())
|
||||
|
|
@ -13,8 +13,6 @@ from lib.objects import ail_objects
|
|||
|
||||
from lib.objects import Items
|
||||
|
||||
from export import AILObjects
|
||||
|
||||
# MISP
|
||||
from pymisp import MISPEvent, MISPObject, PyMISP
|
||||
|
||||
|
|
@ -218,10 +216,6 @@ def create_obj_relationships(map_uuid_global_id, misp_obj):
|
|||
|
||||
# TODO CREATE OBJ RELATIONSHIP
|
||||
|
||||
def create_map_all_obj_uuid_golbal_id(map_uuid_global_id):
|
||||
for obj_uuid in map_uuid_global_id:
|
||||
AILObjects.create_map_obj_uuid_golbal_id(obj_uuid, map_uuid_global_id[obj_uuid])
|
||||
|
||||
def import_objs_from_file(filepath):
|
||||
map_uuid_global_id = {}
|
||||
|
||||
|
|
@ -237,7 +231,6 @@ def import_objs_from_file(filepath):
|
|||
for misp_obj in event_to_import.objects:
|
||||
create_obj_relationships(map_uuid_global_id, misp_obj)
|
||||
|
||||
create_map_all_obj_uuid_golbal_id(map_uuid_global_id)
|
||||
return map_uuid_global_id
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -12,12 +12,16 @@ import sys
|
|||
import datetime
|
||||
import uuid
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
|
||||
import item_basic
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'import', 'ail_json_importer'))
|
||||
from Default_json import Default_json
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from lib.objects.Items import Item
|
||||
|
||||
|
||||
class Ail_feeder_urlextract(Default_json):
|
||||
"""urlextract Feeder functions"""
|
||||
|
||||
|
|
@ -46,13 +50,14 @@ class Ail_feeder_urlextract(Default_json):
|
|||
|
||||
# # TODO:
|
||||
def process_json_meta(self, process, item_id):
|
||||
'''
|
||||
"""
|
||||
Process JSON meta filed.
|
||||
'''
|
||||
"""
|
||||
json_meta = self.get_json_meta()
|
||||
parent_id = str(json_meta['parent:twitter:tweet_id']) # TODO SEARCH IN CACHE !!!
|
||||
item = Item(item_id)
|
||||
item.set_parent(parent_id)
|
||||
|
||||
# # TODO: change me
|
||||
parent_type = 'twitter_id'
|
||||
|
||||
parent_id = str(json_meta['parent:twitter:tweet_id'])
|
||||
item_basic.add_item_parent_by_parent_id(parent_type, parent_id, item_id)
|
||||
# parent_type = 'twitter_id'
|
||||
# item_basic.add_item_parent_by_parent_id(parent_type, parent_id, item_id)
|
||||
|
|
|
|||
|
|
@ -200,8 +200,8 @@ class Investigation(object):
|
|||
else:
|
||||
raise UpdateInvestigationError(f'Invalid analysis: {analysis}')
|
||||
|
||||
def add_misp_events(self, misp_url):
|
||||
r_tracking.sadd(f'investigations:misp:{self.uuid}', misp_url)
|
||||
def add_misp_events(self, event_uuid):
|
||||
r_tracking.sadd(f'investigations:misp:{self.uuid}', event_uuid)
|
||||
|
||||
def set_tags(self, tags):
|
||||
# delete previous tags
|
||||
|
|
|
|||
|
|
@ -12,13 +12,13 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
from lib.ConfigLoader import ConfigLoader
|
||||
|
||||
config_loader = ConfigLoader()
|
||||
|
||||
r_serv_db = config_loader.get_db_conn("Kvrocks_DB")
|
||||
config_loader = None
|
||||
|
||||
AIL_OBJECTS = {'cve', 'cryptocurrency', 'decoded', 'domain', 'item', 'pgp', 'screenshot', 'username'}
|
||||
AIL_OBJECTS = sorted({'cve', 'cryptocurrency', 'decoded', 'domain', 'item', 'pgp', 'screenshot', 'username'})
|
||||
|
||||
def get_ail_uuid():
|
||||
pass
|
||||
return r_serv_db.get('ail:uuid')
|
||||
|
||||
#### AIL OBJECTS ####
|
||||
|
||||
|
|
@ -33,11 +33,21 @@ def get_object_all_subtypes(obj_type):
|
|||
if obj_type == 'username':
|
||||
return ['telegram', 'twitter', 'jabber']
|
||||
|
||||
def get_all_objects_with_subtypes_tuple():
|
||||
str_objs = []
|
||||
for obj_type in get_all_objects():
|
||||
subtypes = get_object_all_subtypes(obj_type)
|
||||
if subtypes:
|
||||
for subtype in subtypes:
|
||||
str_objs.append((obj_type, subtype))
|
||||
else:
|
||||
str_objs.append((obj_type, ''))
|
||||
return str_objs
|
||||
|
||||
##-- AIL OBJECTS --##
|
||||
|
||||
def paginate_iterator(iter_elems, nb_obj=50, page=1):
|
||||
dict_page = {}
|
||||
dict_page['nb_all_elem'] = len(iter_elems)
|
||||
dict_page = {'nb_all_elem': len(iter_elems)}
|
||||
nb_pages = dict_page['nb_all_elem'] / nb_obj
|
||||
if not nb_pages.is_integer():
|
||||
nb_pages = int(nb_pages)+1
|
||||
|
|
|
|||
|
|
@ -96,4 +96,3 @@ def update_obj_date(date, obj_type, subtype=''):
|
|||
if __name__ == '__main__':
|
||||
print(r_obj.hgetall(f'date:first'))
|
||||
print(r_obj.hgetall(f'date:last'))
|
||||
|
||||
|
|
|
|||
|
|
@ -420,6 +420,10 @@ class Domain(AbstractObject):
|
|||
def add_history(self, epoch, root_item=None, date=None):
|
||||
if not date:
|
||||
date = time.strftime('%Y%m%d', time.gmtime(epoch))
|
||||
if not root_item:
|
||||
root_item = int(epoch)
|
||||
status = False
|
||||
else:
|
||||
try:
|
||||
root_item = int(root_item)
|
||||
status = False
|
||||
|
|
|
|||
|
|
@ -88,8 +88,11 @@ class Item(AbstractObject):
|
|||
else:
|
||||
return item_basic.get_item_content(self.id)
|
||||
|
||||
def get_raw_content(self):
|
||||
def get_raw_content(self, decompress=False):
|
||||
filepath = self.get_filename()
|
||||
if decompress:
|
||||
raw_content = BytesIO(self.get_content(binary=True))
|
||||
else:
|
||||
with open(filepath, 'rb') as f:
|
||||
raw_content = BytesIO(f.read())
|
||||
return raw_content
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ from lib.ConfigLoader import ConfigLoader
|
|||
from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id
|
||||
|
||||
config_loader = ConfigLoader()
|
||||
|
||||
baseurl = config_loader.get_config_str("Notifications", "ail_domain")
|
||||
config_loader = None
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -243,6 +243,14 @@ class AbstractObject(ABC):
|
|||
"""
|
||||
return is_obj_correlated(self.type, self.subtype, self.id, type2, subtype2, id2)
|
||||
|
||||
def are_correlated(self, object2):
|
||||
"""
|
||||
Check if an object is correlated with another Object
|
||||
:type object2 AbstractObject
|
||||
"""
|
||||
return is_obj_correlated(self.type, self.subtype, self.id,
|
||||
object2.get_type(), object2.get_subtype(r_str=True), object2.get_id())
|
||||
|
||||
def delete_correlation(self, type2, subtype2, id2):
|
||||
"""
|
||||
Get object correlations
|
||||
|
|
|
|||
|
|
@ -4,15 +4,12 @@
|
|||
import os
|
||||
import sys
|
||||
|
||||
from abc import ABC
|
||||
from flask import url_for
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from lib.ConfigLoader import ConfigLoader
|
||||
from lib.ail_core import get_all_objects
|
||||
from lib.ail_core import get_all_objects, get_object_all_subtypes, get_all_objects_with_subtypes_tuple
|
||||
from lib import correlations_engine
|
||||
from lib import btc_ail
|
||||
from lib import Tag
|
||||
|
|
@ -26,26 +23,18 @@ from lib.objects import Pgps
|
|||
from lib.objects.Screenshots import Screenshot
|
||||
from lib.objects import Usernames
|
||||
|
||||
|
||||
config_loader = ConfigLoader()
|
||||
|
||||
config_loader = None
|
||||
|
||||
class AILObjects(object): ## ??????????????????????
|
||||
initial = 0
|
||||
ongoing = 1
|
||||
completed = 2
|
||||
|
||||
|
||||
|
||||
def is_valid_object_type(obj_type):
|
||||
return obj_type in get_all_objects()
|
||||
|
||||
def is_valid_object_subtype(obj_type, subtype):
|
||||
if obj_type == 'cryptocurrency':
|
||||
return subtype in CryptoCurrencies.get_all_subtypes()
|
||||
elif obj_type == 'pgp':
|
||||
return subtype in Pgps.get_all_subtypes()
|
||||
elif obj_type == 'username':
|
||||
return subtype in CryptoCurrencies.get_all_subtypes()
|
||||
return subtype in get_object_all_subtypes(obj_type)
|
||||
|
||||
def sanitize_objs_types(objs):
|
||||
l_types = []
|
||||
|
|
@ -56,6 +45,7 @@ def sanitize_objs_types(objs):
|
|||
l_types = get_all_objects()
|
||||
return l_types
|
||||
|
||||
|
||||
def get_object(obj_type, subtype, id):
|
||||
if obj_type == 'item':
|
||||
return Item(id)
|
||||
|
|
@ -74,6 +64,25 @@ def get_object(obj_type, subtype, id):
|
|||
elif obj_type == 'username':
|
||||
return Usernames.Username(id, subtype)
|
||||
|
||||
def get_objects(objects):
|
||||
objs = set()
|
||||
for obj in objects:
|
||||
if isinstance(obj, dict):
|
||||
obj_type = obj['type']
|
||||
obj_subtype = obj['subtype']
|
||||
obj_id = obj['id']
|
||||
if 'lvl' in obj:
|
||||
correl_objs = get_obj_correlations_objs(obj_type, obj_subtype, obj_id, lvl=obj['lvl'])
|
||||
objs = objs.union(correl_objs)
|
||||
else:
|
||||
obj_type, obj_subtype, obj_id = obj
|
||||
objs.add((obj_type, obj_subtype, obj_id))
|
||||
ail_objects = []
|
||||
for obj in objs:
|
||||
ail_objects.append(get_object(obj[0], obj[1], obj[2]))
|
||||
return ail_objects
|
||||
|
||||
|
||||
def exists_obj(obj_type, subtype, obj_id):
|
||||
obj = get_object(obj_type, subtype, obj_id)
|
||||
if obj:
|
||||
|
|
@ -81,35 +90,44 @@ def exists_obj(obj_type, subtype, obj_id):
|
|||
else:
|
||||
return False
|
||||
|
||||
|
||||
def get_obj_global_id(obj_type, subtype, obj_id):
|
||||
obj = get_object(obj_type, subtype, obj_id)
|
||||
return obj.get_global_id()
|
||||
|
||||
|
||||
def get_obj_from_global_id(global_id):
|
||||
obj = global_id.split(':', 3)
|
||||
return get_object(obj[0], obj[1], obj[2])
|
||||
|
||||
|
||||
def get_object_link(obj_type, subtype, id, flask_context=False):
|
||||
obj = get_object(obj_type, subtype, id)
|
||||
return obj.get_link(flask_context=flask_context)
|
||||
|
||||
|
||||
def get_object_svg(obj_type, subtype, id):
|
||||
obj = get_object(obj_type, subtype, id)
|
||||
return obj.get_svg_icon()
|
||||
|
||||
|
||||
## TAGS ##
|
||||
def get_obj_tags(obj_type, subtype, id):
|
||||
obj = get_object(obj_type, subtype, id)
|
||||
return obj.get_tags()
|
||||
|
||||
|
||||
def add_obj_tag(obj_type, subtype, id, tag):
|
||||
obj = get_object(obj_type, subtype, id)
|
||||
obj.add_tag(tag)
|
||||
|
||||
|
||||
def add_obj_tags(obj_type, subtype, id, tags):
|
||||
obj = get_object(obj_type, subtype, id)
|
||||
for tag in tags:
|
||||
obj.add_tag(tag)
|
||||
|
||||
|
||||
# -TAGS- #
|
||||
|
||||
def get_object_meta(obj_type, subtype, id, options=[], flask_context=False):
|
||||
|
|
@ -119,6 +137,7 @@ def get_object_meta(obj_type, subtype, id, options=[], flask_context=False):
|
|||
meta['link'] = obj.get_link(flask_context=flask_context)
|
||||
return meta
|
||||
|
||||
|
||||
def get_objects_meta(objs, options=[], flask_context=False):
|
||||
metas = []
|
||||
for obj_dict in objs:
|
||||
|
|
@ -126,12 +145,14 @@ def get_objects_meta(objs, options=[], flask_context=False):
|
|||
flask_context=flask_context))
|
||||
return metas
|
||||
|
||||
|
||||
def get_object_card_meta(obj_type, subtype, id, related_btc=False):
|
||||
obj = get_object(obj_type, subtype, id)
|
||||
meta = obj.get_meta()
|
||||
meta['icon'] = obj.get_svg_icon()
|
||||
if subtype or obj_type == 'cve':
|
||||
meta['sparkline'] = obj.get_sparkline()
|
||||
if obj_type == 'cve':
|
||||
meta['cve_search'] = obj.get_cve_search()
|
||||
if subtype == 'bitcoin' and related_btc:
|
||||
meta["related_btc"] = btc_ail.get_bitcoin_info(obj.id)
|
||||
|
|
@ -143,6 +164,7 @@ def get_object_card_meta(obj_type, subtype, id, related_btc=False):
|
|||
meta["add_tags_modal"] = Tag.get_modal_add_tags(obj.id, obj.get_type(), obj.get_subtype(r_str=True))
|
||||
return meta
|
||||
|
||||
|
||||
def get_ui_obj_tag_table_keys(obj_type):
|
||||
'''
|
||||
Warning: use only in flask (dynamic templates)
|
||||
|
|
@ -150,15 +172,58 @@ def get_ui_obj_tag_table_keys(obj_type):
|
|||
if obj_type == "domain":
|
||||
return ['id', 'first_seen', 'last_check', 'status'] # # TODO: add root screenshot
|
||||
|
||||
|
||||
# # # # MISP OBJECTS # # # #
|
||||
|
||||
# # TODO: CHECK IF object already have an UUID
|
||||
def get_misp_object(obj_type, subtype, id):
|
||||
obj = get_object(obj_type, subtype, id)
|
||||
return obj.get_misp_object()
|
||||
|
||||
|
||||
def get_misp_objects(objs):
|
||||
misp_objects = {}
|
||||
for obj in objs:
|
||||
misp_objects[obj] = obj.get_misp_object()
|
||||
for relation in get_objects_relationships(objs):
|
||||
obj_src = misp_objects[relation['src']]
|
||||
obj_dest = misp_objects[relation['dest']]
|
||||
# print(relation['src'].get_id(), relation['dest'].get_id())
|
||||
obj_src.add_reference(obj_dest.uuid, relation['relationship'], 'ail correlation')
|
||||
return misp_objects.values()
|
||||
|
||||
# get misp relationship
|
||||
def get_objects_relationship(obj_1, obj2):
|
||||
relationship = {}
|
||||
obj_types = (obj_1.get_type(), obj2.get_type())
|
||||
def get_objects_relationships(objs):
|
||||
relation = []
|
||||
if len(objs) == 2:
|
||||
if objs[0].are_correlated(objs[1]):
|
||||
relationship = get_objects_relationship(objs[0], objs[1])
|
||||
if relationship:
|
||||
relation.append(relationship)
|
||||
else:
|
||||
iterator = objs.copy() # [obj1, obj2, obj3, obj4]
|
||||
for obj in objs[:-1]: # [obj1, obj2, obj3]
|
||||
iterator.pop(0) # [obj2, obj3, obj4] obj1 correlation already checked
|
||||
for obj2 in iterator:
|
||||
# CHECK CORRELATION obj - > obj2
|
||||
if obj.are_correlated(obj2):
|
||||
relationship = get_objects_relationship(obj, obj2)
|
||||
if relationship:
|
||||
relation.append(relationship)
|
||||
return relation
|
||||
|
||||
def get_relationship_src_dest(src_type, obj1, obj2):
|
||||
if obj1.get_type() == src_type:
|
||||
src = obj1
|
||||
dest = obj2
|
||||
else:
|
||||
src = obj2
|
||||
dest = obj1
|
||||
return src, dest
|
||||
|
||||
# get misp relationship
|
||||
def get_objects_relationship(obj1, obj2):
|
||||
obj_types = (obj1.get_type(), obj2.get_type())
|
||||
|
||||
##############################################################
|
||||
# if ['cryptocurrency', 'pgp', 'username', 'decoded', 'screenshot']:
|
||||
|
|
@ -166,63 +231,79 @@ def get_objects_relationship(obj_1, obj2):
|
|||
# relationship[relation] =
|
||||
##############################################################
|
||||
if 'cryptocurrency' in obj_types:
|
||||
relationship['relation'] = 'extracted-from'
|
||||
if obj_1.get_type() == 'cryptocurrency':
|
||||
relationship['src'] = obj_1.get_id()
|
||||
relationship['dest'] = obj2.get_id()
|
||||
else:
|
||||
relationship['src'] = obj2.get_id()
|
||||
relationship['dest'] = obj_1.get_id()
|
||||
|
||||
relationship = 'extracted-from'
|
||||
src, dest = get_relationship_src_dest('cryptocurrency', obj1, obj2)
|
||||
elif 'cve' in obj_types:
|
||||
relationship = 'extracted-from'
|
||||
src, dest = get_relationship_src_dest('cve', obj1, obj2)
|
||||
elif 'pgp' in obj_types:
|
||||
relationship['relation'] = 'extracted-from'
|
||||
|
||||
relationship = 'extracted-from'
|
||||
src, dest = get_relationship_src_dest('pgp', obj1, obj2)
|
||||
elif 'username' in obj_types:
|
||||
relationship['relation'] = 'extracted-from'
|
||||
|
||||
relationship = 'extracted-from'
|
||||
src, dest = get_relationship_src_dest('username', obj1, obj2)
|
||||
elif 'decoded' in obj_types:
|
||||
relationship['relation'] = 'included-in'
|
||||
|
||||
relationship = 'included-in'
|
||||
src, dest = get_relationship_src_dest('decoded', obj1, obj2)
|
||||
elif 'screenshot' in obj_types:
|
||||
relationship['relation'] = 'screenshot-of'
|
||||
|
||||
relationship = 'screenshot-of'
|
||||
src, dest = get_relationship_src_dest('screenshot', obj1, obj2)
|
||||
elif 'domain' in obj_types:
|
||||
relationship['relation'] = 'extracted-from'
|
||||
|
||||
relationship = 'extracted-from'
|
||||
src, dest = get_relationship_src_dest('domain', obj1, obj2)
|
||||
# default
|
||||
else:
|
||||
pass
|
||||
relationship = None
|
||||
src = None
|
||||
dest = None
|
||||
if not relationship:
|
||||
return {}
|
||||
else:
|
||||
return {'src': src, 'dest': dest, 'relationship': relationship}
|
||||
|
||||
# - - - MISP OBJECTS - - - #
|
||||
|
||||
return relationship
|
||||
|
||||
def api_sanitize_object_type(obj_type):
|
||||
if not is_valid_object_type(obj_type):
|
||||
return {'status': 'error', 'reason': 'Incorrect object type'}, 400
|
||||
|
||||
def get_obj_correlations(obj_type, subtype, id):
|
||||
obj = get_object(obj_type, subtype, id)
|
||||
|
||||
def get_obj_correlations(obj_type, subtype, obj_id):
|
||||
obj = get_object(obj_type, subtype, obj_id)
|
||||
return obj.get_correlations()
|
||||
|
||||
################################################################################
|
||||
# DATA RETENTION
|
||||
# # TODO: TO ADD ??????????????????????
|
||||
# def get_first_objects_date():
|
||||
# return r_object.zrange('objs:first_date', 0, -1)
|
||||
#
|
||||
# def get_first_object_date(obj_type, subtype):
|
||||
# return r_object.zscore('objs:first_date', f'{obj_type}:{subtype}')
|
||||
#
|
||||
# def set_first_object_date(obj_type, subtype, date):
|
||||
# return r_object.zadd('objs:first_date', f'{obj_type}:{subtype}', date)
|
||||
def _get_obj_correlations_objs(objs, obj_type, subtype, obj_id, lvl=0):
|
||||
if lvl > 0 and (obj_type, subtype, obj_id) not in objs: # Avoid looking for the same correlation
|
||||
objs.add((obj_type, subtype, obj_id))
|
||||
lvl = lvl - 1
|
||||
obj = get_object(obj_type, subtype, obj_id)
|
||||
correlations = obj.get_correlations()
|
||||
# print('--------------------------')
|
||||
# print( obj_id, correlations)
|
||||
# print(lvl)
|
||||
# print('--------------------------')
|
||||
for obj2_type in correlations:
|
||||
for str_obj in correlations[obj2_type]:
|
||||
obj2_subtype, obj2_id = str_obj.split(':', 1)
|
||||
_get_obj_correlations_objs(objs, obj2_type, obj2_subtype, obj2_id, lvl=lvl)
|
||||
# print(len(objs))
|
||||
objs.add((obj_type, subtype, obj_id))
|
||||
return objs
|
||||
|
||||
def get_obj_correlations_objs(obj_type, subtype, obj_id, lvl=0):
|
||||
objs = set()
|
||||
_get_obj_correlations_objs(objs, obj_type, subtype, obj_id, lvl=lvl)
|
||||
return objs
|
||||
|
||||
################################################################################
|
||||
################################################################################
|
||||
################################################################################
|
||||
|
||||
def delete_obj(obj_type, subtype, id):
|
||||
object = get_object(obj_type, subtype, id)
|
||||
return object.delete()
|
||||
def delete_obj(obj_type, subtype, obj_id):
|
||||
obj = get_object(obj_type, subtype, obj_id)
|
||||
return obj.delete()
|
||||
|
||||
|
||||
################################################################################
|
||||
################################################################################
|
||||
|
|
@ -236,11 +317,12 @@ def create_correlation_graph_links(links_set):
|
|||
links.append({"source": link[0], "target": link[1]})
|
||||
return links
|
||||
|
||||
|
||||
def create_correlation_graph_nodes(nodes_set, obj_str_id, flask_context=True):
|
||||
graph_nodes_list = []
|
||||
for node_id in nodes_set:
|
||||
obj_type, subtype, obj_id = node_id.split(';', 2)
|
||||
dict_node = {"id": node_id}
|
||||
dict_node = {'id': node_id}
|
||||
dict_node['style'] = get_object_svg(obj_type, subtype, obj_id)
|
||||
|
||||
# # TODO: # FIXME: in UI
|
||||
|
|
@ -258,11 +340,27 @@ def create_correlation_graph_nodes(nodes_set, obj_str_id, flask_context=True):
|
|||
graph_nodes_list.append(dict_node)
|
||||
return graph_nodes_list
|
||||
|
||||
def get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=[], max_nodes=300, level=1, flask_context=False):
|
||||
obj_str_id, nodes, links = correlations_engine.get_correlations_graph_nodes_links(obj_type, subtype, obj_id, filter_types=filter_types, max_nodes=max_nodes, level=level, flask_context=flask_context)
|
||||
return {"nodes": create_correlation_graph_nodes(nodes, obj_str_id, flask_context=flask_context), "links": create_correlation_graph_links(links)}
|
||||
|
||||
|
||||
def get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=[], max_nodes=300, level=1,
|
||||
flask_context=False):
|
||||
obj_str_id, nodes, links = correlations_engine.get_correlations_graph_nodes_links(obj_type, subtype, obj_id,
|
||||
filter_types=filter_types,
|
||||
max_nodes=max_nodes, level=level,
|
||||
flask_context=flask_context)
|
||||
return {"nodes": create_correlation_graph_nodes(nodes, obj_str_id, flask_context=flask_context),
|
||||
"links": create_correlation_graph_links(links)}
|
||||
|
||||
|
||||
###############
|
||||
|
||||
|
||||
# if __name__ == '__main__':
|
||||
# r = get_objects([{'lvl': 1, 'type': 'item', 'subtype': '', 'id': 'crawled/2020/09/14/circl.lu0f4976a4-dda4-4189-ba11-6618c4a8c951'}])
|
||||
# r = get_misp_objects([Item('crawled/2020/09/14/circl.lu0f4976a4-dda4-4189-ba11-6618c4a8c951'),
|
||||
# Cve('CVE-2020-16856'), Cve('CVE-2014-6585'), Cve('CVE-2015-0383'),
|
||||
# Cve('CVE-2015-0410')])
|
||||
# print()
|
||||
# print(r)
|
||||
|
||||
# res = get_obj_correlations_objs('username', 'telegram', 'corona', lvl=100)
|
||||
# print(res)
|
||||
|
|
@ -6,7 +6,7 @@ d4-pyclient>=0.1.6
|
|||
thehive4py
|
||||
|
||||
# Core
|
||||
redis==3.0.0
|
||||
redis>4.4.0
|
||||
python-magic>0.4.15
|
||||
yara-python>4.0.2
|
||||
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ from lib import Tag
|
|||
|
||||
from packages import Date
|
||||
|
||||
#import Config_DB
|
||||
# import Config_DB
|
||||
bootstrap_label = Flask_config.bootstrap_label
|
||||
|
||||
# ============ BLUEPRINT ============
|
||||
|
|
|
|||
|
|
@ -4,17 +4,16 @@
|
|||
'''
|
||||
Blueprint Flask: MISP format import export
|
||||
'''
|
||||
|
||||
import io
|
||||
import os
|
||||
import sys
|
||||
import uuid
|
||||
import json
|
||||
|
||||
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, send_file
|
||||
from flask import render_template, jsonify, request, Blueprint, redirect, url_for, Response, send_file, abort
|
||||
from flask_login import login_required, current_user, login_user, logout_user
|
||||
|
||||
sys.path.append('modules')
|
||||
import Flask_config
|
||||
|
||||
# Import Role_Manager
|
||||
from Role_Manager import login_admin, login_analyst, login_read_only
|
||||
|
|
@ -23,20 +22,21 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from export import MispExport
|
||||
from export import MispImport
|
||||
from export import AILObjects ####################### # # # # ########################## ## # ####################################
|
||||
|
||||
from export import Export
|
||||
from lib.objects import ail_objects
|
||||
|
||||
from export import MispImport # TODO REMOVE ME
|
||||
|
||||
# TODO REMOVE ME
|
||||
|
||||
# ============ BLUEPRINT ============
|
||||
import_export = Blueprint('import_export', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/import_export'))
|
||||
import_export = Blueprint('import_export', __name__,
|
||||
template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/import_export'))
|
||||
|
||||
|
||||
# ============ VARIABLES ============
|
||||
|
||||
|
||||
|
||||
# ============ FUNCTIONS ============
|
||||
|
||||
|
||||
|
|
@ -47,6 +47,7 @@ import_export = Blueprint('import_export', __name__, template_folder=os.path.joi
|
|||
def import_object():
|
||||
return render_template("import_object.html")
|
||||
|
||||
# TODO
|
||||
@import_export.route("/import_export/import_file", methods=['POST'])
|
||||
@login_required
|
||||
@login_analyst
|
||||
|
|
@ -79,137 +80,129 @@ def import_object_file():
|
|||
|
||||
return render_template("import_object.html", all_imported_obj=all_imported_obj, error=error)
|
||||
|
||||
@import_export.route('/import_export/export')
|
||||
|
||||
@import_export.route("/objects/misp/export", methods=['GET'])
|
||||
@login_required
|
||||
@login_analyst
|
||||
def export_object():
|
||||
def objects_misp_export():
|
||||
user_id = current_user.get_id()
|
||||
object_types = ail_objects.get_all_objects_with_subtypes_tuple()
|
||||
to_export = Export.get_user_misp_objects_to_export(user_id)
|
||||
return render_template("export_object.html", object_types=object_types, to_export=to_export)
|
||||
|
||||
# get user saved obj to export
|
||||
l_obj_to_export = AILObjects.get_user_list_of_obj_to_export(user_id)
|
||||
|
||||
return render_template("export_object.html", l_obj_to_export=l_obj_to_export)
|
||||
|
||||
@import_export.route("/import_export/export_file", methods=['POST'])
|
||||
@import_export.route("/objects/misp/export/post", methods=['POST'])
|
||||
@login_required
|
||||
@login_analyst
|
||||
def export_object_file():
|
||||
def objects_misp_export_post():
|
||||
user_id = current_user.get_id()
|
||||
|
||||
l_obj_to_export = []
|
||||
l_obj_invalid = []
|
||||
|
||||
export_to_misp = False
|
||||
dict_misp_event_export = {}
|
||||
|
||||
# Get new added Object
|
||||
new_export = []
|
||||
user_lvl_export = {}
|
||||
for obj_tuple in list(request.form):
|
||||
# GET NEW LEVEL
|
||||
if obj_tuple[0] == '{':
|
||||
obj_j = obj_tuple.replace("'", "\"")
|
||||
obj_dict = json.loads(obj_j) # TODO sanitize
|
||||
lvl = int(request.form.getlist(obj_tuple)[0])
|
||||
user_lvl_export[f"{obj_dict['type']}:{obj_dict['subtype']}:{obj_dict['id']}"] = lvl
|
||||
|
||||
l_input = request.form.getlist(obj_tuple)
|
||||
if len(l_input) == 3:
|
||||
obj_type = l_input[0]
|
||||
obj_id = l_input[1]
|
||||
lvl = l_input[2]
|
||||
lvl = MispExport.sanitize_obj_export_lvl(lvl)
|
||||
if l_input[0] != 'Object type...':
|
||||
new_type, new_subtype = l_input[0].split(':', 1)
|
||||
if not new_subtype:
|
||||
new_subtype = ''
|
||||
new_export.append({'type': new_type, 'subtype': new_subtype, 'id': l_input[1], 'lvl': l_input[2]})
|
||||
|
||||
obj_subtype = obj_type.split(';')
|
||||
if len(obj_subtype) == 2:
|
||||
obj_type = obj_subtype[0]
|
||||
obj_subtype = obj_subtype[1]
|
||||
objects = []
|
||||
invalid_obj = []
|
||||
for obj in new_export:
|
||||
if not ail_objects.exists_obj(obj['type'], obj['subtype'], obj['id']):
|
||||
invalid_obj.append(obj)
|
||||
else:
|
||||
obj_subtype = None
|
||||
|
||||
obj_dict = {'id': obj_id, 'type': obj_type, 'lvl': lvl}
|
||||
if obj_subtype:
|
||||
obj_dict['subtype'] = obj_subtype
|
||||
|
||||
if MispExport.is_valid_obj_to_export(obj_type, obj_subtype, obj_id):
|
||||
l_obj_to_export.append(obj_dict)
|
||||
AILObjects.add_user_object_to_export(user_id, obj_dict['type'], obj_dict['id'], obj_dict['lvl'], obj_subtype=obj_dict.get('subtype', None))
|
||||
objects.append(obj)
|
||||
for obj in Export.get_user_misp_objects_to_export(user_id):
|
||||
if not ail_objects.exists_obj(obj['type'], obj['subtype'], obj['id']):
|
||||
invalid_obj.append(obj)
|
||||
else:
|
||||
if obj_id:
|
||||
l_obj_invalid.append(obj_dict)
|
||||
str_id = f"{obj['type']}:{obj['subtype']}:{obj['id']}"
|
||||
if str_id in user_lvl_export:
|
||||
obj['lvl'] = user_lvl_export[str_id]
|
||||
objects.append(obj)
|
||||
|
||||
if invalid_obj:
|
||||
object_types = ail_objects.get_all_objects_with_subtypes_tuple()
|
||||
return render_template("export_object.html", object_types=object_types,
|
||||
to_export=objects, l_obj_invalid=invalid_obj)
|
||||
|
||||
export = request.form.get('export_to_misp', False)
|
||||
distribution = request.form.get('misp_event_distribution')
|
||||
threat_level_id = request.form.get('threat_level_id')
|
||||
analysis = request.form.get('misp_event_analysis')
|
||||
info = request.form.get('misp_event_info')
|
||||
publish = request.form.get('misp_event_info', False)
|
||||
|
||||
objs = ail_objects.get_objects(objects)
|
||||
event = Export.create_misp_event(objs, distribution=distribution, threat_level_id=threat_level_id,
|
||||
analysis=analysis, info=info, export=export, publish=publish)
|
||||
|
||||
Export.delete_user_misp_objects_to_export(user_id)
|
||||
|
||||
if not export:
|
||||
return send_file(io.BytesIO(event['event'].encode()), as_attachment=True,
|
||||
download_name=f'ail_export_{event["uuid"]}.json')
|
||||
else:
|
||||
dict_misp_event_export[str(obj_tuple)] = request.form.get(obj_tuple)
|
||||
|
||||
# print(dict_misp_event_export)
|
||||
|
||||
if dict_misp_event_export.get('export_to_misp', None):
|
||||
export_to_misp = True
|
||||
else:
|
||||
dict_misp_event_export = None
|
||||
|
||||
if l_obj_invalid:
|
||||
# get user saved obj to export # # TODO: # performance
|
||||
l_obj_to_export = AILObjects.get_user_list_of_obj_to_export(user_id)
|
||||
|
||||
for obj_dict in l_obj_invalid: # set uuid input
|
||||
obj_dict['uuid'] = str(uuid.uuid4())
|
||||
subtype = obj_dict.get('subtype', None)
|
||||
if subtype:
|
||||
obj_dict['type'] = f'{obj_dict["type"]};{subtype}'
|
||||
|
||||
return render_template("export_object.html", l_obj_to_export=l_obj_to_export,
|
||||
l_obj_invalid=l_obj_invalid, dict_misp_event_export=dict_misp_event_export)
|
||||
else:
|
||||
if export_to_misp and MispExport.ping_misp():
|
||||
event = MispExport.create_list_of_objs_to_export(l_obj_to_export, r_type='event')
|
||||
|
||||
event_metadata = MispExport.create_misp_event(event, distribution=dict_misp_event_export.get('export_to_misp', None),
|
||||
threat_level_id=dict_misp_event_export.get('misp_threat_level_id', None),
|
||||
publish=dict_misp_event_export.get('misp_publish', None),
|
||||
analysis=dict_misp_event_export.get('misp_event_analysis', None),
|
||||
event_info=dict_misp_event_export.get('misp_event_info', None))
|
||||
|
||||
AILObjects.delete_all_user_object_to_export(user_id)
|
||||
return render_template("export_object.html", l_obj_to_export=l_obj_to_export,
|
||||
event_metadata=event_metadata,
|
||||
l_obj_invalid=[], dict_misp_event_export=[])
|
||||
else:
|
||||
# get user saved obj to export # # TODO: # performance
|
||||
json_export = MispExport.create_list_of_objs_to_export(l_obj_to_export)
|
||||
export_filename = MispExport.get_export_filename(json_export)
|
||||
json_export = MispExport.create_in_memory_file(json_export.to_json())
|
||||
AILObjects.delete_all_user_object_to_export(user_id)
|
||||
return send_file(json_export, as_attachment=True, attachment_filename=export_filename)
|
||||
object_types = ail_objects.get_all_objects_with_subtypes_tuple()
|
||||
return render_template("export_object.html", object_types=object_types,
|
||||
misp_url=event['url'])
|
||||
|
||||
|
||||
@import_export.route("/import_export/add_object_id_to_export", methods=['GET'])
|
||||
@import_export.route("/objects/misp/export/add", methods=['GET'])
|
||||
@login_required
|
||||
@login_analyst
|
||||
def add_object_id_to_export():
|
||||
user_id = current_user.get_id()
|
||||
user_id = current_user.get_id()
|
||||
obj_type = request.args.get('obj_type')
|
||||
obj_id = request.args.get('obj_id')
|
||||
obj_subtype = request.args.get('obj_subtype')
|
||||
obj_lvl = request.args.get('obj_lvl')
|
||||
AILObjects.add_user_object_to_export(user_id, obj_type, obj_id, obj_lvl, obj_subtype=obj_subtype)
|
||||
obj_type = request.args.get('type')
|
||||
obj_id = request.args.get('id')
|
||||
obj_subtype = request.args.get('subtype')
|
||||
obj_lvl = request.args.get('lvl')
|
||||
|
||||
try:
|
||||
obj_lvl = int(obj_lvl)
|
||||
except:
|
||||
obj_lvl = 0
|
||||
|
||||
if not ail_objects.exists_obj(obj_type, obj_subtype, obj_id):
|
||||
abort(404)
|
||||
Export.add_user_misp_object_to_export(user_id, obj_type, obj_subtype, obj_id, lvl=obj_lvl)
|
||||
# redirect
|
||||
return redirect(url_for('import_export.export_object'))
|
||||
return redirect(url_for('import_export.objects_misp_export'))
|
||||
|
||||
|
||||
@import_export.route("/objects/misp/export/delete", methods=['GET'])
|
||||
@login_required
|
||||
@login_analyst
|
||||
def delete_object_id_to_export():
|
||||
user_id = current_user.get_id()
|
||||
obj_type = request.args.get('type')
|
||||
obj_id = request.args.get('id')
|
||||
obj_subtype = request.args.get('subtype')
|
||||
|
||||
Export.delete_user_misp_object_to_export(user_id, obj_type, obj_subtype, obj_id)
|
||||
return jsonify(success=True)
|
||||
|
||||
|
||||
@import_export.route("/import_export/investigation", methods=['GET'])
|
||||
@login_required
|
||||
@login_analyst
|
||||
def export_investigation():
|
||||
investigation_uuid = request.args.get("uuid")
|
||||
|
||||
if MispExport.ping_misp():
|
||||
event_metadata = MispExport.create_investigation_event(investigation_uuid)
|
||||
if Export.ping_misp():
|
||||
event = Export.create_investigation_misp_event(investigation_uuid)
|
||||
print(event)
|
||||
else:
|
||||
return Response(json.dumps({"error": "Can't reach MISP Instance"}, indent=2, sort_keys=True), mimetype='application/json'), 400
|
||||
|
||||
return Response(json.dumps({"error": "Can't reach MISP Instance"}, indent=2, sort_keys=True),
|
||||
mimetype='application/json'), 400
|
||||
return redirect(url_for('investigations_b.show_investigation', uuid=investigation_uuid))
|
||||
|
||||
|
||||
# @import_export.route("/import_export/delete_object_id_to_export", methods=['GET'])
|
||||
# @login_required
|
||||
# @login_analyst
|
||||
# def delete_object_id_to_export():
|
||||
# user_id = current_user.get_id()
|
||||
# obj_type = request.args.get('obj_type')
|
||||
# obj_id = request.args.get('obj_id')
|
||||
# obj_subtype = request.args.get('obj_subtype')
|
||||
# obj_lvl = request.args.get('obj_lvl')
|
||||
# AILObjects.delete_user_object_to_export(user_id, object_type, object_id, obj_lvl, obj_subtype=obj_subtype)
|
||||
# # redirect
|
||||
# return 'ok'
|
||||
|
|
|
|||
|
|
@ -1,16 +1,15 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
'''
|
||||
Flask functions and routes for the trending modules page
|
||||
'''
|
||||
"""
|
||||
Flask functions and routes for the Item Submit modules page
|
||||
"""
|
||||
##################################
|
||||
# Import External packages
|
||||
##################################
|
||||
import re
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import string
|
||||
import datetime
|
||||
import unicodedata
|
||||
|
|
@ -20,7 +19,7 @@ from io import BytesIO
|
|||
from functools import wraps
|
||||
|
||||
# Flask
|
||||
from flask import Flask, render_template, jsonify, request, Blueprint, url_for, redirect, abort
|
||||
from flask import render_template, jsonify, request, Blueprint, url_for, redirect, abort
|
||||
from Role_Manager import login_admin, login_analyst
|
||||
from flask_login import login_required
|
||||
|
||||
|
|
@ -28,56 +27,39 @@ from flask_login import login_required
|
|||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from export import Export
|
||||
from lib import Tag
|
||||
from lib.objects.Items import Item
|
||||
|
||||
from packages import Import_helper
|
||||
|
||||
from pytaxonomies import Taxonomies
|
||||
from pytaxonomies import Taxonomies # TODO REMOVE ME
|
||||
|
||||
try:
|
||||
from pymisp.mispevent import MISPObject
|
||||
flag_misp = True
|
||||
except:
|
||||
flag_misp = False
|
||||
try:
|
||||
from thehive4py.models import Case, CaseTask, CustomFieldHelper, CaseObservable
|
||||
flag_hive = True
|
||||
except:
|
||||
flag_hive = False
|
||||
|
||||
# ============ VARIABLES ============
|
||||
import Flask_config
|
||||
|
||||
app = Flask_config.app
|
||||
baseUrl = Flask_config.baseUrl
|
||||
r_serv_metadata = Flask_config.r_serv_metadata
|
||||
r_serv_db = Flask_config.r_serv_db
|
||||
r_serv_log_submit = Flask_config.r_serv_log_submit
|
||||
|
||||
|
||||
|
||||
r_serv_metadata = Flask_config.r_serv_metadata # TODO REMOVE ME
|
||||
r_serv_db = Flask_config.r_serv_db # TODO REMOVE ME
|
||||
r_serv_log_submit = Flask_config.r_serv_log_submit # TODO REMOVE ME
|
||||
|
||||
logger = Flask_config.redis_logger
|
||||
|
||||
pymisp = Flask_config.pymisp
|
||||
if pymisp is False:
|
||||
flag_misp = False
|
||||
|
||||
HiveApi = Flask_config.HiveApi
|
||||
if HiveApi is False:
|
||||
flag_hive = False
|
||||
|
||||
PasteSubmit = Blueprint('PasteSubmit', __name__, template_folder='templates')
|
||||
|
||||
valid_filename_chars = "-_ %s%s" % (string.ascii_letters, string.digits)
|
||||
|
||||
UPLOAD_FOLDER = Flask_config.UPLOAD_FOLDER
|
||||
|
||||
misp_event_url = Flask_config.misp_event_url
|
||||
hive_case_url = Flask_config.hive_case_url
|
||||
|
||||
text_max_size = int(Flask_config.SUBMIT_PASTE_TEXT_MAX_SIZE) / (1000*1000)
|
||||
file_max_size = int(Flask_config.SUBMIT_PASTE_FILE_MAX_SIZE) / (1000*1000*1000)
|
||||
allowed_extensions = ", ". join(Flask_config.SUBMIT_PASTE_FILE_ALLOWED_EXTENSIONS)
|
||||
|
||||
PasteSubmit = Blueprint('PasteSubmit', __name__, template_folder='templates')
|
||||
|
||||
# ============ Validators ============
|
||||
def limit_content_length():
|
||||
|
|
@ -117,133 +99,6 @@ def clean_filename(filename, whitelist=valid_filename_chars, replace=' '):
|
|||
# keep only whitelisted chars
|
||||
return ''.join(c for c in cleaned_filename if c in whitelist)
|
||||
|
||||
def date_to_str(date):
|
||||
return "{0}-{1}-{2}".format(date.year, date.month, date.day)
|
||||
|
||||
def misp_create_event(distribution, threat_level_id, analysis, info, l_tags, publish, path):
|
||||
|
||||
item = Item(path)
|
||||
source = item.get_source()
|
||||
ail_uuid = r_serv_db.get('ail:uuid')
|
||||
pseudofile = BytesIO(item.get_content(binary=True))
|
||||
|
||||
temp = item.get_duplicates()
|
||||
|
||||
# beautifier
|
||||
if not temp:
|
||||
temp = ''
|
||||
|
||||
p_duplicate_number = len(temp) if len(temp) >= 0 else 0
|
||||
|
||||
to_ret = ""
|
||||
for dup in temp[:10]:
|
||||
dup = dup.replace('\'','\"').replace('(','[').replace(')',']')
|
||||
dup = json.loads(dup)
|
||||
algo = dup[0]
|
||||
path = dup[1].split('/')[-6:]
|
||||
path = '/'.join(path)[:-3] # -3 removes .gz
|
||||
if algo == 'tlsh':
|
||||
perc = 100 - int(dup[2])
|
||||
else:
|
||||
perc = dup[2]
|
||||
to_ret += "{}: {} [{}%]\n".format(path, algo, perc)
|
||||
p_duplicate = to_ret
|
||||
|
||||
today = datetime.date.today()
|
||||
# [0-3]
|
||||
if publish == 'True':
|
||||
published = True
|
||||
else:
|
||||
published = False
|
||||
org_id = None
|
||||
orgc_id = None
|
||||
sharing_group_id = None
|
||||
date = today
|
||||
event = pymisp.new_event(distribution, threat_level_id,
|
||||
analysis, info, date,
|
||||
published, orgc_id, org_id, sharing_group_id)
|
||||
eventUuid = event['Event']['uuid']
|
||||
eventid = event['Event']['id']
|
||||
|
||||
r_serv_metadata.set('misp_events:'+path, eventid)
|
||||
|
||||
# add tags
|
||||
for tag in l_tags:
|
||||
pymisp.tag(eventUuid, tag)
|
||||
|
||||
# create attributes
|
||||
obj_name = 'ail-leak'
|
||||
leak_obj = MISPObject(obj_name)
|
||||
leak_obj.add_attribute('sensor', value=ail_uuid, type="text")
|
||||
leak_obj.add_attribute('origin', value=source, type='text')
|
||||
leak_obj.add_attribute('last-seen', value=date_to_str(item.get_date()), type='datetime')
|
||||
leak_obj.add_attribute('raw-data', value=source, data=pseudofile, type="attachment")
|
||||
|
||||
if p_duplicate_number > 0:
|
||||
leak_obj.add_attribute('duplicate', value=p_duplicate, type='text')
|
||||
leak_obj.add_attribute('duplicate_number', value=p_duplicate_number, type='counter')
|
||||
|
||||
try:
|
||||
templateID = [x['ObjectTemplate']['id'] for x in pymisp.get_object_templates_list()['response'] if x['ObjectTemplate']['name'] == obj_name][0]
|
||||
except IndexError:
|
||||
valid_types = ", ".join([x['ObjectTemplate']['name'] for x in pymisp.get_object_templates_list()])
|
||||
print (f"Template for type {obj_name} not found! Valid types are: {valid_types}")
|
||||
return False
|
||||
r = pymisp.add_object(eventid, templateID, leak_obj)
|
||||
if 'errors' in r:
|
||||
print(r)
|
||||
return False
|
||||
else:
|
||||
event_url = misp_event_url + eventid
|
||||
return event_url
|
||||
|
||||
def hive_create_case(hive_tlp, threat_level, hive_description, hive_case_title, l_tags, path):
|
||||
|
||||
ail_uuid = r_serv_db.get('ail:uuid')
|
||||
source = path.split('/')[-6:]
|
||||
source = '/'.join(source)[:-3]
|
||||
# get item date
|
||||
var = path.split('/')
|
||||
last_seen = "{0}-{1}-{2}".format(var[-4], var[-3], var[-2])
|
||||
|
||||
case = Case(title=hive_case_title,
|
||||
tlp=hive_tlp,
|
||||
severity=threat_level,
|
||||
flag=False,
|
||||
tags=l_tags,
|
||||
description='hive_description')
|
||||
|
||||
# Create the case
|
||||
id = None
|
||||
response = HiveApi.create_case(case)
|
||||
if response.status_code == 201:
|
||||
id = response.json()['id']
|
||||
|
||||
observ_sensor = CaseObservable(dataType="other", data=[ail_uuid], message="sensor")
|
||||
observ_file = CaseObservable(dataType="file", data=[path], tags=l_tags)
|
||||
observ_source = CaseObservable(dataType="other", data=[source], message="source")
|
||||
observ_last_seen = CaseObservable(dataType="other", data=[last_seen], message="last-seen")
|
||||
|
||||
res = HiveApi.create_case_observable(id,observ_sensor)
|
||||
if res.status_code != 201:
|
||||
logger.info(f'ko sensor: {res.status_code}/{res.text}')
|
||||
res = HiveApi.create_case_observable(id, observ_source)
|
||||
if res.status_code != 201:
|
||||
logger.info(f'ko source: {res.status_code}/{res.text}')
|
||||
res = HiveApi.create_case_observable(id, observ_file)
|
||||
if res.status_code != 201:
|
||||
logger.info(f'ko file: {res.status_code}/{res.text}')
|
||||
res = HiveApi.create_case_observable(id, observ_last_seen)
|
||||
if res.status_code != 201:
|
||||
logger.info(f'ko last_seen: {res.status_code}/{res.text}')
|
||||
|
||||
r_serv_metadata.set('hive_cases:'+path, id)
|
||||
|
||||
return hive_case_url.replace('id_here', id)
|
||||
else:
|
||||
logger.info(f'ko: {response.status_code}/{response.text}')
|
||||
return False
|
||||
|
||||
# ============= ROUTES ==============
|
||||
|
||||
@PasteSubmit.route("/PasteSubmit/", methods=['GET'])
|
||||
|
|
@ -462,50 +317,31 @@ def submit_status():
|
|||
else:
|
||||
return 'INVALID UUID'
|
||||
|
||||
|
||||
@PasteSubmit.route("/PasteSubmit/create_misp_event", methods=['POST'])
|
||||
@login_required
|
||||
@login_analyst
|
||||
def create_misp_event():
|
||||
|
||||
distribution = int(request.form['misp_data[Event][distribution]'])
|
||||
threat_level_id = int(request.form['misp_data[Event][threat_level_id]'])
|
||||
analysis = int(request.form['misp_data[Event][analysis]'])
|
||||
info = request.form['misp_data[Event][info]']
|
||||
path = request.form['paste']
|
||||
publish = request.form.get('misp_publish')
|
||||
|
||||
#verify input
|
||||
if (0 <= distribution <= 3) and (1 <= threat_level_id <= 4) and (0 <= analysis <= 2):
|
||||
|
||||
l_tags = list(r_serv_metadata.smembers('tag:'+path))
|
||||
event = misp_create_event(distribution, threat_level_id, analysis, info, l_tags, publish, path)
|
||||
|
||||
if event != False:
|
||||
return redirect(event)
|
||||
else:
|
||||
return 'error, event creation'
|
||||
return 'error0'
|
||||
|
||||
# TODO MOVE ME IN import_export blueprint
|
||||
@PasteSubmit.route("/PasteSubmit/create_hive_case", methods=['POST'])
|
||||
@login_required
|
||||
@login_analyst
|
||||
def create_hive_case():
|
||||
|
||||
hive_tlp = int(request.form['hive_tlp'])
|
||||
threat_level = int(request.form['threat_level_hive'])
|
||||
hive_description = request.form['hive_description']
|
||||
hive_case_title = request.form['hive_case_title']
|
||||
path = os.environ['AIL_HOME'] + "/PASTES/"+ request.form['paste']
|
||||
tlp = request.form['hive_tlp']
|
||||
if tlp:
|
||||
tlp = int(tlp)
|
||||
else:
|
||||
tlp = 2
|
||||
threat_level = request.form['threat_level_hive']
|
||||
if threat_level:
|
||||
threat_level = int(threat_level)
|
||||
else:
|
||||
threat_level = 2
|
||||
description = request.form['hive_description']
|
||||
title = request.form['hive_case_title']
|
||||
item_id = request.form['obj_id']
|
||||
|
||||
#verify input
|
||||
if (0 <= hive_tlp <= 3) and (1 <= threat_level <= 4):
|
||||
if (0 <= tlp <= 3) and (1 <= threat_level <= 4):
|
||||
|
||||
l_tags = list(r_serv_metadata.smembers('tag:'+path))
|
||||
case = hive_create_case(hive_tlp, threat_level, hive_description, hive_case_title, l_tags, path)
|
||||
|
||||
if case != False:
|
||||
return redirect(case)
|
||||
case_id = Export.create_thehive_case(item_id, title=title, tlp=tlp, threat_level=threat_level, description=description)
|
||||
if case_id:
|
||||
return redirect(get_case_url(case_id))
|
||||
else:
|
||||
return 'error'
|
||||
|
||||
|
|
|
|||
|
|
@ -131,7 +131,7 @@ def get_all_types_id(correlation_type):
|
|||
|
||||
def get_key_id_metadata(obj_type, subtype, obj_id):
|
||||
obj = ail_objects.get_object_meta(obj_type, subtype, obj_id)
|
||||
return obj._get_meta()
|
||||
return obj
|
||||
|
||||
def list_sparkline_type_id_values(date_range_sparkline, correlation_type, type_id, key_id):
|
||||
sparklines_value = []
|
||||
|
|
|
|||
|
|
@ -171,7 +171,7 @@
|
|||
{% for key_id in all_metadata %}
|
||||
<tr>
|
||||
<td><i class="{{ all_metadata[key_id]['type_icon'] }}"></i> {{ all_metadata[key_id]['type_id'] }}</td>
|
||||
<td><a target="_blank" href="{{ url_for(show_key_id_endpoint) }}?object_type={{correlation_type_n}}&type_id={{ all_metadata[key_id]['type_id'] }}&correlation_id={{ key_id }}&correlation_objects=paste">{{ key_id }}</a></td>
|
||||
<td><a target="_blank" href="{{ url_for(show_key_id_endpoint) }}?type={{correlation_type_n}}&subtype={{ all_metadata[key_id]['type_id'] }}&id={{ key_id }}">{{ key_id }}</a></td>
|
||||
<td>{{ all_metadata[key_id]['first_seen'] }}</td>
|
||||
<td>{{ all_metadata[key_id]['last_seen'] }}</td>
|
||||
<td>{{ all_metadata[key_id]['nb_seen'] }}</td>
|
||||
|
|
|
|||
|
|
@ -158,7 +158,7 @@
|
|||
{% for b64 in l_64 %}
|
||||
<tr>
|
||||
<td><i class="fas {{ b64[0] }}"></i> {{ b64[1] }}</td>
|
||||
<td><a target="_blank" href="{{ url_for('correlation.show_correlation') }}?object_type=decoded&correlation_id={{ b64[2] }}&correlation_objects=paste">{{ b64[2] }}</a></td>
|
||||
<td><a target="_blank" href="{{ url_for('correlation.show_correlation') }}?type=decoded&id={{ b64[2] }}">{{ b64[2] }}</a></td>
|
||||
<td>{{ b64[5] }}</td>
|
||||
<td>{{ b64[6] }}</td>
|
||||
<td>{{ b64[3] }}</td>
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
<a class="btn btn-lg btn-outline-dark" target="_blank" href="{{ url_for('import_export.add_object_id_to_export')}}?obj_type={{obj_type}}&obj_id={{obj_id}}&obj_lvl={{obj_lvl}}{%if obj_subtype%}&obj_subtype={{obj_subtype}}{%endif%}">
|
||||
<a class="btn btn-lg btn-outline-dark" target="_blank" href="{{ url_for('import_export.add_object_id_to_export')}}?type={{obj_type}}{%if obj_subtype%}&subtype={{obj_subtype}}{%endif%}&id={{obj_id}}&lvl={{obj_lvl}}">
|
||||
Add to
|
||||
<img id="misp-logo" src="{{ url_for('static', filename='image/misp-logo.png')}}" height="32">
|
||||
Export
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
<div class="d-flex flex-row-reverse bd-highlight">
|
||||
<div>
|
||||
<a class="btn" target="_blank" href="{{ url_for('import_export.add_object_id_to_export')}}?obj_type={{obj_type}}&obj_id={{obj_id}}&obj_lvl={{obj_lvl}}{%if obj_subtype%}&obj_subtype={{obj_subtype}}{%endif%}">
|
||||
<a class="btn" target="_blank" href="{{ url_for('import_export.add_object_id_to_export')}}?type={{obj_type}}{%if obj_subtype%}&subtype={{obj_subtype}}{%endif%}&id={{obj_id}}&lvl={{obj_lvl}}">
|
||||
<img id="misp-logo" src="{{ url_for('static', filename='image/misp-logo.png')}}" height="25">
|
||||
</a>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -1,38 +1,21 @@
|
|||
<div class="input-group mb-1">
|
||||
<select class="custom-select col-2" name="{{input_uuid}}" value="{{obj_type}}">
|
||||
{% if not obj_type %}
|
||||
<option selected="">Object type...</option>
|
||||
<select class="custom-select col-2" name="{{input_uuid}}" disabled>
|
||||
<option selected="" value="{{ obj_type }}:{{ obj_subtype }}">
|
||||
{% if obj_subtype %}
|
||||
{{ obj_type }} - {{ obj_subtype }}
|
||||
{% else %}
|
||||
<option>Object type...</option>
|
||||
{{ obj_type }}
|
||||
{% endif %}
|
||||
<option value="item" {%if obj_type=="item"%}selected{%endif%}>Item</option>
|
||||
<option value="domain" {%if obj_type=="domain"%}selected{%endif%}>Domain</option>
|
||||
<option value="image" {%if obj_type=="image"%}selected{%endif%}>Image</option>
|
||||
<option value="decoded" {%if obj_type=="decoded"%}selected{%endif%}>Decoded</option>
|
||||
<option value="pgp;key" {%if obj_type=="pgp" and obj_subtype=="key"%}selected{%endif%}>PGP - Key</option>
|
||||
<option value="pgp;name" {%if obj_type=="pgp" and obj_subtype=="name"%}selected{%endif%}>PGP - Name</option>
|
||||
<option value="pgp;mail" {%if obj_type=="pgp" and obj_subtype=="mail"%}selected{%endif%}>PGP - Mail</option>
|
||||
<option value="cryptocurrency;bitcoin" {%if obj_type=="cryptocurrency" and obj_subtype=="bitcoin"%}selected{%endif%}>Cryptocurrency - Bitcoin</option>
|
||||
<option value="cryptocurrency;bitcoin-cash" {%if obj_type=="cryptocurrency" and obj_subtype=="bitcoin-cash"%}selected{%endif%}>Cryptocurrency - Bitcoin Cash</option>
|
||||
<option value="cryptocurrency;dash" {%if obj_type=="cryptocurrency" and obj_subtype=="dash"%}selected{%endif%}>Cryptocurrency - Dash</option>
|
||||
<option value="cryptocurrency;etherum" {%if obj_type=="cryptocurrency" and obj_subtype=="etherum"%}selected{%endif%}>Cryptocurrency - Etherum</option>
|
||||
<option value="cryptocurrency;litecoin" {%if obj_type=="cryptocurrency" and obj_subtype=="litecoin"%}selected{%endif%}>Cryptocurrency - Litecoin</option>
|
||||
<option value="cryptocurrency;monero" {%if obj_type=="cryptocurrency" and obj_subtype=="monero"%}selected{%endif%}>Cryptocurrency - Monero</option>
|
||||
<option value="cryptocurrency;zcash" {%if obj_type=="cryptocurrency" and obj_subtype=="zcash"%}selected{%endif%}>Cryptocurrency - Zcash</option>
|
||||
|
||||
<option value="username;telegram" {%if obj_type=="username" and obj_subtype=="telegram"%}selected{%endif%}>Username - telegram</option>
|
||||
<option value="username;twitter" {%if obj_type=="username" and obj_subtype=="twitter"%}selected{%endif%}>Username - twitter</option>
|
||||
<option value="username;jabber" {%if obj_type=="username" and obj_subtype=="jabber"%}selected{%endif%}>Username - jabber</option>
|
||||
|
||||
</option>
|
||||
</select>
|
||||
<input type="text" class="form-control col-8 {%if obj_error%}is-invalid{%else%}is-valid{%endif%}" name="{{input_uuid}}" value="{{obj_id}}">
|
||||
<input type="text" class="form-control col-8 {%if obj_error%}is-invalid{%else%}is-valid{%endif%}" name="{{input_uuid}}" value="{{obj_id}}" disabled>
|
||||
<input class="form-control col-1" type="number" min="0" name="{{input_uuid}}" {%if obj_lvl%}value="{{obj_lvl}}"{%else%}value="0"{%endif%}>
|
||||
<span class="btn btn-danger input-group-addon delete-field col-1"><i class="fas fa-trash-alt"></i></span>
|
||||
<span class="btn btn-danger input-group-addon delete-field col-1" id="{{ obj_type }}:{{ obj_subtype }}:{{ obj_id }}"><i class="fas fa-trash-alt"></i></span>
|
||||
<div class="invalid-feedback">
|
||||
{%if obj_type=="Object type..."%}
|
||||
Please select an object type.
|
||||
{%else%}
|
||||
Unknow or Invalid ID.
|
||||
Unknown or Invalid ID.
|
||||
{%endif%}
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@
|
|||
</div>
|
||||
<div class="card-body">
|
||||
|
||||
<form action="{{ url_for('import_export.export_object_file') }}" method="post" enctype=multipart/form-data onsubmit="submitPaste()">
|
||||
<form action="{{ url_for('import_export.objects_misp_export_post') }}" method="post" enctype=multipart/form-data onsubmit="submitPaste()">
|
||||
|
||||
<div id="container-id-to-import">
|
||||
|
||||
|
|
@ -53,30 +53,24 @@
|
|||
<div class="input-group mb-1">
|
||||
<select class="custom-select col-2" name="first_obj_to_export" id="obj_input_type">
|
||||
<option selected>Object type...</option>
|
||||
<option value="item">Item</option>
|
||||
<option value="domain">Domain</option>
|
||||
<option value="image">Image</option>
|
||||
<option value="decoded">Decoded</option>
|
||||
<option value="pgp;key">PGP - Key</option>
|
||||
<option value="pgp;name">PGP - Name</option>
|
||||
<option value="pgp;mail">PGP - Mail</option>
|
||||
<option value="cryptocurrency;bitcoin">Cryptocurrency - Bitcoin</option>
|
||||
<option value="cryptocurrency;bitcoin-cash">Cryptocurrency - Bitcoin Cash</option>
|
||||
<option value="cryptocurrency;dash">Cryptocurrency - Dash</option>
|
||||
<option value="cryptocurrency;etherum">Cryptocurrency - Etherum</option>
|
||||
<option value="cryptocurrency;litecoin">Cryptocurrency - Litecoin</option>
|
||||
<option value="cryptocurrency;monero">Cryptocurrency - Monero</option>
|
||||
<option value="cryptocurrency;zcash">Cryptocurrency - Zcash</option>
|
||||
<option value="username;telegram">Username - telegram</option>
|
||||
<option value="username;twitter">Username - twitter</option>
|
||||
<option value="username;jabber">Username - jabber</option>
|
||||
{% for object_type in object_types %}
|
||||
<option value="{{ object_type[0] }}:{{ object_type[1] }}">
|
||||
{% if object_type[1] %}
|
||||
{{ object_type[0] }} - {{ object_type[1] }}
|
||||
{% else %}
|
||||
{{ object_type[0] }}
|
||||
{% endif %}
|
||||
</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
|
||||
<input type="text" class="form-control col-8" name="first_obj_to_export" id="obj_input_id">
|
||||
<input class="form-control col-1" type="number" min="0" value="0" name="first_obj_to_export" id="obj_input_lvl">
|
||||
<span class="btn btn-info input-group-addon add-field col-1"><i class="fas fa-plus"></i></span>
|
||||
</div>
|
||||
{% for obj_dict in l_obj_to_export %}
|
||||
{% with obj_type=obj_dict['type'], obj_subtype=obj_dict.get('subtype', None), obj_id=obj_dict['id'], obj_lvl=obj_dict['lvl'], input_uuid=obj_dict, obj_error=False%}
|
||||
|
||||
{% for obj_dict in to_export %}
|
||||
{% with obj_type=obj_dict['type'], obj_subtype=obj_dict['subtype'], obj_id=obj_dict['id'], obj_lvl=obj_dict['lvl'], input_uuid=obj_dict, obj_error=False%}
|
||||
{% include 'import_export/block_to_export_input.html' %}
|
||||
{% endwith %}
|
||||
{% endfor %}
|
||||
|
|
@ -84,7 +78,7 @@
|
|||
<br>
|
||||
|
||||
{% for obj_dict in l_obj_invalid %}
|
||||
{% with obj_type=obj_dict['type'], obj_subtype=obj_dict.get('subtype', None), obj_id=obj_dict['id'], obj_lvl=obj_dict['lvl'], input_uuid=obj_dict['uuid'], obj_error=True%}
|
||||
{% with obj_type=obj_dict['type'], obj_subtype=obj_dict.get('subtype', None), obj_id=obj_dict['id'], obj_lvl=obj_dict['lvl'], input_uuid=obj_dict, obj_error=True%}
|
||||
{% include 'import_export/block_to_export_input.html' %}
|
||||
{% endwith %}
|
||||
{% endfor %}
|
||||
|
|
@ -119,10 +113,10 @@
|
|||
</div>
|
||||
</div>
|
||||
|
||||
{% if event_metadata %}
|
||||
{% if misp_url %}
|
||||
MISP Event Created:
|
||||
<a target="_blank" href="{{ event_metadata['url'] }}">
|
||||
{{ event_metadata['url'] }}
|
||||
<a target="_blank" href="{{ misp_url }}">
|
||||
{{ misp_url }}
|
||||
</a>
|
||||
{% endif %}
|
||||
|
||||
|
|
@ -155,7 +149,17 @@ function toggle_sidebar(){
|
|||
}
|
||||
|
||||
var input_1 = '<div class="input-group mb-1"><select class="custom-select col-2" name="'
|
||||
var input_2 = '"><option selected>Object type...</option><option value="item">Item</option><option value="domain">Domain</option><option value="image">Image</option><option value="decoded">Decoded</option><option value="pgp;key">PGP - Key</option><option value="pgp;name">PGP - Name</option><option value="pgp;mail">PGP - Mail</option><option value="cryptocurrency;bitcoin">Cryptocurrency - Bitcoin</option><option value="cryptocurrency;bitcoin-cash">Cryptocurrency - Bitcoin Cash</option><option value="cryptocurrency;dash">Cryptocurrency - Dash</option><option value="cryptocurrency;etherum">Cryptocurrency - Etherum</option><option value="cryptocurrency;litecoin">Cryptocurrency - Litecoin</option><option value="cryptocurrency;monero">Cryptocurrency - Monero</option><option value="cryptocurrency;zcash">Cryptocurrency - Zcash</option></select><input type="text" class="form-control col-8" name="'
|
||||
var input_2 = '"><option selected>Object type...</option>' +
|
||||
{% for object_type in object_types %}
|
||||
'<option value="{{ object_type[0] }}:{{ object_type[1] }}">' +
|
||||
{% if object_type[1] %}
|
||||
'{{ object_type[0] }} - {{ object_type[1] }}' +
|
||||
{% else %}
|
||||
'{{ object_type[0] }}' +
|
||||
{% endif %}
|
||||
'</option>' +
|
||||
{% endfor %}
|
||||
'</select><input type="text" class="form-control col-8" name="'
|
||||
var input_3 = '"><input class="form-control col-1" type="number" min="0" value="0" name="'
|
||||
var input_4 = '"></div>';
|
||||
var minusButton = '<span class="btn btn-danger input-group-addon delete-field col-1"><i class="fas fa-trash-alt"></i></span>';
|
||||
|
|
@ -168,8 +172,10 @@ $('.add-field').click(function() {
|
|||
});
|
||||
|
||||
$('.fields').on('click', '.delete-field', function(){
|
||||
var obj = $(this).attr('id').split(":", 3);
|
||||
var params = "?type=" + obj[0] + "&subtype=" + obj[1] +"&id=" + obj[2];
|
||||
$.ajax({url: "{{ url_for('import_export.delete_object_id_to_export') }}" + params })
|
||||
$(this).parent().remove();
|
||||
//$.get( "#")
|
||||
});
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -107,8 +107,8 @@
|
|||
|
||||
{% if metadata['misp_events'] %}
|
||||
{% for misp_url in metadata['misp_events'] %}
|
||||
<a target="_blank" href="{{misp_url}}/events/view/{{metadata['uuid'][:8]}}-{{metadata['uuid'][8:12]}}-{{metadata['uuid'][12:16]}}-{{metadata['uuid'][16:20]}}-{{metadata['uuid'][20:]}}">
|
||||
{{misp_url}}/events/view/{{metadata['uuid'][:8]}}-{{metadata['uuid'][8:12]}}-{{metadata['uuid'][12:16]}}-{{metadata['uuid'][16:20]}}-{{metadata['uuid'][20:]}}</br>
|
||||
<a target="_blank" href="{{ misp_url }}">
|
||||
{{ misp_url }}</br>
|
||||
</a>
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
|
|
|||
|
|
@ -6,10 +6,10 @@
|
|||
<img id="hive-logo" src="{{url_for('static', filename='image/thehive-logo.png') }}" width="500" >
|
||||
</div>
|
||||
|
||||
<div class="modal-body">
|
||||
|
||||
<form method="post" action="{{ url_for('PasteSubmit.create_hive_case') }}" target="_blank">
|
||||
|
||||
<div class="modal-body">
|
||||
|
||||
<div class="input clear required">
|
||||
<label for="EventThreatLevelId">Threat Level</label>
|
||||
<select name="threat_level_hive" id="EventThreatLevelId" required="required">
|
||||
|
|
@ -19,7 +19,7 @@
|
|||
</select>
|
||||
</div>
|
||||
<div class="input select required">
|
||||
<label for="TLP">TLP</label>
|
||||
<label for="hive_tlp">TLP</label>
|
||||
<select name="hive_tlp" id="hive_tlp" required="required" class="selectpicker">
|
||||
<option value="0">White</option>
|
||||
<option value="1">Green</option>
|
||||
|
|
@ -27,18 +27,15 @@
|
|||
<option value="3">Red</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="clear required">
|
||||
<div class="clear">
|
||||
<label for="hive_case_title">Title</label>
|
||||
<input name="hive_case_title" class="form-control span6" placeholder="Title" type="text" id="hive_case_title" required="required"/>
|
||||
<input name="hive_case_title" class="form-control span6" placeholder="Title" type="text" id="hive_case_title"/>
|
||||
</div>
|
||||
<div class="clear required">
|
||||
<div class="clear">
|
||||
<label for="hive_description">Description</label>
|
||||
<input name="hive_description" class="form-control span6" placeholder="Quick Case Description" type="text" id="hive_description" required="required"/>
|
||||
<input name="hive_description" class="form-control span6" placeholder="Quick Case Description" type="text" id="hive_description"/>
|
||||
</div>
|
||||
<input type="hidden" id="paste" name="paste" value="{{ dict_item['id'] }}">
|
||||
|
||||
|
||||
|
||||
<input type="hidden" id="obj_id" name="obj_id" value="{{ dict_item['id'] }}">
|
||||
</div>
|
||||
|
||||
<div class="modal-footer">
|
||||
|
|
@ -47,74 +44,11 @@
|
|||
Create Case
|
||||
</button>
|
||||
|
||||
</form>
|
||||
<button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
|
||||
</div>
|
||||
|
||||
</form>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script> // TODO: add tags to PASTE or DOMAIN
|
||||
var ltags;
|
||||
var ltagsgalaxies;
|
||||
|
||||
$.getJSON("{{ url_for('Tags.get_all_tags_taxonomies') }}",
|
||||
function(data) {
|
||||
|
||||
ltags = $('#ltags').tagSuggest({
|
||||
data: data,
|
||||
maxDropHeight: 200,
|
||||
name: 'ltags'
|
||||
});
|
||||
});
|
||||
|
||||
$.getJSON("{{ url_for('Tags.get_all_tags_galaxy') }}",
|
||||
function(data) {
|
||||
|
||||
ltagsgalaxies = $('#ltagsgalaxies').tagSuggest({
|
||||
data: data,
|
||||
maxDropHeight: 200,
|
||||
name: 'ltagsgalaxies'
|
||||
});
|
||||
});
|
||||
|
||||
jQuery("#all-tags-taxonomies").click(function(e){
|
||||
//change input tags list
|
||||
$.getJSON("{{ url_for('Tags.get_all_tags_taxonomies') }}",
|
||||
function(data) {
|
||||
ltags.setData(data)
|
||||
});
|
||||
});
|
||||
|
||||
jQuery("#all-tags-galaxies").click(function(e){
|
||||
$.getJSON("{{ url_for('Tags.get_all_tags_galaxy') }}",
|
||||
function(data) {
|
||||
ltagsgalaxies.setData(data)
|
||||
});
|
||||
});
|
||||
|
||||
{% for taxo in modal_add_tags['active_taxonomies'] %}
|
||||
jQuery("#{{ taxo }}-id{{ loop.index0 }}").click(function(e){
|
||||
$.getJSON("{{ url_for('Tags.get_tags_taxonomie') }}?taxonomie={{ taxo }}",
|
||||
function(data) {
|
||||
ltags.setData(data)
|
||||
});
|
||||
});
|
||||
{% endfor %}
|
||||
|
||||
{% for galaxy in modal_add_tags['active_galaxies'] %}
|
||||
jQuery("#{{ galaxy }}-idgalax{{ loop.index0 }}").click(function(e){
|
||||
$.getJSON("{{ url_for('Tags.get_tags_galaxy') }}?galaxy={{ galaxy }}",
|
||||
function(data) {
|
||||
ltagsgalaxies.setData(data)
|
||||
});
|
||||
});
|
||||
{% endfor %}
|
||||
|
||||
function addTags() {
|
||||
var tags = ltags.getValue()
|
||||
var tagsgalaxy = ltagsgalaxies.getValue()
|
||||
window.location.replace("{{ url_for('tags_ui.add_tags') }}?tags=" + tags + "&tagsgalaxies=" + tagsgalaxy + "&object_id={{ modal_add_tags['object_id'] }}&object_type={{ modal_add_tags['object_type'] }}");
|
||||
}
|
||||
</script>
|
||||
|
|
|
|||
|
|
@ -75,7 +75,7 @@
|
|||
</a>
|
||||
</li>
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href="{{url_for('import_export.export_object')}}" id="nav_misp_export">
|
||||
<a class="nav-link" href="{{url_for('import_export.objects_misp_export')}}" id="nav_misp_export">
|
||||
<b>Export</b>
|
||||
</a>
|
||||
</li>
|
||||
|
|
|
|||
Loading…
Reference in a new issue