chg: [add MISP import] correlation

TODO: create correlation
This commit is contained in:
Terrtia 2020-02-07 15:08:41 +01:00
parent 73f98c0897
commit 6bbcef024b
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
5 changed files with 109 additions and 123 deletions

View file

@ -67,7 +67,7 @@ def unpack_item_obj(map_uuid_global_id, misp_obj):
res = Item.create_item(obj_id, obj_meta, io_content)
print(res)
map_uuid_global_id[misp_obj.uuid] = get_global_id('item', obj_id, io_content)
map_uuid_global_id[misp_obj.uuid] = get_global_id('item', obj_id)
def get_obj_relationship(misp_obj):
for item in misp_obj.ObjectReference:
@ -87,34 +87,38 @@ def unpack_obj_pgp(map_uuid_global_id, misp_obj):
elif obj_attr.object_relation == 'user-id-email':
obj_subtype = 'mail'
obj_meta = get_object_metadata(misp_obj)
if obj_id and io_content:
res = Pgp.pgp.create_item(obj_subtype, obj_id, obj_meta)
print(res)
map_uuid_global_id[misp_obj.uuid] = get_global_id('pgp', obj_id, obj_subtype=obj_subtype)
get_obj_relationship(misp_obj)
def unpack_obj_cryptocurrency(map_uuid_global_id, misp_obj):
obj_id = None
crypto_symbol = None
for attribute in misp_obj.attributes:
if attribute.object_relation == 'address':
obj_id = attribute.value
elif attribute.object_relation == 'symbol':
pass
if obj_id and obj_subtype:
obj_meta = get_object_metadata(misp_obj)
if obj_id and io_content:
res = Pgp.pgp.create_item(obj_subtype, obj_id, obj_meta)
print(res)
res = Pgp.pgp.create_correlation(obj_subtype, obj_id, obj_meta)
print(res)
map_uuid_global_id[misp_obj.uuid] = get_global_id('pgp', obj_id, obj_subtype=obj_subtype)
get_obj_relationship(misp_obj)
#get_obj_relationship(misp_obj)
def unpack_obj_cryptocurrency(map_uuid_global_id, misp_obj):
obj_id = None
obj_subtype = None
for attribute in misp_obj.attributes:
if attribute.object_relation == 'address': # # TODO: handle xmr address field
obj_id = attribute.value
elif attribute.object_relation == 'symbol':
obj_subtype = Cryptocurrency.get_cryptocurrency_type(attribute.value)
# valid cryptocurrency type
if obj_subtype and obj_id:
print('crypto')
print(obj_id)
print(obj_subtype)
obj_meta = get_object_metadata(misp_obj)
print(obj_meta)
res = Cryptocurrency.cryptocurrency.create_correlation(obj_subtype, obj_id, obj_meta)
print(res)
map_uuid_global_id[misp_obj.uuid] = get_global_id('pgp', obj_id, obj_subtype=obj_subtype)
get_obj_relationship(misp_obj)
def get_misp_import_fct(map_uuid_global_id, misp_obj):
#print(misp_obj.ObjectReference)
@ -155,4 +159,5 @@ if __name__ == '__main__':
# misp = PyMISP('https://127.0.0.1:8443/', 'uXgcN42b7xuL88XqK5hubwD8Q8596VrrBvkHQzB0', False)
import_objs_from_file('test_import_item.json')
#import_objs_from_file('test_import_item.json')
import_objs_from_file('test_export.json')

View file

@ -9,7 +9,6 @@ from hashlib import sha256
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
import correlation
import Item
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
@ -56,39 +55,6 @@ def get_cryptocurrency(request_dict, cryptocurrency_type):
return cryptocurrency.get_correlation(request_dict, cryptocurrency_type, field_name)
# # TODO: refractor/move me in Correlation
def save_cryptocurrency_data(cryptocurrency_name, date, item_path, cryptocurrency_address):
# create basic medata
if not r_serv_metadata.exists('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address)):
r_serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'first_seen', date)
r_serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen', date)
else:
last_seen = r_serv_metadata.hget('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen')
if not last_seen:
r_serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen', date)
else:
if int(last_seen) < int(date):
r_serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen', date)
## global set
# item
r_serv_metadata.sadd('set_cryptocurrency_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), item_path)
# daily
r_serv_metadata.hincrby('cryptocurrency:{}:{}'.format(cryptocurrency_name, date), cryptocurrency_address, 1)
# all type
r_serv_metadata.zincrby('cryptocurrency_all:{}'.format(cryptocurrency_name), cryptocurrency_address, 1)
## object_metadata
# item
r_serv_metadata.sadd('item_cryptocurrency_{}:{}'.format(cryptocurrency_name, item_path), cryptocurrency_address)
# domain
if Item.is_crawled(item_path): # # TODO: use save_domain_correlation
domain = Item.get_item_domain(item_path)
r_serv_metadata.sadd('domain_cryptocurrency_{}:{}'.format(cryptocurrency_name, domain), cryptocurrency_address)
r_serv_metadata.sadd('set_domain_cryptocurrency_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), domain)
def get_cryptocurrency_symbol(crypto_type):
if crypto_type=='bitcoin':

View file

@ -129,15 +129,32 @@ def validate_str_date(str_date, separator=''):
except TypeError:
return False
def sanitise_date_range(date_from, date_to, separator=''):
def sanitise_date_range(date_from, date_to, separator='', date_type='str'):
'''
Check/Return a correct date_form and date_to
'''
if not validate_str_date(date_from, separator=separator):
date_from = datetime.date.today().strftime("%Y%m%d")
if not validate_str_date(date_to, separator=separator):
date_to = datetime.date.today().strftime("%Y%m%d")
if not date_from and date_to:
date_from = date_to
elif not date_to and date_from:
date_to = date_from
if date_type=='str':
if not validate_str_date(date_from, separator=separator):
date_from = datetime.date.today().strftime("%Y%m%d")
if not validate_str_date(date_to, separator=separator):
date_to = datetime.date.today().strftime("%Y%m%d")
else: # datetime
if isinstance(date_from, datetime.datetime):
date_from = date_from.strftime("%Y%m%d")
else:
date_from = datetime.date.today().strftime("%Y%m%d")
if isinstance(date_to, datetime.datetime):
date_to = date_to.strftime("%Y%m%d")
else:
date_to = datetime.date.today().strftime("%Y%m%d")
if int(date_from) > int(date_to):
res = date_from
date_from = date_to
date_to = res
return {"date_from": date_from, "date_to": date_to}

View file

@ -7,7 +7,6 @@ import redis
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
import correlation
import Item
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
@ -27,36 +26,3 @@ def get_pgp(request_dict, pgp_type):
field_name = request_dict.get(pgp_type)
return pgpdump.get_correlation(request_dict, pgp_type, field_name)
def save_pgp_data(type_pgp, date, item_path, data):
# create basic medata
if not serv_metadata.exists('pgpdump_metadata_{}:{}'.format(type_pgp, data)):
serv_metadata.hset('pgpdump_metadata_{}:{}'.format(type_pgp, data), 'first_seen', date)
serv_metadata.hset('pgpdump_metadata_{}:{}'.format(type_pgp, data), 'last_seen', date)
else:
last_seen = serv_metadata.hget('pgpdump_metadata_{}:{}'.format(type_pgp, data), 'last_seen')
if not last_seen:
serv_metadata.hset('pgpdump_metadata_{}:{}'.format(type_pgp, data), 'last_seen', date)
else:
if int(last_seen) < int(date):
serv_metadata.hset('pgpdump_metadata_{}:{}'.format(type_pgp, data), 'last_seen', date)
# global set
serv_metadata.sadd('set_pgpdump_{}:{}'.format(type_pgp, data), item_path)
# daily
serv_metadata.hincrby('pgpdump:{}:{}'.format(type_pgp, date), data, 1)
# all type
serv_metadata.zincrby('pgpdump_all:{}'.format(type_pgp), data, 1)
## object_metadata
# paste
serv_metadata.sadd('item_pgpdump_{}:{}'.format(type_pgp, item_path), data)
# domain object
if Item.is_crawled(item_path):
domain = Item.get_item_domain(item_path)
serv_metadata.sadd('domain_pgpdump_{}:{}'.format(type_pgp, domain), data)
serv_metadata.sadd('set_domain_pgpdump_{}:{}'.format(type_pgp, data), domain)

View file

@ -10,6 +10,8 @@ import ConfigLoader
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
import Date
import Item
import Tag
config_loader = ConfigLoader.ConfigLoader()
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
@ -47,11 +49,31 @@ class Correlation(object):
else:
return []
def _get_metadata(self, correlation_type, field_name):
def get_correlation_first_seen(self, subtype, obj_id, r_int=False):
res = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, subtype, obj_id), 'first_seen')
if r_int:
if res:
return int(res)
else:
return 99999999
else:
return res
def get_correlation_last_seen(self, subtype, obj_id, r_int=False):
res = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, subtype, obj_id), 'last_seen')
if r_int:
if res:
return int(res)
else:
return 0
else:
return res
def _get_metadata(self, subtype, obj_id):
meta_dict = {}
meta_dict['first_seen'] = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, correlation_type, field_name), 'first_seen')
meta_dict['last_seen'] = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, correlation_type, field_name), 'last_seen')
meta_dict['nb_seen'] = r_serv_metadata.scard('set_{}_{}:{}'.format(self.correlation_name, correlation_type, field_name))
meta_dict['first_seen'] = self.get_correlation_first_seen(subtype, obj_id)
meta_dict['last_seen'] = self.get_correlation_last_seen(subtype, obj_id)
meta_dict['nb_seen'] = r_serv_metadata.scard('set_{}_{}:{}'.format(self.correlation_name, subtype, obj_id))
return meta_dict
def get_metadata(self, correlation_type, field_name, date_format='str_date'):
@ -276,18 +298,19 @@ class Correlation(object):
correlation_obj[correlation_object] = res
return correlation_obj
def update_correlation_daterange(self, subtype, obj_id, date): # # TODO: update fisrt_seen
def update_correlation_daterange(self, subtype, obj_id, date):
date = int(date)
# obj_id don't exit
if not r_serv_metadata.exists('{}_metadata_{}:{}'.format(self.correlation_name, subtype, obj_id)):
r_serv_metadata.hset('{}_metadata_{}:{}'.format(self.correlation_name, subtype, obj_id), 'first_seen', date)
r_serv_metadata.hset('{}_metadata_{}:{}'.format(self.correlation_name, subtype, obj_id), 'last_seen', date)
else:
last_seen = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, subtype, obj_id), 'last_seen')
if not last_seen:
first_seen = self.get_correlation_last_seen(subtype, obj_id, r_int=True)
last_seen = self.get_correlation_first_seen(subtype, obj_id, r_int=True)
if date < first_seen:
r_serv_metadata.hset('{}_metadata_{}:{}'.format(self.correlation_name, subtype, obj_id), 'first_seen', date)
if date > last_seen:
r_serv_metadata.hset('{}_metadata_{}:{}'.format(self.correlation_name, subtype, obj_id), 'last_seen', date)
else:
if int(last_seen) < int(date):
r_serv_metadata.hset('{}_metadata_{}:{}'.format(self.correlation_name, subtype, obj_id), 'last_seen', date)
def save_item_correlation(self, subtype, date, obj_id, item_id, item_date):
update_correlation_daterange(subtype, obj_id, item_date)
@ -305,28 +328,37 @@ class Correlation(object):
# item
r_serv_metadata.sadd('item_{}_{}:{}'.format(self.correlation_name, subtype, item_id), obj_id)
def save_domain_correlation(self, domain, correlation_type, correlation_value):
r_serv_metadata.sadd('domain_{}_{}:{}'.format(self.correlation_name, correlation_type, domain), correlation_value)
r_serv_metadata.sadd('set_domain_{}_{}:{}'.format(self.correlation_name, correlation_type, correlation_value), domain)
# domain
if Item.is_crawled(item_id):
domain = Item.get_item_domain(item_id)
self.save_domain_correlation(domain, subtype, obj_id)
def save_domain_correlation(self, domain, subtype, obj_id):
r_serv_metadata.sadd('domain_{}_{}:{}'.format(self.correlation_name, subtype, domain), obj_id)
r_serv_metadata.sadd('set_domain_{}_{}:{}'.format(self.correlation_name, subtype, obj_id), domain)
def save_correlation(self, subtype, obj_id): # # TODO: add first_seen/last_seen
def save_correlation(self, subtype, obj_id, date_range):
r_serv_metadata.zincrby('{}_all:{}'.format(self.correlation_name, subtype), obj_id, 0)
self.update_correlation_daterange(subtype, obj_id, date_range['date_from'])
if date_range['date_from'] != date_range['date_to']:
self.update_correlation_daterange(subtype, obj_id, date_range['date_to'])
return True
def create_correlation(self, subtype, obj_id, obj_meta):
res = self.sanythise_correlation_types(correlation_type, r_boolean=True)
res = self.sanythise_correlation_types([subtype], r_boolean=True)
if not res:
print('invalid subtype')
return False
if not exist_correlation(subtype, obj_id):
res = save_correlation(subtype, obj_id)
if res:
if 'tags' in obj_metadata:
# # TODO: handle mixed tags: taxonomies and Galaxies
Tag.api_add_obj_tags(tags=obj_metadata['tags'], object_id=obj_id, object_type=self.get_correlation_obj_type())
return True
return False
first_seen = obj_meta.get('first_seen', None)
last_seen = obj_meta.get('last_seen', None)
date_range = Date.sanitise_date_range(first_seen, last_seen, separator='', date_type='datetime')
print(date_range)
res = self.save_correlation(subtype, obj_id, date_range)
if res and 'tags' in obj_meta:
# # TODO: handle mixed tags: taxonomies and Galaxies
Tag.api_add_obj_tags(tags=obj_meta['tags'], object_id=obj_id, object_type=self.get_correlation_obj_type())
return True
######## API EXPOSED ########