mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-22 22:27:17 +00:00
chg: [add MISP import] correlation
TODO: create correlation
This commit is contained in:
Terrtia
parent
73f98c0897
commit
6bbcef024b
5 changed files with 109 additions and 123 deletions
|
|
@ -67,7 +67,7 @@ def unpack_item_obj(map_uuid_global_id, misp_obj):
|
|||
res = Item.create_item(obj_id, obj_meta, io_content)
|
||||
print(res)
|
||||
|
||||
map_uuid_global_id[misp_obj.uuid] = get_global_id('item', obj_id, io_content)
|
||||
map_uuid_global_id[misp_obj.uuid] = get_global_id('item', obj_id)
|
||||
|
||||
def get_obj_relationship(misp_obj):
|
||||
for item in misp_obj.ObjectReference:
|
||||
|
|
@ -87,34 +87,38 @@ def unpack_obj_pgp(map_uuid_global_id, misp_obj):
|
|||
elif obj_attr.object_relation == 'user-id-email':
|
||||
obj_subtype = 'mail'
|
||||
|
||||
obj_meta = get_object_metadata(misp_obj)
|
||||
if obj_id and io_content:
|
||||
res = Pgp.pgp.create_item(obj_subtype, obj_id, obj_meta)
|
||||
print(res)
|
||||
|
||||
map_uuid_global_id[misp_obj.uuid] = get_global_id('pgp', obj_id, obj_subtype=obj_subtype)
|
||||
|
||||
get_obj_relationship(misp_obj)
|
||||
|
||||
def unpack_obj_cryptocurrency(map_uuid_global_id, misp_obj):
|
||||
obj_id = None
|
||||
crypto_symbol = None
|
||||
for attribute in misp_obj.attributes:
|
||||
if attribute.object_relation == 'address':
|
||||
obj_id = attribute.value
|
||||
elif attribute.object_relation == 'symbol':
|
||||
pass
|
||||
|
||||
|
||||
|
||||
if obj_id and obj_subtype:
|
||||
obj_meta = get_object_metadata(misp_obj)
|
||||
if obj_id and io_content:
|
||||
res = Pgp.pgp.create_item(obj_subtype, obj_id, obj_meta)
|
||||
print(res)
|
||||
res = Pgp.pgp.create_correlation(obj_subtype, obj_id, obj_meta)
|
||||
print(res)
|
||||
|
||||
map_uuid_global_id[misp_obj.uuid] = get_global_id('pgp', obj_id, obj_subtype=obj_subtype)
|
||||
|
||||
get_obj_relationship(misp_obj)
|
||||
#get_obj_relationship(misp_obj)
|
||||
|
||||
def unpack_obj_cryptocurrency(map_uuid_global_id, misp_obj):
|
||||
obj_id = None
|
||||
obj_subtype = None
|
||||
for attribute in misp_obj.attributes:
|
||||
if attribute.object_relation == 'address': # # TODO: handle xmr address field
|
||||
obj_id = attribute.value
|
||||
elif attribute.object_relation == 'symbol':
|
||||
obj_subtype = Cryptocurrency.get_cryptocurrency_type(attribute.value)
|
||||
|
||||
# valid cryptocurrency type
|
||||
if obj_subtype and obj_id:
|
||||
print('crypto')
|
||||
print(obj_id)
|
||||
print(obj_subtype)
|
||||
|
||||
obj_meta = get_object_metadata(misp_obj)
|
||||
print(obj_meta)
|
||||
res = Cryptocurrency.cryptocurrency.create_correlation(obj_subtype, obj_id, obj_meta)
|
||||
print(res)
|
||||
|
||||
map_uuid_global_id[misp_obj.uuid] = get_global_id('pgp', obj_id, obj_subtype=obj_subtype)
|
||||
|
||||
get_obj_relationship(misp_obj)
|
||||
|
||||
def get_misp_import_fct(map_uuid_global_id, misp_obj):
|
||||
#print(misp_obj.ObjectReference)
|
||||
|
|
@ -155,4 +159,5 @@ if __name__ == '__main__':
|
|||
|
||||
# misp = PyMISP('https://127.0.0.1:8443/', 'uXgcN42b7xuL88XqK5hubwD8Q8596VrrBvkHQzB0', False)
|
||||
|
||||
import_objs_from_file('test_import_item.json')
|
||||
#import_objs_from_file('test_import_item.json')
|
||||
import_objs_from_file('test_export.json')
|
||||
|
|
|
|||
|
|
@ -9,7 +9,6 @@ from hashlib import sha256
|
|||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
|
||||
import correlation
|
||||
import Item
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
|
|
@ -56,39 +55,6 @@ def get_cryptocurrency(request_dict, cryptocurrency_type):
|
|||
|
||||
return cryptocurrency.get_correlation(request_dict, cryptocurrency_type, field_name)
|
||||
|
||||
# # TODO: refractor/move me in Correlation
|
||||
def save_cryptocurrency_data(cryptocurrency_name, date, item_path, cryptocurrency_address):
|
||||
# create basic medata
|
||||
if not r_serv_metadata.exists('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address)):
|
||||
r_serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'first_seen', date)
|
||||
r_serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen', date)
|
||||
else:
|
||||
last_seen = r_serv_metadata.hget('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen')
|
||||
if not last_seen:
|
||||
r_serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen', date)
|
||||
else:
|
||||
if int(last_seen) < int(date):
|
||||
r_serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen', date)
|
||||
|
||||
## global set
|
||||
# item
|
||||
r_serv_metadata.sadd('set_cryptocurrency_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), item_path)
|
||||
|
||||
# daily
|
||||
r_serv_metadata.hincrby('cryptocurrency:{}:{}'.format(cryptocurrency_name, date), cryptocurrency_address, 1)
|
||||
|
||||
# all type
|
||||
r_serv_metadata.zincrby('cryptocurrency_all:{}'.format(cryptocurrency_name), cryptocurrency_address, 1)
|
||||
|
||||
## object_metadata
|
||||
# item
|
||||
r_serv_metadata.sadd('item_cryptocurrency_{}:{}'.format(cryptocurrency_name, item_path), cryptocurrency_address)
|
||||
|
||||
# domain
|
||||
if Item.is_crawled(item_path): # # TODO: use save_domain_correlation
|
||||
domain = Item.get_item_domain(item_path)
|
||||
r_serv_metadata.sadd('domain_cryptocurrency_{}:{}'.format(cryptocurrency_name, domain), cryptocurrency_address)
|
||||
r_serv_metadata.sadd('set_domain_cryptocurrency_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), domain)
|
||||
|
||||
def get_cryptocurrency_symbol(crypto_type):
|
||||
if crypto_type=='bitcoin':
|
||||
|
|
|
|||
|
|
@ -129,15 +129,32 @@ def validate_str_date(str_date, separator=''):
|
|||
except TypeError:
|
||||
return False
|
||||
|
||||
def sanitise_date_range(date_from, date_to, separator=''):
|
||||
def sanitise_date_range(date_from, date_to, separator='', date_type='str'):
|
||||
'''
|
||||
Check/Return a correct date_form and date_to
|
||||
'''
|
||||
if not validate_str_date(date_from, separator=separator):
|
||||
date_from = datetime.date.today().strftime("%Y%m%d")
|
||||
if not validate_str_date(date_to, separator=separator):
|
||||
date_to = datetime.date.today().strftime("%Y%m%d")
|
||||
if not date_from and date_to:
|
||||
date_from = date_to
|
||||
elif not date_to and date_from:
|
||||
date_to = date_from
|
||||
|
||||
if date_type=='str':
|
||||
if not validate_str_date(date_from, separator=separator):
|
||||
date_from = datetime.date.today().strftime("%Y%m%d")
|
||||
if not validate_str_date(date_to, separator=separator):
|
||||
date_to = datetime.date.today().strftime("%Y%m%d")
|
||||
else: # datetime
|
||||
if isinstance(date_from, datetime.datetime):
|
||||
date_from = date_from.strftime("%Y%m%d")
|
||||
else:
|
||||
date_from = datetime.date.today().strftime("%Y%m%d")
|
||||
if isinstance(date_to, datetime.datetime):
|
||||
date_to = date_to.strftime("%Y%m%d")
|
||||
else:
|
||||
date_to = datetime.date.today().strftime("%Y%m%d")
|
||||
|
||||
if int(date_from) > int(date_to):
|
||||
res = date_from
|
||||
date_from = date_to
|
||||
date_to = res
|
||||
return {"date_from": date_from, "date_to": date_to}
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ import redis
|
|||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
|
||||
import correlation
|
||||
import Item
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
|
|
@ -27,36 +26,3 @@ def get_pgp(request_dict, pgp_type):
|
|||
field_name = request_dict.get(pgp_type)
|
||||
|
||||
return pgpdump.get_correlation(request_dict, pgp_type, field_name)
|
||||
|
||||
def save_pgp_data(type_pgp, date, item_path, data):
|
||||
# create basic medata
|
||||
if not serv_metadata.exists('pgpdump_metadata_{}:{}'.format(type_pgp, data)):
|
||||
serv_metadata.hset('pgpdump_metadata_{}:{}'.format(type_pgp, data), 'first_seen', date)
|
||||
serv_metadata.hset('pgpdump_metadata_{}:{}'.format(type_pgp, data), 'last_seen', date)
|
||||
else:
|
||||
last_seen = serv_metadata.hget('pgpdump_metadata_{}:{}'.format(type_pgp, data), 'last_seen')
|
||||
if not last_seen:
|
||||
serv_metadata.hset('pgpdump_metadata_{}:{}'.format(type_pgp, data), 'last_seen', date)
|
||||
else:
|
||||
if int(last_seen) < int(date):
|
||||
serv_metadata.hset('pgpdump_metadata_{}:{}'.format(type_pgp, data), 'last_seen', date)
|
||||
|
||||
# global set
|
||||
serv_metadata.sadd('set_pgpdump_{}:{}'.format(type_pgp, data), item_path)
|
||||
|
||||
# daily
|
||||
serv_metadata.hincrby('pgpdump:{}:{}'.format(type_pgp, date), data, 1)
|
||||
|
||||
# all type
|
||||
serv_metadata.zincrby('pgpdump_all:{}'.format(type_pgp), data, 1)
|
||||
|
||||
## object_metadata
|
||||
# paste
|
||||
serv_metadata.sadd('item_pgpdump_{}:{}'.format(type_pgp, item_path), data)
|
||||
|
||||
|
||||
# domain object
|
||||
if Item.is_crawled(item_path):
|
||||
domain = Item.get_item_domain(item_path)
|
||||
serv_metadata.sadd('domain_pgpdump_{}:{}'.format(type_pgp, domain), data)
|
||||
serv_metadata.sadd('set_domain_pgpdump_{}:{}'.format(type_pgp, data), domain)
|
||||
|
|
|
|||
|
|
@ -10,6 +10,8 @@ import ConfigLoader
|
|||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
|
||||
import Date
|
||||
import Item
|
||||
import Tag
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
|
||||
|
|
@ -47,11 +49,31 @@ class Correlation(object):
|
|||
else:
|
||||
return []
|
||||
|
||||
def _get_metadata(self, correlation_type, field_name):
|
||||
def get_correlation_first_seen(self, subtype, obj_id, r_int=False):
|
||||
res = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, subtype, obj_id), 'first_seen')
|
||||
if r_int:
|
||||
if res:
|
||||
return int(res)
|
||||
else:
|
||||
return 99999999
|
||||
else:
|
||||
return res
|
||||
|
||||
def get_correlation_last_seen(self, subtype, obj_id, r_int=False):
|
||||
res = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, subtype, obj_id), 'last_seen')
|
||||
if r_int:
|
||||
if res:
|
||||
return int(res)
|
||||
else:
|
||||
return 0
|
||||
else:
|
||||
return res
|
||||
|
||||
def _get_metadata(self, subtype, obj_id):
|
||||
meta_dict = {}
|
||||
meta_dict['first_seen'] = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, correlation_type, field_name), 'first_seen')
|
||||
meta_dict['last_seen'] = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, correlation_type, field_name), 'last_seen')
|
||||
meta_dict['nb_seen'] = r_serv_metadata.scard('set_{}_{}:{}'.format(self.correlation_name, correlation_type, field_name))
|
||||
meta_dict['first_seen'] = self.get_correlation_first_seen(subtype, obj_id)
|
||||
meta_dict['last_seen'] = self.get_correlation_last_seen(subtype, obj_id)
|
||||
meta_dict['nb_seen'] = r_serv_metadata.scard('set_{}_{}:{}'.format(self.correlation_name, subtype, obj_id))
|
||||
return meta_dict
|
||||
|
||||
def get_metadata(self, correlation_type, field_name, date_format='str_date'):
|
||||
|
|
@ -276,18 +298,19 @@ class Correlation(object):
|
|||
correlation_obj[correlation_object] = res
|
||||
return correlation_obj
|
||||
|
||||
def update_correlation_daterange(self, subtype, obj_id, date): # # TODO: update fisrt_seen
|
||||
def update_correlation_daterange(self, subtype, obj_id, date):
|
||||
date = int(date)
|
||||
# obj_id don't exit
|
||||
if not r_serv_metadata.exists('{}_metadata_{}:{}'.format(self.correlation_name, subtype, obj_id)):
|
||||
r_serv_metadata.hset('{}_metadata_{}:{}'.format(self.correlation_name, subtype, obj_id), 'first_seen', date)
|
||||
r_serv_metadata.hset('{}_metadata_{}:{}'.format(self.correlation_name, subtype, obj_id), 'last_seen', date)
|
||||
else:
|
||||
last_seen = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, subtype, obj_id), 'last_seen')
|
||||
if not last_seen:
|
||||
first_seen = self.get_correlation_last_seen(subtype, obj_id, r_int=True)
|
||||
last_seen = self.get_correlation_first_seen(subtype, obj_id, r_int=True)
|
||||
if date < first_seen:
|
||||
r_serv_metadata.hset('{}_metadata_{}:{}'.format(self.correlation_name, subtype, obj_id), 'first_seen', date)
|
||||
if date > last_seen:
|
||||
r_serv_metadata.hset('{}_metadata_{}:{}'.format(self.correlation_name, subtype, obj_id), 'last_seen', date)
|
||||
else:
|
||||
if int(last_seen) < int(date):
|
||||
r_serv_metadata.hset('{}_metadata_{}:{}'.format(self.correlation_name, subtype, obj_id), 'last_seen', date)
|
||||
|
||||
def save_item_correlation(self, subtype, date, obj_id, item_id, item_date):
|
||||
update_correlation_daterange(subtype, obj_id, item_date)
|
||||
|
|
@ -305,28 +328,37 @@ class Correlation(object):
|
|||
# item
|
||||
r_serv_metadata.sadd('item_{}_{}:{}'.format(self.correlation_name, subtype, item_id), obj_id)
|
||||
|
||||
def save_domain_correlation(self, domain, correlation_type, correlation_value):
|
||||
r_serv_metadata.sadd('domain_{}_{}:{}'.format(self.correlation_name, correlation_type, domain), correlation_value)
|
||||
r_serv_metadata.sadd('set_domain_{}_{}:{}'.format(self.correlation_name, correlation_type, correlation_value), domain)
|
||||
# domain
|
||||
if Item.is_crawled(item_id):
|
||||
domain = Item.get_item_domain(item_id)
|
||||
self.save_domain_correlation(domain, subtype, obj_id)
|
||||
|
||||
def save_domain_correlation(self, domain, subtype, obj_id):
|
||||
r_serv_metadata.sadd('domain_{}_{}:{}'.format(self.correlation_name, subtype, domain), obj_id)
|
||||
r_serv_metadata.sadd('set_domain_{}_{}:{}'.format(self.correlation_name, subtype, obj_id), domain)
|
||||
|
||||
|
||||
def save_correlation(self, subtype, obj_id): # # TODO: add first_seen/last_seen
|
||||
def save_correlation(self, subtype, obj_id, date_range):
|
||||
r_serv_metadata.zincrby('{}_all:{}'.format(self.correlation_name, subtype), obj_id, 0)
|
||||
self.update_correlation_daterange(subtype, obj_id, date_range['date_from'])
|
||||
if date_range['date_from'] != date_range['date_to']:
|
||||
self.update_correlation_daterange(subtype, obj_id, date_range['date_to'])
|
||||
return True
|
||||
|
||||
def create_correlation(self, subtype, obj_id, obj_meta):
|
||||
res = self.sanythise_correlation_types(correlation_type, r_boolean=True)
|
||||
res = self.sanythise_correlation_types([subtype], r_boolean=True)
|
||||
if not res:
|
||||
print('invalid subtype')
|
||||
return False
|
||||
|
||||
if not exist_correlation(subtype, obj_id):
|
||||
res = save_correlation(subtype, obj_id)
|
||||
if res:
|
||||
if 'tags' in obj_metadata:
|
||||
# # TODO: handle mixed tags: taxonomies and Galaxies
|
||||
Tag.api_add_obj_tags(tags=obj_metadata['tags'], object_id=obj_id, object_type=self.get_correlation_obj_type())
|
||||
return True
|
||||
return False
|
||||
first_seen = obj_meta.get('first_seen', None)
|
||||
last_seen = obj_meta.get('last_seen', None)
|
||||
date_range = Date.sanitise_date_range(first_seen, last_seen, separator='', date_type='datetime')
|
||||
print(date_range)
|
||||
res = self.save_correlation(subtype, obj_id, date_range)
|
||||
if res and 'tags' in obj_meta:
|
||||
# # TODO: handle mixed tags: taxonomies and Galaxies
|
||||
Tag.api_add_obj_tags(tags=obj_meta['tags'], object_id=obj_id, object_type=self.get_correlation_obj_type())
|
||||
return True
|
||||
|
||||
######## API EXPOSED ########
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue