From b31fffa728f8bfe759e26d3b974f8dd33a42f20b Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 4 Feb 2020 13:04:09 +0100 Subject: [PATCH] chg: [MISP export] add pgp object + add relationship: item + pgp + cryptocurrency --- bin/export/MispExport.py | 105 ++++++++++++------ bin/lib/Decoded.py | 5 +- bin/lib/Domain.py | 8 +- bin/packages/Cryptocurrency.py | 9 +- bin/packages/Item.py | 5 +- bin/packages/Pgp.py | 9 +- .../{Correlation.py => correlation.py} | 9 +- 7 files changed, 89 insertions(+), 61 deletions(-) rename bin/packages/{Correlation.py => correlation.py} (98%) diff --git a/bin/export/MispExport.py b/bin/export/MispExport.py index f9ca2721..9880ee07 100755 --- a/bin/export/MispExport.py +++ b/bin/export/MispExport.py @@ -8,7 +8,8 @@ import redis sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages')) -#from Cryptocurrency import cryptocurrency +import Cryptocurrency +import Pgp import Decoded import Domain import Item @@ -45,6 +46,7 @@ def export_ail_item(item_id): def export_domain(domain): domain_obj = Domain.Domain(domain) dict_metadata = domain_obj.get_domain_metadata(tags=True) + dict_metadata['ports'] = ['80', '223', '443'] # create domain-ip obj obj = MISPObject('domain-ip', standalone=True) @@ -55,6 +57,8 @@ def export_domain(domain): l_obj_attr.append( obj.add_attribute('first-seen', value=dict_metadata['first_seen']) ) l_obj_attr.append( obj.add_attribute('last-seen', value=dict_metadata['last_check']) ) l_obj_attr.append( obj.add_attribute('domain', value=domain) ) + for port in dict_metadata['ports']: + l_obj_attr.append( obj.add_attribute('port', value=port) ) # add tags if dict_metadata['tags']: @@ -100,7 +104,7 @@ def export_screenshot(sha256_string): # TODO: add tags def export_cryptocurrency(crypto_type, crypto_address): - dict_metadata = cryptocurrency.get_metadata(crypto_address, crypto_type) + dict_metadata = Cryptocurrency.cryptocurrency.get_metadata(crypto_type, crypto_address) obj = MISPObject('coin-address') obj.first_seen = dict_metadata['first_seen'] @@ -114,6 +118,26 @@ def export_cryptocurrency(crypto_type, crypto_address): return obj +# TODO: add tags +def export_pgp(pgp_type, pgp_value): + dict_metadata = Pgp.pgp.get_metadata(pgp_type, pgp_value) + + obj = MISPObject('pgp-meta', misp_objects_path_custom='../../../misp-objects/objects') + obj.first_seen = dict_metadata['first_seen'] + obj.last_seen = dict_metadata['last_seen'] + + l_obj_attr = [] + if pgp_type=='key': + l_obj_attr.append( obj.add_attribute('key-id', value=pgp_value) ) + elif pgp_type=='name': + #l_obj_attr.append( obj.add_attribute('key-id', value='debug') ) + l_obj_attr.append( obj.add_attribute('user-id-name', value=pgp_value) ) + else: # mail + #l_obj_attr.append( obj.add_attribute('key-id', value='debug') ) + l_obj_attr.append( obj.add_attribute('user-id-email', value=pgp_value) ) + return obj + + # filter objects to export, export only object who correlect which each other def filter_obj_linked(l_obj): for obj in l_obj: @@ -149,7 +173,7 @@ def add_obj_to_create_by_lvl(all_obj_to_export, set_relationship, dict_obj, lvl) lvl = lvl - 1 # # TODO: filter by correlation types - obj_correlations = Correlate_object.get_object_correlation(dict_obj['type'], dict_obj['id'], dict_obj.get('subtype', None)) + obj_correlations = Correlate_object.get_object_correlation(dict_obj['type'], dict_obj['id'], requested_correl_type=dict_obj.get('subtype', None)) for obj_type in obj_correlations: dict_new_obj = {'type': obj_type} if obj_type=='pgp' or obj_type=='cryptocurrency': @@ -195,10 +219,10 @@ def create_list_of_objs_to_export(l_obj, mode='union'): # add object to event event.add_object(dict_misp_obj[obj_global_id]) - #print(event.to_json()) + print(event.to_json()) - misp = PyMISP('https://127.0.0.1:8443/', 'uXgcN42b7xuL88XqK5hubwD8Q8596VrrBvkHQzB0', False) - misp.add_event(event, pythonify=True) + #misp = PyMISP('https://127.0.0.1:8443/', 'uXgcN42b7xuL88XqK5hubwD8Q8596VrrBvkHQzB0', False) + #misp.add_event(event, pythonify=True) def create_all_misp_obj(all_obj_to_export, set_relationship): @@ -221,7 +245,7 @@ def create_misp_obj(obj_type, obj_id): return export_cryptocurrency(obj_subtype, obj_id) elif obj_type == 'pgp': obj_subtype, obj_id = obj_id.split(':', 1) - pass + return export_pgp(obj_subtype, obj_id) elif obj_type == 'domain': return export_domain(obj_id) @@ -246,12 +270,41 @@ def get_relationship_between_global_obj(obj_global_id_1, obj_global_id_2): src = obj_global_id_2 dest = obj_global_id_1 return {'relation': 'included-in', 'src': src, 'dest': dest} - elif 'pgp': - return None + elif 'pgp' in type_tuple: + if obj_type_1 == 'pgp': + src = obj_global_id_1 + dest = obj_global_id_2 + else: + src = obj_global_id_2 + dest = obj_global_id_1 + return {'relation': 'extracted-from', 'src': src, 'dest': dest} elif 'cryptocurrency': - return None - elif 'domain': - return None + if obj_type_1 == 'cryptocurrency': + src = obj_global_id_1 + dest = obj_global_id_2 + else: + src = obj_global_id_2 + dest = obj_global_id_1 + return {'relation': 'extracted-from', 'src': src, 'dest': dest} + elif 'domain' in type_tuple: + if 'item' in type_tuple: + if obj_type_1 == 'item': + src = obj_global_id_1 + dest = obj_global_id_2 + else: + src = obj_global_id_2 + dest = obj_global_id_1 + return {'relation': 'extracted-from', 'src': src, 'dest': dest} # replave by crawled-from + elif 'item' in type_tuple: + if 'domain' in type_tuple: + if obj_type_1 == 'item': + src = obj_global_id_1 + dest = obj_global_id_2 + else: + src = obj_global_id_2 + dest = obj_global_id_1 + return {'relation': 'extracted-from', 'src': src, 'dest': dest} # replave by crawled-from + return None ###### # @@ -263,31 +316,11 @@ def get_relationship_between_global_obj(obj_global_id_1, obj_global_id_2): if __name__ == '__main__': - l_obj = [{'id': 'crawled/2019/11/08/6d3zimnpbwbzdgnp.onionf58258c8-c990-4707-b236-762a2b881183', 'type': 'item', 'lvl': 3}, - {'id': '6d3zimnpbwbzdgnp.onion', 'type': 'domain', 'lvl': 0}, - #{'id': '0xA4BB02A75E6AF448', 'type': 'pgp', 'subtype': 'key', 'lvl': 0}, - {'id': 'a92d459f70c4dea8a14688f585a5e2364be8b91fbf924290ead361d9b909dcf1', 'type': 'image', 'lvl': 3}] + l_obj = [#{'id': 'crawled/2019/11/08/6d3zimnpbwbzdgnp.onionf58258c8-c990-4707-b236-762a2b881183', 'type': 'item', 'lvl': 3}, + #{'id': '6d3zimnpbwbzdgnp.onion', 'type': 'domain', 'lvl': 0}, + #{'id': 'a92d459f70c4dea8a14688f585a5e2364be8b91fbf924290ead361d9b909dcf1', 'type': 'image', 'lvl': 3}, + {'id': '15efuhpw5V9B1opHAgNXKPBPqdYALXP4hc', 'type': 'cryptocurrency', 'subtype': 'bitcoin', 'lvl': 1}] create_list_of_objs_to_export(l_obj, mode='union') - - - - #event = MISPEvent() - #event.info = 'AIL framework export' - # - # obj_item = export_ail_item('crawled/2019/11/08/6d3zimnpbwbzdgnp.onionf58258c8-c990-4707-b236-762a2b881183') - # event.add_object(obj_item) - # - # obj_domain = export_domain('2222222222xkrmay.onion') - # event.add_object(obj_domain) - # - # obj_decoded = export_decoded('fc351baadefce6f702155fb908a9e84dd5dd0fa7') - # obj_decoded.add_reference(obj_domain.uuid, 'injected-into', 'add a comment') - # event.add_object(obj_decoded) - - # obj_screenshot = export_screenshot('5fcc292ea8a699aa7a9ce93a704b78b8f493620ccdb2a5cebacb1069a4327211') - # obj_screenshot.add_reference(obj_domain.uuid, 'screenshot-of') - # event.add_object(obj_screenshot) - #print(event.to_json()) diff --git a/bin/lib/Decoded.py b/bin/lib/Decoded.py index 5c5cffac..b47b7038 100755 --- a/bin/lib/Decoded.py +++ b/bin/lib/Decoded.py @@ -12,6 +12,9 @@ import Item import Date import Tag +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) +import correlation + import ConfigLoader @@ -144,7 +147,7 @@ def get_decoded_correlated_object(sha1_string, correlation_objects=[]): :rtype: dict ''' if correlation_objects is None: - correlation_objects = Correlation.get_all_correlation_objects() + correlation_objects = correlation.get_all_correlation_objects() decoded_correlation = {} for correlation_object in correlation_objects: if correlation_object == 'paste': diff --git a/bin/lib/Domain.py b/bin/lib/Domain.py index aa1b51cf..8f2500e6 100755 --- a/bin/lib/Domain.py +++ b/bin/lib/Domain.py @@ -15,14 +15,12 @@ import random sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) import Cryptocurrency -from Pgp import pgp +import Pgp import Date import Decoded import Item import Tag -cryptocurrency = Cryptocurrency.cryptocurrency - sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) import ConfigLoader import Correlate_object @@ -523,7 +521,7 @@ def get_domain_cryptocurrency(domain, currencies_type=None, get_nb=False): :param currencies_type: list of cryptocurrencies type :type currencies_type: list, optional ''' - return cryptocurrency.get_domain_correlation_dict(domain, correlation_type=currencies_type, get_nb=get_nb) + return Cryptocurrency.cryptocurrency.get_domain_correlation_dict(domain, correlation_type=currencies_type, get_nb=get_nb) def get_domain_pgp(domain, currencies_type=None, get_nb=False): ''' @@ -533,7 +531,7 @@ def get_domain_pgp(domain, currencies_type=None, get_nb=False): :param currencies_type: list of pgp type :type currencies_type: list, optional ''' - return pgp.get_domain_correlation_dict(domain, correlation_type=currencies_type, get_nb=get_nb) + return Pgp.pgp.get_domain_correlation_dict(domain, correlation_type=currencies_type, get_nb=get_nb) def get_domain_decoded(domain): ''' diff --git a/bin/packages/Cryptocurrency.py b/bin/packages/Cryptocurrency.py index 5ef97425..1c0ca141 100755 --- a/bin/packages/Cryptocurrency.py +++ b/bin/packages/Cryptocurrency.py @@ -8,7 +8,7 @@ import redis from hashlib import sha256 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages')) -from Correlation import Correlation +import correlation import Item sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) @@ -20,12 +20,7 @@ config_loader = None digits58 = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' - -class Cryptocurrency(Correlation): - def __init__(self): - super().__init__('cryptocurrency', ['bitcoin', 'ethereum', 'bitcoin-cash', 'litecoin', 'monero', 'zcash', 'dash']) - -cryptocurrency = Cryptocurrency() +cryptocurrency = correlation.Correlation('cryptocurrency', ['bitcoin', 'ethereum', 'bitcoin-cash', 'litecoin', 'monero', 'zcash', 'dash']) # http://rosettacode.org/wiki/Bitcoin/address_validation#Python def decode_base58(bc, length): diff --git a/bin/packages/Item.py b/bin/packages/Item.py index d55ab332..7aac2457 100755 --- a/bin/packages/Item.py +++ b/bin/packages/Item.py @@ -11,9 +11,8 @@ from io import BytesIO sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) import Date import Tag -import Correlation import Cryptocurrency -from Pgp import pgp +import Pgp sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) import ConfigLoader @@ -166,7 +165,7 @@ def get_item_pgp(item_id, currencies_type=None, get_nb=False): :param currencies_type: list of cryptocurrencies type :type currencies_type: list, optional ''' - return pgp.get_item_correlation_dict(item_id, correlation_type=currencies_type, get_nb=get_nb) + return Pgp.pgp.get_item_correlation_dict(item_id, correlation_type=currencies_type, get_nb=get_nb) def get_item_decoded(item_id): ''' diff --git a/bin/packages/Pgp.py b/bin/packages/Pgp.py index 986c78d3..a5ffb901 100755 --- a/bin/packages/Pgp.py +++ b/bin/packages/Pgp.py @@ -6,7 +6,7 @@ import sys import redis sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages')) -from Correlation import Correlation +import correlation import Item sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) @@ -16,12 +16,7 @@ config_loader = ConfigLoader.ConfigLoader() serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") config_loader = None - -class Pgp(Correlation): - def __init__(self): - super().__init__('pgpdump', ['key', 'mail', 'name']) - -pgp = Pgp() +pgp = correlation.Correlation('pgpdump', ['key', 'mail', 'name']) def get_pgp(request_dict, pgp_type): # basic verification diff --git a/bin/packages/Correlation.py b/bin/packages/correlation.py similarity index 98% rename from bin/packages/Correlation.py rename to bin/packages/correlation.py index faea376d..8b5a70da 100755 --- a/bin/packages/Correlation.py +++ b/bin/packages/correlation.py @@ -7,7 +7,6 @@ import redis sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) import ConfigLoader -import Correlate_object sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) import Date @@ -16,6 +15,12 @@ config_loader = ConfigLoader.ConfigLoader() r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") config_loader = None +def get_all_correlation_objects(): + ''' + Return a list of all correllated objects + ''' + return ['domain', 'paste'] + class Correlation(object): def __init__(self, correlation_name, all_correlation_types): @@ -236,7 +241,7 @@ class Correlation(object): def get_correlation_all_object(self, correlation_type, correlation_value, correlation_objects=[]): if correlation_objects is None: - correlation_objects = Correlate_object.get_all_correlation_objects() + correlation_objects = get_all_correlation_objects() correlation_obj = {} for correlation_object in correlation_objects: if correlation_object == 'paste':