From 44bb18a8be7aac01a529f08f6a57d600337a8f4a Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 29 Oct 2019 16:52:33 +0100 Subject: [PATCH] chg: [Correlation] get correlation (crypto + pgp) by domain --- bin/lib/Domain.py | 76 ++++++++++++++++---------- bin/packages/Correlation.py | 99 ++++++++++++++++++++++------------ bin/packages/Cryptocurrency.py | 45 +++++----------- bin/packages/Pgp.py | 19 ++++--- bin/packages/Tag.py | 17 ++++-- 5 files changed, 149 insertions(+), 107 deletions(-) diff --git a/bin/lib/Domain.py b/bin/lib/Domain.py index 233fc574..8da5d960 100755 --- a/bin/lib/Domain.py +++ b/bin/lib/Domain.py @@ -14,8 +14,10 @@ import redis sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) import Correlation -import Cryptocurrency +from Cryptocurrency import cryptocurrency +from Pgp import pgp import Item +import Tag sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) import ConfigLoader @@ -54,35 +56,53 @@ def get_link_tree(): pass -### -### correlation -### -""" -def _get_domain_correlation(domain, correlation_name=None, correlation_type=None): - res = r_serv_metadata.smembers('item_{}_{}:{}'.format(correlation_name, correlation_type, item_id)) +def get_domain_tags(domain): + ''' + Retun all tags of a given domain. + + :param domain: crawled domain + ''' + return Tag.get_item_tags(domain) + +def get_domain_cryptocurrency(domain, currencies_type=None): + ''' + Retun all cryptocurrencies of a given domain. + + :param domain: crawled domain + :param currencies_type: list of cryptocurrencies type + :type currencies_type: list, optional + ''' + return cryptocurrency.get_domain_correlation_dict(domain, correlation_type=currencies_type) + +def get_domain_pgp(domain, currencies_type=None): + ''' + Retun all pgp of a given domain. + + :param domain: crawled domain + :param currencies_type: list of pgp type + :type currencies_type: list, optional + ''' + return pgp.get_domain_correlation_dict(domain, correlation_type=currencies_type) + +def get_domain_all_correlation(domain, correlation_type=None): + ''' + Retun all correlation of a given domain. + + :param domain: crawled domain + :type domain: str + + :return: a dict of all correlation for a given domain + :rtype: dict + ''' + domain_correl = {} + res = get_domain_cryptocurrency(domain) if res: - return list(res) - else: - return [] + domain_correl['cryptocurrency'] = res + res = get_domain_pgp(domain) + if res: + domain_correl['pgp'] = res + return domain_correl -def get_item_bitcoin(item_id): - return _get_item_correlation('cryptocurrency', 'bitcoin', item_id) - -def get_item_pgp_key(item_id): - return _get_item_correlation('pgpdump', 'key', item_id) - -def get_item_pgp_name(item_id): - return _get_item_correlation('pgpdump', 'name', item_id) - -def get_item_pgp_mail(item_id): - return _get_item_correlation('pgpdump', 'mail', item_id) - -def get_item_pgp_correlation(item_id): - pass -""" - -def _get_domain_correlation(domain, correlation_list): - return Cryptocurrency.get_cryptocurrency_domain(domain) class Domain(object): """docstring for Domain.""" diff --git a/bin/packages/Correlation.py b/bin/packages/Correlation.py index dbef4d8d..27b6dc24 100755 --- a/bin/packages/Correlation.py +++ b/bin/packages/Correlation.py @@ -5,16 +5,18 @@ import os import sys import redis -sys.path.append(os.path.join(os.environ['AIL_FLASK'], 'modules/')) -import Flask_config - -r_serv_metadata = Flask_config.r_serv_metadata +sys.path.append(os.path.join(os.environ['AIL_FLASK'], 'lib/')) +import ConfigLoader +config_loader = ConfigLoader.ConfigLoader() +r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") +config_loader = None class Correlation(object): - def __init__(self, correlation_name): + def __init__(self, correlation_name, all_correlation_types): self.correlation_name = correlation_name + self.all_correlation_types = all_correlation_types def _exist_corelation_field(self, correlation_type, field_name, item_type='paste'): if type=='paste': @@ -29,13 +31,6 @@ class Correlation(object): else: return [] - def _get_domains(self, correlation_type, field_name): - res = r_serv_metadata.smembers('set_domain_{}_{}:{}'.format(self.correlation_name, correlation_type, field_name)) - if res: - return list(res) - else: - return [] - def _get_metadata(self, correlation_type, field_name): meta_dict = {} meta_dict['first_seen'] = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, correlation_type, field_name), 'first_seen') @@ -49,7 +44,6 @@ class Correlation(object): if not request_dict: return ({'status': 'error', 'reason': 'Malformed JSON'}, 400) - print(correlation_type) field_name = request_dict.get(correlation_type, None) if not field_name: return ( {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400 ) @@ -69,37 +63,72 @@ class Correlation(object): return (dict_resp, 200) - def get_correlation_domain(self, request_dict, correlation_type, field_name): - dict_resp = {} + def get_all_correlation_types(self): + ''' + Gel all correlation types - dict_resp['domain'] = self._get_domains(correlation_type, field_name) + :return: A list of all the correlation types + :rtype: list + ''' + return self.all_correlation_types - #if request_dict.get('metadata'): - # dict_resp['metadata'] = self._get_metadata(correlation_type, field_name) + def sanythise_correlation_types(self, correlation_types): + ''' + Check if all correlation types in the list are valid. - dict_resp[correlation_type] = field_name + :param correlation_types: list of correlation type + :type currency_type: list - return (dict_resp, 200) + :return: If a type is invalid, return the full list of correlation types else return the provided list + :rtype: list + ''' + if correlation_types is None: + return self.get_all_correlation_types() + for correl in correlation_types: # # TODO: # OPTIMIZE: + if correl not in self.get_all_correlation_types(): + return self.get_all_correlation_types() + return correlation_types -######## INTERNAL ######## -def _get_domain_correlation_obj(correlation_name, correlation_type, domain): - print('domain_{}_{}:{}'.format(correlation_name, correlation_type, domain)) - res = r_serv_metadata.smembers('domain_{}_{}:{}'.format(correlation_name, correlation_type, domain)) - if res: - return list(res) - else: - return [] + def _get_domain_correlation_obj(self, domain, correlation_type): + ''' + Return correlation of a given domain. + + :param domain: crawled domain + :type domain: str + :param correlation_type: correlation type + :type correlation_type: str + + :return: a list of correlation + :rtype: list + ''' + res = r_serv_metadata.smembers('domain_{}_{}:{}'.format(self.correlation_name, correlation_type, domain)) + if res: + return list(res) + else: + return [] + + def get_domain_correlation_dict(self, domain, correlation_type=None): + ''' + Return all correlation of a given domain. + + :param domain: crawled domain + :param correlation_type: list of correlation types + :type correlation_type: list, optional + + :return: a dictionnary of all the requested correlations + :rtype: dict + ''' + correlation_type = self.sanythise_correlation_types(correlation_type) + dict_correlation = {} + for correl in correlation_type: + res = self._get_domain_correlation_obj(domain, correl) + if res: + dict_correlation[correl] = res + return dict_correlation -######## ######## ######## API EXPOSED ######## -def get_domain_correlation_obj(request_dict, correlation_name, correlation_type, domain): - dict_resp = {} - dict_resp[correlation_type] = _get_domain_correlation_obj(correlation_name, correlation_type, domain) - dict_resp['domain'] = domain - - return (dict_resp, 200) ######## ######## diff --git a/bin/packages/Cryptocurrency.py b/bin/packages/Cryptocurrency.py index d9f657b9..d83f91dd 100755 --- a/bin/packages/Cryptocurrency.py +++ b/bin/packages/Cryptocurrency.py @@ -7,18 +7,25 @@ import redis from hashlib import sha256 -sys.path.append(os.path.join(os.environ['AIL_FLASK'], 'modules')) -import Flask_config +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages')) from Correlation import Correlation import Item -r_serv_metadata = Flask_config.r_serv_metadata +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) +import ConfigLoader -all_cryptocurrency = ['bitcoin', 'ethereum', 'bitcoin-cash', 'litecoin', 'monero', 'zcash', 'dash'] +config_loader = ConfigLoader.ConfigLoader() +r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") +config_loader = None digits58 = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' -cryptocurrency = Correlation('cryptocurrency') + +class Cryptocurrency(Correlation): + def __init__(self): + super().__init__('cryptocurrency', ['bitcoin', 'ethereum', 'bitcoin-cash', 'litecoin', 'monero', 'zcash', 'dash']) + +cryptocurrency = Cryptocurrency() # http://rosettacode.org/wiki/Bitcoin/address_validation#Python def decode_base58(bc, length): @@ -41,18 +48,6 @@ def verify_cryptocurrency_address(cryptocurrency_type, cryptocurrency_address): else: return True -def get_all_all_cryptocurrency(): - return all_cryptocurrency - -# check if all crypto type in the list are valid -# if a type is invalid, return the full list of currency types -def sanythise_cryptocurrency_types(cryptocurrency_types): - if cryptocurrency_types is None: - return get_all_all_cryptocurrency() - for currency in cryptocurrency_types: # # TODO: # OPTIMIZE: - if currency not in all_cryptocurrency: - return get_all_all_cryptocurrency() - return cryptocurrency_types def get_cryptocurrency(request_dict, cryptocurrency_type): # basic verification @@ -66,22 +61,6 @@ def get_cryptocurrency(request_dict, cryptocurrency_type): return cryptocurrency.get_correlation(request_dict, cryptocurrency_type, field_name) -def get_cryptocurrency_domain(request_dict, cryptocurrency_type=None): - currency_types = sanythise_cryptocurrency_types(cryptocurrency_type) - - res = cryptocurrency.verify_correlation_field_request(request_dict, currency_types, item_type='domain') - if res: - return res - field_name = request_dict.get(cryptocurrency_type) - if not verify_cryptocurrency_address(cryptocurrency_type, field_name): - return ( {'status': 'error', 'reason': 'Invalid Cryptocurrency address'}, 400 ) - - return cryptocurrency.get_correlation_domain(request_dict, cryptocurrency_type, field_name) - -def get_domain_cryptocurrency(request_dict, cryptocurrency_type): - return cryptocurrency.get_domain_correlation_obj(self, request_dict, cryptocurrency_type, domain) - - def save_cryptocurrency_data(cryptocurrency_name, date, item_path, cryptocurrency_address): # create basic medata if not r_serv_metadata.exists('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address)): diff --git a/bin/packages/Pgp.py b/bin/packages/Pgp.py index 12ff34fa..986c78d3 100755 --- a/bin/packages/Pgp.py +++ b/bin/packages/Pgp.py @@ -5,19 +5,24 @@ import os import sys import redis -from hashlib import sha256 - -sys.path.append(os.path.join(os.environ['AIL_FLASK'], 'modules')) -import Flask_config - +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages')) from Correlation import Correlation import Item -serv_metadata = Flask_config.r_serv_metadata +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) +import ConfigLoader -pgpdump = Correlation('pgpdump') +config_loader = ConfigLoader.ConfigLoader() +serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") +config_loader = None +class Pgp(Correlation): + def __init__(self): + super().__init__('pgpdump', ['key', 'mail', 'name']) + +pgp = Pgp() + def get_pgp(request_dict, pgp_type): # basic verification res = pgpdump.verify_correlation_field_request(request_dict, pgp_type) diff --git a/bin/packages/Tag.py b/bin/packages/Tag.py index f1147715..00c59cfa 100755 --- a/bin/packages/Tag.py +++ b/bin/packages/Tag.py @@ -2,17 +2,22 @@ # -*-coding:UTF-8 -* import os +import sys import redis -import Flask_config import Date import Item +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) +import ConfigLoader + from pytaxonomies import Taxonomies from pymispgalaxies import Galaxies, Clusters -r_serv_tags = Flask_config.r_serv_tags -r_serv_metadata = Flask_config.r_serv_metadata +config_loader = ConfigLoader.ConfigLoader() +r_serv_tags = config_loader.get_redis_conn("ARDB_Tags") +r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") +config_loader = None def get_taxonomie_from_tag(tag): return tag.split(':')[0] @@ -77,8 +82,12 @@ def is_tag_in_all_tag(tag): def get_all_tags(): return list(r_serv_tags.smembers('list_tags')) +''' +Retun all the tags of a given item. +:param item_id: (Paste or domain) +''' def get_item_tags(item_id): - tags = r_serv_metadata.smembers('tag:'+item_id) + tags = r_serv_metadata.smembers('tag:{}'.format(item_id)) if tags: return list(tags) else: