mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 08:38:28 +00:00
chg: [Correlation] get correlation (crypto + pgp) by domain
This commit is contained in:
Terrtia
parent
4b389559ab
commit
44bb18a8be
5 changed files with 149 additions and 107 deletions
|
|
@ -14,8 +14,10 @@ import redis
|
|||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
|
||||
import Correlation
|
||||
import Cryptocurrency
|
||||
from Cryptocurrency import cryptocurrency
|
||||
from Pgp import pgp
|
||||
import Item
|
||||
import Tag
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
|
|
@ -54,35 +56,53 @@ def get_link_tree():
|
|||
pass
|
||||
|
||||
|
||||
###
|
||||
### correlation
|
||||
###
|
||||
"""
|
||||
def _get_domain_correlation(domain, correlation_name=None, correlation_type=None):
|
||||
res = r_serv_metadata.smembers('item_{}_{}:{}'.format(correlation_name, correlation_type, item_id))
|
||||
def get_domain_tags(domain):
|
||||
'''
|
||||
Retun all tags of a given domain.
|
||||
|
||||
:param domain: crawled domain
|
||||
'''
|
||||
return Tag.get_item_tags(domain)
|
||||
|
||||
def get_domain_cryptocurrency(domain, currencies_type=None):
|
||||
'''
|
||||
Retun all cryptocurrencies of a given domain.
|
||||
|
||||
:param domain: crawled domain
|
||||
:param currencies_type: list of cryptocurrencies type
|
||||
:type currencies_type: list, optional
|
||||
'''
|
||||
return cryptocurrency.get_domain_correlation_dict(domain, correlation_type=currencies_type)
|
||||
|
||||
def get_domain_pgp(domain, currencies_type=None):
|
||||
'''
|
||||
Retun all pgp of a given domain.
|
||||
|
||||
:param domain: crawled domain
|
||||
:param currencies_type: list of pgp type
|
||||
:type currencies_type: list, optional
|
||||
'''
|
||||
return pgp.get_domain_correlation_dict(domain, correlation_type=currencies_type)
|
||||
|
||||
def get_domain_all_correlation(domain, correlation_type=None):
|
||||
'''
|
||||
Retun all correlation of a given domain.
|
||||
|
||||
:param domain: crawled domain
|
||||
:type domain: str
|
||||
|
||||
:return: a dict of all correlation for a given domain
|
||||
:rtype: dict
|
||||
'''
|
||||
domain_correl = {}
|
||||
res = get_domain_cryptocurrency(domain)
|
||||
if res:
|
||||
return list(res)
|
||||
else:
|
||||
return []
|
||||
domain_correl['cryptocurrency'] = res
|
||||
res = get_domain_pgp(domain)
|
||||
if res:
|
||||
domain_correl['pgp'] = res
|
||||
return domain_correl
|
||||
|
||||
def get_item_bitcoin(item_id):
|
||||
return _get_item_correlation('cryptocurrency', 'bitcoin', item_id)
|
||||
|
||||
def get_item_pgp_key(item_id):
|
||||
return _get_item_correlation('pgpdump', 'key', item_id)
|
||||
|
||||
def get_item_pgp_name(item_id):
|
||||
return _get_item_correlation('pgpdump', 'name', item_id)
|
||||
|
||||
def get_item_pgp_mail(item_id):
|
||||
return _get_item_correlation('pgpdump', 'mail', item_id)
|
||||
|
||||
def get_item_pgp_correlation(item_id):
|
||||
pass
|
||||
"""
|
||||
|
||||
def _get_domain_correlation(domain, correlation_list):
|
||||
return Cryptocurrency.get_cryptocurrency_domain(domain)
|
||||
|
||||
class Domain(object):
|
||||
"""docstring for Domain."""
|
||||
|
|
|
|||
|
|
@ -5,16 +5,18 @@ import os
|
|||
import sys
|
||||
import redis
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_FLASK'], 'modules/'))
|
||||
import Flask_config
|
||||
|
||||
r_serv_metadata = Flask_config.r_serv_metadata
|
||||
sys.path.append(os.path.join(os.environ['AIL_FLASK'], 'lib/'))
|
||||
import ConfigLoader
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
|
||||
config_loader = None
|
||||
|
||||
class Correlation(object):
|
||||
|
||||
def __init__(self, correlation_name):
|
||||
def __init__(self, correlation_name, all_correlation_types):
|
||||
self.correlation_name = correlation_name
|
||||
self.all_correlation_types = all_correlation_types
|
||||
|
||||
def _exist_corelation_field(self, correlation_type, field_name, item_type='paste'):
|
||||
if type=='paste':
|
||||
|
|
@ -29,13 +31,6 @@ class Correlation(object):
|
|||
else:
|
||||
return []
|
||||
|
||||
def _get_domains(self, correlation_type, field_name):
|
||||
res = r_serv_metadata.smembers('set_domain_{}_{}:{}'.format(self.correlation_name, correlation_type, field_name))
|
||||
if res:
|
||||
return list(res)
|
||||
else:
|
||||
return []
|
||||
|
||||
def _get_metadata(self, correlation_type, field_name):
|
||||
meta_dict = {}
|
||||
meta_dict['first_seen'] = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, correlation_type, field_name), 'first_seen')
|
||||
|
|
@ -49,7 +44,6 @@ class Correlation(object):
|
|||
if not request_dict:
|
||||
return ({'status': 'error', 'reason': 'Malformed JSON'}, 400)
|
||||
|
||||
print(correlation_type)
|
||||
field_name = request_dict.get(correlation_type, None)
|
||||
if not field_name:
|
||||
return ( {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400 )
|
||||
|
|
@ -69,37 +63,72 @@ class Correlation(object):
|
|||
|
||||
return (dict_resp, 200)
|
||||
|
||||
def get_correlation_domain(self, request_dict, correlation_type, field_name):
|
||||
dict_resp = {}
|
||||
def get_all_correlation_types(self):
|
||||
'''
|
||||
Gel all correlation types
|
||||
|
||||
dict_resp['domain'] = self._get_domains(correlation_type, field_name)
|
||||
:return: A list of all the correlation types
|
||||
:rtype: list
|
||||
'''
|
||||
return self.all_correlation_types
|
||||
|
||||
#if request_dict.get('metadata'):
|
||||
# dict_resp['metadata'] = self._get_metadata(correlation_type, field_name)
|
||||
def sanythise_correlation_types(self, correlation_types):
|
||||
'''
|
||||
Check if all correlation types in the list are valid.
|
||||
|
||||
dict_resp[correlation_type] = field_name
|
||||
:param correlation_types: list of correlation type
|
||||
:type currency_type: list
|
||||
|
||||
return (dict_resp, 200)
|
||||
:return: If a type is invalid, return the full list of correlation types else return the provided list
|
||||
:rtype: list
|
||||
'''
|
||||
if correlation_types is None:
|
||||
return self.get_all_correlation_types()
|
||||
for correl in correlation_types: # # TODO: # OPTIMIZE:
|
||||
if correl not in self.get_all_correlation_types():
|
||||
return self.get_all_correlation_types()
|
||||
return correlation_types
|
||||
|
||||
######## INTERNAL ########
|
||||
|
||||
def _get_domain_correlation_obj(correlation_name, correlation_type, domain):
|
||||
print('domain_{}_{}:{}'.format(correlation_name, correlation_type, domain))
|
||||
res = r_serv_metadata.smembers('domain_{}_{}:{}'.format(correlation_name, correlation_type, domain))
|
||||
def _get_domain_correlation_obj(self, domain, correlation_type):
|
||||
'''
|
||||
Return correlation of a given domain.
|
||||
|
||||
:param domain: crawled domain
|
||||
:type domain: str
|
||||
:param correlation_type: correlation type
|
||||
:type correlation_type: str
|
||||
|
||||
:return: a list of correlation
|
||||
:rtype: list
|
||||
'''
|
||||
res = r_serv_metadata.smembers('domain_{}_{}:{}'.format(self.correlation_name, correlation_type, domain))
|
||||
if res:
|
||||
return list(res)
|
||||
else:
|
||||
return []
|
||||
|
||||
######## ########
|
||||
def get_domain_correlation_dict(self, domain, correlation_type=None):
|
||||
'''
|
||||
Return all correlation of a given domain.
|
||||
|
||||
:param domain: crawled domain
|
||||
:param correlation_type: list of correlation types
|
||||
:type correlation_type: list, optional
|
||||
|
||||
:return: a dictionnary of all the requested correlations
|
||||
:rtype: dict
|
||||
'''
|
||||
correlation_type = self.sanythise_correlation_types(correlation_type)
|
||||
dict_correlation = {}
|
||||
for correl in correlation_type:
|
||||
res = self._get_domain_correlation_obj(domain, correl)
|
||||
if res:
|
||||
dict_correlation[correl] = res
|
||||
return dict_correlation
|
||||
|
||||
|
||||
######## API EXPOSED ########
|
||||
|
||||
def get_domain_correlation_obj(request_dict, correlation_name, correlation_type, domain):
|
||||
dict_resp = {}
|
||||
dict_resp[correlation_type] = _get_domain_correlation_obj(correlation_name, correlation_type, domain)
|
||||
dict_resp['domain'] = domain
|
||||
|
||||
return (dict_resp, 200)
|
||||
|
||||
######## ########
|
||||
|
|
|
|||
|
|
@ -7,18 +7,25 @@ import redis
|
|||
|
||||
from hashlib import sha256
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_FLASK'], 'modules'))
|
||||
import Flask_config
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
|
||||
from Correlation import Correlation
|
||||
import Item
|
||||
|
||||
r_serv_metadata = Flask_config.r_serv_metadata
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
|
||||
all_cryptocurrency = ['bitcoin', 'ethereum', 'bitcoin-cash', 'litecoin', 'monero', 'zcash', 'dash']
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
|
||||
config_loader = None
|
||||
|
||||
digits58 = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
|
||||
|
||||
cryptocurrency = Correlation('cryptocurrency')
|
||||
|
||||
class Cryptocurrency(Correlation):
|
||||
def __init__(self):
|
||||
super().__init__('cryptocurrency', ['bitcoin', 'ethereum', 'bitcoin-cash', 'litecoin', 'monero', 'zcash', 'dash'])
|
||||
|
||||
cryptocurrency = Cryptocurrency()
|
||||
|
||||
# http://rosettacode.org/wiki/Bitcoin/address_validation#Python
|
||||
def decode_base58(bc, length):
|
||||
|
|
@ -41,18 +48,6 @@ def verify_cryptocurrency_address(cryptocurrency_type, cryptocurrency_address):
|
|||
else:
|
||||
return True
|
||||
|
||||
def get_all_all_cryptocurrency():
|
||||
return all_cryptocurrency
|
||||
|
||||
# check if all crypto type in the list are valid
|
||||
# if a type is invalid, return the full list of currency types
|
||||
def sanythise_cryptocurrency_types(cryptocurrency_types):
|
||||
if cryptocurrency_types is None:
|
||||
return get_all_all_cryptocurrency()
|
||||
for currency in cryptocurrency_types: # # TODO: # OPTIMIZE:
|
||||
if currency not in all_cryptocurrency:
|
||||
return get_all_all_cryptocurrency()
|
||||
return cryptocurrency_types
|
||||
|
||||
def get_cryptocurrency(request_dict, cryptocurrency_type):
|
||||
# basic verification
|
||||
|
|
@ -66,22 +61,6 @@ def get_cryptocurrency(request_dict, cryptocurrency_type):
|
|||
|
||||
return cryptocurrency.get_correlation(request_dict, cryptocurrency_type, field_name)
|
||||
|
||||
def get_cryptocurrency_domain(request_dict, cryptocurrency_type=None):
|
||||
currency_types = sanythise_cryptocurrency_types(cryptocurrency_type)
|
||||
|
||||
res = cryptocurrency.verify_correlation_field_request(request_dict, currency_types, item_type='domain')
|
||||
if res:
|
||||
return res
|
||||
field_name = request_dict.get(cryptocurrency_type)
|
||||
if not verify_cryptocurrency_address(cryptocurrency_type, field_name):
|
||||
return ( {'status': 'error', 'reason': 'Invalid Cryptocurrency address'}, 400 )
|
||||
|
||||
return cryptocurrency.get_correlation_domain(request_dict, cryptocurrency_type, field_name)
|
||||
|
||||
def get_domain_cryptocurrency(request_dict, cryptocurrency_type):
|
||||
return cryptocurrency.get_domain_correlation_obj(self, request_dict, cryptocurrency_type, domain)
|
||||
|
||||
|
||||
def save_cryptocurrency_data(cryptocurrency_name, date, item_path, cryptocurrency_address):
|
||||
# create basic medata
|
||||
if not r_serv_metadata.exists('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address)):
|
||||
|
|
|
|||
|
|
@ -5,19 +5,24 @@ import os
|
|||
import sys
|
||||
import redis
|
||||
|
||||
from hashlib import sha256
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_FLASK'], 'modules'))
|
||||
import Flask_config
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
|
||||
from Correlation import Correlation
|
||||
import Item
|
||||
|
||||
serv_metadata = Flask_config.r_serv_metadata
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
|
||||
pgpdump = Correlation('pgpdump')
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
|
||||
config_loader = None
|
||||
|
||||
|
||||
class Pgp(Correlation):
|
||||
def __init__(self):
|
||||
super().__init__('pgpdump', ['key', 'mail', 'name'])
|
||||
|
||||
pgp = Pgp()
|
||||
|
||||
def get_pgp(request_dict, pgp_type):
|
||||
# basic verification
|
||||
res = pgpdump.verify_correlation_field_request(request_dict, pgp_type)
|
||||
|
|
|
|||
|
|
@ -2,17 +2,22 @@
|
|||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
import redis
|
||||
|
||||
import Flask_config
|
||||
import Date
|
||||
import Item
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
|
||||
from pytaxonomies import Taxonomies
|
||||
from pymispgalaxies import Galaxies, Clusters
|
||||
|
||||
r_serv_tags = Flask_config.r_serv_tags
|
||||
r_serv_metadata = Flask_config.r_serv_metadata
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
r_serv_tags = config_loader.get_redis_conn("ARDB_Tags")
|
||||
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
|
||||
config_loader = None
|
||||
|
||||
def get_taxonomie_from_tag(tag):
|
||||
return tag.split(':')[0]
|
||||
|
|
@ -77,8 +82,12 @@ def is_tag_in_all_tag(tag):
|
|||
def get_all_tags():
|
||||
return list(r_serv_tags.smembers('list_tags'))
|
||||
|
||||
'''
|
||||
Retun all the tags of a given item.
|
||||
:param item_id: (Paste or domain)
|
||||
'''
|
||||
def get_item_tags(item_id):
|
||||
tags = r_serv_metadata.smembers('tag:'+item_id)
|
||||
tags = r_serv_metadata.smembers('tag:{}'.format(item_id))
|
||||
if tags:
|
||||
return list(tags)
|
||||
else:
|
||||
|
|
|
|||
Loading…
Reference in a new issue