chg: [Correlation] get correlation (crypto + pgp) by domain

bin/lib/Domain.py

@@ -14,8 +14,10 @@ import redis
 
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
 import Correlation
-import Cryptocurrency
+from Cryptocurrency import cryptocurrency
+from Pgp import pgp
 import Item
+import Tag
 
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
 import ConfigLoader
@@ -54,35 +56,53 @@ def get_link_tree():
     pass
 
 
-###
+def get_domain_tags(domain):
-### correlation
+    '''
-###
+    Retun all tags of a given domain.
-"""
+
-def _get_domain_correlation(domain, correlation_name=None, correlation_type=None):
+    :param domain: crawled domain
-    res = r_serv_metadata.smembers('item_{}_{}:{}'.format(correlation_name, correlation_type, item_id))
+    '''
+    return Tag.get_item_tags(domain)
+
+def get_domain_cryptocurrency(domain, currencies_type=None):
+    '''
+    Retun all cryptocurrencies of a given domain.
+
+    :param domain: crawled domain
+    :param currencies_type: list of cryptocurrencies type
+    :type currencies_type: list, optional
+    '''
+    return cryptocurrency.get_domain_correlation_dict(domain, correlation_type=currencies_type)
+
+def get_domain_pgp(domain, currencies_type=None):
+    '''
+    Retun all pgp of a given domain.
+
+    :param domain: crawled domain
+    :param currencies_type: list of pgp type
+    :type currencies_type: list, optional
+    '''
+    return pgp.get_domain_correlation_dict(domain, correlation_type=currencies_type)
+
+def get_domain_all_correlation(domain, correlation_type=None):
+    '''
+    Retun all correlation of a given domain.
+
+    :param domain: crawled domain
+    :type domain: str
+
+    :return: a dict of all correlation for a given domain
+    :rtype: dict
+    '''
+    domain_correl = {}
+    res = get_domain_cryptocurrency(domain)
     if res:
-        return list(res)
+        domain_correl['cryptocurrency'] = res
-    else:
+    res = get_domain_pgp(domain)
-        return []
+    if res:
+        domain_correl['pgp'] = res
+    return domain_correl
 
-def get_item_bitcoin(item_id):
-    return _get_item_correlation('cryptocurrency', 'bitcoin', item_id)
-
-def get_item_pgp_key(item_id):
-    return _get_item_correlation('pgpdump', 'key', item_id)
-
-def get_item_pgp_name(item_id):
-    return _get_item_correlation('pgpdump', 'name', item_id)
-
-def get_item_pgp_mail(item_id):
-    return _get_item_correlation('pgpdump', 'mail', item_id)
-
-def get_item_pgp_correlation(item_id):
-    pass
-"""
-
-def _get_domain_correlation(domain, correlation_list):
-    return Cryptocurrency.get_cryptocurrency_domain(domain)
 
 class Domain(object):
     """docstring for Domain."""

bin/packages/Correlation.py

@@ -5,16 +5,18 @@ import os
 import sys
 import redis
 
-sys.path.append(os.path.join(os.environ['AIL_FLASK'], 'modules/'))
+sys.path.append(os.path.join(os.environ['AIL_FLASK'], 'lib/'))
-import Flask_config
+import ConfigLoader
-
-r_serv_metadata = Flask_config.r_serv_metadata
 
+config_loader = ConfigLoader.ConfigLoader()
+r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
+config_loader = None
 
 class Correlation(object):
 
-    def __init__(self, correlation_name):
+    def __init__(self, correlation_name, all_correlation_types):
         self.correlation_name = correlation_name
+        self.all_correlation_types = all_correlation_types
 
     def _exist_corelation_field(self, correlation_type, field_name, item_type='paste'):
         if type=='paste':
@@ -29,13 +31,6 @@ class Correlation(object):
         else:
             return []
 
-    def _get_domains(self, correlation_type, field_name):
-        res =  r_serv_metadata.smembers('set_domain_{}_{}:{}'.format(self.correlation_name, correlation_type, field_name))
-        if res:
-            return list(res)
-        else:
-            return []
-
     def _get_metadata(self, correlation_type, field_name):
         meta_dict = {}
         meta_dict['first_seen'] = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, correlation_type, field_name), 'first_seen')
@@ -49,7 +44,6 @@ class Correlation(object):
         if not request_dict:
             return ({'status': 'error', 'reason': 'Malformed JSON'}, 400)
 
-        print(correlation_type)
         field_name = request_dict.get(correlation_type, None)
         if not field_name:
             return ( {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400 )
@@ -69,37 +63,72 @@ class Correlation(object):
 
         return (dict_resp, 200)
 
-    def get_correlation_domain(self, request_dict, correlation_type, field_name):
+    def get_all_correlation_types(self):
-        dict_resp = {}
+        '''
+        Gel all correlation types
 
-        dict_resp['domain'] = self._get_domains(correlation_type, field_name)
+        :return: A list of all the correlation types
+        :rtype: list
+        '''
+        return self.all_correlation_types
 
-        #if request_dict.get('metadata'):
+    def sanythise_correlation_types(self, correlation_types):
-        #    dict_resp['metadata'] = self._get_metadata(correlation_type, field_name)
+        '''
+        Check if all correlation types in the list are valid.
 
-        dict_resp[correlation_type] = field_name
+        :param correlation_types: list of correlation type
+        :type currency_type: list
 
-        return (dict_resp, 200)
+        :return: If a type is invalid, return the full list of correlation types else return the provided list
+        :rtype: list
+        '''
+        if correlation_types is None:
+            return self.get_all_correlation_types()
+        for correl in correlation_types: # # TODO: # OPTIMIZE:
+            if correl not in self.get_all_correlation_types():
+                return self.get_all_correlation_types()
+        return correlation_types
 
-######## INTERNAL ########
 
-def _get_domain_correlation_obj(correlation_name, correlation_type, domain):
+    def _get_domain_correlation_obj(self, domain, correlation_type):
-    print('domain_{}_{}:{}'.format(correlation_name, correlation_type, domain))
+        '''
-    res = r_serv_metadata.smembers('domain_{}_{}:{}'.format(correlation_name, correlation_type, domain))
+        Return correlation of a given domain.
+
+        :param domain: crawled domain
+        :type domain: str
+        :param correlation_type: correlation type
+        :type correlation_type: str
+
+        :return: a list of correlation
+        :rtype: list
+        '''
+        res = r_serv_metadata.smembers('domain_{}_{}:{}'.format(self.correlation_name, correlation_type, domain))
         if res:
             return list(res)
         else:
             return []
 
-########  ########
+    def get_domain_correlation_dict(self, domain, correlation_type=None):
+        '''
+        Return all correlation of a given domain.
+
+        :param domain: crawled domain
+        :param correlation_type: list of correlation types
+        :type correlation_type: list, optional
+
+        :return: a dictionnary of all the requested correlations
+        :rtype: dict
+        '''
+        correlation_type = self.sanythise_correlation_types(correlation_type)
+        dict_correlation = {}
+        for correl in correlation_type:
+            res = self._get_domain_correlation_obj(domain, correl)
+            if res:
+                dict_correlation[correl] = res
+        return dict_correlation
+
 
 ######## API EXPOSED ########
 
-def get_domain_correlation_obj(request_dict, correlation_name, correlation_type, domain):
-    dict_resp = {}
-    dict_resp[correlation_type] = _get_domain_correlation_obj(correlation_name, correlation_type, domain)
-    dict_resp['domain'] = domain
-
-    return (dict_resp, 200)
 
 ########  ########

bin/packages/Cryptocurrency.py

@@ -7,18 +7,25 @@ import redis
 
 from hashlib import sha256
 
-sys.path.append(os.path.join(os.environ['AIL_FLASK'], 'modules'))
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
-import Flask_config
 from Correlation import Correlation
 import Item
 
-r_serv_metadata = Flask_config.r_serv_metadata
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
+import ConfigLoader
 
-all_cryptocurrency = ['bitcoin', 'ethereum', 'bitcoin-cash', 'litecoin', 'monero', 'zcash', 'dash']
+config_loader = ConfigLoader.ConfigLoader()
+r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
+config_loader = None
 
 digits58 = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
 
-cryptocurrency = Correlation('cryptocurrency')
+
+class Cryptocurrency(Correlation):
+    def __init__(self):
+        super().__init__('cryptocurrency', ['bitcoin', 'ethereum', 'bitcoin-cash', 'litecoin', 'monero', 'zcash', 'dash'])
+
+cryptocurrency = Cryptocurrency()
 
 # http://rosettacode.org/wiki/Bitcoin/address_validation#Python
 def decode_base58(bc, length):
@@ -41,18 +48,6 @@ def verify_cryptocurrency_address(cryptocurrency_type, cryptocurrency_address):
     else:
         return True
 
-def get_all_all_cryptocurrency():
-    return all_cryptocurrency
-
-# check if all crypto type in the list are valid
-# if a type is invalid, return the full list of currency types
-def sanythise_cryptocurrency_types(cryptocurrency_types):
-    if cryptocurrency_types is None:
-        return get_all_all_cryptocurrency()
-    for currency in cryptocurrency_types: # # TODO: # OPTIMIZE:
-        if currency not in all_cryptocurrency:
-            return get_all_all_cryptocurrency()
-    return cryptocurrency_types
 
 def get_cryptocurrency(request_dict, cryptocurrency_type):
     # basic verification
@@ -66,22 +61,6 @@ def get_cryptocurrency(request_dict, cryptocurrency_type):
 
     return cryptocurrency.get_correlation(request_dict, cryptocurrency_type, field_name)
 
-def get_cryptocurrency_domain(request_dict, cryptocurrency_type=None):
-    currency_types = sanythise_cryptocurrency_types(cryptocurrency_type)
-
-    res = cryptocurrency.verify_correlation_field_request(request_dict, currency_types, item_type='domain')
-    if res:
-        return res
-    field_name = request_dict.get(cryptocurrency_type)
-    if not verify_cryptocurrency_address(cryptocurrency_type, field_name):
-        return ( {'status': 'error', 'reason': 'Invalid Cryptocurrency address'}, 400 )
-
-    return cryptocurrency.get_correlation_domain(request_dict, cryptocurrency_type, field_name)
-
-def get_domain_cryptocurrency(request_dict, cryptocurrency_type):
-    return cryptocurrency.get_domain_correlation_obj(self, request_dict, cryptocurrency_type, domain)
-
-
 def save_cryptocurrency_data(cryptocurrency_name, date, item_path, cryptocurrency_address):
     # create basic medata
     if not r_serv_metadata.exists('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address)):

bin/packages/Pgp.py

@@ -5,19 +5,24 @@ import os
 import sys
 import redis
 
-from hashlib import sha256
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
-
-sys.path.append(os.path.join(os.environ['AIL_FLASK'], 'modules'))
-import Flask_config
-
 from Correlation import Correlation
 import Item
 
-serv_metadata = Flask_config.r_serv_metadata
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
+import ConfigLoader
 
-pgpdump = Correlation('pgpdump')
+config_loader = ConfigLoader.ConfigLoader()
+serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
+config_loader = None
 
 
+class Pgp(Correlation):
+    def __init__(self):
+        super().__init__('pgpdump', ['key', 'mail', 'name'])
+
+pgp = Pgp()
+
 def get_pgp(request_dict, pgp_type):
     # basic verification
     res = pgpdump.verify_correlation_field_request(request_dict, pgp_type)

bin/packages/Tag.py

@@ -2,17 +2,22 @@
 # -*-coding:UTF-8 -*
 
 import os
+import sys
 import redis
 
-import Flask_config
 import Date
 import Item
 
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
+import ConfigLoader
+
 from pytaxonomies import Taxonomies
 from pymispgalaxies import Galaxies, Clusters
 
-r_serv_tags = Flask_config.r_serv_tags
+config_loader = ConfigLoader.ConfigLoader()
-r_serv_metadata = Flask_config.r_serv_metadata
+r_serv_tags = config_loader.get_redis_conn("ARDB_Tags")
+r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
+config_loader = None
 
 def get_taxonomie_from_tag(tag):
     return tag.split(':')[0]
@@ -77,8 +82,12 @@ def is_tag_in_all_tag(tag):
 def get_all_tags():
     return list(r_serv_tags.smembers('list_tags'))
 
+'''
+Retun all the tags of a given item.
+:param item_id: (Paste or domain)
+'''
 def get_item_tags(item_id):
-    tags = r_serv_metadata.smembers('tag:'+item_id)
+    tags = r_serv_metadata.smembers('tag:{}'.format(item_id))
     if tags:
         return list(tags)
     else: