chg: [add MISP import] correlation

TODO: create correlation

bin/export/MispImport.py

@@ -67,7 +67,7 @@ def unpack_item_obj(map_uuid_global_id, misp_obj):
         res = Item.create_item(obj_id, obj_meta, io_content)
         print(res)
 
-    map_uuid_global_id[misp_obj.uuid] = get_global_id('item', obj_id, io_content)
+    map_uuid_global_id[misp_obj.uuid] = get_global_id('item', obj_id)
 
 def get_obj_relationship(misp_obj):
     for item in misp_obj.ObjectReference:
@@ -87,29 +87,33 @@ def unpack_obj_pgp(map_uuid_global_id, misp_obj):
     elif obj_attr.object_relation == 'user-id-email':
         obj_subtype = 'mail'
 
+    if obj_id and obj_subtype:
         obj_meta = get_object_metadata(misp_obj)
-    if obj_id and io_content:
+        res = Pgp.pgp.create_correlation(obj_subtype, obj_id, obj_meta)
-        res = Pgp.pgp.create_item(obj_subtype, obj_id, obj_meta)
         print(res)
 
         map_uuid_global_id[misp_obj.uuid] = get_global_id('pgp', obj_id, obj_subtype=obj_subtype)
 
-    get_obj_relationship(misp_obj)
+        #get_obj_relationship(misp_obj)
 
 def unpack_obj_cryptocurrency(map_uuid_global_id, misp_obj):
     obj_id = None
-    crypto_symbol = None
+    obj_subtype = None
     for attribute in misp_obj.attributes:
-        if attribute.object_relation == 'address':
+        if attribute.object_relation == 'address': # # TODO: handle xmr address field
             obj_id = attribute.value
         elif attribute.object_relation == 'symbol':
-            pass
+            obj_subtype = Cryptocurrency.get_cryptocurrency_type(attribute.value)
-
 
+    # valid cryptocurrency type
+    if obj_subtype and obj_id:
+        print('crypto')
+        print(obj_id)
+        print(obj_subtype)
 
         obj_meta = get_object_metadata(misp_obj)
-        if obj_id and io_content:
+        print(obj_meta)
-            res = Pgp.pgp.create_item(obj_subtype, obj_id, obj_meta)
+        res = Cryptocurrency.cryptocurrency.create_correlation(obj_subtype, obj_id, obj_meta)
         print(res)
 
         map_uuid_global_id[misp_obj.uuid] = get_global_id('pgp', obj_id, obj_subtype=obj_subtype)
@@ -155,4 +159,5 @@ if __name__ == '__main__':
 
     # misp = PyMISP('https://127.0.0.1:8443/', 'uXgcN42b7xuL88XqK5hubwD8Q8596VrrBvkHQzB0', False)
 
-    import_objs_from_file('test_import_item.json')
+    #import_objs_from_file('test_import_item.json')
+    import_objs_from_file('test_export.json')

bin/packages/Cryptocurrency.py

@@ -9,7 +9,6 @@ from hashlib import sha256
 
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
 import correlation
-import Item
 
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
 import ConfigLoader
@@ -56,39 +55,6 @@ def get_cryptocurrency(request_dict, cryptocurrency_type):
 
     return cryptocurrency.get_correlation(request_dict, cryptocurrency_type, field_name)
 
-# # TODO: refractor/move me in Correlation
-def save_cryptocurrency_data(cryptocurrency_name, date, item_path, cryptocurrency_address):
-    # create basic medata
-    if not r_serv_metadata.exists('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address)):
-        r_serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'first_seen', date)
-        r_serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen', date)
-    else:
-        last_seen = r_serv_metadata.hget('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen')
-        if not last_seen:
-            r_serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen', date)
-        else:
-            if int(last_seen) < int(date):
-                r_serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen', date)
-
-    ## global set
-    # item
-    r_serv_metadata.sadd('set_cryptocurrency_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), item_path)
-
-    # daily
-    r_serv_metadata.hincrby('cryptocurrency:{}:{}'.format(cryptocurrency_name, date), cryptocurrency_address, 1)
-
-    # all type
-    r_serv_metadata.zincrby('cryptocurrency_all:{}'.format(cryptocurrency_name), cryptocurrency_address, 1)
-
-    ## object_metadata
-    # item
-    r_serv_metadata.sadd('item_cryptocurrency_{}:{}'.format(cryptocurrency_name, item_path), cryptocurrency_address)
-
-    # domain
-    if Item.is_crawled(item_path): # # TODO:  use save_domain_correlation
-        domain = Item.get_item_domain(item_path)
-        r_serv_metadata.sadd('domain_cryptocurrency_{}:{}'.format(cryptocurrency_name, domain), cryptocurrency_address)
-        r_serv_metadata.sadd('set_domain_cryptocurrency_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), domain)
 
 def get_cryptocurrency_symbol(crypto_type):
     if crypto_type=='bitcoin':

bin/packages/Date.py

@@ -129,15 +129,32 @@ def validate_str_date(str_date, separator=''):
     except TypeError:
         return False
 
-def sanitise_date_range(date_from, date_to, separator=''):
+def sanitise_date_range(date_from, date_to, separator='', date_type='str'):
     '''
     Check/Return a correct date_form and date_to
     '''
+    if not date_from and date_to:
+        date_from = date_to
+    elif not date_to and date_from:
+        date_to = date_from
+
+    if date_type=='str':
         if not validate_str_date(date_from, separator=separator):
             date_from = datetime.date.today().strftime("%Y%m%d")
         if not validate_str_date(date_to, separator=separator):
             date_to = datetime.date.today().strftime("%Y%m%d")
+    else: # datetime
+        if isinstance(date_from, datetime.datetime):
+            date_from = date_from.strftime("%Y%m%d")
+        else:
+            date_from = datetime.date.today().strftime("%Y%m%d")
+        if isinstance(date_to, datetime.datetime):
+            date_to = date_to.strftime("%Y%m%d")
+        else:
+            date_to = datetime.date.today().strftime("%Y%m%d")
 
     if int(date_from) > int(date_to):
+        res = date_from
         date_from = date_to
+        date_to = res
     return {"date_from": date_from, "date_to": date_to}

bin/packages/Pgp.py

@@ -7,7 +7,6 @@ import redis
 
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
 import correlation
-import Item
 
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
 import ConfigLoader
@@ -27,36 +26,3 @@ def get_pgp(request_dict, pgp_type):
     field_name = request_dict.get(pgp_type)
 
     return pgpdump.get_correlation(request_dict, pgp_type, field_name)
-
-def save_pgp_data(type_pgp, date, item_path, data):
-    # create basic medata
-    if not serv_metadata.exists('pgpdump_metadata_{}:{}'.format(type_pgp, data)):
-        serv_metadata.hset('pgpdump_metadata_{}:{}'.format(type_pgp, data), 'first_seen', date)
-        serv_metadata.hset('pgpdump_metadata_{}:{}'.format(type_pgp, data), 'last_seen', date)
-    else:
-        last_seen = serv_metadata.hget('pgpdump_metadata_{}:{}'.format(type_pgp, data), 'last_seen')
-        if not last_seen:
-            serv_metadata.hset('pgpdump_metadata_{}:{}'.format(type_pgp, data), 'last_seen', date)
-        else:
-            if int(last_seen) < int(date):
-                serv_metadata.hset('pgpdump_metadata_{}:{}'.format(type_pgp, data), 'last_seen', date)
-
-    # global set
-    serv_metadata.sadd('set_pgpdump_{}:{}'.format(type_pgp, data), item_path)
-
-    # daily
-    serv_metadata.hincrby('pgpdump:{}:{}'.format(type_pgp, date), data, 1)
-
-    # all type
-    serv_metadata.zincrby('pgpdump_all:{}'.format(type_pgp), data, 1)
-
-    ## object_metadata
-    # paste
-    serv_metadata.sadd('item_pgpdump_{}:{}'.format(type_pgp, item_path), data)
-
-
-    # domain object
-    if Item.is_crawled(item_path):
-        domain = Item.get_item_domain(item_path)
-        serv_metadata.sadd('domain_pgpdump_{}:{}'.format(type_pgp, domain), data)
-        serv_metadata.sadd('set_domain_pgpdump_{}:{}'.format(type_pgp, data), domain)

bin/packages/correlation.py

@@ -10,6 +10,8 @@ import ConfigLoader
 
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
 import Date
+import Item
+import Tag
 
 config_loader = ConfigLoader.ConfigLoader()
 r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
@@ -47,11 +49,31 @@ class Correlation(object):
         else:
             return []
 
-    def _get_metadata(self, correlation_type, field_name):
+    def get_correlation_first_seen(self, subtype, obj_id, r_int=False):
+        res = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, subtype, obj_id), 'first_seen')
+        if r_int:
+            if res:
+                return int(res)
+            else:
+                return 99999999
+        else:
+            return res
+
+    def get_correlation_last_seen(self, subtype, obj_id, r_int=False):
+        res = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, subtype, obj_id), 'last_seen')
+        if r_int:
+            if res:
+                return int(res)
+            else:
+                return 0
+        else:
+            return res
+
+    def _get_metadata(self, subtype, obj_id):
         meta_dict = {}
-        meta_dict['first_seen'] = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, correlation_type, field_name), 'first_seen')
+        meta_dict['first_seen'] = self.get_correlation_first_seen(subtype, obj_id)
-        meta_dict['last_seen'] = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, correlation_type, field_name), 'last_seen')
+        meta_dict['last_seen'] = self.get_correlation_last_seen(subtype, obj_id)
-        meta_dict['nb_seen'] = r_serv_metadata.scard('set_{}_{}:{}'.format(self.correlation_name, correlation_type, field_name))
+        meta_dict['nb_seen'] = r_serv_metadata.scard('set_{}_{}:{}'.format(self.correlation_name, subtype, obj_id))
         return meta_dict
 
     def get_metadata(self, correlation_type, field_name, date_format='str_date'):
@@ -276,17 +298,18 @@ class Correlation(object):
                 correlation_obj[correlation_object] = res
         return correlation_obj
 
-    def update_correlation_daterange(self, subtype, obj_id, date): # # TODO:  update fisrt_seen
+    def update_correlation_daterange(self, subtype, obj_id, date):
+        date = int(date)
         # obj_id don't exit
         if not r_serv_metadata.exists('{}_metadata_{}:{}'.format(self.correlation_name, subtype, obj_id)):
             r_serv_metadata.hset('{}_metadata_{}:{}'.format(self.correlation_name, subtype, obj_id), 'first_seen', date)
             r_serv_metadata.hset('{}_metadata_{}:{}'.format(self.correlation_name, subtype, obj_id), 'last_seen', date)
         else:
-            last_seen = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, subtype, obj_id), 'last_seen')
+            first_seen = self.get_correlation_last_seen(subtype, obj_id, r_int=True)
-            if not last_seen:
+            last_seen = self.get_correlation_first_seen(subtype, obj_id, r_int=True)
-                r_serv_metadata.hset('{}_metadata_{}:{}'.format(self.correlation_name, subtype, obj_id), 'last_seen', date)
+            if date < first_seen:
-            else:
+                r_serv_metadata.hset('{}_metadata_{}:{}'.format(self.correlation_name, subtype, obj_id), 'first_seen', date)
-                if int(last_seen) < int(date):
+            if date > last_seen:
                 r_serv_metadata.hset('{}_metadata_{}:{}'.format(self.correlation_name, subtype, obj_id), 'last_seen', date)
 
     def save_item_correlation(self, subtype, date, obj_id, item_id, item_date):
@@ -305,28 +328,37 @@ class Correlation(object):
         # item
         r_serv_metadata.sadd('item_{}_{}:{}'.format(self.correlation_name, subtype, item_id), obj_id)
 
-    def save_domain_correlation(self, domain, correlation_type, correlation_value):
+        # domain
-        r_serv_metadata.sadd('domain_{}_{}:{}'.format(self.correlation_name, correlation_type, domain), correlation_value)
+        if Item.is_crawled(item_id):
-        r_serv_metadata.sadd('set_domain_{}_{}:{}'.format(self.correlation_name, correlation_type, correlation_value), domain)
+            domain = Item.get_item_domain(item_id)
+            self.save_domain_correlation(domain, subtype, obj_id)
+
+    def save_domain_correlation(self, domain, subtype, obj_id):
+        r_serv_metadata.sadd('domain_{}_{}:{}'.format(self.correlation_name, subtype, domain), obj_id)
+        r_serv_metadata.sadd('set_domain_{}_{}:{}'.format(self.correlation_name, subtype, obj_id), domain)
 
 
-    def save_correlation(self, subtype, obj_id): # # TODO: add first_seen/last_seen
+    def save_correlation(self, subtype, obj_id, date_range):
         r_serv_metadata.zincrby('{}_all:{}'.format(self.correlation_name, subtype), obj_id, 0)
+        self.update_correlation_daterange(subtype, obj_id, date_range['date_from'])
+        if date_range['date_from'] != date_range['date_to']:
+            self.update_correlation_daterange(subtype, obj_id, date_range['date_to'])
+        return True
 
     def create_correlation(self, subtype, obj_id, obj_meta):
-        res = self.sanythise_correlation_types(correlation_type, r_boolean=True)
+        res = self.sanythise_correlation_types([subtype], r_boolean=True)
         if not res:
             print('invalid subtype')
             return False
-
+        first_seen = obj_meta.get('first_seen', None)
-        if not exist_correlation(subtype, obj_id):
+        last_seen = obj_meta.get('last_seen', None)
-            res = save_correlation(subtype, obj_id)
+        date_range = Date.sanitise_date_range(first_seen, last_seen, separator='', date_type='datetime')
-            if res:
+        print(date_range)
-                if 'tags' in obj_metadata:
+        res = self.save_correlation(subtype, obj_id, date_range)
+        if res and 'tags' in obj_meta:
             # # TODO: handle mixed tags: taxonomies and Galaxies
-                    Tag.api_add_obj_tags(tags=obj_metadata['tags'], object_id=obj_id, object_type=self.get_correlation_obj_type())
+            Tag.api_add_obj_tags(tags=obj_meta['tags'], object_id=obj_id, object_type=self.get_correlation_obj_type())
         return True
-        return False
 
 ######## API EXPOSED ########