chg: [CVE] migrate CVE + get CVEs by daterange

bin/DB_KVROCKS_MIGRATION.py

@@ -840,23 +840,43 @@ def statistics_migration():
     pass
 
 
+
+###############################
+#                             #
+#       CVES MIGRATION        #
+#                             #
+###############################
+
+from modules.CveModule import CveModule
+
+def cves_migration():
+    module = CveModule()
+    tag = 'infoleak:automatic-detection="cve"'
+    first = Tag.get_tag_first_seen(tag)
+    last = Tag.get_tag_last_seen(tag)
+    if first and last:
+        for date in Date.substract_date(first, last):
+            for item_id in Tag.get_tag_objects(tag, 'item', date=date):
+                module.compute(f'{item_id} 0')
+
+
 if __name__ == '__main__':
 
     #core_migration()
     #user_migration()
     #tags_migration()
-    items_migration()
+    # items_migration()
     #crawler_migration()
     # domain_migration()                      # TO TEST ###########################
     #decodeds_migration()
     # screenshots_migration()
-    subtypes_obj_migration()
+    # subtypes_obj_migration()
     # ail_2_ail_migration()
     # trackers_migration()
     # investigations_migration()
     # statistics_migration()
 
-
+    cves_migration()
 
     # custom tags
     # crawler queues + auto_crawlers

bin/lib/objects/Cves.py

@@ -5,6 +5,7 @@ import os
 import sys
 
 from flask import url_for
+from pymisp import MISPObject
 
 sys.path.append(os.environ['AIL_BIN'])
 ##################################
@@ -20,12 +21,6 @@ baseurl = config_loader.get_config_str("Notifications", "ail_domain")
 config_loader = None
 
 
-################################################################################
-################################################################################
-################################################################################
-
-# # TODO: COMPLETE CLASS
-
 class Cve(AbstractDaterangeObject):
     """
     AIL Cve Object.
@@ -55,14 +50,21 @@ class Cve(AbstractDaterangeObject):
     def get_svg_icon(self):
         return {'style': 'fas', 'icon': '\uf188', 'color': '#1E88E5', 'radius': 5}
 
-    # TODO # TODO # TODO # TODO # TODO # TODO # TODO # TODO
     def get_misp_object(self):
-        pass
+        obj_attrs = []
+        obj = MISPObject('vulnerability')
+        obj.first_seen = self.get_first_seen()
+        obj.last_seen = self.get_last_seen()
+
+        obj_attrs.append(obj.add_attribute('id', value=self.id))
+        for obj_attr in obj_attrs:
+            for tag in self.get_tags():
+                obj_attr.add_tag(tag)
+        return obj
 
     def get_meta(self, options=set()):
         meta = self._get_meta(options=options)
         meta['id'] = self.id
-        meta['subtype'] = self.subtype
         meta['tags'] = self.get_tags(r_list=True)
         return meta
 
@@ -70,19 +72,21 @@ class Cve(AbstractDaterangeObject):
         self._add(date, item_id)
 
 
-# TODO # TODO # TODO # TODO # TODO # TODO # TODO # TODO # TODO # TODO
+# TODO  ADD SEARCH FUNCTION
+
 def get_all_cves():
-    cves = []
+    return r_objects.smembers(f'cve:all')
-    return cves
 
 def get_cves_by_date(date):
-    # return r_objects.zrange(f'cve:date:{date}', 0, -1)
+    return r_objects.zrange(f'cve:date:{date}', 0, -1)
-    return set(r_objects.hkeys(f'cve:date:{date}'))
+
+def get_nb_cves_by_date(date):
+    return r_objects.zcard(f'cve:date:{date}')
 
 def get_cves_by_daterange(date_from, date_to):
     cves = set()
     for date in Date.substract_date(date_from, date_to):
-        cves | get_cves_by_date(date)
+        cves = cves | set(get_cves_by_date(date))
     return cves
 
 def get_cves_meta(cves_id, options=set()):
@@ -92,6 +96,14 @@ def get_cves_meta(cves_id, options=set()):
         dict_cve[cve_id] = cve.get_meta(options=options)
     return dict_cve
 
+def api_get_cves_range_by_daterange(date_from, date_to):
+    cves = []
+    for date in Date.substract_date(date_from, date_to):
+        d = {'date': f'{date[0:4]}-{date[4:6]}-{date[6:8]}',
+             'CVE': get_nb_cves_by_date(date)}
+        cves.append(d)
+    return cves
+
 def api_get_cves_meta_by_daterange(date_from, date_to):
     date = Date.sanitise_date_range(date_from, date_to)
     return get_cves_meta(get_cves_by_daterange(date['date_from'], date['date_to']), options=['sparkline'])

bin/lib/objects/Pgps.py

@@ -3,18 +3,19 @@
 
 import os
 import sys
-import redis
 
-# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
-
-sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
-import ConfigLoader
-
-from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id
 from flask import url_for
+from pymisp import MISPObject
 
-config_loader = ConfigLoader.ConfigLoader()
+sys.path.append(os.environ['AIL_BIN'])
+##################################
+# Import Project packages
+##################################
+from lib.ConfigLoader import ConfigLoader
+from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id
 
+config_loader = ConfigLoader()
+baseurl = config_loader.get_config_str("Notifications", "ail_domain")
 config_loader = None
 
 
@@ -64,7 +65,7 @@ class Pgp(AbstractSubtypeObject):
             icon = '\uf1fa'
         else:
             icon = 'times'
-        return {'style': 'fas', 'icon': icon, 'color': '#44AA99', 'radius':5}
+        return {'style': 'fas', 'icon': icon, 'color': '#44AA99', 'radius': 5}
 
     def get_misp_object(self):
         obj_attrs = []
@@ -72,12 +73,12 @@ class Pgp(AbstractSubtypeObject):
         obj.first_seen = self.get_first_seen()
         obj.last_seen = self.get_last_seen()
 
-        if self.subtype=='key':
+        if self.subtype == 'key':
-            obj_attrs.append( obj.add_attribute('key-id', value=self.id) )
+            obj_attrs.append(obj.add_attribute('key-id', value=self.id))
-        elif self.subtype=='name':
+        elif self.subtype == 'name':
-            obj_attrs.append( obj.add_attribute('user-id-name', value=self.id) )
+            obj_attrs.append(obj.add_attribute('user-id-name', value=self.id))
         else:  # mail
-            obj_attrs.append( obj.add_attribute('user-id-email', value=self.id) )
+            obj_attrs.append(obj.add_attribute('user-id-email', value=self.id))
 
         for obj_attr in obj_attrs:
             for tag in self.get_tags():
@@ -88,7 +89,6 @@ class Pgp(AbstractSubtypeObject):
     ############################################################################
 
 def get_all_subtypes():
-    #return get_object_all_subtypes(self.type)
     return ['key', 'mail', 'name']
 
 def get_all_pgps():
@@ -101,5 +101,4 @@ def get_all_pgps_by_subtype(subtype):
     return get_all_id('pgp', subtype)
 
 
-
+# if __name__ == '__main__':
-#if __name__ == '__main__':

bin/lib/objects/abstract_daterange_object.py

@@ -68,7 +68,7 @@ class AbstractDaterangeObject(AbstractObject, ABC):
         return r_object.hget(f'meta:{self.type}:{self.id}', 'nb')
 
     def get_nb_seen_by_date(self, date):
-        nb = r_object.hget(f'{self.type}:date:{date}', self.id)
+        nb = r_object.zscore(f'{self.type}:date:{date}', self.id)
         if nb is None:
             return 0
         else:
@@ -118,8 +118,8 @@ class AbstractDaterangeObject(AbstractObject, ABC):
         update_obj_date(date, self.type)
 
         # NB Object seen by day
-        r_object.hincrby(f'{self.type}:date:{date}', self.id, 1)
+        print(f'{self.type}:date:{date}', 1, self.id)
-        r_object.zincrby(f'{self.type}:date:{date}', 1, self.id) # # # # # # # # # #
+        r_object.zincrby(f'{self.type}:date:{date}', 1, self.id)
         # NB Object seen
         r_object.hincrby(f'meta:{self.type}:{self.id}', 'nb', 1)
 

bin/modules/CveModule.py

@@ -51,7 +51,7 @@ class CveModule(AbstractModule):
 
         cves = self.regex_findall(self.reg_cve, item_id, item.get_content())
         if cves:
-            print(cves)
+            # print(cves)
             date = item.get_date()
             for cve_id in cves:
                 cve = Cves.Cve(cve_id)

var/www/blueprints/objects_cve.py

@@ -41,11 +41,11 @@ def objects_cves():
     date_from = date['date_from']
     date_to = date['date_to']
 
-    # barchart_type
+    if show_objects:
-    # correlation_type_search_endpoint
-
         dict_objects = Cves.api_get_cves_meta_by_daterange(date_from, date_to)
-    print(date_from, date_to, dict_objects)
+    else:
+        dict_objects = {}
+
     return render_template("CveDaterange.html", date_from=date_from, date_to=date_to,
                            dict_objects=dict_objects, show_objects=show_objects)
 
@@ -62,7 +62,12 @@ def objects_cves_post():
 @login_required
 @login_read_only
 def objects_cve_range_json():
-    return None
+    date_from = request.args.get('date_from')
+    date_to = request.args.get('date_to')
+    date = Date.sanitise_date_range(date_from, date_to)
+    date_from = date['date_from']
+    date_to = date['date_to']
+    return jsonify(Cves.api_get_cves_range_by_daterange(date_from, date_to))
 
 @objects_cve.route("/objects/cve/search", methods=['POST'])
 @login_required