chg: [CVE] migrate CVE + get CVEs by daterange

This commit is contained in:
Terrtia 2022-12-21 14:20:13 +01:00
parent bf71c9ba99
commit 82ff568feb
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
6 changed files with 81 additions and 45 deletions

View file

@ -840,23 +840,43 @@ def statistics_migration():
pass
###############################
# #
# CVES MIGRATION #
# #
###############################
from modules.CveModule import CveModule
def cves_migration():
module = CveModule()
tag = 'infoleak:automatic-detection="cve"'
first = Tag.get_tag_first_seen(tag)
last = Tag.get_tag_last_seen(tag)
if first and last:
for date in Date.substract_date(first, last):
for item_id in Tag.get_tag_objects(tag, 'item', date=date):
module.compute(f'{item_id} 0')
if __name__ == '__main__':
#core_migration()
#user_migration()
#tags_migration()
items_migration()
# items_migration()
#crawler_migration()
# domain_migration() # TO TEST ###########################
#decodeds_migration()
# screenshots_migration()
subtypes_obj_migration()
# subtypes_obj_migration()
# ail_2_ail_migration()
# trackers_migration()
# investigations_migration()
# statistics_migration()
cves_migration()
# custom tags
# crawler queues + auto_crawlers

View file

@ -5,6 +5,7 @@ import os
import sys
from flask import url_for
from pymisp import MISPObject
sys.path.append(os.environ['AIL_BIN'])
##################################
@ -20,12 +21,6 @@ baseurl = config_loader.get_config_str("Notifications", "ail_domain")
config_loader = None
################################################################################
################################################################################
################################################################################
# # TODO: COMPLETE CLASS
class Cve(AbstractDaterangeObject):
"""
AIL Cve Object.
@ -55,14 +50,21 @@ class Cve(AbstractDaterangeObject):
def get_svg_icon(self):
return {'style': 'fas', 'icon': '\uf188', 'color': '#1E88E5', 'radius': 5}
# TODO # TODO # TODO # TODO # TODO # TODO # TODO # TODO
def get_misp_object(self):
pass
obj_attrs = []
obj = MISPObject('vulnerability')
obj.first_seen = self.get_first_seen()
obj.last_seen = self.get_last_seen()
obj_attrs.append(obj.add_attribute('id', value=self.id))
for obj_attr in obj_attrs:
for tag in self.get_tags():
obj_attr.add_tag(tag)
return obj
def get_meta(self, options=set()):
meta = self._get_meta(options=options)
meta['id'] = self.id
meta['subtype'] = self.subtype
meta['tags'] = self.get_tags(r_list=True)
return meta
@ -70,19 +72,21 @@ class Cve(AbstractDaterangeObject):
self._add(date, item_id)
# TODO # TODO # TODO # TODO # TODO # TODO # TODO # TODO # TODO # TODO
# TODO ADD SEARCH FUNCTION
def get_all_cves():
cves = []
return cves
return r_objects.smembers(f'cve:all')
def get_cves_by_date(date):
# return r_objects.zrange(f'cve:date:{date}', 0, -1)
return set(r_objects.hkeys(f'cve:date:{date}'))
return r_objects.zrange(f'cve:date:{date}', 0, -1)
def get_nb_cves_by_date(date):
return r_objects.zcard(f'cve:date:{date}')
def get_cves_by_daterange(date_from, date_to):
cves = set()
for date in Date.substract_date(date_from, date_to):
cves | get_cves_by_date(date)
cves = cves | set(get_cves_by_date(date))
return cves
def get_cves_meta(cves_id, options=set()):
@ -92,6 +96,14 @@ def get_cves_meta(cves_id, options=set()):
dict_cve[cve_id] = cve.get_meta(options=options)
return dict_cve
def api_get_cves_range_by_daterange(date_from, date_to):
cves = []
for date in Date.substract_date(date_from, date_to):
d = {'date': f'{date[0:4]}-{date[4:6]}-{date[6:8]}',
'CVE': get_nb_cves_by_date(date)}
cves.append(d)
return cves
def api_get_cves_meta_by_daterange(date_from, date_to):
date = Date.sanitise_date_range(date_from, date_to)
return get_cves_meta(get_cves_by_daterange(date['date_from'], date['date_to']), options=['sparkline'])

View file

@ -3,18 +3,19 @@
import os
import sys
import redis
# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id
from flask import url_for
from pymisp import MISPObject
config_loader = ConfigLoader.ConfigLoader()
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from lib.ConfigLoader import ConfigLoader
from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id
config_loader = ConfigLoader()
baseurl = config_loader.get_config_str("Notifications", "ail_domain")
config_loader = None
@ -64,7 +65,7 @@ class Pgp(AbstractSubtypeObject):
icon = '\uf1fa'
else:
icon = 'times'
return {'style': 'fas', 'icon': icon, 'color': '#44AA99', 'radius':5}
return {'style': 'fas', 'icon': icon, 'color': '#44AA99', 'radius': 5}
def get_misp_object(self):
obj_attrs = []
@ -72,12 +73,12 @@ class Pgp(AbstractSubtypeObject):
obj.first_seen = self.get_first_seen()
obj.last_seen = self.get_last_seen()
if self.subtype=='key':
obj_attrs.append( obj.add_attribute('key-id', value=self.id) )
elif self.subtype=='name':
obj_attrs.append( obj.add_attribute('user-id-name', value=self.id) )
else: # mail
obj_attrs.append( obj.add_attribute('user-id-email', value=self.id) )
if self.subtype == 'key':
obj_attrs.append(obj.add_attribute('key-id', value=self.id))
elif self.subtype == 'name':
obj_attrs.append(obj.add_attribute('user-id-name', value=self.id))
else: # mail
obj_attrs.append(obj.add_attribute('user-id-email', value=self.id))
for obj_attr in obj_attrs:
for tag in self.get_tags():
@ -88,7 +89,6 @@ class Pgp(AbstractSubtypeObject):
############################################################################
def get_all_subtypes():
#return get_object_all_subtypes(self.type)
return ['key', 'mail', 'name']
def get_all_pgps():
@ -101,5 +101,4 @@ def get_all_pgps_by_subtype(subtype):
return get_all_id('pgp', subtype)
#if __name__ == '__main__':
# if __name__ == '__main__':

View file

@ -68,7 +68,7 @@ class AbstractDaterangeObject(AbstractObject, ABC):
return r_object.hget(f'meta:{self.type}:{self.id}', 'nb')
def get_nb_seen_by_date(self, date):
nb = r_object.hget(f'{self.type}:date:{date}', self.id)
nb = r_object.zscore(f'{self.type}:date:{date}', self.id)
if nb is None:
return 0
else:
@ -118,8 +118,8 @@ class AbstractDaterangeObject(AbstractObject, ABC):
update_obj_date(date, self.type)
# NB Object seen by day
r_object.hincrby(f'{self.type}:date:{date}', self.id, 1)
r_object.zincrby(f'{self.type}:date:{date}', 1, self.id) # # # # # # # # # #
print(f'{self.type}:date:{date}', 1, self.id)
r_object.zincrby(f'{self.type}:date:{date}', 1, self.id)
# NB Object seen
r_object.hincrby(f'meta:{self.type}:{self.id}', 'nb', 1)

View file

@ -51,7 +51,7 @@ class CveModule(AbstractModule):
cves = self.regex_findall(self.reg_cve, item_id, item.get_content())
if cves:
print(cves)
# print(cves)
date = item.get_date()
for cve_id in cves:
cve = Cves.Cve(cve_id)

View file

@ -41,11 +41,11 @@ def objects_cves():
date_from = date['date_from']
date_to = date['date_to']
# barchart_type
# correlation_type_search_endpoint
if show_objects:
dict_objects = Cves.api_get_cves_meta_by_daterange(date_from, date_to)
else:
dict_objects = {}
dict_objects = Cves.api_get_cves_meta_by_daterange(date_from, date_to)
print(date_from, date_to, dict_objects)
return render_template("CveDaterange.html", date_from=date_from, date_to=date_to,
dict_objects=dict_objects, show_objects=show_objects)
@ -62,7 +62,12 @@ def objects_cves_post():
@login_required
@login_read_only
def objects_cve_range_json():
return None
date_from = request.args.get('date_from')
date_to = request.args.get('date_to')
date = Date.sanitise_date_range(date_from, date_to)
date_from = date['date_from']
date_to = date['date_to']
return jsonify(Cves.api_get_cves_range_by_daterange(date_from, date_to))
@objects_cve.route("/objects/cve/search", methods=['POST'])
@login_required