mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-27 00:07:16 +00:00
chg: [CVE] migrate CVE + get CVEs by daterange
This commit is contained in:
parent
bf71c9ba99
commit
82ff568feb
6 changed files with 81 additions and 45 deletions
|
@ -840,23 +840,43 @@ def statistics_migration():
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
###############################
|
||||||
|
# #
|
||||||
|
# CVES MIGRATION #
|
||||||
|
# #
|
||||||
|
###############################
|
||||||
|
|
||||||
|
from modules.CveModule import CveModule
|
||||||
|
|
||||||
|
def cves_migration():
|
||||||
|
module = CveModule()
|
||||||
|
tag = 'infoleak:automatic-detection="cve"'
|
||||||
|
first = Tag.get_tag_first_seen(tag)
|
||||||
|
last = Tag.get_tag_last_seen(tag)
|
||||||
|
if first and last:
|
||||||
|
for date in Date.substract_date(first, last):
|
||||||
|
for item_id in Tag.get_tag_objects(tag, 'item', date=date):
|
||||||
|
module.compute(f'{item_id} 0')
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
|
|
||||||
#core_migration()
|
#core_migration()
|
||||||
#user_migration()
|
#user_migration()
|
||||||
#tags_migration()
|
#tags_migration()
|
||||||
items_migration()
|
# items_migration()
|
||||||
#crawler_migration()
|
#crawler_migration()
|
||||||
# domain_migration() # TO TEST ###########################
|
# domain_migration() # TO TEST ###########################
|
||||||
#decodeds_migration()
|
#decodeds_migration()
|
||||||
# screenshots_migration()
|
# screenshots_migration()
|
||||||
subtypes_obj_migration()
|
# subtypes_obj_migration()
|
||||||
# ail_2_ail_migration()
|
# ail_2_ail_migration()
|
||||||
# trackers_migration()
|
# trackers_migration()
|
||||||
# investigations_migration()
|
# investigations_migration()
|
||||||
# statistics_migration()
|
# statistics_migration()
|
||||||
|
|
||||||
|
cves_migration()
|
||||||
|
|
||||||
# custom tags
|
# custom tags
|
||||||
# crawler queues + auto_crawlers
|
# crawler queues + auto_crawlers
|
||||||
|
|
|
@ -5,6 +5,7 @@ import os
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
from flask import url_for
|
from flask import url_for
|
||||||
|
from pymisp import MISPObject
|
||||||
|
|
||||||
sys.path.append(os.environ['AIL_BIN'])
|
sys.path.append(os.environ['AIL_BIN'])
|
||||||
##################################
|
##################################
|
||||||
|
@ -20,12 +21,6 @@ baseurl = config_loader.get_config_str("Notifications", "ail_domain")
|
||||||
config_loader = None
|
config_loader = None
|
||||||
|
|
||||||
|
|
||||||
################################################################################
|
|
||||||
################################################################################
|
|
||||||
################################################################################
|
|
||||||
|
|
||||||
# # TODO: COMPLETE CLASS
|
|
||||||
|
|
||||||
class Cve(AbstractDaterangeObject):
|
class Cve(AbstractDaterangeObject):
|
||||||
"""
|
"""
|
||||||
AIL Cve Object.
|
AIL Cve Object.
|
||||||
|
@ -55,14 +50,21 @@ class Cve(AbstractDaterangeObject):
|
||||||
def get_svg_icon(self):
|
def get_svg_icon(self):
|
||||||
return {'style': 'fas', 'icon': '\uf188', 'color': '#1E88E5', 'radius': 5}
|
return {'style': 'fas', 'icon': '\uf188', 'color': '#1E88E5', 'radius': 5}
|
||||||
|
|
||||||
# TODO # TODO # TODO # TODO # TODO # TODO # TODO # TODO
|
|
||||||
def get_misp_object(self):
|
def get_misp_object(self):
|
||||||
pass
|
obj_attrs = []
|
||||||
|
obj = MISPObject('vulnerability')
|
||||||
|
obj.first_seen = self.get_first_seen()
|
||||||
|
obj.last_seen = self.get_last_seen()
|
||||||
|
|
||||||
|
obj_attrs.append(obj.add_attribute('id', value=self.id))
|
||||||
|
for obj_attr in obj_attrs:
|
||||||
|
for tag in self.get_tags():
|
||||||
|
obj_attr.add_tag(tag)
|
||||||
|
return obj
|
||||||
|
|
||||||
def get_meta(self, options=set()):
|
def get_meta(self, options=set()):
|
||||||
meta = self._get_meta(options=options)
|
meta = self._get_meta(options=options)
|
||||||
meta['id'] = self.id
|
meta['id'] = self.id
|
||||||
meta['subtype'] = self.subtype
|
|
||||||
meta['tags'] = self.get_tags(r_list=True)
|
meta['tags'] = self.get_tags(r_list=True)
|
||||||
return meta
|
return meta
|
||||||
|
|
||||||
|
@ -70,19 +72,21 @@ class Cve(AbstractDaterangeObject):
|
||||||
self._add(date, item_id)
|
self._add(date, item_id)
|
||||||
|
|
||||||
|
|
||||||
# TODO # TODO # TODO # TODO # TODO # TODO # TODO # TODO # TODO # TODO
|
# TODO ADD SEARCH FUNCTION
|
||||||
|
|
||||||
def get_all_cves():
|
def get_all_cves():
|
||||||
cves = []
|
return r_objects.smembers(f'cve:all')
|
||||||
return cves
|
|
||||||
|
|
||||||
def get_cves_by_date(date):
|
def get_cves_by_date(date):
|
||||||
# return r_objects.zrange(f'cve:date:{date}', 0, -1)
|
return r_objects.zrange(f'cve:date:{date}', 0, -1)
|
||||||
return set(r_objects.hkeys(f'cve:date:{date}'))
|
|
||||||
|
def get_nb_cves_by_date(date):
|
||||||
|
return r_objects.zcard(f'cve:date:{date}')
|
||||||
|
|
||||||
def get_cves_by_daterange(date_from, date_to):
|
def get_cves_by_daterange(date_from, date_to):
|
||||||
cves = set()
|
cves = set()
|
||||||
for date in Date.substract_date(date_from, date_to):
|
for date in Date.substract_date(date_from, date_to):
|
||||||
cves | get_cves_by_date(date)
|
cves = cves | set(get_cves_by_date(date))
|
||||||
return cves
|
return cves
|
||||||
|
|
||||||
def get_cves_meta(cves_id, options=set()):
|
def get_cves_meta(cves_id, options=set()):
|
||||||
|
@ -92,6 +96,14 @@ def get_cves_meta(cves_id, options=set()):
|
||||||
dict_cve[cve_id] = cve.get_meta(options=options)
|
dict_cve[cve_id] = cve.get_meta(options=options)
|
||||||
return dict_cve
|
return dict_cve
|
||||||
|
|
||||||
|
def api_get_cves_range_by_daterange(date_from, date_to):
|
||||||
|
cves = []
|
||||||
|
for date in Date.substract_date(date_from, date_to):
|
||||||
|
d = {'date': f'{date[0:4]}-{date[4:6]}-{date[6:8]}',
|
||||||
|
'CVE': get_nb_cves_by_date(date)}
|
||||||
|
cves.append(d)
|
||||||
|
return cves
|
||||||
|
|
||||||
def api_get_cves_meta_by_daterange(date_from, date_to):
|
def api_get_cves_meta_by_daterange(date_from, date_to):
|
||||||
date = Date.sanitise_date_range(date_from, date_to)
|
date = Date.sanitise_date_range(date_from, date_to)
|
||||||
return get_cves_meta(get_cves_by_daterange(date['date_from'], date['date_to']), options=['sparkline'])
|
return get_cves_meta(get_cves_by_daterange(date['date_from'], date['date_to']), options=['sparkline'])
|
||||||
|
|
|
@ -3,18 +3,19 @@
|
||||||
|
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
import redis
|
|
||||||
|
|
||||||
# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
|
|
||||||
|
|
||||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
|
||||||
import ConfigLoader
|
|
||||||
|
|
||||||
from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id
|
|
||||||
from flask import url_for
|
from flask import url_for
|
||||||
|
from pymisp import MISPObject
|
||||||
|
|
||||||
config_loader = ConfigLoader.ConfigLoader()
|
sys.path.append(os.environ['AIL_BIN'])
|
||||||
|
##################################
|
||||||
|
# Import Project packages
|
||||||
|
##################################
|
||||||
|
from lib.ConfigLoader import ConfigLoader
|
||||||
|
from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id
|
||||||
|
|
||||||
|
config_loader = ConfigLoader()
|
||||||
|
baseurl = config_loader.get_config_str("Notifications", "ail_domain")
|
||||||
config_loader = None
|
config_loader = None
|
||||||
|
|
||||||
|
|
||||||
|
@ -64,7 +65,7 @@ class Pgp(AbstractSubtypeObject):
|
||||||
icon = '\uf1fa'
|
icon = '\uf1fa'
|
||||||
else:
|
else:
|
||||||
icon = 'times'
|
icon = 'times'
|
||||||
return {'style': 'fas', 'icon': icon, 'color': '#44AA99', 'radius':5}
|
return {'style': 'fas', 'icon': icon, 'color': '#44AA99', 'radius': 5}
|
||||||
|
|
||||||
def get_misp_object(self):
|
def get_misp_object(self):
|
||||||
obj_attrs = []
|
obj_attrs = []
|
||||||
|
@ -72,12 +73,12 @@ class Pgp(AbstractSubtypeObject):
|
||||||
obj.first_seen = self.get_first_seen()
|
obj.first_seen = self.get_first_seen()
|
||||||
obj.last_seen = self.get_last_seen()
|
obj.last_seen = self.get_last_seen()
|
||||||
|
|
||||||
if self.subtype=='key':
|
if self.subtype == 'key':
|
||||||
obj_attrs.append( obj.add_attribute('key-id', value=self.id) )
|
obj_attrs.append(obj.add_attribute('key-id', value=self.id))
|
||||||
elif self.subtype=='name':
|
elif self.subtype == 'name':
|
||||||
obj_attrs.append( obj.add_attribute('user-id-name', value=self.id) )
|
obj_attrs.append(obj.add_attribute('user-id-name', value=self.id))
|
||||||
else: # mail
|
else: # mail
|
||||||
obj_attrs.append( obj.add_attribute('user-id-email', value=self.id) )
|
obj_attrs.append(obj.add_attribute('user-id-email', value=self.id))
|
||||||
|
|
||||||
for obj_attr in obj_attrs:
|
for obj_attr in obj_attrs:
|
||||||
for tag in self.get_tags():
|
for tag in self.get_tags():
|
||||||
|
@ -88,7 +89,6 @@ class Pgp(AbstractSubtypeObject):
|
||||||
############################################################################
|
############################################################################
|
||||||
|
|
||||||
def get_all_subtypes():
|
def get_all_subtypes():
|
||||||
#return get_object_all_subtypes(self.type)
|
|
||||||
return ['key', 'mail', 'name']
|
return ['key', 'mail', 'name']
|
||||||
|
|
||||||
def get_all_pgps():
|
def get_all_pgps():
|
||||||
|
@ -101,5 +101,4 @@ def get_all_pgps_by_subtype(subtype):
|
||||||
return get_all_id('pgp', subtype)
|
return get_all_id('pgp', subtype)
|
||||||
|
|
||||||
|
|
||||||
|
# if __name__ == '__main__':
|
||||||
#if __name__ == '__main__':
|
|
||||||
|
|
|
@ -68,7 +68,7 @@ class AbstractDaterangeObject(AbstractObject, ABC):
|
||||||
return r_object.hget(f'meta:{self.type}:{self.id}', 'nb')
|
return r_object.hget(f'meta:{self.type}:{self.id}', 'nb')
|
||||||
|
|
||||||
def get_nb_seen_by_date(self, date):
|
def get_nb_seen_by_date(self, date):
|
||||||
nb = r_object.hget(f'{self.type}:date:{date}', self.id)
|
nb = r_object.zscore(f'{self.type}:date:{date}', self.id)
|
||||||
if nb is None:
|
if nb is None:
|
||||||
return 0
|
return 0
|
||||||
else:
|
else:
|
||||||
|
@ -118,8 +118,8 @@ class AbstractDaterangeObject(AbstractObject, ABC):
|
||||||
update_obj_date(date, self.type)
|
update_obj_date(date, self.type)
|
||||||
|
|
||||||
# NB Object seen by day
|
# NB Object seen by day
|
||||||
r_object.hincrby(f'{self.type}:date:{date}', self.id, 1)
|
print(f'{self.type}:date:{date}', 1, self.id)
|
||||||
r_object.zincrby(f'{self.type}:date:{date}', 1, self.id) # # # # # # # # # #
|
r_object.zincrby(f'{self.type}:date:{date}', 1, self.id)
|
||||||
# NB Object seen
|
# NB Object seen
|
||||||
r_object.hincrby(f'meta:{self.type}:{self.id}', 'nb', 1)
|
r_object.hincrby(f'meta:{self.type}:{self.id}', 'nb', 1)
|
||||||
|
|
||||||
|
|
|
@ -51,7 +51,7 @@ class CveModule(AbstractModule):
|
||||||
|
|
||||||
cves = self.regex_findall(self.reg_cve, item_id, item.get_content())
|
cves = self.regex_findall(self.reg_cve, item_id, item.get_content())
|
||||||
if cves:
|
if cves:
|
||||||
print(cves)
|
# print(cves)
|
||||||
date = item.get_date()
|
date = item.get_date()
|
||||||
for cve_id in cves:
|
for cve_id in cves:
|
||||||
cve = Cves.Cve(cve_id)
|
cve = Cves.Cve(cve_id)
|
||||||
|
|
|
@ -41,11 +41,11 @@ def objects_cves():
|
||||||
date_from = date['date_from']
|
date_from = date['date_from']
|
||||||
date_to = date['date_to']
|
date_to = date['date_to']
|
||||||
|
|
||||||
# barchart_type
|
if show_objects:
|
||||||
# correlation_type_search_endpoint
|
dict_objects = Cves.api_get_cves_meta_by_daterange(date_from, date_to)
|
||||||
|
else:
|
||||||
|
dict_objects = {}
|
||||||
|
|
||||||
dict_objects = Cves.api_get_cves_meta_by_daterange(date_from, date_to)
|
|
||||||
print(date_from, date_to, dict_objects)
|
|
||||||
return render_template("CveDaterange.html", date_from=date_from, date_to=date_to,
|
return render_template("CveDaterange.html", date_from=date_from, date_to=date_to,
|
||||||
dict_objects=dict_objects, show_objects=show_objects)
|
dict_objects=dict_objects, show_objects=show_objects)
|
||||||
|
|
||||||
|
@ -62,7 +62,12 @@ def objects_cves_post():
|
||||||
@login_required
|
@login_required
|
||||||
@login_read_only
|
@login_read_only
|
||||||
def objects_cve_range_json():
|
def objects_cve_range_json():
|
||||||
return None
|
date_from = request.args.get('date_from')
|
||||||
|
date_to = request.args.get('date_to')
|
||||||
|
date = Date.sanitise_date_range(date_from, date_to)
|
||||||
|
date_from = date['date_from']
|
||||||
|
date_to = date['date_to']
|
||||||
|
return jsonify(Cves.api_get_cves_range_by_daterange(date_from, date_to))
|
||||||
|
|
||||||
@objects_cve.route("/objects/cve/search", methods=['POST'])
|
@objects_cve.route("/objects/cve/search", methods=['POST'])
|
||||||
@login_required
|
@login_required
|
||||||
|
|
Loading…
Reference in a new issue