mirror of
https://github.com/cve-search/cpe-guesser.git
synced 2024-11-14 19:08:27 +00:00
chg: [format] black -S
on all files
This commit is contained in:
parent
3d40abfd5c
commit
80f0542d22
4 changed files with 66 additions and 23 deletions
|
@ -13,10 +13,13 @@ import time
|
||||||
|
|
||||||
# Configuration
|
# Configuration
|
||||||
cpe_path = '../data/official-cpe-dictionary_v2.3.xml'
|
cpe_path = '../data/official-cpe-dictionary_v2.3.xml'
|
||||||
cpe_source = 'https://nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml.gz'
|
cpe_source = (
|
||||||
|
'https://nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml.gz'
|
||||||
|
)
|
||||||
rdb = redis.Redis(host='127.0.0.1', port=6379, db=8)
|
rdb = redis.Redis(host='127.0.0.1', port=6379, db=8)
|
||||||
|
|
||||||
class CPEHandler( xml.sax.ContentHandler ):
|
|
||||||
|
class CPEHandler(xml.sax.ContentHandler):
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.cpe = ""
|
self.cpe = ""
|
||||||
self.title = ""
|
self.title = ""
|
||||||
|
@ -52,19 +55,21 @@ class CPEHandler( xml.sax.ContentHandler ):
|
||||||
if tag == 'cpe-item':
|
if tag == 'cpe-item':
|
||||||
to_insert = CPEExtractor(cpe=self.record['cpe-23'])
|
to_insert = CPEExtractor(cpe=self.record['cpe-23'])
|
||||||
for word in canonize(to_insert['vendor']):
|
for word in canonize(to_insert['vendor']):
|
||||||
insert( word=word, cpe=to_insert['cpeline'] )
|
insert(word=word, cpe=to_insert['cpeline'])
|
||||||
self.wordcount += 1
|
self.wordcount += 1
|
||||||
for word in canonize(to_insert['product']):
|
for word in canonize(to_insert['product']):
|
||||||
insert( word=word, cpe=to_insert['cpeline'] )
|
insert(word=word, cpe=to_insert['cpeline'])
|
||||||
self.wordcount += 1
|
self.wordcount += 1
|
||||||
self.record = {}
|
self.record = {}
|
||||||
self.itemcount += 1
|
self.itemcount += 1
|
||||||
if self.itemcount % 5000 == 0:
|
if self.itemcount % 5000 == 0:
|
||||||
time_elapsed = round( time.time() - self.start_time )
|
time_elapsed = round(time.time() - self.start_time)
|
||||||
print (f"... {self.itemcount} items processed ({self.wordcount} words) in {time_elapsed} seconds")
|
print(
|
||||||
|
f"... {self.itemcount} items processed ({self.wordcount} words) in {time_elapsed} seconds"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def CPEExtractor( cpe=None ):
|
def CPEExtractor(cpe=None):
|
||||||
if cpe is None:
|
if cpe is None:
|
||||||
return False
|
return False
|
||||||
record = {}
|
record = {}
|
||||||
|
@ -77,12 +82,14 @@ def CPEExtractor( cpe=None ):
|
||||||
record['cpeline'] = cpeline[1:]
|
record['cpeline'] = cpeline[1:]
|
||||||
return record
|
return record
|
||||||
|
|
||||||
def canonize( value=None ):
|
|
||||||
|
def canonize(value=None):
|
||||||
value = value.lower()
|
value = value.lower()
|
||||||
words = value.split('_')
|
words = value.split('_')
|
||||||
return words
|
return words
|
||||||
|
|
||||||
def insert( word=None, cpe=None):
|
|
||||||
|
def insert(word=None, cpe=None):
|
||||||
if cpe is None or word is None:
|
if cpe is None or word is None:
|
||||||
return False
|
return False
|
||||||
rdb.sadd(f"w:{word}", cpe)
|
rdb.sadd(f"w:{word}", cpe)
|
||||||
|
@ -91,10 +98,30 @@ def insert( word=None, cpe=None):
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
argparser = argparse.ArgumentParser(description='Initializes the Redis database with CPE dictionary.')
|
argparser = argparse.ArgumentParser(
|
||||||
argparser.add_argument('--download', '-d', action='count', default=0, help='Download the CPE dictionary even if it already exists.')
|
description='Initializes the Redis database with CPE dictionary.'
|
||||||
argparser.add_argument('--replace', '-r', action='count', default=0, help='Flush and repopulated the CPE database.')
|
)
|
||||||
argparser.add_argument('--update', '-u', action='store_true', default=False, help='Update the CPE database without flushing')
|
argparser.add_argument(
|
||||||
|
'--download',
|
||||||
|
'-d',
|
||||||
|
action='count',
|
||||||
|
default=0,
|
||||||
|
help='Download the CPE dictionary even if it already exists.',
|
||||||
|
)
|
||||||
|
argparser.add_argument(
|
||||||
|
'--replace',
|
||||||
|
'-r',
|
||||||
|
action='count',
|
||||||
|
default=0,
|
||||||
|
help='Flush and repopulated the CPE database.',
|
||||||
|
)
|
||||||
|
argparser.add_argument(
|
||||||
|
'--update',
|
||||||
|
'-u',
|
||||||
|
action='store_true',
|
||||||
|
default=False,
|
||||||
|
help='Update the CPE database without flushing',
|
||||||
|
)
|
||||||
args = argparser.parse_args()
|
args = argparser.parse_args()
|
||||||
|
|
||||||
if args.replace == 0 and rdb.dbsize() > 0 and not args.update:
|
if args.replace == 0 and rdb.dbsize() > 0 and not args.update:
|
||||||
|
@ -106,7 +133,12 @@ if __name__ == '__main__':
|
||||||
print(f"Downloading CPE data from {cpe_source} ...")
|
print(f"Downloading CPE data from {cpe_source} ...")
|
||||||
try:
|
try:
|
||||||
urllib.request.urlretrieve(cpe_source, f"{cpe_path}.gz")
|
urllib.request.urlretrieve(cpe_source, f"{cpe_path}.gz")
|
||||||
except (urllib.error.HTTPError, urllib.error.URLError, FileNotFoundError, PermissionError) as e:
|
except (
|
||||||
|
urllib.error.HTTPError,
|
||||||
|
urllib.error.URLError,
|
||||||
|
FileNotFoundError,
|
||||||
|
PermissionError,
|
||||||
|
) as e:
|
||||||
print(e)
|
print(e)
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
|
@ -130,6 +162,6 @@ if __name__ == '__main__':
|
||||||
print("Populating the database (please be patient)...")
|
print("Populating the database (please be patient)...")
|
||||||
parser = xml.sax.make_parser()
|
parser = xml.sax.make_parser()
|
||||||
Handler = CPEHandler()
|
Handler = CPEHandler()
|
||||||
parser.setContentHandler( Handler )
|
parser.setContentHandler(Handler)
|
||||||
parser.parse(cpe_path)
|
parser.parse(cpe_path)
|
||||||
print(f"Done! {rdb.dbsize()} keys inserted.")
|
print(f"Done! {rdb.dbsize()} keys inserted.")
|
||||||
|
|
|
@ -11,8 +11,16 @@ sys.path.append(os.path.join(runPath, ".."))
|
||||||
from lib.cpeguesser import CPEGuesser
|
from lib.cpeguesser import CPEGuesser
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
parser = argparse.ArgumentParser(description='Find potential CPE names from a list of keyword(s) and return a JSON of the results')
|
parser = argparse.ArgumentParser(
|
||||||
parser.add_argument('word', metavar='WORD', type=str, nargs='+', help='One or more keyword(s) to lookup')
|
description='Find potential CPE names from a list of keyword(s) and return a JSON of the results'
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
'word',
|
||||||
|
metavar='WORD',
|
||||||
|
type=str,
|
||||||
|
nargs='+',
|
||||||
|
help='One or more keyword(s) to lookup',
|
||||||
|
)
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
cpeGuesser = CPEGuesser()
|
cpeGuesser = CPEGuesser()
|
||||||
|
|
|
@ -14,7 +14,8 @@ runPath = os.path.dirname(os.path.realpath(__file__))
|
||||||
sys.path.append(os.path.join(runPath, ".."))
|
sys.path.append(os.path.join(runPath, ".."))
|
||||||
from lib.cpeguesser import CPEGuesser
|
from lib.cpeguesser import CPEGuesser
|
||||||
|
|
||||||
class Search():
|
|
||||||
|
class Search:
|
||||||
def on_post(self, req, resp):
|
def on_post(self, req, resp):
|
||||||
data_post = req.bounded_stream.read()
|
data_post = req.bounded_stream.read()
|
||||||
js = data_post.decode('utf-8')
|
js = data_post.decode('utf-8')
|
||||||
|
@ -33,7 +34,8 @@ class Search():
|
||||||
return
|
return
|
||||||
|
|
||||||
cpeGuesser = CPEGuesser()
|
cpeGuesser = CPEGuesser()
|
||||||
resp.media=cpeGuesser.guessCpe(q['query'])
|
resp.media = cpeGuesser.guessCpe(q['query'])
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
app = falcon.App()
|
app = falcon.App()
|
||||||
|
@ -44,7 +46,7 @@ if __name__ == '__main__':
|
||||||
print(f"Serving on port {port}...")
|
print(f"Serving on port {port}...")
|
||||||
httpd.serve_forever()
|
httpd.serve_forever()
|
||||||
except OSError as e:
|
except OSError as e:
|
||||||
print (e)
|
print(e)
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
except KeyboardInterrupt:
|
except KeyboardInterrupt:
|
||||||
sys.exit(0)
|
sys.exit(0)
|
||||||
|
|
|
@ -3,12 +3,13 @@
|
||||||
|
|
||||||
import redis
|
import redis
|
||||||
|
|
||||||
class CPEGuesser():
|
|
||||||
|
class CPEGuesser:
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.rdb = redis.Redis(host='127.0.0.1', port=6379, db=8, decode_responses=True)
|
self.rdb = redis.Redis(host='127.0.0.1', port=6379, db=8, decode_responses=True)
|
||||||
|
|
||||||
def guessCpe(self, words):
|
def guessCpe(self, words):
|
||||||
k=[]
|
k = []
|
||||||
for keyword in words:
|
for keyword in words:
|
||||||
k.append(f"w:{keyword.lower()}")
|
k.append(f"w:{keyword.lower()}")
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue