mirror of
https://github.com/MISP/misp-galaxy.git
synced 2024-11-30 10:47:17 +00:00
52 lines
1.4 KiB
Python
52 lines
1.4 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
# coding=utf-8
|
||
|
|
"""
|
||
|
|
Tools to find duplicate in galaxies
|
||
|
|
"""
|
||
|
|
import json
|
||
|
|
import os
|
||
|
|
import collections
|
||
|
|
|
||
|
|
|
||
|
|
def loadjsons(path):
|
||
|
|
"""
|
||
|
|
Find all Jsons and load them in a dict
|
||
|
|
"""
|
||
|
|
files = []
|
||
|
|
data = []
|
||
|
|
for name in os.listdir(path):
|
||
|
|
if os.path.isfile(os.path.join(path, name)) and name.endswith('.json'):
|
||
|
|
files.append(name)
|
||
|
|
for jfile in files:
|
||
|
|
data.append(json.load(open("%s/%s" % (path, jfile))))
|
||
|
|
return data
|
||
|
|
|
||
|
|
if __name__ == '__main__':
|
||
|
|
"""
|
||
|
|
Iterate all name + synonyms
|
||
|
|
tell what is duplicated.
|
||
|
|
"""
|
||
|
|
jsons = loadjsons("../clusters")
|
||
|
|
counter = collections.Counter()
|
||
|
|
namespace = []
|
||
|
|
for djson in jsons:
|
||
|
|
items = djson.get('values')
|
||
|
|
for entry in items:
|
||
|
|
name = entry.get('value').strip().lower()
|
||
|
|
counter[name]+=1
|
||
|
|
namespace.append([name, djson.get('name')])
|
||
|
|
try:
|
||
|
|
for synonym in entry.get('meta').get('synonyms'):
|
||
|
|
name = synonym.strip().lower()
|
||
|
|
counter[name]+=1
|
||
|
|
namespace.append([name, djson.get('name')])
|
||
|
|
except (AttributeError, TypeError):
|
||
|
|
pass
|
||
|
|
counter = dict(counter)
|
||
|
|
for key, val in counter.items():
|
||
|
|
if val>1:
|
||
|
|
print ("Warning duplicate %s" % key)
|
||
|
|
for item in namespace:
|
||
|
|
if item[0]==key:
|
||
|
|
print (item)
|