mirror of
https://github.com/MISP/misp-galaxy.git
synced 2024-11-23 07:17:17 +00:00
Add [tool] statistics
This commit is contained in:
niclas
parent
9a0fca647b
commit
94e0b855d1
5 changed files with 210 additions and 246 deletions
|
|
@ -1,68 +1,26 @@
|
||||||
#!/usr/bin/python
|
from modules.universe import Universe
|
||||||
|
from modules.site import IndexSite, StatisticsSite
|
||||||
|
|
||||||
from modules.galaxy import Galaxy
|
import multiprocessing
|
||||||
from modules.statistics import Statistics
|
from multiprocessing import Pool
|
||||||
|
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import time
|
import time
|
||||||
|
import sys
|
||||||
|
|
||||||
|
sys.setrecursionlimit(10000)
|
||||||
|
|
||||||
|
FILES_TO_IGNORE = []
|
||||||
CLUSTER_PATH = "../../clusters"
|
CLUSTER_PATH = "../../clusters"
|
||||||
SITE_PATH = "./site/docs"
|
SITE_PATH = "./site/docs"
|
||||||
GALAXY_PATH = "../../galaxies"
|
GALAXY_PATH = "../../galaxies"
|
||||||
|
|
||||||
|
def get_cluster_relationships(cluster_data):
|
||||||
FILES_TO_IGNORE = [] # if you want to skip a specific cluster in the generation
|
galaxy, cluster = cluster_data
|
||||||
|
relationships = universe.get_relationships_with_levels(universe.galaxies[galaxy].clusters[cluster])
|
||||||
|
print(f"Processed {galaxy}, {cluster}")
|
||||||
INTRO = """
|
return cluster, galaxy, relationships
|
||||||
# MISP Galaxy
|
|
||||||
|
|
||||||
The MISP galaxy offers a streamlined approach for representing large entities, known as clusters, which can be linked to MISP events or attributes. Each cluster consists of one or more elements, represented as key-value pairs. MISP galaxy comes with a default knowledge base, encompassing areas like Threat Actors, Tools, Ransomware, and ATT&CK matrices. However, users have the flexibility to modify, update, replace, or share these elements according to their needs.
|
|
||||||
|
|
||||||
Clusters and vocabularies within MISP galaxy can be utilized in their original form or as a foundational knowledge base. The distribution settings for each cluster can be adjusted, allowing for either restricted or wide dissemination.
|
|
||||||
|
|
||||||
Additionally, MISP galaxies enable the representation of existing standards like the MITRE ATT&CK™ framework, as well as custom matrices.
|
|
||||||
|
|
||||||
The aim is to provide a core set of clusters for organizations embarking on analysis, which can be further tailored to include localized, private information or additional, shareable data.
|
|
||||||
|
|
||||||
Clusters serve as an open and freely accessible knowledge base, which can be utilized and expanded within [MISP](https://www.misp-project.org/) or other threat intelligence platforms.
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
## Publicly available clusters
|
|
||||||
|
|
||||||
"""
|
|
||||||
|
|
||||||
STATISTICS = """
|
|
||||||
## Statistics
|
|
||||||
|
|
||||||
You can find some statistics about MISP galaxies [here](./statistics.md).
|
|
||||||
|
|
||||||
"""
|
|
||||||
|
|
||||||
CONTRIBUTING = """
|
|
||||||
|
|
||||||
# Contributing
|
|
||||||
|
|
||||||
In the dynamic realm of threat intelligence, a variety of models and approaches exist to systematically organize, categorize, and delineate threat actors, hazards, or activity groups. We embrace innovative methodologies for articulating threat intelligence. The galaxy model is particularly versatile, enabling you to leverage and integrate methodologies that you trust and are already utilizing within your organization or community.
|
|
||||||
|
|
||||||
We encourage collaboration and contributions to the [MISP Galaxy JSON files](https://github.com/MISP/misp-galaxy/). Feel free to fork the project, enhance existing elements or clusters, or introduce new ones. Your insights are valuable - share them with us through a pull-request.
|
|
||||||
|
|
||||||
"""
|
|
||||||
|
|
||||||
def write_galaxy_entry(galaxy, site_path, cluster_dict):
|
|
||||||
galaxy.write_entry(site_path, cluster_dict)
|
|
||||||
return f"Finished writing entry for {galaxy.name}"
|
|
||||||
|
|
||||||
def create_index(galaxies):
|
|
||||||
index_output = INTRO
|
|
||||||
for galaxy in galaxies:
|
|
||||||
index_output += f"- [{galaxy.name}](./{galaxy.json_file_name}/index.md)\n"
|
|
||||||
index_output += STATISTICS
|
|
||||||
index_output += CONTRIBUTING
|
|
||||||
return index_output
|
|
||||||
|
|
||||||
|
|
||||||
def get_deprecated_galaxy_files():
|
def get_deprecated_galaxy_files():
|
||||||
deprecated_galaxy_files = []
|
deprecated_galaxy_files = []
|
||||||
|
|
@ -74,9 +32,39 @@ def get_deprecated_galaxy_files():
|
||||||
|
|
||||||
return deprecated_galaxy_files
|
return deprecated_galaxy_files
|
||||||
|
|
||||||
|
def cluster_transform_to_link(cluster):
|
||||||
|
placeholder = "__TMP__"
|
||||||
|
section = (
|
||||||
|
cluster
|
||||||
|
.value.lower()
|
||||||
|
.replace(" - ", placeholder) # Replace " - " first
|
||||||
|
.replace(" ", "-")
|
||||||
|
.replace("/", "")
|
||||||
|
.replace(":", "")
|
||||||
|
.replace(placeholder, "-")
|
||||||
|
)
|
||||||
|
galaxy_folder = cluster.galaxy.json_file_name.replace(".json", "")
|
||||||
|
return f"[{cluster.value} ({cluster.uuid})](../../{galaxy_folder}/index.md#{section})"
|
||||||
|
|
||||||
def main():
|
def galaxy_transform_to_link(galaxy):
|
||||||
|
galaxy_folder = galaxy.json_file_name.replace(".json", "")
|
||||||
|
return f"[{galaxy.galaxy_name}](../../{galaxy_folder}/index.md)"
|
||||||
|
|
||||||
|
def generate_relations_table(relationships):
|
||||||
|
markdown = "|Cluster A | Galaxy A | Cluster B | Galaxy B | Level { .graph } |\n"
|
||||||
|
markdown += "| --- | --- | --- | --- | --- |\n"
|
||||||
|
for from_cluster, to_cluster, level in relationships:
|
||||||
|
from_galaxy = from_cluster.galaxy
|
||||||
|
if to_cluster.value != "Private Cluster":
|
||||||
|
to_galaxy = to_cluster.galaxy
|
||||||
|
markdown += f"{cluster_transform_to_link(from_cluster)} | {galaxy_transform_to_link(from_galaxy)} | {cluster_transform_to_link(to_cluster)} | {galaxy_transform_to_link(to_galaxy)} | {level}\n"
|
||||||
|
else:
|
||||||
|
markdown += f"{cluster_transform_to_link(from_cluster)} | {galaxy_transform_to_link(from_galaxy)} | {to_cluster.value} | Unknown | {level}\n"
|
||||||
|
return markdown
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
start_time = time.time()
|
start_time = time.time()
|
||||||
|
universe = Universe()
|
||||||
|
|
||||||
FILES_TO_IGNORE.extend(get_deprecated_galaxy_files())
|
FILES_TO_IGNORE.extend(get_deprecated_galaxy_files())
|
||||||
galaxies_fnames = []
|
galaxies_fnames = []
|
||||||
|
|
@ -85,45 +73,78 @@ def main():
|
||||||
galaxies_fnames.append(f)
|
galaxies_fnames.append(f)
|
||||||
galaxies_fnames.sort()
|
galaxies_fnames.sort()
|
||||||
|
|
||||||
galaxies = []
|
# Create the universe of clusters and galaxies
|
||||||
for galaxy in galaxies_fnames:
|
for galaxy in galaxies_fnames:
|
||||||
with open(os.path.join(CLUSTER_PATH, galaxy)) as fr:
|
with open(os.path.join(CLUSTER_PATH, galaxy)) as fr:
|
||||||
galaxy_json = json.load(fr)
|
galaxy_json = json.load(fr)
|
||||||
galaxies.append(
|
universe.add_galaxy(galaxy_name=galaxy_json["name"], json_file_name=galaxy, authors=galaxy_json["authors"], description=galaxy_json["description"])
|
||||||
Galaxy(
|
for cluster in galaxy_json["values"]:
|
||||||
cluster_list=galaxy_json["values"],
|
universe.add_cluster(
|
||||||
authors=galaxy_json["authors"],
|
galaxy_name=galaxy_json.get("name", None),
|
||||||
description=galaxy_json["description"],
|
uuid=cluster.get("uuid", None),
|
||||||
name=galaxy_json["name"],
|
description=cluster.get("description", None),
|
||||||
json_file_name=galaxy.split(".")[0],
|
value=cluster.get("value", None),
|
||||||
|
meta=cluster.get("meta", None)
|
||||||
)
|
)
|
||||||
)
|
|
||||||
cluster_dict = {}
|
|
||||||
for galaxy in galaxies:
|
|
||||||
for cluster in galaxy.clusters:
|
|
||||||
cluster_dict[cluster.uuid] = cluster
|
|
||||||
|
|
||||||
statistics = Statistics(cluster_dict=cluster_dict)
|
# Define the relationships between clusters
|
||||||
for galaxy in galaxies:
|
for galaxy in galaxies_fnames:
|
||||||
for cluster in galaxy.clusters:
|
with open(os.path.join(CLUSTER_PATH, galaxy)) as fr:
|
||||||
statistics.add_cluster(cluster)
|
galaxy_json = json.load(fr)
|
||||||
|
for cluster in galaxy_json["values"]:
|
||||||
|
if "related" in cluster:
|
||||||
|
for related in cluster["related"]:
|
||||||
|
universe.define_relationship(cluster["uuid"], related["dest-uuid"])
|
||||||
|
|
||||||
# Write files
|
tasks = []
|
||||||
|
for galaxy_name, galaxy in universe.galaxies.items():
|
||||||
|
for cluster_name, cluster in galaxy.clusters.items():
|
||||||
|
tasks.append((galaxy_name, cluster_name))
|
||||||
|
|
||||||
|
with Pool(processes=multiprocessing.cpu_count()) as pool:
|
||||||
|
result = pool.map(get_cluster_relationships, tasks)
|
||||||
|
|
||||||
|
for cluster, galaxy, relationships in result:
|
||||||
|
universe.galaxies[galaxy].clusters[cluster].relationships = relationships
|
||||||
|
|
||||||
|
print("All clusters processed.")
|
||||||
|
|
||||||
|
print(f"Finished relations in {time.time() - start_time} seconds")
|
||||||
|
|
||||||
|
# Write output
|
||||||
if not os.path.exists(SITE_PATH):
|
if not os.path.exists(SITE_PATH):
|
||||||
os.mkdir(SITE_PATH)
|
os.mkdir(SITE_PATH)
|
||||||
|
|
||||||
for galaxy in galaxies:
|
index = IndexSite(SITE_PATH)
|
||||||
galaxy.write_entry(SITE_PATH, cluster_dict)
|
index.add_content("# MISP Galaxy\n\nThe MISP galaxy offers a streamlined approach for representing large entities, known as clusters, which can be linked to MISP events or attributes. Each cluster consists of one or more elements, represented as key-value pairs. MISP galaxy comes with a default knowledge base, encompassing areas like Threat Actors, Tools, Ransomware, and ATT&CK matrices. However, users have the flexibility to modify, update, replace, or share these elements according to their needs.\n\nClusters and vocabularies within MISP galaxy can be utilized in their original form or as a foundational knowledge base. The distribution settings for each cluster can be adjusted, allowing for either restricted or wide dissemination.\n\nAdditionally, MISP galaxies enable the representation of existing standards like the MITRE ATT&CK™ framework, as well as custom matrices.\n\nThe aim is to provide a core set of clusters for organizations embarking on analysis, which can be further tailored to include localized, private information or additional, shareable data.\n\nClusters serve as an open and freely accessible knowledge base, which can be utilized and expanded within [MISP](https://www.misp-project.org/) or other threat intelligence platforms.\n\n\n\n## Publicly available clusters\n")
|
||||||
|
index.add_toc(universe.galaxies.values())
|
||||||
|
index.add_content("## Statistics\n\nYou can find some statistics about MISP galaxies [here](./statistics.md).\n\n")
|
||||||
|
index.add_content("# Contributing\n\nIn the dynamic realm of threat intelligence, a variety of models and approaches exist to systematically organize, categorize, and delineate threat actors, hazards, or activity groups. We embrace innovative methodologies for articulating threat intelligence. The galaxy model is particularly versatile, enabling you to leverage and integrate methodologies that you trust and are already utilizing within your organization or community.\n\nWe encourage collaboration and contributions to the [MISP Galaxy JSON files](https://github.com/MISP/misp-galaxy/). Feel free to fork the project, enhance existing elements or clusters, or introduce new ones. Your insights are valuable - share them with us through a pull-request.\n")
|
||||||
|
index.write_entry()
|
||||||
|
|
||||||
index_output = create_index(galaxies)
|
statistics = StatisticsSite(SITE_PATH)
|
||||||
|
statistics.add_galaxy_statistics(universe.galaxies.values())
|
||||||
|
statistics.add_cluster_statistics([cluster for galaxy in universe.galaxies.values() for cluster in galaxy.clusters.values()])
|
||||||
|
statistics.add_relation_statistics([cluster for galaxy in universe.galaxies.values() for cluster in galaxy.clusters.values()])
|
||||||
|
statistics.write_entry()
|
||||||
|
|
||||||
statistics.write_entry(SITE_PATH)
|
for galaxy in universe.galaxies.values():
|
||||||
|
galaxy.write_entry(SITE_PATH)
|
||||||
|
|
||||||
with open(os.path.join(SITE_PATH, "index.md"), "w") as index:
|
for galaxy in universe.galaxies.values():
|
||||||
index.write(index_output)
|
galaxy_path = os.path.join(SITE_PATH, f"{galaxy.json_file_name}".replace(".json", ""))
|
||||||
|
if not os.path.exists(galaxy_path):
|
||||||
|
os.mkdir(galaxy_path)
|
||||||
|
relation_path = os.path.join(galaxy_path, "relations")
|
||||||
|
if not os.path.exists(relation_path):
|
||||||
|
os.mkdir(relation_path)
|
||||||
|
with open(os.path.join(relation_path, ".pages"), "w") as index:
|
||||||
|
index.write(f"hide: true\n")
|
||||||
|
|
||||||
print(f"Finished file creation in {time.time() - start_time} seconds")
|
for cluster in galaxy.clusters.values():
|
||||||
|
if cluster.relationships:
|
||||||
|
print(f"Writing {cluster.uuid}.md")
|
||||||
|
with open(os.path.join(relation_path, f"{cluster.uuid}.md"), "w") as index:
|
||||||
|
index.write(generate_relations_table(cluster.relationships))
|
||||||
|
|
||||||
|
print(f"Finished in {time.time() - start_time} seconds")
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
|
|
|
||||||
|
|
@ -1,150 +0,0 @@
|
||||||
from modules.universe import Universe
|
|
||||||
|
|
||||||
import multiprocessing
|
|
||||||
from multiprocessing import Pool
|
|
||||||
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import time
|
|
||||||
import sys
|
|
||||||
|
|
||||||
sys.setrecursionlimit(10000)
|
|
||||||
|
|
||||||
FILES_TO_IGNORE = []
|
|
||||||
CLUSTER_PATH = "../../clusters"
|
|
||||||
SITE_PATH = "./site/docs"
|
|
||||||
GALAXY_PATH = "../../galaxies"
|
|
||||||
|
|
||||||
def get_cluster_relationships(cluster_data):
|
|
||||||
# Unpack cluster data
|
|
||||||
galaxy, cluster = cluster_data
|
|
||||||
relationships = universe.get_relationships_with_levels(universe.galaxies[galaxy].clusters[cluster])
|
|
||||||
|
|
||||||
print(f"Processed {galaxy}, {cluster}")
|
|
||||||
return cluster, galaxy, relationships
|
|
||||||
|
|
||||||
def get_deprecated_galaxy_files():
|
|
||||||
deprecated_galaxy_files = []
|
|
||||||
for f in os.listdir(GALAXY_PATH):
|
|
||||||
with open(os.path.join(GALAXY_PATH, f)) as fr:
|
|
||||||
galaxy_json = json.load(fr)
|
|
||||||
if "namespace" in galaxy_json and galaxy_json["namespace"] == "deprecated":
|
|
||||||
deprecated_galaxy_files.append(f)
|
|
||||||
|
|
||||||
return deprecated_galaxy_files
|
|
||||||
|
|
||||||
def cluster_transform_to_link(cluster):
|
|
||||||
placeholder = "__TMP__"
|
|
||||||
section = (
|
|
||||||
cluster
|
|
||||||
.value.lower()
|
|
||||||
.replace(" - ", placeholder) # Replace " - " first
|
|
||||||
.replace(" ", "-")
|
|
||||||
.replace("/", "")
|
|
||||||
.replace(":", "")
|
|
||||||
.replace(placeholder, "-")
|
|
||||||
)
|
|
||||||
galaxy_folder = cluster.galaxy.json_file_name.replace(".json", "")
|
|
||||||
return f"[{cluster.value} ({cluster.uuid})](../../{galaxy_folder}/index.md#{section})"
|
|
||||||
|
|
||||||
def galaxy_transform_to_link(galaxy):
|
|
||||||
galaxy_folder = galaxy.json_file_name.replace(".json", "")
|
|
||||||
return f"[{galaxy.galaxy_name}](../../{galaxy_folder}/index.md)"
|
|
||||||
|
|
||||||
def generate_relations_table(relationships):
|
|
||||||
markdown = "|Cluster A | Galaxy A | Cluster B | Galaxy B | Level { .graph } |\n"
|
|
||||||
markdown += "| --- | --- | --- | --- | --- |\n"
|
|
||||||
for from_cluster, to_cluster, level in relationships:
|
|
||||||
from_galaxy = from_cluster.galaxy
|
|
||||||
if to_cluster.value != "Private Cluster":
|
|
||||||
to_galaxy = to_cluster.galaxy
|
|
||||||
markdown += f"{cluster_transform_to_link(from_cluster)} | {galaxy_transform_to_link(from_galaxy)} | {cluster_transform_to_link(to_cluster)} | {galaxy_transform_to_link(to_galaxy)} | {level}\n"
|
|
||||||
else:
|
|
||||||
markdown += f"{cluster_transform_to_link(from_cluster)} | {galaxy_transform_to_link(from_galaxy)} | {to_cluster.value} | Unknown | {level}\n"
|
|
||||||
return markdown
|
|
||||||
|
|
||||||
def generate_index_page(galaxies):
|
|
||||||
index_output = "# MISP Galaxy\n\nThe MISP galaxy offers a streamlined approach for representing large entities, known as clusters, which can be linked to MISP events or attributes. Each cluster consists of one or more elements, represented as key-value pairs. MISP galaxy comes with a default knowledge base, encompassing areas like Threat Actors, Tools, Ransomware, and ATT&CK matrices. However, users have the flexibility to modify, update, replace, or share these elements according to their needs.\n\nClusters and vocabularies within MISP galaxy can be utilized in their original form or as a foundational knowledge base. The distribution settings for each cluster can be adjusted, allowing for either restricted or wide dissemination.\n\nAdditionally, MISP galaxies enable the representation of existing standards like the MITRE ATT&CK™ framework, as well as custom matrices.\n\nThe aim is to provide a core set of clusters for organizations embarking on analysis, which can be further tailored to include localized, private information or additional, shareable data.\n\nClusters serve as an open and freely accessible knowledge base, which can be utilized and expanded within [MISP](https://www.misp-project.org/) or other threat intelligence platforms.\n\n\n\n## Publicly available clusters\n"
|
|
||||||
for galaxy in galaxies:
|
|
||||||
galaxy_folder = galaxy.json_file_name.replace(".json", "")
|
|
||||||
index_output += f"- [{galaxy.galaxy_name}](./{galaxy_folder}/index.md)\n"
|
|
||||||
index_output += "## Statistics\n\nYou can find some statistics about MISP galaxies [here](./statistics.md).\n"
|
|
||||||
index_output += "# Contributing\n\nIn the dynamic realm of threat intelligence, a variety of models and approaches exist to systematically organize, categorize, and delineate threat actors, hazards, or activity groups. We embrace innovative methodologies for articulating threat intelligence. The galaxy model is particularly versatile, enabling you to leverage and integrate methodologies that you trust and are already utilizing within your organization or community.\n\nWe encourage collaboration and contributions to the [MISP Galaxy JSON files](https://github.com/MISP/misp-galaxy/). Feel free to fork the project, enhance existing elements or clusters, or introduce new ones. Your insights are valuable - share them with us through a pull-request.\n"
|
|
||||||
return index_output
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
start_time = time.time()
|
|
||||||
universe = Universe()
|
|
||||||
|
|
||||||
FILES_TO_IGNORE.extend(get_deprecated_galaxy_files())
|
|
||||||
galaxies_fnames = []
|
|
||||||
for f in os.listdir(CLUSTER_PATH):
|
|
||||||
if ".json" in f and f not in FILES_TO_IGNORE:
|
|
||||||
galaxies_fnames.append(f)
|
|
||||||
galaxies_fnames.sort()
|
|
||||||
|
|
||||||
# Create the universe of clusters and galaxies
|
|
||||||
for galaxy in galaxies_fnames:
|
|
||||||
with open(os.path.join(CLUSTER_PATH, galaxy)) as fr:
|
|
||||||
galaxy_json = json.load(fr)
|
|
||||||
universe.add_galaxy(galaxy_name=galaxy_json["name"], json_file_name=galaxy, authors=galaxy_json["authors"], description=galaxy_json["description"])
|
|
||||||
for cluster in galaxy_json["values"]:
|
|
||||||
universe.add_cluster(
|
|
||||||
galaxy_name=galaxy_json.get("name", None),
|
|
||||||
uuid=cluster.get("uuid", None),
|
|
||||||
description=cluster.get("description", None),
|
|
||||||
value=cluster.get("value", None),
|
|
||||||
meta=cluster.get("meta", None)
|
|
||||||
)
|
|
||||||
|
|
||||||
# Define the relationships between clusters
|
|
||||||
for galaxy in galaxies_fnames:
|
|
||||||
with open(os.path.join(CLUSTER_PATH, galaxy)) as fr:
|
|
||||||
galaxy_json = json.load(fr)
|
|
||||||
for cluster in galaxy_json["values"]:
|
|
||||||
if "related" in cluster:
|
|
||||||
for related in cluster["related"]:
|
|
||||||
universe.define_relationship(cluster["uuid"], related["dest-uuid"])
|
|
||||||
|
|
||||||
tasks = []
|
|
||||||
for galaxy_name, galaxy in universe.galaxies.items():
|
|
||||||
for cluster_name, cluster in galaxy.clusters.items():
|
|
||||||
tasks.append((galaxy_name, cluster_name))
|
|
||||||
|
|
||||||
with Pool(processes=multiprocessing.cpu_count()) as pool:
|
|
||||||
result = pool.map(get_cluster_relationships, tasks)
|
|
||||||
|
|
||||||
for cluster, galaxy, relationships in result:
|
|
||||||
universe.galaxies[galaxy].clusters[cluster].relationships = relationships
|
|
||||||
|
|
||||||
print("All clusters processed.")
|
|
||||||
|
|
||||||
print(f"Finished relations in {time.time() - start_time} seconds")
|
|
||||||
|
|
||||||
# Write output
|
|
||||||
if not os.path.exists(SITE_PATH):
|
|
||||||
os.mkdir(SITE_PATH)
|
|
||||||
|
|
||||||
with open(os.path.join(SITE_PATH, "index.md"), "w") as index:
|
|
||||||
index.write(generate_index_page(universe.galaxies.values()))
|
|
||||||
|
|
||||||
for galaxy in universe.galaxies.values():
|
|
||||||
galaxy.write_entry(SITE_PATH)
|
|
||||||
|
|
||||||
for galaxy in universe.galaxies.values():
|
|
||||||
galaxy_path = os.path.join(SITE_PATH, f"{galaxy.json_file_name}".replace(".json", ""))
|
|
||||||
if not os.path.exists(galaxy_path):
|
|
||||||
os.mkdir(galaxy_path)
|
|
||||||
relation_path = os.path.join(galaxy_path, "relations")
|
|
||||||
if not os.path.exists(relation_path):
|
|
||||||
os.mkdir(relation_path)
|
|
||||||
with open(os.path.join(relation_path, ".pages"), "w") as index:
|
|
||||||
index.write(f"hide: true\n")
|
|
||||||
|
|
||||||
for cluster in galaxy.clusters.values():
|
|
||||||
if cluster.relationships:
|
|
||||||
print(f"Writing {cluster.uuid}.md")
|
|
||||||
with open(os.path.join(relation_path, f"{cluster.uuid}.md"), "w") as index:
|
|
||||||
index.write(generate_relations_table(cluster.relationships))
|
|
||||||
|
|
||||||
print(f"Finished in {time.time() - start_time} seconds")
|
|
||||||
81
tools/mkdocs/modules/site.py
Normal file
81
tools/mkdocs/modules/site.py
Normal file
|
|
@ -0,0 +1,81 @@
|
||||||
|
import os
|
||||||
|
|
||||||
|
from utils.helper import create_bar_chart, get_top_x, create_pie_chart
|
||||||
|
|
||||||
|
class Site:
|
||||||
|
def __init__(self, path, name) -> None:
|
||||||
|
self.path = path
|
||||||
|
self.name = name
|
||||||
|
self.content = ""
|
||||||
|
|
||||||
|
def add_content(self, content):
|
||||||
|
self.content += content
|
||||||
|
|
||||||
|
def write_entry(self):
|
||||||
|
with open(os.path.join(self.path, self.name), "w") as index:
|
||||||
|
index.write(self.content)
|
||||||
|
|
||||||
|
class IndexSite(Site):
|
||||||
|
def __init__(self, path) -> None:
|
||||||
|
super().__init__(path=path, name="index.md")
|
||||||
|
|
||||||
|
def add_toc(self, galaxies):
|
||||||
|
for galaxy in galaxies:
|
||||||
|
galaxy_folder = galaxy.json_file_name.replace(".json", "")
|
||||||
|
self.add_content(f"- [{galaxy.galaxy_name}](./{galaxy_folder}/index.md)\n")
|
||||||
|
self.add_content("\n")
|
||||||
|
|
||||||
|
class StatisticsSite(Site):
|
||||||
|
def __init__(self, path) -> None:
|
||||||
|
super().__init__(path=path, name="statistics.md")
|
||||||
|
|
||||||
|
def add_galaxy_statistics(self, galaxies):
|
||||||
|
galaxy_cluster_count = {galaxy.galaxy_name: len(galaxy.clusters) for galaxy in galaxies}
|
||||||
|
top_20 = get_top_x(galaxy_cluster_count, 20)
|
||||||
|
flop_20 = get_top_x(galaxy_cluster_count, 20, False)
|
||||||
|
self.add_content(f"# Galaxy statistics\n")
|
||||||
|
self.add_content(f"## Galaxies with the most clusters\n")
|
||||||
|
self.add_content(create_bar_chart(x_axis="Galaxy", y_axis="Count", values=top_20))
|
||||||
|
self.add_content(f"## Galaxies with the least clusters\n")
|
||||||
|
self.add_content(create_bar_chart(x_axis="Galaxy", y_axis="Count", values=flop_20))
|
||||||
|
|
||||||
|
def add_cluster_statistics(self, clusters):
|
||||||
|
public_clusters = 0
|
||||||
|
private_clusters = 0
|
||||||
|
for cluster in clusters:
|
||||||
|
if cluster.value == "Private Cluster":
|
||||||
|
private_clusters += 1
|
||||||
|
else:
|
||||||
|
public_clusters += 1
|
||||||
|
values = {"Public clusters": public_clusters, "Private clusters": private_clusters}
|
||||||
|
self.add_content(f"# Cluster statistics\n")
|
||||||
|
self.add_content(f"## Number of clusters\n")
|
||||||
|
self.add_content(f"Here you can find the total number of clusters including public and private clusters.The number of public clusters has been calculated based on the number of unique Clusters in the MISP galaxy JSON files. The number of private clusters could only be approximated based on the number of relations to non-existing clusters. Therefore the number of private clusters is not accurate and only an approximation.\n\n")
|
||||||
|
self.add_content(create_pie_chart(sector="Type", unit="Count", values=values))
|
||||||
|
|
||||||
|
def add_relation_statistics(self, clusters):
|
||||||
|
cluster_relations = {}
|
||||||
|
private_relations = 0
|
||||||
|
public_relations = 0
|
||||||
|
for cluster in clusters:
|
||||||
|
cluster_relations[cluster.uuid] = len(cluster.relations)
|
||||||
|
for relation in cluster.relations:
|
||||||
|
if relation.to_cluster.value == "Private Cluster":
|
||||||
|
private_relations += 1
|
||||||
|
else:
|
||||||
|
public_relations += 1
|
||||||
|
top_20 = get_top_x(cluster_relations, 20)
|
||||||
|
flop_20 = get_top_x(cluster_relations, 20, False)
|
||||||
|
self.add_content(f"# Relation statistics\n")
|
||||||
|
self.add_content(f"Here you can find the total number of relations including public and private relations. The number includes relations between public clusters and relations between public and private clusters. Therefore relatons between private clusters are not included in the statistics.\n\n")
|
||||||
|
self.add_content(f"## Number of relations\n")
|
||||||
|
self.add_content(create_pie_chart(sector="Type", unit="Count", values={"Public relations": public_relations, "Private relations": private_relations}))
|
||||||
|
self.add_content(f"**Average number of relations per cluster**: {int(sum(cluster_relations.values()) / len(cluster_relations))}\n")
|
||||||
|
self.add_content(f"## Cluster with the most relations\n")
|
||||||
|
self.add_content(create_bar_chart(x_axis="Cluster", y_axis="Count", values=top_20))
|
||||||
|
self.add_content(f"## Cluster with the least relations\n")
|
||||||
|
self.add_content(create_bar_chart(x_axis="Cluster", y_axis="Count", values=flop_20))
|
||||||
|
|
||||||
|
def add_synonym_statistics(self, clusters):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
@ -21,13 +21,16 @@ class Universe:
|
||||||
cluster_a = None
|
cluster_a = None
|
||||||
cluster_b = None
|
cluster_b = None
|
||||||
|
|
||||||
|
if cluster_a_id == cluster_b_id:
|
||||||
|
return
|
||||||
|
|
||||||
# Search for Cluster A and Cluster B in all galaxies
|
# Search for Cluster A and Cluster B in all galaxies
|
||||||
for galaxy in self.galaxies.values():
|
for galaxy in self.galaxies.values():
|
||||||
if cluster_a_id in galaxy.clusters:
|
if cluster_a_id in galaxy.clusters:
|
||||||
cluster_a = galaxy.clusters[cluster_a_id]
|
cluster_a = galaxy.clusters[cluster_a_id]
|
||||||
if cluster_b_id in galaxy.clusters:
|
if cluster_b_id in galaxy.clusters:
|
||||||
cluster_b = galaxy.clusters[cluster_b_id]
|
cluster_b = galaxy.clusters[cluster_b_id]
|
||||||
if cluster_a and cluster_b: # Both clusters found
|
if cluster_a and cluster_b:
|
||||||
break
|
break
|
||||||
|
|
||||||
# If both clusters are found, define the relationship
|
# If both clusters are found, define the relationship
|
||||||
|
|
@ -35,7 +38,6 @@ class Universe:
|
||||||
cluster_a.add_outbound_relationship(cluster_b)
|
cluster_a.add_outbound_relationship(cluster_b)
|
||||||
cluster_b.add_inbound_relationship(cluster_a)
|
cluster_b.add_inbound_relationship(cluster_a)
|
||||||
else:
|
else:
|
||||||
# If Cluster B is not found, create a private cluster relationship for Cluster A
|
|
||||||
if cluster_a:
|
if cluster_a:
|
||||||
private_cluster = Cluster(uuid=cluster_b_id, galaxy=None, description=None, value="Private Cluster", meta=None)
|
private_cluster = Cluster(uuid=cluster_b_id, galaxy=None, description=None, value="Private Cluster", meta=None)
|
||||||
cluster_a.add_outbound_relationship(private_cluster)
|
cluster_a.add_outbound_relationship(private_cluster)
|
||||||
|
|
@ -67,26 +69,18 @@ class Universe:
|
||||||
if neighbor not in visited and neighbor.value != "Private Cluster":
|
if neighbor not in visited and neighbor.value != "Private Cluster":
|
||||||
queue.append((neighbor, level + 1))
|
queue.append((neighbor, level + 1))
|
||||||
|
|
||||||
# count = 0
|
|
||||||
# Convert the defaultdict to a list of tuples, ignoring direction
|
# Convert the defaultdict to a list of tuples, ignoring direction
|
||||||
processed_relationships = []
|
processed_relationships = []
|
||||||
for link, lvl in relationships.items():
|
for link, lvl in relationships.items():
|
||||||
# Extract clusters from the frozenset; direction is irrelevant
|
# Extract clusters from the frozenset; direction is irrelevant
|
||||||
clusters = list(link)
|
clusters = list(link)
|
||||||
if len(clusters) != 2:
|
|
||||||
# count += 1
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Arbitrarily choose the first cluster as 'source' for consistency
|
# Arbitrarily choose the first cluster as 'source' for consistency
|
||||||
if clusters[0].value == "Private Cluster":
|
if clusters[0].value == "Private Cluster":
|
||||||
processed_relationships.append((clusters[1], clusters[0], lvl))
|
processed_relationships.append((clusters[1], clusters[0], lvl))
|
||||||
else:
|
else:
|
||||||
|
|
||||||
processed_relationships.append((clusters[0], clusters[1], lvl))
|
processed_relationships.append((clusters[0], clusters[1], lvl))
|
||||||
# except:
|
|
||||||
# processed_relationships.append((clusters[0], clusters[0], lvl)) # This is wrong just for testing!!!
|
|
||||||
|
|
||||||
# print(f"Count: {count}")
|
|
||||||
return processed_relationships
|
return processed_relationships
|
||||||
|
|
||||||
return bfs_with_undirected_relationships(start_cluster)
|
return bfs_with_undirected_relationships(start_cluster)
|
||||||
|
|
@ -19,3 +19,21 @@ def name_to_section(name):
|
||||||
.replace(":", "")
|
.replace(":", "")
|
||||||
.replace(placeholder, "-")
|
.replace(placeholder, "-")
|
||||||
) # Replace the placeholder with "-"
|
) # Replace the placeholder with "-"
|
||||||
|
|
||||||
|
|
||||||
|
def create_bar_chart(x_axis, y_axis, values, log=False):
|
||||||
|
if not log:
|
||||||
|
chart = f"| No. | {x_axis} | {y_axis} {{ .bar-chart }}|\n"
|
||||||
|
else:
|
||||||
|
chart = f"| No. | {x_axis} | {y_axis} {{ .log-bar-chart }}|\n"
|
||||||
|
chart += f"|----|--------|-------|\n"
|
||||||
|
for i, x, y in enumerate(values):
|
||||||
|
chart += f"| {i+1} | {x} | {y} |\n"
|
||||||
|
return chart
|
||||||
|
|
||||||
|
def create_pie_chart(sector, unit, values):
|
||||||
|
chart = f"| No. | {sector} | {unit} {{ .pie-chart }}|\n"
|
||||||
|
chart += f"|----|--------|-------|\n"
|
||||||
|
for i, x, y in enumerate(values):
|
||||||
|
chart += f"| {i+1} | {x} | {y} |\n"
|
||||||
|
return chart
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue