mirror of
https://github.com/MISP/misp-galaxy.git
synced 2024-11-30 02:37:17 +00:00
Add [graph] basic graph
This commit is contained in:
parent
5c87f0d720
commit
d357075432
5 changed files with 171 additions and 62 deletions
1
tools/mkdocs/.gitignore
vendored
1
tools/mkdocs/.gitignore
vendored
|
@ -2,3 +2,4 @@
|
||||||
!/site/docs/javascripts
|
!/site/docs/javascripts
|
||||||
|
|
||||||
/site/site
|
/site/site
|
||||||
|
/javascripts
|
|
@ -103,7 +103,8 @@ class Galaxy():
|
||||||
date=cluster.get('date', None),
|
date=cluster.get('date', None),
|
||||||
related_list=cluster.get('related', None),
|
related_list=cluster.get('related', None),
|
||||||
meta=cluster.get('meta', None),
|
meta=cluster.get('meta', None),
|
||||||
galaxie=self.json_file_name
|
galaxie=self.name,
|
||||||
|
galaxie_file_name=self.json_file_name
|
||||||
))
|
))
|
||||||
return clusters
|
return clusters
|
||||||
|
|
||||||
|
@ -128,7 +129,7 @@ class Galaxy():
|
||||||
index.write(self.entry)
|
index.write(self.entry)
|
||||||
|
|
||||||
class Cluster():
|
class Cluster():
|
||||||
def __init__(self, description, uuid, date, value, related_list, meta, galaxie):
|
def __init__(self, description, uuid, date, value, related_list, meta, galaxie, galaxie_file_name):
|
||||||
self.description = description
|
self.description = description
|
||||||
self.uuid = uuid
|
self.uuid = uuid
|
||||||
self.date = date
|
self.date = date
|
||||||
|
@ -137,6 +138,7 @@ class Cluster():
|
||||||
self.meta = meta
|
self.meta = meta
|
||||||
self.entry = ""
|
self.entry = ""
|
||||||
self.galaxie = galaxie
|
self.galaxie = galaxie
|
||||||
|
self.galaxie_file_name = galaxie_file_name
|
||||||
|
|
||||||
global public_clusters_dict
|
global public_clusters_dict
|
||||||
if self.galaxie:
|
if self.galaxie:
|
||||||
|
@ -193,11 +195,17 @@ class Cluster():
|
||||||
self.entry += f'\n'
|
self.entry += f'\n'
|
||||||
self.entry += f'??? info "Associated metadata"\n'
|
self.entry += f'??? info "Associated metadata"\n'
|
||||||
self.entry += f'\n'
|
self.entry += f'\n'
|
||||||
self.entry += f' |Metadata key |Value|\n'
|
self.entry += f' <div class="no-filter">\n'
|
||||||
self.entry += f' |------------------|-----|\n'
|
self.entry += f' \n'
|
||||||
|
self.entry += f' |Metadata key {{ .no-filter }} |Value|\n'
|
||||||
|
self.entry += f' |-----------------------------------|-----|\n'
|
||||||
for meta in sorted(self.meta.keys()):
|
for meta in sorted(self.meta.keys()):
|
||||||
if meta not in excluded_meta:
|
if meta not in excluded_meta:
|
||||||
self.entry += f' | {meta} | {self.meta[meta]} |\n'
|
self.entry += f' | {meta} | {self.meta[meta]} |\n'
|
||||||
|
# self.entry += " {: .no-filter }\n"
|
||||||
|
self.entry += f' \n'
|
||||||
|
self.entry += f' </div>\n'
|
||||||
|
self.entry += f'\n'
|
||||||
|
|
||||||
|
|
||||||
def get_related_clusters(self, cluster_dict, depth=-1, visited=None, level=1):
|
def get_related_clusters(self, cluster_dict, depth=-1, visited=None, level=1):
|
||||||
|
@ -228,7 +236,7 @@ class Cluster():
|
||||||
private_relations_count += 1
|
private_relations_count += 1
|
||||||
if dest_uuid not in private_clusters:
|
if dest_uuid not in private_clusters:
|
||||||
private_clusters.append(dest_uuid)
|
private_clusters.append(dest_uuid)
|
||||||
related_clusters.append((self, Cluster(value="Private Cluster", uuid=dest_uuid, date=None, description=None, related_list=None, meta=None, galaxie=None), level))
|
related_clusters.append((self, Cluster(value="Private Cluster", uuid=dest_uuid, date=None, description=None, related_list=None, meta=None, galaxie=None, galaxie_file_name=None), level))
|
||||||
continue
|
continue
|
||||||
|
|
||||||
related_cluster = cluster_dict[dest_uuid]
|
related_cluster = cluster_dict[dest_uuid]
|
||||||
|
@ -261,10 +269,12 @@ class Cluster():
|
||||||
output = ""
|
output = ""
|
||||||
output += f'## Related clusters for {self.value}\n'
|
output += f'## Related clusters for {self.value}\n'
|
||||||
output += f'\n'
|
output += f'\n'
|
||||||
output += f'| Cluster A | Cluster B | Level |\n'
|
output += f'| Cluster A | Cluster B | Level {{ .graph }} |\n'
|
||||||
output += f'|-----------|-----------|-------|\n'
|
output += f'|-----------|-----------|-------|\n'
|
||||||
for relation in relations:
|
for relation in relations:
|
||||||
output += f'| {relation[0].value} ({relation[0].uuid}) | {relation[1].value} ({relation[1].uuid}) | {relation[2]} |\n'
|
cluster_a_section = relation[0].value.lower().replace(" ", "+").replace("/", "").replace(":", "")
|
||||||
|
cluster_b_section = relation[1].value.lower().replace(" ", "+").replace("/", "").replace(":", "")
|
||||||
|
output += f'| [{relation[0].value} ({relation[0].uuid})](../../{relation[0].galaxie_file_name}/index.md#{cluster_a_section}) | [{relation[1].value} ({relation[1].uuid})](../../{relation[1].galaxie_file_name}/index.md#{cluster_b_section}) | {relation[2]} |\n'
|
||||||
return output
|
return output
|
||||||
|
|
||||||
def create_entry(self, cluster_dict):
|
def create_entry(self, cluster_dict):
|
||||||
|
@ -283,7 +293,7 @@ class Cluster():
|
||||||
related_clusters = self.get_related_clusters(cluster_dict)
|
related_clusters = self.get_related_clusters(cluster_dict)
|
||||||
global relation_count_dict
|
global relation_count_dict
|
||||||
relation_count_dict[self.value] = len(related_clusters)
|
relation_count_dict[self.value] = len(related_clusters)
|
||||||
galaxy_path = os.path.join(path, self.galaxie)
|
galaxy_path = os.path.join(path, self.galaxie_file_name)
|
||||||
if not os.path.exists(galaxy_path):
|
if not os.path.exists(galaxy_path):
|
||||||
os.mkdir(galaxy_path)
|
os.mkdir(galaxy_path)
|
||||||
relation_path = os.path.join(galaxy_path, 'relations')
|
relation_path = os.path.join(galaxy_path, 'relations')
|
||||||
|
@ -356,16 +366,23 @@ def create_statistics():
|
||||||
for galaxy in public_clusters_dict.values():
|
for galaxy in public_clusters_dict.values():
|
||||||
galaxy_counts[galaxy] = galaxy_counts.get(galaxy, 0) + 1
|
galaxy_counts[galaxy] = galaxy_counts.get(galaxy, 0) + 1
|
||||||
top_galaxies, top_galaxies_values = get_top_x(galaxy_counts, 25)
|
top_galaxies, top_galaxies_values = get_top_x(galaxy_counts, 25)
|
||||||
statistic_output += create_xy_chart("Galaxies with the most clusters", 2500, 500, top_galaxies, "Number of clusters", top_galaxies_values)
|
statistic_output += create_xy_chart("Galaxies with the most clusters", 3000, 1000, top_galaxies, "Number of clusters", top_galaxies_values)
|
||||||
|
for i, galaxy in enumerate(top_galaxies.split(", "), 1):
|
||||||
|
statistic_output += f'{i}. [{galaxy}](./{galaxy}/index.md)\n'
|
||||||
|
statistic_output += f'\n'
|
||||||
|
|
||||||
statistic_output += f'## Galaxies with the least clusters\n'
|
statistic_output += f'## Galaxies with the least clusters\n'
|
||||||
flop_galaxies, flop_galaxies_values = get_top_x(galaxy_counts, 25, False)
|
flop_galaxies, flop_galaxies_values = get_top_x(galaxy_counts, 25, False)
|
||||||
statistic_output += create_xy_chart("Galaxies with the least clusters", 2500, 500, flop_galaxies, "Number of clusters", flop_galaxies_values)
|
statistic_output += create_xy_chart("Galaxies with the least clusters", 3000, 1000, flop_galaxies, "Number of clusters", flop_galaxies_values)
|
||||||
|
for i, galaxy in enumerate(flop_galaxies.split(", "), 1):
|
||||||
|
statistic_output += f'{i}. [{galaxy}](./{galaxy}/index.md)\n'
|
||||||
|
statistic_output += f'\n'
|
||||||
|
|
||||||
galaxy_number = 0
|
# galaxy_number = 0
|
||||||
for galaxy in public_clusters_dict.values():
|
# global galaxies
|
||||||
galaxy_number += 1
|
# for galaxy in galaxies:
|
||||||
statistic_output += f'**Average number of clusters per galaxy**: {len(public_clusters_dict) / galaxy_number}\n'
|
# galaxy_number += 1
|
||||||
|
# statistic_output += f'**Average number of clusters per galaxy**: {len(public_clusters_dict) / galaxy_number}\n'
|
||||||
|
|
||||||
statistic_output += f'# Relation statistics\n'
|
statistic_output += f'# Relation statistics\n'
|
||||||
statistic_output += f'## Number of relations\n'
|
statistic_output += f'## Number of relations\n'
|
||||||
|
@ -375,20 +392,20 @@ def create_statistics():
|
||||||
|
|
||||||
statistic_output += f'## Cluster with the most relations\n'
|
statistic_output += f'## Cluster with the most relations\n'
|
||||||
top_25_relation, top_25_relation_values = get_top_x(relation_count_dict, 25)
|
top_25_relation, top_25_relation_values = get_top_x(relation_count_dict, 25)
|
||||||
statistic_output += create_xy_chart("Cluster with the most relations", 2500, 500, top_25_relation, "Number of relations", top_25_relation_values)
|
statistic_output += create_xy_chart("Cluster with the most relations", 3000, 1000, top_25_relation, "Number of relations", top_25_relation_values)
|
||||||
|
|
||||||
statistic_output += f'## Cluster with the least relations\n'
|
statistic_output += f'## Cluster with the least relations\n'
|
||||||
top_25_relation, top_25_relation_values = get_top_x(relation_count_dict, 25, False)
|
top_25_relation, top_25_relation_values = get_top_x(relation_count_dict, 25, False)
|
||||||
statistic_output += create_xy_chart("Cluster with the least relations", 2500, 500, top_25_relation, "Number of relations", top_25_relation_values)
|
statistic_output += create_xy_chart("Cluster with the least relations", 3000, 1000, top_25_relation, "Number of relations", top_25_relation_values)
|
||||||
|
|
||||||
statistic_output += f'# Synonyms statistics\n'
|
statistic_output += f'# Synonyms statistics\n'
|
||||||
statistic_output += f'## Cluster with the most synonyms\n'
|
statistic_output += f'## Cluster with the most synonyms\n'
|
||||||
top_25_synonyms, top_25_synonyms_values = get_top_x(synonyms_count_dict, 25)
|
top_25_synonyms, top_25_synonyms_values = get_top_x(synonyms_count_dict, 25)
|
||||||
statistic_output += create_xy_chart("Cluster with the most synonyms", 2500, 500, top_25_synonyms, "Number of synonyms", top_25_synonyms_values)
|
statistic_output += create_xy_chart("Cluster with the most synonyms", 3000, 1000, top_25_synonyms, "Number of synonyms", top_25_synonyms_values)
|
||||||
|
|
||||||
statistic_output += f'## Cluster with the least synonyms\n'
|
statistic_output += f'## Cluster with the least synonyms\n'
|
||||||
top_25_synonyms, top_25_synonyms_values = get_top_x(synonyms_count_dict, 25, False)
|
top_25_synonyms, top_25_synonyms_values = get_top_x(synonyms_count_dict, 25, False)
|
||||||
statistic_output += create_xy_chart("Cluster with the least synonyms", 2500, 500, top_25_synonyms, "Number of synonyms", top_25_synonyms_values)
|
statistic_output += create_xy_chart("Cluster with the least synonyms", 3000, 1000, top_25_synonyms, "Number of synonyms", top_25_synonyms_values)
|
||||||
|
|
||||||
statistic_output += f'# Empty UUIDs statistics\n'
|
statistic_output += f'# Empty UUIDs statistics\n'
|
||||||
statistic_output += f'**Number of empty UUIDs**: {sum(empty_uuids_dict.values())}\n'
|
statistic_output += f'**Number of empty UUIDs**: {sum(empty_uuids_dict.values())}\n'
|
||||||
|
@ -441,7 +458,7 @@ def main():
|
||||||
# for galaxy in galaxies:
|
# for galaxy in galaxies:
|
||||||
# galaxy.write_entry(SITE_PATH, cluster_dict)
|
# galaxy.write_entry(SITE_PATH, cluster_dict)
|
||||||
|
|
||||||
count = 3
|
count = 10
|
||||||
for galaxy in galaxies:
|
for galaxy in galaxies:
|
||||||
galaxy.write_entry(SITE_PATH, cluster_dict)
|
galaxy.write_entry(SITE_PATH, cluster_dict)
|
||||||
count -= 1
|
count -= 1
|
||||||
|
|
84
tools/mkdocs/site/docs/javascripts/d3.js
vendored
Normal file
84
tools/mkdocs/site/docs/javascripts/d3.js
vendored
Normal file
|
@ -0,0 +1,84 @@
|
||||||
|
document$.subscribe(function () {
|
||||||
|
// Function to parse table and return data
|
||||||
|
function parseTable(table) {
|
||||||
|
var data = [];
|
||||||
|
var rows = table.querySelectorAll("tr");
|
||||||
|
rows.forEach((row, i) => {
|
||||||
|
// Skipping header row and filter row
|
||||||
|
if (i > 1) {
|
||||||
|
var cells = row.querySelectorAll("td");
|
||||||
|
data.push({ source: cells[0].textContent, target: cells[1].textContent, level: cells[2].textContent });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Function to create graph
|
||||||
|
function createGraph(data, elementId) {
|
||||||
|
// Extract nodes and links
|
||||||
|
var nodes = Array.from(new Set(data.flatMap(d => [d.source, d.target])))
|
||||||
|
.map(id => ({ id }));
|
||||||
|
|
||||||
|
var links = data.map(d => ({ source: d.source, target: d.target }));
|
||||||
|
|
||||||
|
// Set up the dimensions of the graph
|
||||||
|
var width = 800, height = 600;
|
||||||
|
|
||||||
|
// Append SVG for the graph
|
||||||
|
var svg = d3.select(elementId).append("svg")
|
||||||
|
.attr("width", width)
|
||||||
|
.attr("height", height);
|
||||||
|
|
||||||
|
// Create a force simulation
|
||||||
|
var simulation = d3.forceSimulation(nodes)
|
||||||
|
.force("link", d3.forceLink(links).id(d => d.id))
|
||||||
|
.force("charge", d3.forceManyBody())
|
||||||
|
.force("center", d3.forceCenter(width / 2, height / 2));
|
||||||
|
|
||||||
|
// Create links
|
||||||
|
var link = svg.append("g")
|
||||||
|
.attr("stroke", "#999")
|
||||||
|
.attr("stroke-opacity", 0.6)
|
||||||
|
.selectAll("line")
|
||||||
|
.data(links)
|
||||||
|
.enter().append("line")
|
||||||
|
.attr("stroke-width", d => Math.sqrt(d.value));
|
||||||
|
|
||||||
|
// Create nodes
|
||||||
|
var node = svg.append("g")
|
||||||
|
.attr("stroke", "#fff")
|
||||||
|
.attr("stroke-width", 1.5)
|
||||||
|
.selectAll("circle")
|
||||||
|
.data(nodes)
|
||||||
|
.enter().append("circle")
|
||||||
|
.attr("r", 5)
|
||||||
|
.attr("fill", "#69b3a2");
|
||||||
|
|
||||||
|
// Update positions on each simulation 'tick'
|
||||||
|
simulation.on("tick", () => {
|
||||||
|
link
|
||||||
|
.attr("x1", d => d.source.x)
|
||||||
|
.attr("y1", d => d.source.y)
|
||||||
|
.attr("x2", d => d.target.x)
|
||||||
|
.attr("y2", d => d.target.y);
|
||||||
|
|
||||||
|
node
|
||||||
|
.attr("cx", d => d.x)
|
||||||
|
.attr("cy", d => d.y);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Find all tables that have a th with the class .graph and generate graphs
|
||||||
|
document.querySelectorAll("table").forEach((table, index) => {
|
||||||
|
var graphHeader = table.querySelector("th.graph");
|
||||||
|
if (graphHeader) {
|
||||||
|
var data = parseTable(table);
|
||||||
|
console.log(data);
|
||||||
|
var graphId = "graph" + index;
|
||||||
|
var div = document.createElement("div");
|
||||||
|
div.id = graphId;
|
||||||
|
table.after(div);
|
||||||
|
createGraph(data, "#" + graphId);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
|
@ -1,48 +1,53 @@
|
||||||
document$.subscribe(function () {
|
document$.subscribe(function () {
|
||||||
var tables = document.querySelectorAll("article table:not([class])")
|
var tables = document.querySelectorAll("article table")
|
||||||
tables.forEach(function (table) {
|
tables.forEach(function (table) {
|
||||||
var tf = new TableFilter(table, {
|
var excludeTable = table.querySelector("td.no-filter, th.no-filter");
|
||||||
base_path: "https://unpkg.com/tablefilter@0.7.3/dist/tablefilter/",
|
if (!excludeTable) {
|
||||||
highlight_keywords: true,
|
var tf = new TableFilter(table, {
|
||||||
// col_0: "select",
|
base_path: "https://unpkg.com/tablefilter@0.7.3/dist/tablefilter/",
|
||||||
// col_1: "select",
|
highlight_keywords: true,
|
||||||
col_2: "checklist",
|
// col_0: "select",
|
||||||
col_widths: ["350px", "350px", "100px"],
|
// col_1: "select",
|
||||||
col_types: ["string", "string", "number"],
|
col_2: "checklist",
|
||||||
grid_layout: false,
|
col_widths: ["350px", "350px", "100px"],
|
||||||
responsive: false,
|
col_types: ["string", "string", "number"],
|
||||||
watermark: ["Filter table ...", "Filter table ..."],
|
grid_layout: false,
|
||||||
|
responsive: false,
|
||||||
|
watermark: ["Filter table ...", "Filter table ..."],
|
||||||
|
|
||||||
auto_filter: {
|
auto_filter: {
|
||||||
delay: 100 //milliseconds
|
delay: 100 //milliseconds
|
||||||
},
|
},
|
||||||
filters_row_index: 1,
|
filters_row_index: 1,
|
||||||
state: true,
|
state: true,
|
||||||
// alternate_rows: true,
|
// alternate_rows: true,
|
||||||
rows_counter: true,
|
rows_counter: true,
|
||||||
status_bar: true,
|
status_bar: true,
|
||||||
|
|
||||||
themes: [{
|
themes: [{
|
||||||
name: "transparent",
|
name: "transparent",
|
||||||
}],
|
}],
|
||||||
|
|
||||||
btn_reset: {
|
btn_reset: {
|
||||||
tooltip: "Reset",
|
tooltip: "Reset",
|
||||||
toolbar_position: "right",
|
toolbar_position: "right",
|
||||||
},
|
},
|
||||||
no_results_message: {
|
no_results_message: {
|
||||||
content: "No matching records found",
|
content: "No matching records found",
|
||||||
},
|
},
|
||||||
toolbar: true,
|
toolbar: true,
|
||||||
extensions: [{
|
extensions: [{
|
||||||
name: "sort",
|
name: "sort",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: 'filtersVisibility',
|
name: 'filtersVisibility',
|
||||||
description: 'Sichtbarkeit der Filter',
|
description: 'Sichtbarkeit der Filter',
|
||||||
toolbar_position: 'right',
|
toolbar_position: 'right',
|
||||||
},],
|
},],
|
||||||
})
|
})
|
||||||
tf.init()
|
tf.init()
|
||||||
|
}
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -68,6 +68,8 @@ extra:
|
||||||
extra_javascript:
|
extra_javascript:
|
||||||
- javascripts/tablefilter.js
|
- javascripts/tablefilter.js
|
||||||
- "https://unpkg.com/tablefilter@0.7.3/dist/tablefilter/tablefilter.js"
|
- "https://unpkg.com/tablefilter@0.7.3/dist/tablefilter/tablefilter.js"
|
||||||
|
- "https://d3js.org/d3.v6.min.js"
|
||||||
|
- javascripts/d3.js
|
||||||
|
|
||||||
plugins:
|
plugins:
|
||||||
- search
|
- search
|
||||||
|
|
Loading…
Reference in a new issue