new: [malpedia] remove duplicate UUIDs objects (coming from Malpedia API)

clusters/malpedia.json

@@ -28522,6 +28522,7 @@
           "https://www.bleepingcomputer.com/news/security/lockbit-ransomware-gang-claims-attack-on-bridgestone-americas/",
           "https://www.sentinelone.com/labs/lockbit-ransomware-side-loads-cobalt-strike-beacon-with-legitimate-vmware-utility/",
           "https://docs.google.com/spreadsheets/d/1MI8Z2tBhmqQ5X8Wf_ozv3dVjz5sJOs-3",
+          "https://cybergeeks.tech/a-technical-analysis-of-the-leaked-lockbit-3-0-builder/",
           "https://news.sophos.com/en-us/2020/10/21/lockbit-attackers-uses-automated-attack-tools-to-identify-tasty-targets",
           "https://www.splunk.com/en_us/pdfs/resources/whitepaper/an-empirically-comparative-analysis-of-ransomware-binaries.pdf",
           "https://www.dr.dk/nyheder/viden/teknologi/frygtede-skulle-lukke-alle-vindmoeller-nu-aabner-vestas-op-om-hacking-angreb",
@@ -28595,8 +28596,8 @@
           "https://www.trendmicro.com/vinfo/us/security/news/ransomware-by-the-numbers/lockbit-conti-and-blackcat-lead-pack-amid-rise-in-active-raas-and-extortion-groups-ransomware-in-q1-2022",
           "https://techcommunity.microsoft.com/t5/security-compliance-and-identity/part-2-lockbit-2-0-ransomware-bugs-and-database-recovery/ba-p/3254421",
           "https://www.crowdstrike.com/blog/big-game-hunting-on-the-rise-again-according-to-ecrime-index/",
-          "https://intel471.com/blog/privateloader-malware",
           "https://asec.ahnlab.com/en/35822/",
+          "https://intel471.com/blog/privateloader-malware",
           "https://intel471.com/blog/conti-ransomware-cooperation-maze-lockbit-ragnar-locker",
           "https://news.sophos.com/en-us/2022/04/12/attackers-linger-on-government-agency-computers-before-deploying-lockbit-ransomware/",
           "https://www.bleepingcomputer.com/news/security/lockbit-ransomware-now-encrypts-windows-domains-using-group-policies/",
@@ -41811,19 +41812,6 @@
       "uuid": "129163aa-8539-40ee-a627-0ac6775697b5",
       "value": "SUGARRUSH"
     },
-    {
-      "description": "According to Mandiant, SUGARUSH is a backdoor written to establish a connection with an embedded C2 and to execute CMD commands.",
-      "meta": {
-        "refs": [
-          "https://malpedia.caad.fkie.fraunhofer.de/details/win.sugarush",
-          "https://www.mandiant.com/resources/suspected-iranian-actor-targeting-israeli-shipping"
-        ],
-        "synonyms": [],
-        "type": []
-      },
-      "uuid": "129163aa-8539-40ee-a627-0ac6775697b5",
-      "value": "SUGARUSH"
-    },
     {
       "description": "FireEye describes SUNBURST as a trojanized SolarWinds digitally-signed component of the Orion software framework that contains a backdoor that communicates via HTTP to third party servers. After an initial dormant period of up to two weeks, it uses a DGA to generate specific subdomains for a set C&C domain. The backdoor retrieves and executes commands, that include the ability to transfer files, execute files, profile the system, reboot the machine, and disable system services. The C2 traffic to the malicious domains is designed to mimic normal SolarWinds API communications: Orion Improvement Program (OIP) protocol. The backdoor uses multiple obfuscated blocklists to identify forensic and anti-virus tools running as processes, services, and drivers. Multiple trojanzied updates were digitally signed from March - May 2020 and posted to the SolarWinds updates website.",
       "meta": {
@@ -44520,19 +44508,6 @@
       "uuid": "f2979fee-603d-496e-a526-d622e9cba84f",
       "value": "Unidentified 072 (Metamorfo Loader)"
     },
-    {
-      "description": "",
-      "meta": {
-        "refs": [
-          "https://malpedia.caad.fkie.fraunhofer.de/details/win.unidentified_073",
-          "https://blog.certfa.com/posts/fake-interview-the-new-activity-of-charming-kitten/"
-        ],
-        "synonyms": [],
-        "type": []
-      },
-      "uuid": "f049e626-7de2-4648-81db-53dfd34f2fab",
-      "value": "Unidentified 073 (Charming Kitten)"
-    },
     {
       "description": "",
       "meta": {
@@ -47684,5 +47659,5 @@
       "value": "Zyklon"
     }
   ],
-  "version": 15970
+  "version": 15975
 }

tools/del_duplicate_uuids.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+# coding=utf-8
+"""
+    Tool to remove duplicates in cluster references
+"""
+import sys
+import json
+
+with open(sys.argv[1], 'r') as f:
+    data = json.load(f)
+
+unique_uuid = set()
+values = []
+for c in data['values']:
+    if c['uuid'] in unique_uuid:
+        sys.stderr.write(f"Duplicate UUID - {c['uuid']}\n")
+        continue
+    unique_uuid.add(c['uuid'])
+    values.append(c)
+
+data['values'] = []
+data['values'] = values
+
+with open(sys.argv[1], 'w') as f:
+    json.dump(data, f)
+

tools/fetch_malpedia.sh

@@ -3,4 +3,5 @@ cd "${0%/*}"
 wget -O malpedia.json https://malpedia.caad.fkie.fraunhofer.de/api/get/misp
 mv malpedia.json ../clusters/malpedia.json
 ./del_duplicate_refs.py ../clusters/malpedia.json
+./del_duplicate_uuids.py ../clusters/malpedia.json
 (cd ..; ./jq_all_the_things.sh)