MISP/SkillAegis
MISP/SkillAegis
6
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Compare commits

base: MISP:3e6c2e5561094f94829829f0f9b247e089e2e556
Branches Tags
MISP:main
MISP:scenario-designer
MISP:feature/chart
MISP:feature/aiohttp
...
compare: MISP:1a2ff4c3c9fc51843761908afa4770bc9148fd35
Branches Tags
MISP:scenario-designer
MISP:main
MISP:feature/chart
MISP:feature/aiohttp

2 commits
3e6c2e5561 ... 1a2ff4c3c9

Author SHA1 Message Date
Sami Mokaddem
1a2ff4c3c9 chg: [app] Improved data selection, condition checking and evaluation definition 2024-06-28 17:14:05 +02:00
Sami Mokaddem
d13577212b chg: [exercises] Correct usage of JQ query 2024-06-27 07:34:21 +02:00
10 changed files with 584 additions and 50 deletions
Show stats Expand all files Collapse all files

0
active_exercises/basic-event-creation.json → active_exercises/basic-event-creation.json.bak
View file

0
active_exercises/basic-filtering.json → active_exercises/basic-filtering.json.bak
View file

1
active_exercises/ransomware-exercise.json Symbolic link
View file

@ -0,0 +1 @@
../exercises/ransomware-exercise.json

24
exercise.py
View file

@ -134,9 +134,10 @@ def get_model_action(data: dict):
def is_accepted_query(data: dict) -> bool:
model, action = get_model_action(data)
if model in ['Event', 'Attribute', 'Object', 'Tag',]:
if action in ['add', 'edit', 'delete',]:
if data['Log']['change'].startswith('attribute_count'):
return False
if action in ['add', 'edit', 'delete', 'publish']:
# # improved condition below. It blocks some queries
# if data['Log']['change'].startswith('attribute_count'):
# return False
if data['Log']['change'].startswith('Validation errors:'):
return False
return True
@ -218,7 +219,7 @@ def check_inject(user_id: int, inject: dict, data: dict, context: dict) -> bool:
def is_valid_evaluation_context(user_id: int, inject_evaluation: dict, data: dict, context: dict) -> bool:
if 'evaluation_context' not in inject_evaluation:
if 'evaluation_context' not in inject_evaluation or len(inject_evaluation['evaluation_context']) == 0:
return True
if 'request_is_rest' in inject_evaluation['evaluation_context']:
if 'request_is_rest' in context:
@ -266,14 +267,21 @@ def get_data_to_validate(user_id: int, inject_evaluation: dict, data: dict) -> U
def parse_event_id_from_log(data: dict) -> Union[int, None]:
event_id_from_change_field_regex = r".*event_id \(.*\) => \((\d+)\).*"
event_id_from_title_field_regex = r".*from Event \((\d+)\).*"
if 'Log' in data:
log = data['Log']
if 'model' in log and 'model_id' in log and log['model'] == 'Event':
return int(log['model_id'])
if 'change' in log:
event_id_search = re.search(event_id_from_change_field_regex, log['change'], re.IGNORECASE)
if event_id_search is None:
return None
event_id = event_id_search.group(1)
return event_id
if event_id_search is not None:
event_id = event_id_search.group(1)
return event_id
if 'title' in log:
event_id_search = re.search(event_id_from_title_field_regex, log['title'], re.IGNORECASE)
if event_id_search is not None:
event_id = event_id_search.group(1)
return event_id
return None

24
exercises/basic-event-creation.json
View file

@ -138,7 +138,7 @@
{
"parameters": [
{
"Event.info": {
".Event.info": {
"comparison": "contains",
"values": [
"event",
@ -168,7 +168,7 @@
{
"parameters": [
{
"Event.info": {
".Event.info": {
"comparison": "contains",
"values": [
"event",
@ -177,7 +177,7 @@
}
},
{
"Event.Attribute": {
".Event.Attribute": {
"comparison": "contains",
"values": [
{
@ -217,7 +217,7 @@
{
"parameters": [
{
"Event.info": {
".Event.info": {
"comparison": "contains",
"values": [
"event",
@ -226,7 +226,7 @@
}
},
{
"Event.Object": {
".Event.Object": {
"comparison": "contains",
"values": [
{
@ -250,7 +250,7 @@
{
"parameters": [
{
"Event.info": {
".Event.info": {
"comparison": "contains",
"values": [
"event",
@ -259,7 +259,7 @@
}
},
{
"Event.Object[name=\"domain-ip\"].Attribute": {
".Event.Object[name=\"domain-ip\"].Attribute": {
"comparison": "contains",
"values": [
{
@ -299,7 +299,7 @@
{
"parameters": [
{
"Event.info": {
".Event.info": {
"comparison": "contains",
"values": [
"event",
@ -308,7 +308,7 @@
}
},
{
"Event.Attribute": {
".Event.Attribute": {
"comparison": "contains",
"values": [
{
@ -341,7 +341,7 @@
{
"parameters": [
{
"Event.info": {
".Event.info": {
"comparison": "contains",
"values": [
"event",
@ -350,7 +350,7 @@
}
},
{
"Event.Attribute": {
".Event.Attribute": {
"comparison": "contains",
"values": [
{
@ -362,7 +362,7 @@
}
},
{
"Event.Attribute[value=\"1.2.3.4\"].Tag": {
".Event.Attribute[value=\"1.2.3.4\"].Tag": {
"JQ": "jq '.Event.Attribute[] | select(.value == \"1.2.3.4\") | .Tag'",
"comparison": "contains",
"values": [

484
exercises/ransomware-exercise.json Normal file
View file

@ -0,0 +1,484 @@
{
"exercise": {
"description": "MISP Encoding Exercise : Ransomware infection via e-mail",
"expanded": "MISP Encoding Exercise : Ransomware infection via e-mail",
"meta": {
"author": "MISP Project",
"level": "beginner",
"priority": 0
},
"name": "MISP Encoding Exercise : Ransomware infection via e-mail",
"namespace": "data-model",
"tags": [
"exercise:software-scope=\"misp\"",
"state:production"
],
"total_duration": "7200",
"uuid": "29324587-db6c-4a73-a209-cf8c79871629",
"version": "20240624"
},
"inject_flow": [
{
"description": "event-creation",
"inject_uuid": "8e8dbda2-0f5e-4101-83ff-63c1ddda2cae",
"reporting_callback": [],
"requirements": {},
"sequence": {
"completion_trigger": [
"time_expiration",
"completion"
],
"followed_by": [
"8f636640-e4f0-4ffb-abff-4e85597aa1bd"
],
"trigger": [
"startex"
]
},
"timing": {
"triggered_at": null
}
},
{
"description": "infection-email",
"inject_uuid": "8f636640-e4f0-4ffb-abff-4e85597aa1bd",
"reporting_callback": [],
"requirements": {
"inject_uuid": "8e8dbda2-0f5e-4101-83ff-63c1ddda2cae"
},
"sequence": {
"completion_trigger": [
"time_expiration",
"completion"
],
"followed_by": [
"3e61a340-0314-4622-91cc-042f3ff8543a"
],
"trigger": [
]
},
"timing": {
"triggered_at": null
}
},
{
"description": "malicious-payload",
"inject_uuid": "3e61a340-0314-4622-91cc-042f3ff8543a",
"reporting_callback": [],
"requirements": {
"inject_uuid": "8f636640-e4f0-4ffb-abff-4e85597aa1bd"
},
"sequence": {
"completion_trigger": [
"time_expiration",
"completion"
],
"followed_by": [
"8a2d58c8-2b3a-4ba2-bb77-15bcfa704828"
],
"trigger": [
]
},
"timing": {
"triggered_at": null
}
},
{
"description": "c2-ip-address",
"inject_uuid": "8a2d58c8-2b3a-4ba2-bb77-15bcfa704828",
"reporting_callback": [],
"requirements": {
"inject_uuid": "3e61a340-0314-4622-91cc-042f3ff8543a"
},
"sequence": {
"completion_trigger": [
"time_expiration",
"completion"
],
"followed_by": [
"9df13cc8-b61b-4c9f-a1a8-66def8b64439"
],
"trigger": [
]
},
"timing": {
"triggered_at": null
}
},
{
"description": "registry-keys",
"inject_uuid": "9df13cc8-b61b-4c9f-a1a8-66def8b64439",
"reporting_callback": [],
"requirements": {
"inject_uuid": "8a2d58c8-2b3a-4ba2-bb77-15bcfa704828"
},
"sequence": {
"completion_trigger": [
"time_expiration",
"completion"
],
"followed_by": [
"c5c03af1-7ef3-44e7-819a-6c4fd402148a"
],
"trigger": [
]
},
"timing": {
"triggered_at": null
}
},
{
"description": "asym-encryption-key",
"inject_uuid": "c5c03af1-7ef3-44e7-819a-6c4fd402148a",
"reporting_callback": [],
"requirements": {
"inject_uuid": "9df13cc8-b61b-4c9f-a1a8-66def8b64439"
},
"sequence": {
"completion_trigger": [
"time_expiration",
"completion"
],
"followed_by": [
"11f6f0c2-8813-42ee-a312-136649d3f077"
],
"trigger": [
]
},
"timing": {
"triggered_at": null
}
},
{
"description": "context",
"inject_uuid": "11f6f0c2-8813-42ee-a312-136649d3f077",
"reporting_callback": [],
"requirements": {
"inject_uuid": "c5c03af1-7ef3-44e7-819a-6c4fd402148a"
},
"sequence": {
"completion_trigger": [
"time_expiration",
"completion"
],
"followed_by": [
"e3ef4e5f-454a-48c8-a5d7-b3d1d25ecc9f"
],
"trigger": [
]
},
"timing": {
"triggered_at": null
}
},
{
"description": "published",
"inject_uuid": "e3ef4e5f-454a-48c8-a5d7-b3d1d25ecc9f",
"reporting_callback": [],
"requirements": {
"inject_uuid": "11f6f0c2-8813-42ee-a312-136649d3f077"
},
"sequence": {
"completion_trigger": [
"time_expiration",
"completion"
],
"trigger": [
]
},
"timing": {
"triggered_at": null
}
}
],
"inject_payloads": [
],
"injects": [
{
"action": "event-creation",
"inject_evaluation": [
{
"parameters": [
{
".Event.info": {
"comparison": "contains",
"values": [
"ransomware"
]
}
}
],
"result": "MISP Event created",
"evaluation_strategy": "data_filtering",
"evaluation_context": {
},
"score_range": [
0,
20
]
}
],
"name": "Event Creation",
"target_tool": "MISP",
"uuid": "8e8dbda2-0f5e-4101-83ff-63c1ddda2cae"
},
{
"action": "infection-email",
"inject_evaluation": [
{
"parameters": [
{
".Event.info": {
"comparison": "contains",
"values": [
"ransomware"
]
}
},
{
".Event.Object[].Attribute[] | select(.object_relation == \"email-body\")": {
"extract_type": "all",
"comparison": "count",
"values": [
">=1"
]
}
}
],
"result": "Infection Email added",
"evaluation_strategy": "data_filtering",
"evaluation_context": {
},
"score_range": [
0,
20
]
}
],
"name": "Infection Email",
"target_tool": "MISP",
"uuid": "8f636640-e4f0-4ffb-abff-4e85597aa1bd"
},
{
"action": "malicious-payload",
"inject_evaluation": [
{
"parameters": [
{
".Event.info": {
"comparison": "contains",
"values": [
"ransomware"
]
}
},
{
".Event.Object[].Attribute[] | select((.type == \"filename\")).value": {
"extract_type": "all",
"comparison": "equals",
"values": [
"cryptolocker.exe"
]
}
}
],
"result": "Malicious payload added",
"evaluation_strategy": "data_filtering",
"evaluation_context": {
},
"score_range": [
0,
20
]
}
],
"name": "Malicious Payload",
"target_tool": "MISP",
"uuid": "3e61a340-0314-4622-91cc-042f3ff8543a"
},
{
"action": "c2-ip",
"inject_evaluation": [
{
"parameters": [
{
".Event.info": {
"comparison": "contains",
"values": [
"ransomware"
]
}
},
{
".Event.Object[] | select((.name == \"domain-ip\") or (.name == \"ip-port\")) | .Attribute[].value": {
"extract_type": "all",
"comparison": "contains",
"values": [
"81.177.170.166"
]
}
}
],
"result": "C2 IP added",
"evaluation_strategy": "data_filtering",
"evaluation_context": {
},
"score_range": [
0,
20
]
}
],
"name": "C2 IP Address",
"target_tool": "MISP",
"uuid": "8a2d58c8-2b3a-4ba2-bb77-15bcfa704828"
},
{
"action": "registry-key",
"inject_evaluation": [
{
"parameters": [
{
".Event.info": {
"comparison": "contains",
"values": [
"ransomware"
]
}
},
{
"[.Event.Object[].Attribute[], .Event.Attribute[]] | .[].value": {
"extract_type": "all",
"comparison": "contains-regex",
"values": [
"HKCU.+SOFTWARE.+CryptoLocker.*"
]
}
}
],
"result": "Registry key added",
"evaluation_strategy": "data_filtering",
"evaluation_context": {
},
"score_range": [
0,
20
]
}
],
"name": "Registry Keys",
"target_tool": "MISP",
"uuid": "9df13cc8-b61b-4c9f-a1a8-66def8b64439"
},
{
"action": "pub-key",
"inject_evaluation": [
{
"parameters": [
{
".Event.info": {
"comparison": "contains",
"values": [
"ransomware"
]
}
},
{
"[.Event.Object[].Attribute[], .Event.Attribute[]] | .[].value": {
"extract_type": "all",
"comparison": "contains-regex",
"values": [
"-----BEGIN PUBLIC KEY-----.*"
]
}
}
],
"result": "Public key added",
"evaluation_strategy": "data_filtering",
"evaluation_context": {
},
"score_range": [
0,
20
]
}
],
"name": "Public Key",
"target_tool": "MISP",
"uuid": "c5c03af1-7ef3-44e7-819a-6c4fd402148a"
},
{
"action": "context",
"inject_evaluation": [
{
"parameters": [
{
".Event.info": {
"comparison": "contains",
"values": [
"ransomware"
]
}
},
{
".Event.Tag | select(length > 0) | .[].name": {
"extract_type": "all",
"comparison": "count",
"values": [
">=3"
]
}
}
],
"result": "Context added",
"evaluation_strategy": "data_filtering",
"evaluation_context": {
},
"score_range": [
0,
20
]
}
],
"name": "Contextualization",
"target_tool": "MISP",
"uuid": "11f6f0c2-8813-42ee-a312-136649d3f077"
},
{
"action": "published",
"inject_evaluation": [
{
"parameters": [
{
".Event.info": {
"comparison": "contains",
"values": [
"ransomware"
]
}
},
{
".Event.published": {
"comparison": "equals",
"values": [
"1"
]
}
}
],
"result": "Event published",
"evaluation_strategy": "data_filtering",
"evaluation_context": {
},
"score_range": [
0,
20
]
}
],
"name": "Published",
"target_tool": "MISP",
"uuid": "e3ef4e5f-454a-48c8-a5d7-b3d1d25ecc9f"
}
]
}

78
inject_evaluator.py
View file

@ -5,9 +5,13 @@ import re
import operator
def jq_extract(path: str, data: dict):
path = '.' + path if not path.startswith('.') else path
return jq.compile(path).input_value(data).first()
# .Event.Attribute[] | select(.value == "evil.exe") | .Tag
def jq_extract(path: str, data: dict, extract_type='first'):
query = jq.compile(path).input_value(data)
try:
return query.first() if extract_type == 'first' else query.all()
except StopIteration:
return None
##
@ -15,6 +19,8 @@ def jq_extract(path: str, data: dict):
##
def condition_satisfied(evaluation_config: dict, data_to_validate: Union[dict, list, str]) -> bool:
if type(data_to_validate) is bool:
data_to_validate = "1" if data_to_validate else "0"
if type(data_to_validate) is str:
return eval_condition_str(evaluation_config, data_to_validate)
elif type(data_to_validate) is list:
@ -61,15 +67,54 @@ def eval_condition_list(evaluation_config: dict, data_to_validate: str) -> bool:
if len(values) == 0:
return False
data_to_validate_set = set(data_to_validate)
values_set = set(values)
if comparison_type == 'contains' or comparison_type == 'equals':
data_to_validate_set = set(data_to_validate)
values_set = set(values)
intersection = data_to_validate_set & values_set
if comparison_type == 'contains':
return len(intersection) == len(values_set)
elif comparison_type == 'equals':
return len(intersection) == len(values_set) and len(intersection) == len(data_to_validate_set)
if comparison_type == 'contains-regex':
regex = re.compile(values[0])
for candidate in data_to_validate:
if regex.match(candidate):
return True
return False
elif comparison_type == 'count':
value = values[0]
if value.isdigit():
return len(data_to_validate) == value
elif value[:2] in comparators.keys():
count = len(data_to_validate)
value_operator = values[0][:2]
value = int(value[2:])
return comparators[value_operator](count, value)
elif value[0] in comparators.keys():
count = len(data_to_validate)
value_operator = value[0]
value = int(value[1:])
return comparators[value_operator](count, value)
return False
def eval_condition_dict(evaluation_config: dict, data_to_validate: str) -> bool:
comparison_type = evaluation_config['comparison']
values = evaluation_config['values']
comparators = {
'<': operator.lt,
'<=': operator.le,
'>': operator.gt,
'>=': operator.ge,
'=': operator.eq,
}
comparison_type = evaluation_config['comparison']
if comparison_type == 'contains':
intersection = data_to_validate_set & values_set
return len(intersection) == len(values_set)
pass
elif comparison_type == 'equals':
intersection = data_to_validate_set & values_set
return len(intersection) == len(values_set) and len(intersection) == len(data_to_validate_set)
pass
elif comparison_type == 'count':
if values[0].isdigit():
return len(data_to_validate) == values[0]
@ -81,23 +126,12 @@ def eval_condition_list(evaluation_config: dict, data_to_validate: str) -> bool:
return False
def eval_condition_dict(evaluation_config: dict, data_to_validate: str) -> bool:
print('Condition on dict not supported yet')
comparison_type = evaluation_config['comparison']
if comparison_type == 'contains':
pass
elif comparison_type == 'equals':
pass
elif comparison_type == 'count':
pass
return False
def eval_data_filtering(user_id: int, inject_evaluation: dict, data: dict) -> bool:
for evaluation_params in inject_evaluation['parameters']:
for evaluation_path, evaluation_config in evaluation_params.items():
data_to_validate = jq_extract(evaluation_path, data)
data_to_validate = jq_extract(evaluation_path, data, evaluation_config.get('extract_type', 'first'))
if data_to_validate is None:
print('Could not extract data')
return False
if not condition_satisfied(evaluation_config, data_to_validate):
return False

4
misp_api.py
View file

@ -20,7 +20,7 @@ def get(url, data={}, api_key=misp_apikey):
}
full_url = urljoin(misp_url, url)
response = requests.get(full_url, data=data, headers=headers, verify=not misp_skipssl)
return response.json() if response.headers['content-type'] == 'application/json' else response.text
return response.json() if response.headers['content-type'].startswith('application/json') else response.text
def post(url, data={}, api_key=misp_apikey):
@ -32,7 +32,7 @@ def post(url, data={}, api_key=misp_apikey):
}
full_url = urljoin(misp_url, url)
response = requests.post(full_url, data=json.dumps(data), headers=headers, verify=not misp_skipssl)
return response.json() if response.headers['content-type'] == 'application/json' else response.text
return response.json() if response.headers['content-type'].startswith('application/json') else response.text
def getEvent(event_id: int) -> Union[None, dict]:

13
notification.py
View file

@ -101,7 +101,7 @@ def get_notification_message(data: dict) -> dict:
response_code = data.get('response_code', '?')
user_agent = data.get('user_agent', '?')
_, action = get_scope_action_from_url(url)
http_method = 'DELETE' if http_method == 'POST' and action == 'delete' else http_method # small override for UI
http_method = 'DELETE' if (http_method == 'POST' or http_method == 'PUT') and action == 'delete' else http_method # small override for UI
payload = get_request_post_body(data)
return {
'user': user,
@ -115,13 +115,18 @@ def get_notification_message(data: dict) -> dict:
}
def get_scope_action_from_url(url) -> str:
def get_scope_action_from_url(url) -> Union[str, None]:
split = url.split('/')
return (split[1], split[2],)
if len(split) > 2:
return (split[1], split[2],)
else:
return (None, None,)
def is_accepted_notification(notification) -> bool:
if notification['user_agent'] == 'misp-exercise-dashboard':
if notification['user_agent'] == 'misp-exercise-dashboard': # Ignore message generated from this app
return False
if '@' not in notification['user']: # Ignore message from system
return False
scope, action = get_scope_action_from_url(notification['url'])

6
server.py
View file

@ -65,7 +65,7 @@ def handleMessage(topic, s, message):
if topic == 'misp_json_audit':
user_id, email = notification_model.get_user_email_id_pair(data)
if user_id is not None:
if user_id is not None and '@' in email:
if user_id not in db.USER_ID_TO_EMAIL_MAPPING:
db.USER_ID_TO_EMAIL_MAPPING[user_id] = email
sio.emit('new_user', email)
@ -107,8 +107,10 @@ def forward_zmq_to_socketio():
while True:
message = zsocket.recv_string()
topic, s, m = message.partition(" ")
handleMessage(topic, s, m)
try:
handleMessage(topic, s, m)
pass
# handleMessage(topic, s, m)
except Exception as e:
print(e)