Compare commits
4 commits
a21549f587
...
99f9751e22
Author | SHA1 | Date | |
---|---|---|---|
|
99f9751e22 | ||
|
8774d70759 | ||
|
b79152a6e5 | ||
|
44d040c70b |
5 changed files with 124 additions and 45 deletions
51
exercise.py
51
exercise.py
|
@ -2,7 +2,6 @@
|
|||
|
||||
import functools
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
import json
|
||||
import re
|
||||
|
@ -10,7 +9,7 @@ from typing import Union
|
|||
import jq
|
||||
|
||||
import db
|
||||
from inject_evaluator import eval_data_filtering, eval_query_mirror
|
||||
from inject_evaluator import eval_data_filtering, eval_query_mirror, eval_query_search
|
||||
import misp_api
|
||||
import config
|
||||
from config import logger
|
||||
|
@ -312,9 +311,9 @@ def get_progress():
|
|||
return progress
|
||||
|
||||
|
||||
def check_inject(user_id: int, inject: dict, data: dict, context: dict) -> bool:
|
||||
async def check_inject(user_id: int, inject: dict, data: dict, context: dict) -> bool:
|
||||
for inject_evaluation in inject['inject_evaluation']:
|
||||
success = inject_checker_router(user_id, inject_evaluation, data, context)
|
||||
success = await inject_checker_router(user_id, inject_evaluation, data, context)
|
||||
if not success:
|
||||
logger.info(f"Task not completed: {inject['uuid']}")
|
||||
return False
|
||||
|
@ -338,35 +337,39 @@ def is_valid_evaluation_context(user_id: int, inject_evaluation: dict, data: dic
|
|||
return False
|
||||
return False
|
||||
|
||||
def inject_checker_router(user_id: int, inject_evaluation: dict, data: dict, context: dict) -> bool:
|
||||
async def inject_checker_router(user_id: int, inject_evaluation: dict, data: dict, context: dict) -> bool:
|
||||
if not is_valid_evaluation_context(user_id, inject_evaluation, data, context):
|
||||
return False
|
||||
|
||||
if 'evaluation_strategy' not in inject_evaluation:
|
||||
return False
|
||||
|
||||
data_to_validate = get_data_to_validate(user_id, inject_evaluation, data)
|
||||
data_to_validate = await get_data_to_validate(user_id, inject_evaluation, data)
|
||||
if data_to_validate is None:
|
||||
logger.debug('Could not fetch data to validate')
|
||||
return False
|
||||
|
||||
if inject_evaluation['evaluation_strategy'] == 'data_filtering':
|
||||
return eval_data_filtering(user_id, inject_evaluation, data_to_validate)
|
||||
return eval_data_filtering(user_id, inject_evaluation, data_to_validate, context)
|
||||
elif inject_evaluation['evaluation_strategy'] == 'query_mirror':
|
||||
expected_data = data_to_validate['expected_data']
|
||||
data_to_validate = data_to_validate['data_to_validate']
|
||||
return eval_query_mirror(user_id, expected_data, data_to_validate)
|
||||
return eval_query_mirror(user_id, expected_data, data_to_validate, context)
|
||||
elif inject_evaluation['evaluation_strategy'] == 'query_search':
|
||||
return eval_query_search(user_id, inject_evaluation, data_to_validate, context)
|
||||
return False
|
||||
|
||||
|
||||
def get_data_to_validate(user_id: int, inject_evaluation: dict, data: dict) -> Union[dict, list, str, None]:
|
||||
async def get_data_to_validate(user_id: int, inject_evaluation: dict, data: dict) -> Union[dict, list, str, None]:
|
||||
data_to_validate = None
|
||||
if inject_evaluation['evaluation_strategy'] == 'data_filtering':
|
||||
event_id = parse_event_id_from_log(data)
|
||||
data_to_validate = fetch_data_for_data_filtering(event_id=event_id)
|
||||
data_to_validate = await fetch_data_for_data_filtering(event_id=event_id)
|
||||
elif inject_evaluation['evaluation_strategy'] == 'query_mirror':
|
||||
perfomed_query = parse_performed_query_from_log(data)
|
||||
data_to_validate = fetch_data_for_query_mirror(user_id, inject_evaluation, perfomed_query)
|
||||
data_to_validate = await fetch_data_for_query_mirror(user_id, inject_evaluation, perfomed_query)
|
||||
elif inject_evaluation['evaluation_strategy'] == 'query_search':
|
||||
data_to_validate = await fetch_data_for_query_search(user_id, inject_evaluation)
|
||||
return data_to_validate
|
||||
|
||||
|
||||
|
@ -412,14 +415,14 @@ def parse_performed_query_from_log(data: dict) -> Union[dict, None]:
|
|||
return None
|
||||
|
||||
|
||||
def fetch_data_for_data_filtering(event_id=None) -> Union[None, dict]:
|
||||
async def fetch_data_for_data_filtering(event_id=None) -> Union[None, dict]:
|
||||
data = None
|
||||
if event_id is not None:
|
||||
data = misp_api.getEvent(event_id)
|
||||
data = await misp_api.getEvent(event_id)
|
||||
return data
|
||||
|
||||
|
||||
def fetch_data_for_query_mirror(user_id: int, inject_evaluation: dict, perfomed_query: dict) -> Union[None, dict]:
|
||||
async def fetch_data_for_query_mirror(user_id: int, inject_evaluation: dict, perfomed_query: dict) -> Union[None, dict]:
|
||||
data = None
|
||||
authkey = db.USER_ID_TO_AUTHKEY_MAPPING[user_id]
|
||||
if perfomed_query is not None:
|
||||
|
@ -429,8 +432,8 @@ def fetch_data_for_query_mirror(user_id: int, inject_evaluation: dict, perfomed_
|
|||
expected_method = query_context['request_method']
|
||||
expected_url = query_context['url']
|
||||
expected_payload = inject_evaluation['parameters'][0]
|
||||
expected_data = misp_api.doRestQuery(authkey, expected_method, expected_url, expected_payload)
|
||||
data_to_validate = misp_api.doRestQuery(authkey, perfomed_query['request_method'], perfomed_query['url'], perfomed_query['payload'])
|
||||
expected_data = await misp_api.doRestQuery(authkey, expected_method, expected_url, expected_payload)
|
||||
data_to_validate = await misp_api.doRestQuery(authkey, perfomed_query['request_method'], perfomed_query['url'], perfomed_query['payload'])
|
||||
data = {
|
||||
'expected_data' : expected_data,
|
||||
'data_to_validate' : data_to_validate,
|
||||
|
@ -438,8 +441,20 @@ def fetch_data_for_query_mirror(user_id: int, inject_evaluation: dict, perfomed_
|
|||
return data
|
||||
|
||||
|
||||
async def fetch_data_for_query_search(user_id: int, inject_evaluation: dict) -> Union[None, dict]:
|
||||
authkey = db.USER_ID_TO_AUTHKEY_MAPPING[user_id]
|
||||
if 'evaluation_context' not in inject_evaluation and 'query_context' not in inject_evaluation['evaluation_context']:
|
||||
return None
|
||||
query_context = inject_evaluation['evaluation_context']['query_context']
|
||||
search_method = query_context['request_method']
|
||||
search_url = query_context['url']
|
||||
search_payload = query_context['payload']
|
||||
search_data = await misp_api.doRestQuery(authkey, search_method, search_url, search_payload)
|
||||
return search_data
|
||||
|
||||
|
||||
@debounce_check_active_tasks(debounce_seconds=2)
|
||||
def check_active_tasks(user_id: int, data: dict, context: dict) -> bool:
|
||||
async def check_active_tasks(user_id: int, data: dict, context: dict) -> bool:
|
||||
succeeded_once = False
|
||||
available_tasks = get_available_tasks_for_user(user_id)
|
||||
for task_uuid in available_tasks:
|
||||
|
@ -447,7 +462,7 @@ def check_active_tasks(user_id: int, data: dict, context: dict) -> bool:
|
|||
if inject['exercise_uuid'] not in db.SELECTED_EXERCISES:
|
||||
continue
|
||||
logger.debug(f"[{task_uuid}] :: checking: {inject['name']}")
|
||||
completed = check_inject(user_id, inject, data, context)
|
||||
completed = await check_inject(user_id, inject, data, context)
|
||||
if completed:
|
||||
succeeded_once = True
|
||||
return succeeded_once
|
|
@ -138,7 +138,15 @@
|
|||
{
|
||||
"parameters": [
|
||||
{
|
||||
".Event.info": {
|
||||
".response[].Event.event_creator_email": {
|
||||
"comparison": "equals",
|
||||
"values": [
|
||||
"{{user_email}}"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
".response[].Event.info": {
|
||||
"comparison": "contains",
|
||||
"values": [
|
||||
"event",
|
||||
|
@ -148,9 +156,17 @@
|
|||
}
|
||||
],
|
||||
"result": "MISP Event created",
|
||||
"evaluation_strategy": "data_filtering",
|
||||
"evaluation_strategy": "query_search",
|
||||
"evaluation_context": {
|
||||
"request_is_rest": true
|
||||
"request_is_rest": true,
|
||||
"query_context": {
|
||||
"url": "/events/restSearch",
|
||||
"request_method": "POST",
|
||||
"payload": {
|
||||
"timestamp": "10d",
|
||||
"eventinfo": "%API%"
|
||||
}
|
||||
}
|
||||
},
|
||||
"score_range": [
|
||||
0,
|
||||
|
|
|
@ -6,7 +6,6 @@ import operator
|
|||
from config import logger
|
||||
|
||||
|
||||
# .Event.Attribute[] | select(.value == "evil.exe") | .Tag
|
||||
def jq_extract(path: str, data: dict, extract_type='first'):
|
||||
query = jq.compile(path).input_value(data)
|
||||
try:
|
||||
|
@ -15,28 +14,42 @@ def jq_extract(path: str, data: dict, extract_type='first'):
|
|||
return None
|
||||
|
||||
|
||||
# Replace the substring `{{variable}}` by context[variable] in the provided string
|
||||
def apply_replacement_from_context(string: str, context: dict) -> str:
|
||||
replacement_regex = r"{{(\w+)}}"
|
||||
if r'{{' not in string and r'}}' not in string:
|
||||
return string
|
||||
matches = re.fullmatch(replacement_regex, string, re.MULTILINE)
|
||||
if not matches:
|
||||
return string
|
||||
subst_str = matches.groups()[0]
|
||||
subst = str(context.get(subst_str, ''))
|
||||
return re.sub(replacement_regex, subst, string)
|
||||
|
||||
|
||||
##
|
||||
## Data Filtering
|
||||
##
|
||||
|
||||
def condition_satisfied(evaluation_config: dict, data_to_validate: Union[dict, list, str]) -> bool:
|
||||
def condition_satisfied(evaluation_config: dict, data_to_validate: Union[dict, list, str], context: dict) -> bool:
|
||||
if type(data_to_validate) is bool:
|
||||
data_to_validate = "1" if data_to_validate else "0"
|
||||
if type(data_to_validate) is str:
|
||||
return eval_condition_str(evaluation_config, data_to_validate)
|
||||
return eval_condition_str(evaluation_config, data_to_validate, context)
|
||||
elif type(data_to_validate) is list:
|
||||
return eval_condition_list(evaluation_config, data_to_validate)
|
||||
return eval_condition_list(evaluation_config, data_to_validate, context)
|
||||
elif type(data_to_validate) is dict:
|
||||
# Not sure how we could have condition on this
|
||||
return eval_condition_dict(evaluation_config, data_to_validate)
|
||||
return eval_condition_dict(evaluation_config, data_to_validate, context)
|
||||
return False
|
||||
|
||||
|
||||
def eval_condition_str(evaluation_config: dict, data_to_validate: str) -> bool:
|
||||
def eval_condition_str(evaluation_config: dict, data_to_validate: str, context: dict) -> bool:
|
||||
comparison_type = evaluation_config['comparison']
|
||||
values = evaluation_config['values']
|
||||
if len(values) == 0:
|
||||
return False
|
||||
values = [apply_replacement_from_context(v, context) for v in values]
|
||||
|
||||
if comparison_type == 'contains':
|
||||
values = [v.lower() for v in values]
|
||||
|
@ -56,7 +69,7 @@ def eval_condition_str(evaluation_config: dict, data_to_validate: str) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
def eval_condition_list(evaluation_config: dict, data_to_validate: str) -> bool:
|
||||
def eval_condition_list(evaluation_config: dict, data_to_validate: str, context: dict) -> bool:
|
||||
comparison_type = evaluation_config['comparison']
|
||||
values = evaluation_config['values']
|
||||
comparators = {
|
||||
|
@ -69,7 +82,7 @@ def eval_condition_list(evaluation_config: dict, data_to_validate: str) -> bool:
|
|||
|
||||
if len(values) == 0:
|
||||
return False
|
||||
|
||||
values = [apply_replacement_from_context(v, context) for v in values]
|
||||
|
||||
if comparison_type == 'contains' or comparison_type == 'equals':
|
||||
data_to_validate_set = set(data_to_validate)
|
||||
|
@ -102,7 +115,7 @@ def eval_condition_list(evaluation_config: dict, data_to_validate: str) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
def eval_condition_dict(evaluation_config: dict, data_to_validate: str) -> bool:
|
||||
def eval_condition_dict(evaluation_config: dict, data_to_validate: str, context: dict) -> bool:
|
||||
comparison_type = evaluation_config['comparison']
|
||||
values = evaluation_config['values']
|
||||
comparators = {
|
||||
|
@ -113,6 +126,10 @@ def eval_condition_dict(evaluation_config: dict, data_to_validate: str) -> bool:
|
|||
'=': operator.eq,
|
||||
}
|
||||
|
||||
if len(values) == 0:
|
||||
return False
|
||||
values = [apply_replacement_from_context(v, context) for v in values]
|
||||
|
||||
comparison_type = evaluation_config['comparison']
|
||||
if comparison_type == 'contains':
|
||||
pass
|
||||
|
@ -129,21 +146,31 @@ def eval_condition_dict(evaluation_config: dict, data_to_validate: str) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
def eval_data_filtering(user_id: int, inject_evaluation: dict, data: dict) -> bool:
|
||||
def eval_data_filtering(user_id: int, inject_evaluation: dict, data: dict, context: dict) -> bool:
|
||||
for evaluation_params in inject_evaluation['parameters']:
|
||||
for evaluation_path, evaluation_config in evaluation_params.items():
|
||||
evaluation_path = apply_replacement_from_context(evaluation_path, context)
|
||||
data_to_validate = jq_extract(evaluation_path, data, evaluation_config.get('extract_type', 'first'))
|
||||
if data_to_validate is None:
|
||||
logger.debug('Could not extract data')
|
||||
return False
|
||||
if not condition_satisfied(evaluation_config, data_to_validate):
|
||||
if not condition_satisfied(evaluation_config, data_to_validate, context):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
##
|
||||
## Query comparison
|
||||
## Query mirror
|
||||
##
|
||||
|
||||
def eval_query_mirror(user_id: int, expected_data, data_to_validate) -> bool:
|
||||
def eval_query_mirror(user_id: int, expected_data, data_to_validate, context: dict) -> bool:
|
||||
return expected_data == data_to_validate
|
||||
|
||||
|
||||
|
||||
##
|
||||
## Query search
|
||||
##
|
||||
|
||||
def eval_query_search(user_id: int, inject_evaluation: dict, data: dict, context: dict) -> bool:
|
||||
return eval_data_filtering(user_id, inject_evaluation, data, context)
|
16
server.py
16
server.py
|
@ -146,8 +146,8 @@ async def handleMessage(topic, s, message):
|
|||
user_id = notification_model.get_user_id(data)
|
||||
if user_id is not None:
|
||||
if exercise_model.is_accepted_query(data):
|
||||
context = get_context(data)
|
||||
succeeded_once = exercise_model.check_active_tasks(user_id, data, context)
|
||||
context = get_context(topic, user_id, data)
|
||||
succeeded_once = await exercise_model.check_active_tasks(user_id, data, context)
|
||||
if succeeded_once:
|
||||
await sendRefreshScore()
|
||||
|
||||
|
@ -157,8 +157,13 @@ async def sendRefreshScore():
|
|||
await sio.emit('refresh_score')
|
||||
|
||||
|
||||
def get_context(data: dict) -> dict:
|
||||
context = {}
|
||||
def get_context(topic: str, user_id: int, data: dict) -> dict:
|
||||
context = {
|
||||
'zmq_topic': topic,
|
||||
'user_id': user_id,
|
||||
'user_email': db.USER_ID_TO_EMAIL_MAPPING.get(user_id, None),
|
||||
'user_authkey': db.USER_ID_TO_AUTHKEY_MAPPING.get(user_id, None),
|
||||
}
|
||||
if 'Log' in data:
|
||||
if 'request_is_rest' in data['Log']:
|
||||
context['request_is_rest'] = data['Log']['request_is_rest']
|
||||
|
@ -200,10 +205,11 @@ async def forward_zmq_to_socketio():
|
|||
while True:
|
||||
message = await zsocket.recv_string()
|
||||
topic, s, m = message.partition(" ")
|
||||
await handleMessage(topic, s, m)
|
||||
try:
|
||||
ZMQ_MESSAGE_COUNT += 1
|
||||
ZMQ_LAST_TIME = time.time()
|
||||
await handleMessage(topic, s, m)
|
||||
# await handleMessage(topic, s, m)
|
||||
except Exception as e:
|
||||
logger.error('Error handling message %s', e)
|
||||
|
||||
|
|
|
@ -1,9 +1,21 @@
|
|||
<script setup>
|
||||
import { computed } from "vue"
|
||||
import { ref, onMounted } from "vue"
|
||||
import { socketConnected, zmqLastTime } from "@/socket";
|
||||
|
||||
const zmqLastTimeSecond = computed(() => {
|
||||
return parseInt(((new Date()).getTime() - zmqLastTime.value * 1000) / 1000)
|
||||
const zmqLastTimeSecond = ref('?')
|
||||
|
||||
function refreshLastTime() {
|
||||
if (zmqLastTime.value !== false) {
|
||||
zmqLastTimeSecond.value = parseInt(((new Date()).getTime() - zmqLastTime.value * 1000) / 1000)
|
||||
} else {
|
||||
zmqLastTimeSecond.value = '?'
|
||||
}
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
setInterval(() => {
|
||||
refreshLastTime()
|
||||
}, 1000)
|
||||
})
|
||||
|
||||
</script>
|
||||
|
@ -13,18 +25,21 @@
|
|||
<span :class="{
|
||||
'px-2 rounded-md inline-block w-48 leading-4': true,
|
||||
'text-slate-900 dark:text-slate-400': socketConnected,
|
||||
'text-slate-50 bg-red-600': !socketConnected,
|
||||
'text-slate-50 bg-red-600 px-2 py-1': !socketConnected,
|
||||
}">
|
||||
<span class="mr-1">Socket.IO:</span>
|
||||
<span v-show="socketConnected" class="font-semibold text-green-600 dark:text-green-400">Connected</span>
|
||||
<span v-show="!socketConnected" class="font-semibold text-slate-50">Disconnected</span>
|
||||
</span>
|
||||
<span class="text-xs font-thin leading-3 inline-block text-center">
|
||||
<span
|
||||
v-if="socketConnected"
|
||||
class="text-xs font-thin leading-3 inline-block text-center"
|
||||
>
|
||||
<template v-if="zmqLastTimeSecond == 0">
|
||||
online
|
||||
</template>
|
||||
<template v-else>
|
||||
{{ zmqLastTimeSecond }}s
|
||||
Last keep-alive: {{ zmqLastTimeSecond }}s ago
|
||||
</template>
|
||||
</span>
|
||||
</span>
|
||||
|
|
Loading…
Reference in a new issue