From abca50d6157aa0f975cf05b81e9ec2dc46492555 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Thu, 4 Jul 2024 19:46:05 +0200 Subject: [PATCH] new: [app:notification_history] Added notification history support --- db.py | 14 +++- notification.py | 8 +++ package-lock.json | 126 ++++++++++++++++++++++++++++++++- package.json | 4 +- server.py | 18 ++++- src/components/TheLiveLogs.vue | 54 +++++++++++++- src/main.js | 6 +- src/socket.js | 6 ++ 8 files changed, 229 insertions(+), 7 deletions(-) diff --git a/db.py b/db.py index fdf99f6..dd69865 100644 --- a/db.py +++ b/db.py @@ -16,6 +16,18 @@ PROGRESS = { NOTIFICATION_BUFFER_SIZE = 30 NOTIFICATION_MESSAGES = collections.deque([], NOTIFICATION_BUFFER_SIZE) +NOTIFICATION_HISTORY_BUFFER_RESOLUTION_PER_MIN = 3 +NOTIFICATION_HISTORY_BUFFER_TIMESPAN_MIN = 60 +NOTIFICATION_HISTORY_FREQUENCY = NOTIFICATION_HISTORY_BUFFER_TIMESPAN_MIN / NOTIFICATION_HISTORY_BUFFER_RESOLUTION_PER_MIN +notification_history_size = NOTIFICATION_HISTORY_BUFFER_RESOLUTION_PER_MIN * NOTIFICATION_HISTORY_BUFFER_TIMESPAN_MIN +NOTIFICATION_HISTORY = collections.deque([], notification_history_size) +NOTIFICATION_HISTORY.extend([0] * notification_history_size) + def resetNotificationMessage(): global NOTIFICATION_MESSAGES - NOTIFICATION_MESSAGES = collections.deque([], NOTIFICATION_BUFFER_SIZE) \ No newline at end of file + NOTIFICATION_MESSAGES = collections.deque([], NOTIFICATION_BUFFER_SIZE) + +def resetNotificationHistory(): + global NOTIFICATION_HISTORY + NOTIFICATION_HISTORY = collections.deque([], notification_history_size) + NOTIFICATION_HISTORY.extend([0] * notification_history_size) \ No newline at end of file diff --git a/notification.py b/notification.py index 4ca19c0..8628419 100644 --- a/notification.py +++ b/notification.py @@ -27,6 +27,10 @@ def get_notifications() -> list[dict]: return list(db.NOTIFICATION_MESSAGES) +def get_notifications_history() -> list[dict]: + return list(db.NOTIFICATION_HISTORY) + + def reset_notifications(): db.resetNotificationMessage() @@ -35,6 +39,10 @@ def record_notification(notification: dict): db.NOTIFICATION_MESSAGES.appendleft(notification) +def record_notification_history(message_count: int): + db.NOTIFICATION_HISTORY.append(message_count) + + def get_user_id(data: dict): if 'user_id' in data: return int(data['user_id']) diff --git a/package-lock.json b/package-lock.json index 42097ea..cd5727a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,8 +13,10 @@ "@fortawesome/free-regular-svg-icons": "^6.5.2", "@fortawesome/free-solid-svg-icons": "^6.5.2", "@fortawesome/vue-fontawesome": "^3.0.8", + "apexcharts": "^3.49.2", "lodash.debounce": "^4.0.8", - "vue": "^3.4.29" + "vue": "^3.4.29", + "vue3-apexcharts": "^1.5.3" }, "devDependencies": { "@rushstack/eslint-patch": "^1.8.0", @@ -1077,6 +1079,12 @@ "resolved": "https://registry.npmjs.org/@vue/shared/-/shared-3.4.29.tgz", "integrity": "sha512-hQ2gAQcBO/CDpC82DCrinJNgOHI2v+FA7BDW4lMSPeBpQ7sRe2OLHWe5cph1s7D8DUQAwRt18dBDfJJ220APEA==" }, + "node_modules/@yr/monotone-cubic-spline": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@yr/monotone-cubic-spline/-/monotone-cubic-spline-1.0.3.tgz", + "integrity": "sha512-FQXkOta0XBSUPHndIKON2Y9JeQz5ZeMqLYZVVK93FliNBFm7LNMIZmY6FrMEB9XPcDbE2bekMbZD6kzDkxwYjA==", + "license": "MIT" + }, "node_modules/acorn": { "version": "8.12.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.12.0.tgz", @@ -1157,6 +1165,21 @@ "node": ">= 8" } }, + "node_modules/apexcharts": { + "version": "3.49.2", + "resolved": "https://registry.npmjs.org/apexcharts/-/apexcharts-3.49.2.tgz", + "integrity": "sha512-vBB8KgwfD9rSObA7s4kY2rU6DeaN67gTR3JN7r32ztgKVf8lKkdFQ6iUhk6oIHrV7W8PoHhr5EwKymn0z5Fz6A==", + "license": "MIT", + "dependencies": { + "@yr/monotone-cubic-spline": "^1.0.3", + "svg.draggable.js": "^2.2.2", + "svg.easing.js": "^2.0.0", + "svg.filter.js": "^2.0.2", + "svg.pathmorphing.js": "^0.1.3", + "svg.resize.js": "^1.4.3", + "svg.select.js": "^3.0.1" + } + }, "node_modules/arg": { "version": "5.0.2", "resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz", @@ -3292,6 +3315,97 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/svg.draggable.js": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/svg.draggable.js/-/svg.draggable.js-2.2.2.tgz", + "integrity": "sha512-JzNHBc2fLQMzYCZ90KZHN2ohXL0BQJGQimK1kGk6AvSeibuKcIdDX9Kr0dT9+UJ5O8nYA0RB839Lhvk4CY4MZw==", + "license": "MIT", + "dependencies": { + "svg.js": "^2.0.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/svg.easing.js": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/svg.easing.js/-/svg.easing.js-2.0.0.tgz", + "integrity": "sha512-//ctPdJMGy22YoYGV+3HEfHbm6/69LJUTAqI2/5qBvaNHZ9uUFVC82B0Pl299HzgH13rKrBgi4+XyXXyVWWthA==", + "license": "MIT", + "dependencies": { + "svg.js": ">=2.3.x" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/svg.filter.js": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/svg.filter.js/-/svg.filter.js-2.0.2.tgz", + "integrity": "sha512-xkGBwU+dKBzqg5PtilaTb0EYPqPfJ9Q6saVldX+5vCRy31P6TlRCP3U9NxH3HEufkKkpNgdTLBJnmhDHeTqAkw==", + "license": "MIT", + "dependencies": { + "svg.js": "^2.2.5" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/svg.js": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/svg.js/-/svg.js-2.7.1.tgz", + "integrity": "sha512-ycbxpizEQktk3FYvn/8BH+6/EuWXg7ZpQREJvgacqn46gIddG24tNNe4Son6omdXCnSOaApnpZw6MPCBA1dODA==", + "license": "MIT" + }, + "node_modules/svg.pathmorphing.js": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/svg.pathmorphing.js/-/svg.pathmorphing.js-0.1.3.tgz", + "integrity": "sha512-49HWI9X4XQR/JG1qXkSDV8xViuTLIWm/B/7YuQELV5KMOPtXjiwH4XPJvr/ghEDibmLQ9Oc22dpWpG0vUDDNww==", + "license": "MIT", + "dependencies": { + "svg.js": "^2.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/svg.resize.js": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/svg.resize.js/-/svg.resize.js-1.4.3.tgz", + "integrity": "sha512-9k5sXJuPKp+mVzXNvxz7U0uC9oVMQrrf7cFsETznzUDDm0x8+77dtZkWdMfRlmbkEEYvUn9btKuZ3n41oNA+uw==", + "license": "MIT", + "dependencies": { + "svg.js": "^2.6.5", + "svg.select.js": "^2.1.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/svg.resize.js/node_modules/svg.select.js": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/svg.select.js/-/svg.select.js-2.1.2.tgz", + "integrity": "sha512-tH6ABEyJsAOVAhwcCjF8mw4crjXSI1aa7j2VQR8ZuJ37H2MBUbyeqYr5nEO7sSN3cy9AR9DUwNg0t/962HlDbQ==", + "license": "MIT", + "dependencies": { + "svg.js": "^2.2.5" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/svg.select.js": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/svg.select.js/-/svg.select.js-3.0.1.tgz", + "integrity": "sha512-h5IS/hKkuVCbKSieR9uQCj9w+zLHoPh+ce19bBYyqF53g6mnPB8sAtIbe1s9dh2S2fCmYX2xel1Ln3PJBbK4kw==", + "license": "MIT", + "dependencies": { + "svg.js": "^2.6.5" + }, + "engines": { + "node": ">= 0.8.0" + } + }, "node_modules/synckit": { "version": "0.8.8", "resolved": "https://registry.npmjs.org/synckit/-/synckit-0.8.8.tgz", @@ -3564,6 +3678,16 @@ "eslint": ">=6.0.0" } }, + "node_modules/vue3-apexcharts": { + "version": "1.5.3", + "resolved": "https://registry.npmjs.org/vue3-apexcharts/-/vue3-apexcharts-1.5.3.tgz", + "integrity": "sha512-yaHTPoj0iVKAtEVg8wEwIwwvf0VG+lPYNufCf3txRzYQOqdKPoZaZ9P3Dj3X+2A1XY9O1kcTk9HVqvLo+rppvQ==", + "license": "MIT", + "peerDependencies": { + "apexcharts": "> 3.0.0", + "vue": "> 3.0.0" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", diff --git a/package.json b/package.json index 4b0f633..57425a7 100644 --- a/package.json +++ b/package.json @@ -16,8 +16,10 @@ "@fortawesome/free-regular-svg-icons": "^6.5.2", "@fortawesome/free-solid-svg-icons": "^6.5.2", "@fortawesome/vue-fontawesome": "^3.0.8", + "apexcharts": "^3.49.2", "lodash.debounce": "^4.0.8", - "vue": "^3.4.29" + "vue": "^3.4.29", + "vue3-apexcharts": "^1.5.3" }, "devDependencies": { "@rushstack/eslint-patch": "^1.8.0", diff --git a/server.py b/server.py index b6da37d..2b3f7ff 100755 --- a/server.py +++ b/server.py @@ -17,6 +17,7 @@ from config import logger import misp_api +ZMQ_MESSAGE_COUNT_LAST_TIMESPAN = 0 ZMQ_MESSAGE_COUNT = 0 ZMQ_LAST_TIME = None @@ -122,6 +123,8 @@ async def any_event(event, sid, data={}): logger.info('>> Unhandled event %s', event) async def handleMessage(topic, s, message): + global ZMQ_MESSAGE_COUNT_LAST_TIMESPAN + data = json.loads(message) if topic == 'misp_json_audit': @@ -141,6 +144,7 @@ async def handleMessage(topic, s, message): notification = notification_model.get_notification_message(data) if notification_model.is_accepted_notification(notification): notification_model.record_notification(notification) + ZMQ_MESSAGE_COUNT_LAST_TIMESPAN += 1 await sio.emit('notification', notification) user_id = notification_model.get_user_id(data) @@ -188,6 +192,16 @@ async def getDiagnostic() -> dict: return diagnostic +async def notification_history(): + global ZMQ_MESSAGE_COUNT_LAST_TIMESPAN + while True: + await sio.sleep(db.NOTIFICATION_HISTORY_FREQUENCY) + notification_model.record_notification_history(ZMQ_MESSAGE_COUNT_LAST_TIMESPAN) + ZMQ_MESSAGE_COUNT_LAST_TIMESPAN = 0 + payload = notification_model.get_notifications_history() + await sio.emit('update_notification_history', payload) + + async def keepalive(): global ZMQ_LAST_TIME while True: @@ -205,10 +219,11 @@ async def forward_zmq_to_socketio(): while True: message = await zsocket.recv_string() topic, s, m = message.partition(" ") + await handleMessage(topic, s, m) try: ZMQ_MESSAGE_COUNT += 1 ZMQ_LAST_TIME = time.time() - await handleMessage(topic, s, m) + # await handleMessage(topic, s, m) except Exception as e: logger.error('Error handling message %s', e) @@ -216,6 +231,7 @@ async def forward_zmq_to_socketio(): async def init_app(): sio.start_background_task(forward_zmq_to_socketio) sio.start_background_task(keepalive) + sio.start_background_task(notification_history) return app diff --git a/src/components/TheLiveLogs.vue b/src/components/TheLiveLogs.vue index b9040ac..9e78b38 100644 --- a/src/components/TheLiveLogs.vue +++ b/src/components/TheLiveLogs.vue @@ -1,12 +1,54 @@