From 4c20f58a522e380a2afaa329fa40793e32993f49 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 1 Aug 2019 13:16:57 +0200 Subject: [PATCH] chg: [api] add advanced get item via POST + use same query for each get item --- bin/packages/Item.py | 74 ++++++++++ doc/README.md | 130 +++++++++++++++++- .../modules/PasteSubmit/Flask_PasteSubmit.py | 10 +- var/www/modules/restApi/Flask_restApi.py | 66 +++++---- 4 files changed, 243 insertions(+), 37 deletions(-) diff --git a/bin/packages/Item.py b/bin/packages/Item.py index a2276fbd..6b24bb18 100755 --- a/bin/packages/Item.py +++ b/bin/packages/Item.py @@ -2,12 +2,15 @@ # -*-coding:UTF-8 -* import os +import gzip import redis import Flask_config import Date +import Tag PASTES_FOLDER = Flask_config.PASTES_FOLDER +r_cache = Flask_config.r_cache def exist_item(item_id): if os.path.isfile(os.path.join(PASTES_FOLDER, item_id)): @@ -18,3 +21,74 @@ def exist_item(item_id): def get_item_date(item_id): l_directory = item_id.split('/') return '{}{}{}'.format(l_directory[-4], l_directory[-3], l_directory[-2]) + +def get_item_size(item_id): + return round(os.path.getsize(os.path.join(PASTES_FOLDER, item_id))/1024.0, 2) + +def get_lines_info(item_id, item_content=None): + if not item_content: + item_content = get_item_content(item_id) + max_length = 0 + line_id = 0 + nb_line = 0 + for line in item_content.splitlines(): + length = len(line) + if length > max_length: + max_length = length + nb_line += 1 + return {'nb': nb_line, 'max_length': max_length} + + +def get_item_content(item_id): + item_full_path = os.path.join(PASTES_FOLDER, item_id) + try: + item_content = r_cache.get(item_full_path) + except UnicodeDecodeError: + item_content = None + except Exception as e: + print("ERROR in: " + item_id) + print(e) + item_content = None + if item_content is None: + try: + with gzip.open(item_full_path, 'r') as f: + item_content = f.read() + r_cache.set(item_full_path, item_content) + r_cache.expire(item_full_path, 300) + except: + item_content = '' + return str(item_content) + +# API +def get_item(request_dict): + if not request_dict: + return Response({'status': 'error', 'reason': 'Malformed JSON'}, 400) + + item_id = request_dict.get('id', None) + if not item_id: + return ( {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400 ) + if not exist_item(item_id): + return ( {'status': 'error', 'reason': 'Item not found'}, 404 ) + + dict_item = {} + dict_item['id'] = item_id + date = request_dict.get('date', True) + if date: + dict_item['date'] = get_item_date(item_id) + tags = request_dict.get('tags', True) + if tags: + dict_item['tags'] = Tag.get_item_tags(item_id) + + size = request_dict.get('size', False) + if size: + dict_item['size'] = get_item_size(item_id) + + content = request_dict.get('content', False) + if content: + dict_item['content'] = get_item_content(item_id) + + lines_info = request_dict.get('lines', False) + if lines_info: + dict_item['lines'] = get_lines_info(item_id, dict_item.get('content', 'None')) + + return (dict_item, 200) diff --git a/doc/README.md b/doc/README.md index 0a819ac6..af651872 100644 --- a/doc/README.md +++ b/doc/README.md @@ -27,10 +27,10 @@ curl --header "Authorization: YOUR_API_KEY" --header "Content-Type: application/ ## Item management -### Get item: `api/get/item/basic/` +### Get item: `api/get/item/default/` #### Description -Get anitem basic information. +Get item default info. **Method** : `GET` @@ -56,7 +56,7 @@ Get anitem basic information. #### Example ``` -curl https://127.0.0.1:7000/api/get/item/info/submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" +curl https://127.0.0.1:7000/api/get/item/default/submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" ``` #### Expected Success Response @@ -81,7 +81,10 @@ curl https://127.0.0.1:7000/api/get/item/info/submitted/2019/07/26/3efb8a79-08e9 #### Expected Fail Response **HTTP Status Code** : `400` - +```json + {"status": "error", "reason": "Mandatory parameter(s) not provided"} +``` +**HTTP Status Code** : `404` ```json {"status": "error", "reason": "Item not found"} ``` @@ -128,7 +131,10 @@ curl https://127.0.0.1:7000/api/get/item/content/submitted/2019/07/26/3efb8a79-0 #### Expected Fail Response **HTTP Status Code** : `400` - +```json + {"status": "error", "reason": "Mandatory parameter(s) not provided"} +``` +**HTTP Status Code** : `404` ```json {"status": "error", "reason": "Item not found"} ``` @@ -181,13 +187,125 @@ curl https://127.0.0.1:7000/api/get/item/tag/submitted/2019/07/26/3efb8a79-08e9- #### Expected Fail Response **HTTP Status Code** : `400` - +```json + {"status": "error", "reason": "Mandatory parameter(s) not provided"} +``` +**HTTP Status Code** : `404` ```json {"status": "error", "reason": "Item not found"} ``` +### Advanced Get item: `api/get/item` + +#### Description +Get item. Filter requested field. + +**Method** : `POST` + +#### Parameters +- `id` + - item id + - *str - relative item path* + - mandatory +- `date` + - get item date + - *boolean* + - default: `true` +- `tags` + - get item tags + - *boolean* + - default: `true` +- `content` + - get item content + - *boolean* + - default: `false` +- `size` + - get item size + - *boolean* + - default: `false` +- `lines` + - get item lines info + - *boolean* + - default: `false` + +#### JSON response +- `content` + - item content + - *str* +- `id` + - item id + - *str* +- `date` + - item date + - *str - YYMMDD* +- `tags` + - item tags list + - *list* +- `size` + - item size (Kb) + - *int* +- `lines` + - item lines info + - *{}* + - `max_length` + - line max length line + - *int* + - `nb` + - nb lines item + - *int* + + +#### Example +``` +curl https://127.0.0.1:7000/api/get/item --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X POST +``` + +#### input.json Example +```json +{ + "id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz", + "content": true, + "lines_info": true, + "tags": true, + "size": true +} +``` + +#### Expected Success Response +**HTTP Status Code** : `200` +```json + { + "content": "b'dsvcdsvcdsc vvvv'", + "date": "20190726", + "id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz", + "lines": { + "max_length": 19, + "nb": 1 + }, + "size": 0.03, + "tags": [ + "misp-galaxy:stealer=\"Vidar\"", + "infoleak:submission=\"manual\"" + ] + } +``` + +#### Expected Fail Response +**HTTP Status Code** : `400` +```json + {"status": "error", "reason": "Mandatory parameter(s) not provided"} +``` +**HTTP Status Code** : `404` +```json + {"status": "error", "reason": "Item not found"} +``` + + + + + ### add item tags: `api/add/item/tag` #### Description diff --git a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py index 71d16de2..76bae898 100644 --- a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py +++ b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py @@ -24,7 +24,7 @@ import json import Paste import Import_helper -import Tags +import Tag from pytaxonomies import Taxonomies from pymispgalaxies import Galaxies, Clusters @@ -224,8 +224,8 @@ def hive_create_case(hive_tlp, threat_level, hive_description, hive_case_title, @login_analyst def PasteSubmit_page(): # Get all active tags/galaxy - active_taxonomies = Tags.get_active_taxonomies() - active_galaxies = Tags.get_active_galaxies() + active_taxonomies = Tag.get_active_taxonomies() + active_galaxies = Tag.get_active_galaxies() return render_template("submit_items.html", active_taxonomies = active_taxonomies, @@ -253,9 +253,9 @@ def submit(): submitted_tag = 'infoleak:submission="manual"' #active taxonomies - active_taxonomies = Tags.get_active_taxonomies() + active_taxonomies = Tag.get_active_taxonomies() #active galaxies - active_galaxies = Tags.get_active_galaxies() + active_galaxies = Tag.get_active_galaxies() if ltags or ltagsgalaxies: diff --git a/var/www/modules/restApi/Flask_restApi.py b/var/www/modules/restApi/Flask_restApi.py index 0bb842aa..719d307d 100644 --- a/var/www/modules/restApi/Flask_restApi.py +++ b/var/www/modules/restApi/Flask_restApi.py @@ -139,11 +139,38 @@ def one(): # def api(): # return 'api doc' -@restApi.route("api/get/item/basic/", methods=['GET']) +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# POST +# +# { +# "id": item_id, mandatory +# "content": true, +# "tags": true, +# +# +# } +# +# response: { +# "id": "item_id", +# "tags": [], +# } +# +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +@restApi.route("api/get/item", methods=['GET', 'POST']) @token_required('admin') -def get_item_id(item_id): +def get_item_id(): + if request.method == 'POST': + data = request.get_json() + res = Item.get_item(data) + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + else: + return 'description API endpoint' + +@restApi.route("api/get/item/default/", methods=['GET']) +@token_required('admin') +def get_item_id_basic(item_id): """ - **GET api/get/item/info/** + **POST api/get/item/default/** **Get item** @@ -155,7 +182,7 @@ def get_item_id(item_id): - Example:: - curl -k https://127.0.0.1:7000/api/get/item/info/submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" + curl -k https://127.0.0.1:7000/api/get/item/default --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json --data @input.json -X POST" - Expected Success Response:: @@ -182,13 +209,10 @@ def get_item_id(item_id): {'status': 'error', 'reason': 'Item not found'} """ - try: - item_object = Paste.Paste(item_id) - except FileNotFoundError: - return Response(json.dumps({'status': 'error', 'reason': 'Item not found'}, indent=2, sort_keys=True), mimetype='application/json'), 404 - data = item_object.get_item_dict() - return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json') + data = {'id': item_id, 'date': True, 'content': True, 'tags': True} + res = Item.get_item(data) + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # GET @@ -244,13 +268,9 @@ def get_item_tag(item_id): {'status': 'error', 'reason': 'Item not found'} """ - if not Item.exist_item(item_id): - return Response(json.dumps({'status': 'error', 'reason': 'Item not found'}, indent=2, sort_keys=True), mimetype='application/json'), 404 - tags = Tag.get_item_tags(item_id) - dict_tags = {} - dict_tags['id'] = item_id - dict_tags['tags'] = tags - return Response(json.dumps(dict_tags, indent=2, sort_keys=True), mimetype='application/json') + data = {'id': item_id, 'date': False, 'tags': True} + res = Item.get_item(data) + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # POST @@ -461,15 +481,9 @@ def get_item_content(item_id): {'status': 'error', 'reason': 'Item not found'} """ - try: - item_object = Paste.Paste(item_id) - except FileNotFoundError: - return Response(json.dumps({'status': 'error', 'reason': 'Item not found'}, indent=2, sort_keys=True), mimetype='application/json'), 404 - item_object = Paste.Paste(item_id) - dict_content = {} - dict_content['id'] = item_id - dict_content['content'] = item_object.get_p_content() - return Response(json.dumps(dict_content, indent=2, sort_keys=True), mimetype='application/json') + data = {'id': item_id, 'date': False, 'content': True, 'tags': False} + res = Item.get_item(data) + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #