chg: [api] add advanced get item via POST + use same query for each get item

bin/packages/Item.py

@@ -2,12 +2,15 @@
 # -*-coding:UTF-8 -*
 
 import os
+import gzip
 import redis
 
 import Flask_config
 import Date
+import Tag
 
 PASTES_FOLDER = Flask_config.PASTES_FOLDER
+r_cache = Flask_config.r_cache
 
 def exist_item(item_id):
     if os.path.isfile(os.path.join(PASTES_FOLDER, item_id)):
@@ -18,3 +21,74 @@ def exist_item(item_id):
 def get_item_date(item_id):
     l_directory = item_id.split('/')
     return '{}{}{}'.format(l_directory[-4], l_directory[-3], l_directory[-2])
+
+def get_item_size(item_id):
+    return round(os.path.getsize(os.path.join(PASTES_FOLDER, item_id))/1024.0, 2)
+
+def get_lines_info(item_id, item_content=None):
+    if not item_content:
+        item_content = get_item_content(item_id)
+    max_length = 0
+    line_id = 0
+    nb_line = 0
+    for line in item_content.splitlines():
+        length = len(line)
+        if length > max_length:
+            max_length = length
+        nb_line += 1
+    return {'nb': nb_line, 'max_length': max_length}
+
+
+def get_item_content(item_id):
+    item_full_path = os.path.join(PASTES_FOLDER, item_id)
+    try:
+        item_content = r_cache.get(item_full_path)
+    except UnicodeDecodeError:
+        item_content = None
+    except Exception as e:
+        print("ERROR in: " + item_id)
+        print(e)
+        item_content = None
+    if item_content is None:
+        try:
+            with gzip.open(item_full_path, 'r') as f:
+                item_content = f.read()
+                r_cache.set(item_full_path, item_content)
+                r_cache.expire(item_full_path, 300)
+        except:
+            item_content = ''
+    return str(item_content)
+
+# API
+def get_item(request_dict):
+    if not request_dict:
+        return Response({'status': 'error', 'reason': 'Malformed JSON'}, 400)
+
+    item_id = request_dict.get('id', None)
+    if not item_id:
+        return ( {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400 )
+    if not exist_item(item_id):
+        return ( {'status': 'error', 'reason': 'Item not found'}, 404 )
+
+    dict_item = {}
+    dict_item['id'] = item_id
+    date = request_dict.get('date', True)
+    if date:
+        dict_item['date'] = get_item_date(item_id)
+    tags = request_dict.get('tags', True)
+    if tags:
+        dict_item['tags'] = Tag.get_item_tags(item_id)
+
+    size = request_dict.get('size', False)
+    if size:
+        dict_item['size'] = get_item_size(item_id)
+
+    content = request_dict.get('content', False)
+    if content:
+        dict_item['content'] = get_item_content(item_id)
+
+    lines_info = request_dict.get('lines', False)
+    if lines_info:
+        dict_item['lines'] = get_lines_info(item_id, dict_item.get('content', 'None'))
+
+    return (dict_item, 200)

doc/README.md

@@ -27,10 +27,10 @@ curl --header "Authorization: YOUR_API_KEY" --header "Content-Type: application/
 
 ## Item management
 
-### Get item: `api/get/item/basic/<path:item_id>`
+### Get item: `api/get/item/default/<path:item_id>`
 
 #### Description
-Get anitem basic information.
+Get item default info.
 
 **Method** : `GET`
 
@@ -56,7 +56,7 @@ Get anitem basic information.
 
 #### Example
 ```
-curl https://127.0.0.1:7000/api/get/item/info/submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json"
+curl https://127.0.0.1:7000/api/get/item/default/submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json"
 ```
 
 #### Expected Success Response
@@ -81,7 +81,10 @@ curl https://127.0.0.1:7000/api/get/item/info/submitted/2019/07/26/3efb8a79-08e9
 #### Expected Fail Response
 
 **HTTP Status Code** : `400`
-
+```json
+  {"status": "error", "reason": "Mandatory parameter(s) not provided"}
+```
+**HTTP Status Code** : `404`
 ```json
   {"status": "error", "reason": "Item not found"}
 ```
@@ -128,7 +131,10 @@ curl https://127.0.0.1:7000/api/get/item/content/submitted/2019/07/26/3efb8a79-0
 #### Expected Fail Response
 
 **HTTP Status Code** : `400`
-
+```json
+  {"status": "error", "reason": "Mandatory parameter(s) not provided"}
+```
+**HTTP Status Code** : `404`
 ```json
   {"status": "error", "reason": "Item not found"}
 ```
@@ -181,13 +187,125 @@ curl https://127.0.0.1:7000/api/get/item/tag/submitted/2019/07/26/3efb8a79-08e9-
 #### Expected Fail Response
 
 **HTTP Status Code** : `400`
-
+```json
+  {"status": "error", "reason": "Mandatory parameter(s) not provided"}
+```
+**HTTP Status Code** : `404`
 ```json
   {"status": "error", "reason": "Item not found"}
 ```
 
 
 
+### Advanced Get item: `api/get/item`
+
+#### Description
+Get item. Filter requested field.
+
+**Method** : `POST`
+
+#### Parameters
+- `id`
+  - item id
+  - *str - relative item path*
+  - mandatory
+- `date`
+  - get item date
+  - *boolean*
+  - default: `true`
+- `tags`
+  - get item tags
+  - *boolean*
+  - default: `true`
+- `content`
+  - get item content
+  - *boolean*
+  - default: `false`
+- `size`
+  - get item size
+  - *boolean*
+  - default: `false`
+- `lines`
+  - get item lines info
+  - *boolean*
+  - default: `false`
+
+#### JSON response
+- `content`
+  - item content
+  - *str*
+- `id`
+  - item id
+  - *str*
+- `date`
+  - item date
+  - *str - YYMMDD*
+- `tags`
+  - item tags list
+  - *list*
+- `size`
+  - item size (Kb)
+  - *int*
+- `lines`
+  - item lines info
+  - *{}*
+      - `max_length`
+        -  line max length line
+        - *int*
+      - `nb`
+        - nb lines item
+        - *int*
+
+
+#### Example
+```
+curl https://127.0.0.1:7000/api/get/item --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X POST
+```
+
+#### input.json Example
+```json
+{
+  "id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz",
+  "content": true,
+  "lines_info": true,
+  "tags": true,
+  "size": true
+}
+```
+
+#### Expected Success Response
+**HTTP Status Code** : `200`
+```json
+  {
+    "content": "b'dsvcdsvcdsc vvvv'",
+    "date": "20190726",
+    "id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz",
+    "lines": {
+      "max_length": 19,
+      "nb": 1
+    },
+    "size": 0.03,
+    "tags": [
+      "misp-galaxy:stealer=\"Vidar\"",
+      "infoleak:submission=\"manual\""
+    ]
+  }
+```
+
+#### Expected Fail Response
+**HTTP Status Code** : `400`
+```json
+  {"status": "error", "reason": "Mandatory parameter(s) not provided"}
+```
+**HTTP Status Code** : `404`
+```json
+  {"status": "error", "reason": "Item not found"}
+```
+
+
+
+
+
 ### add item tags: `api/add/item/tag`
 
 #### Description

var/www/modules/PasteSubmit/Flask_PasteSubmit.py

@@ -24,7 +24,7 @@ import json
 import Paste
 
 import Import_helper
-import Tags
+import Tag
 
 from pytaxonomies import Taxonomies
 from pymispgalaxies import Galaxies, Clusters
@@ -224,8 +224,8 @@ def hive_create_case(hive_tlp, threat_level, hive_description, hive_case_title,
 @login_analyst
 def PasteSubmit_page():
     # Get all active tags/galaxy
-    active_taxonomies = Tags.get_active_taxonomies()
+    active_taxonomies = Tag.get_active_taxonomies()
-    active_galaxies = Tags.get_active_galaxies()
+    active_galaxies = Tag.get_active_galaxies()
 
     return render_template("submit_items.html",
                             active_taxonomies = active_taxonomies,
@@ -253,9 +253,9 @@ def submit():
     submitted_tag = 'infoleak:submission="manual"'
 
     #active taxonomies
-    active_taxonomies = Tags.get_active_taxonomies()
+    active_taxonomies = Tag.get_active_taxonomies()
     #active galaxies
-    active_galaxies = Tags.get_active_galaxies()
+    active_galaxies = Tag.get_active_galaxies()
 
     if ltags or ltagsgalaxies:
 

var/www/modules/restApi/Flask_restApi.py

@@ -139,11 +139,38 @@ def one():
 # def api():
 #     return 'api doc'
 
-@restApi.route("api/get/item/basic/<path:item_id>", methods=['GET'])
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+# POST
+#
+# {
+#   "id": item_id,      mandatory
+#   "content": true,
+#   "tags": true,
+#
+#
+# }
+#
+# response: {
+#               "id": "item_id",
+#               "tags": [],
+#           }
+#
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+@restApi.route("api/get/item", methods=['GET', 'POST'])
 @token_required('admin')
-def get_item_id(item_id):
+def get_item_id():
+    if request.method == 'POST':
+        data = request.get_json()
+        res = Item.get_item(data)
+        return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
+    else:
+        return 'description API endpoint'
+
+@restApi.route("api/get/item/default/<path:item_id>", methods=['GET'])
+@token_required('admin')
+def get_item_id_basic(item_id):
     """
-        **GET api/get/item/info/<item id>**
+        **POST api/get/item/default/<item_id>**
 
         **Get item**
 
@@ -155,7 +182,7 @@ def get_item_id(item_id):
 
         - Example::
 
-            curl -k https://127.0.0.1:7000/api/get/item/info/submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json"
+            curl -k https://127.0.0.1:7000/api/get/item/default --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json --data @input.json -X POST"
 
         - Expected Success Response::
 
@@ -182,13 +209,10 @@ def get_item_id(item_id):
             {'status': 'error', 'reason': 'Item not found'}
 
     """
-    try:
-        item_object = Paste.Paste(item_id)
-    except FileNotFoundError:
-        return Response(json.dumps({'status': 'error', 'reason': 'Item not found'}, indent=2, sort_keys=True), mimetype='application/json'), 404
 
-    data = item_object.get_item_dict()
+    data = {'id': item_id, 'date': True, 'content': True, 'tags': True}
-    return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json')
+    res = Item.get_item(data)
+    return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
 
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 # GET
@@ -244,13 +268,9 @@ def get_item_tag(item_id):
             {'status': 'error', 'reason': 'Item not found'}
 
     """
-    if not Item.exist_item(item_id):
+    data = {'id': item_id, 'date': False, 'tags': True}
-        return Response(json.dumps({'status': 'error', 'reason': 'Item not found'}, indent=2, sort_keys=True), mimetype='application/json'), 404
+    res = Item.get_item(data)
-    tags = Tag.get_item_tags(item_id)
+    return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
-    dict_tags = {}
-    dict_tags['id'] = item_id
-    dict_tags['tags'] = tags
-    return Response(json.dumps(dict_tags, indent=2, sort_keys=True), mimetype='application/json')
 
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 # POST
@@ -461,15 +481,9 @@ def get_item_content(item_id):
             {'status': 'error', 'reason': 'Item not found'}
 
     """
-    try:
+    data = {'id': item_id, 'date': False, 'content': True, 'tags': False}
-        item_object = Paste.Paste(item_id)
+    res = Item.get_item(data)
-    except FileNotFoundError:
+    return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
-        return Response(json.dumps({'status': 'error', 'reason': 'Item not found'}, indent=2, sort_keys=True), mimetype='application/json'), 404
-    item_object = Paste.Paste(item_id)
-    dict_content = {}
-    dict_content['id'] = item_id
-    dict_content['content'] = item_object.get_p_content()
-    return Response(json.dumps(dict_content, indent=2, sort_keys=True), mimetype='application/json')
 
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 #