mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-26 15:57:16 +00:00
chg: [api] add advanced get item via POST + use same query for each get item
This commit is contained in:
parent
8c02c1b00b
commit
4c20f58a52
4 changed files with 243 additions and 37 deletions
|
@ -2,12 +2,15 @@
|
|||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import gzip
|
||||
import redis
|
||||
|
||||
import Flask_config
|
||||
import Date
|
||||
import Tag
|
||||
|
||||
PASTES_FOLDER = Flask_config.PASTES_FOLDER
|
||||
r_cache = Flask_config.r_cache
|
||||
|
||||
def exist_item(item_id):
|
||||
if os.path.isfile(os.path.join(PASTES_FOLDER, item_id)):
|
||||
|
@ -18,3 +21,74 @@ def exist_item(item_id):
|
|||
def get_item_date(item_id):
|
||||
l_directory = item_id.split('/')
|
||||
return '{}{}{}'.format(l_directory[-4], l_directory[-3], l_directory[-2])
|
||||
|
||||
def get_item_size(item_id):
|
||||
return round(os.path.getsize(os.path.join(PASTES_FOLDER, item_id))/1024.0, 2)
|
||||
|
||||
def get_lines_info(item_id, item_content=None):
|
||||
if not item_content:
|
||||
item_content = get_item_content(item_id)
|
||||
max_length = 0
|
||||
line_id = 0
|
||||
nb_line = 0
|
||||
for line in item_content.splitlines():
|
||||
length = len(line)
|
||||
if length > max_length:
|
||||
max_length = length
|
||||
nb_line += 1
|
||||
return {'nb': nb_line, 'max_length': max_length}
|
||||
|
||||
|
||||
def get_item_content(item_id):
|
||||
item_full_path = os.path.join(PASTES_FOLDER, item_id)
|
||||
try:
|
||||
item_content = r_cache.get(item_full_path)
|
||||
except UnicodeDecodeError:
|
||||
item_content = None
|
||||
except Exception as e:
|
||||
print("ERROR in: " + item_id)
|
||||
print(e)
|
||||
item_content = None
|
||||
if item_content is None:
|
||||
try:
|
||||
with gzip.open(item_full_path, 'r') as f:
|
||||
item_content = f.read()
|
||||
r_cache.set(item_full_path, item_content)
|
||||
r_cache.expire(item_full_path, 300)
|
||||
except:
|
||||
item_content = ''
|
||||
return str(item_content)
|
||||
|
||||
# API
|
||||
def get_item(request_dict):
|
||||
if not request_dict:
|
||||
return Response({'status': 'error', 'reason': 'Malformed JSON'}, 400)
|
||||
|
||||
item_id = request_dict.get('id', None)
|
||||
if not item_id:
|
||||
return ( {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400 )
|
||||
if not exist_item(item_id):
|
||||
return ( {'status': 'error', 'reason': 'Item not found'}, 404 )
|
||||
|
||||
dict_item = {}
|
||||
dict_item['id'] = item_id
|
||||
date = request_dict.get('date', True)
|
||||
if date:
|
||||
dict_item['date'] = get_item_date(item_id)
|
||||
tags = request_dict.get('tags', True)
|
||||
if tags:
|
||||
dict_item['tags'] = Tag.get_item_tags(item_id)
|
||||
|
||||
size = request_dict.get('size', False)
|
||||
if size:
|
||||
dict_item['size'] = get_item_size(item_id)
|
||||
|
||||
content = request_dict.get('content', False)
|
||||
if content:
|
||||
dict_item['content'] = get_item_content(item_id)
|
||||
|
||||
lines_info = request_dict.get('lines', False)
|
||||
if lines_info:
|
||||
dict_item['lines'] = get_lines_info(item_id, dict_item.get('content', 'None'))
|
||||
|
||||
return (dict_item, 200)
|
||||
|
|
130
doc/README.md
130
doc/README.md
|
@ -27,10 +27,10 @@ curl --header "Authorization: YOUR_API_KEY" --header "Content-Type: application/
|
|||
|
||||
## Item management
|
||||
|
||||
### Get item: `api/get/item/basic/<path:item_id>`
|
||||
### Get item: `api/get/item/default/<path:item_id>`
|
||||
|
||||
#### Description
|
||||
Get anitem basic information.
|
||||
Get item default info.
|
||||
|
||||
**Method** : `GET`
|
||||
|
||||
|
@ -56,7 +56,7 @@ Get anitem basic information.
|
|||
|
||||
#### Example
|
||||
```
|
||||
curl https://127.0.0.1:7000/api/get/item/info/submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json"
|
||||
curl https://127.0.0.1:7000/api/get/item/default/submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json"
|
||||
```
|
||||
|
||||
#### Expected Success Response
|
||||
|
@ -81,7 +81,10 @@ curl https://127.0.0.1:7000/api/get/item/info/submitted/2019/07/26/3efb8a79-08e9
|
|||
#### Expected Fail Response
|
||||
|
||||
**HTTP Status Code** : `400`
|
||||
|
||||
```json
|
||||
{"status": "error", "reason": "Mandatory parameter(s) not provided"}
|
||||
```
|
||||
**HTTP Status Code** : `404`
|
||||
```json
|
||||
{"status": "error", "reason": "Item not found"}
|
||||
```
|
||||
|
@ -128,7 +131,10 @@ curl https://127.0.0.1:7000/api/get/item/content/submitted/2019/07/26/3efb8a79-0
|
|||
#### Expected Fail Response
|
||||
|
||||
**HTTP Status Code** : `400`
|
||||
|
||||
```json
|
||||
{"status": "error", "reason": "Mandatory parameter(s) not provided"}
|
||||
```
|
||||
**HTTP Status Code** : `404`
|
||||
```json
|
||||
{"status": "error", "reason": "Item not found"}
|
||||
```
|
||||
|
@ -181,13 +187,125 @@ curl https://127.0.0.1:7000/api/get/item/tag/submitted/2019/07/26/3efb8a79-08e9-
|
|||
#### Expected Fail Response
|
||||
|
||||
**HTTP Status Code** : `400`
|
||||
|
||||
```json
|
||||
{"status": "error", "reason": "Mandatory parameter(s) not provided"}
|
||||
```
|
||||
**HTTP Status Code** : `404`
|
||||
```json
|
||||
{"status": "error", "reason": "Item not found"}
|
||||
```
|
||||
|
||||
|
||||
|
||||
### Advanced Get item: `api/get/item`
|
||||
|
||||
#### Description
|
||||
Get item. Filter requested field.
|
||||
|
||||
**Method** : `POST`
|
||||
|
||||
#### Parameters
|
||||
- `id`
|
||||
- item id
|
||||
- *str - relative item path*
|
||||
- mandatory
|
||||
- `date`
|
||||
- get item date
|
||||
- *boolean*
|
||||
- default: `true`
|
||||
- `tags`
|
||||
- get item tags
|
||||
- *boolean*
|
||||
- default: `true`
|
||||
- `content`
|
||||
- get item content
|
||||
- *boolean*
|
||||
- default: `false`
|
||||
- `size`
|
||||
- get item size
|
||||
- *boolean*
|
||||
- default: `false`
|
||||
- `lines`
|
||||
- get item lines info
|
||||
- *boolean*
|
||||
- default: `false`
|
||||
|
||||
#### JSON response
|
||||
- `content`
|
||||
- item content
|
||||
- *str*
|
||||
- `id`
|
||||
- item id
|
||||
- *str*
|
||||
- `date`
|
||||
- item date
|
||||
- *str - YYMMDD*
|
||||
- `tags`
|
||||
- item tags list
|
||||
- *list*
|
||||
- `size`
|
||||
- item size (Kb)
|
||||
- *int*
|
||||
- `lines`
|
||||
- item lines info
|
||||
- *{}*
|
||||
- `max_length`
|
||||
- line max length line
|
||||
- *int*
|
||||
- `nb`
|
||||
- nb lines item
|
||||
- *int*
|
||||
|
||||
|
||||
#### Example
|
||||
```
|
||||
curl https://127.0.0.1:7000/api/get/item --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X POST
|
||||
```
|
||||
|
||||
#### input.json Example
|
||||
```json
|
||||
{
|
||||
"id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz",
|
||||
"content": true,
|
||||
"lines_info": true,
|
||||
"tags": true,
|
||||
"size": true
|
||||
}
|
||||
```
|
||||
|
||||
#### Expected Success Response
|
||||
**HTTP Status Code** : `200`
|
||||
```json
|
||||
{
|
||||
"content": "b'dsvcdsvcdsc vvvv'",
|
||||
"date": "20190726",
|
||||
"id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz",
|
||||
"lines": {
|
||||
"max_length": 19,
|
||||
"nb": 1
|
||||
},
|
||||
"size": 0.03,
|
||||
"tags": [
|
||||
"misp-galaxy:stealer=\"Vidar\"",
|
||||
"infoleak:submission=\"manual\""
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
#### Expected Fail Response
|
||||
**HTTP Status Code** : `400`
|
||||
```json
|
||||
{"status": "error", "reason": "Mandatory parameter(s) not provided"}
|
||||
```
|
||||
**HTTP Status Code** : `404`
|
||||
```json
|
||||
{"status": "error", "reason": "Item not found"}
|
||||
```
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
### add item tags: `api/add/item/tag`
|
||||
|
||||
#### Description
|
||||
|
|
|
@ -24,7 +24,7 @@ import json
|
|||
import Paste
|
||||
|
||||
import Import_helper
|
||||
import Tags
|
||||
import Tag
|
||||
|
||||
from pytaxonomies import Taxonomies
|
||||
from pymispgalaxies import Galaxies, Clusters
|
||||
|
@ -224,8 +224,8 @@ def hive_create_case(hive_tlp, threat_level, hive_description, hive_case_title,
|
|||
@login_analyst
|
||||
def PasteSubmit_page():
|
||||
# Get all active tags/galaxy
|
||||
active_taxonomies = Tags.get_active_taxonomies()
|
||||
active_galaxies = Tags.get_active_galaxies()
|
||||
active_taxonomies = Tag.get_active_taxonomies()
|
||||
active_galaxies = Tag.get_active_galaxies()
|
||||
|
||||
return render_template("submit_items.html",
|
||||
active_taxonomies = active_taxonomies,
|
||||
|
@ -253,9 +253,9 @@ def submit():
|
|||
submitted_tag = 'infoleak:submission="manual"'
|
||||
|
||||
#active taxonomies
|
||||
active_taxonomies = Tags.get_active_taxonomies()
|
||||
active_taxonomies = Tag.get_active_taxonomies()
|
||||
#active galaxies
|
||||
active_galaxies = Tags.get_active_galaxies()
|
||||
active_galaxies = Tag.get_active_galaxies()
|
||||
|
||||
if ltags or ltagsgalaxies:
|
||||
|
||||
|
|
|
@ -139,11 +139,38 @@ def one():
|
|||
# def api():
|
||||
# return 'api doc'
|
||||
|
||||
@restApi.route("api/get/item/basic/<path:item_id>", methods=['GET'])
|
||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||
# POST
|
||||
#
|
||||
# {
|
||||
# "id": item_id, mandatory
|
||||
# "content": true,
|
||||
# "tags": true,
|
||||
#
|
||||
#
|
||||
# }
|
||||
#
|
||||
# response: {
|
||||
# "id": "item_id",
|
||||
# "tags": [],
|
||||
# }
|
||||
#
|
||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||
@restApi.route("api/get/item", methods=['GET', 'POST'])
|
||||
@token_required('admin')
|
||||
def get_item_id(item_id):
|
||||
def get_item_id():
|
||||
if request.method == 'POST':
|
||||
data = request.get_json()
|
||||
res = Item.get_item(data)
|
||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
else:
|
||||
return 'description API endpoint'
|
||||
|
||||
@restApi.route("api/get/item/default/<path:item_id>", methods=['GET'])
|
||||
@token_required('admin')
|
||||
def get_item_id_basic(item_id):
|
||||
"""
|
||||
**GET api/get/item/info/<item id>**
|
||||
**POST api/get/item/default/<item_id>**
|
||||
|
||||
**Get item**
|
||||
|
||||
|
@ -155,7 +182,7 @@ def get_item_id(item_id):
|
|||
|
||||
- Example::
|
||||
|
||||
curl -k https://127.0.0.1:7000/api/get/item/info/submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json"
|
||||
curl -k https://127.0.0.1:7000/api/get/item/default --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json --data @input.json -X POST"
|
||||
|
||||
- Expected Success Response::
|
||||
|
||||
|
@ -182,13 +209,10 @@ def get_item_id(item_id):
|
|||
{'status': 'error', 'reason': 'Item not found'}
|
||||
|
||||
"""
|
||||
try:
|
||||
item_object = Paste.Paste(item_id)
|
||||
except FileNotFoundError:
|
||||
return Response(json.dumps({'status': 'error', 'reason': 'Item not found'}, indent=2, sort_keys=True), mimetype='application/json'), 404
|
||||
|
||||
data = item_object.get_item_dict()
|
||||
return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json')
|
||||
data = {'id': item_id, 'date': True, 'content': True, 'tags': True}
|
||||
res = Item.get_item(data)
|
||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
|
||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||
# GET
|
||||
|
@ -244,13 +268,9 @@ def get_item_tag(item_id):
|
|||
{'status': 'error', 'reason': 'Item not found'}
|
||||
|
||||
"""
|
||||
if not Item.exist_item(item_id):
|
||||
return Response(json.dumps({'status': 'error', 'reason': 'Item not found'}, indent=2, sort_keys=True), mimetype='application/json'), 404
|
||||
tags = Tag.get_item_tags(item_id)
|
||||
dict_tags = {}
|
||||
dict_tags['id'] = item_id
|
||||
dict_tags['tags'] = tags
|
||||
return Response(json.dumps(dict_tags, indent=2, sort_keys=True), mimetype='application/json')
|
||||
data = {'id': item_id, 'date': False, 'tags': True}
|
||||
res = Item.get_item(data)
|
||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
|
||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||
# POST
|
||||
|
@ -461,15 +481,9 @@ def get_item_content(item_id):
|
|||
{'status': 'error', 'reason': 'Item not found'}
|
||||
|
||||
"""
|
||||
try:
|
||||
item_object = Paste.Paste(item_id)
|
||||
except FileNotFoundError:
|
||||
return Response(json.dumps({'status': 'error', 'reason': 'Item not found'}, indent=2, sort_keys=True), mimetype='application/json'), 404
|
||||
item_object = Paste.Paste(item_id)
|
||||
dict_content = {}
|
||||
dict_content['id'] = item_id
|
||||
dict_content['content'] = item_object.get_p_content()
|
||||
return Response(json.dumps(dict_content, indent=2, sort_keys=True), mimetype='application/json')
|
||||
data = {'id': item_id, 'date': False, 'content': True, 'tags': False}
|
||||
res = Item.get_item(data)
|
||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
|
||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||
#
|
||||
|
|
Loading…
Reference in a new issue