mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-26 15:57:16 +00:00
chg: [api] add advanced get item via POST + use same query for each get item
This commit is contained in:
parent
8c02c1b00b
commit
4c20f58a52
4 changed files with 243 additions and 37 deletions
|
@ -2,12 +2,15 @@
|
||||||
# -*-coding:UTF-8 -*
|
# -*-coding:UTF-8 -*
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
import gzip
|
||||||
import redis
|
import redis
|
||||||
|
|
||||||
import Flask_config
|
import Flask_config
|
||||||
import Date
|
import Date
|
||||||
|
import Tag
|
||||||
|
|
||||||
PASTES_FOLDER = Flask_config.PASTES_FOLDER
|
PASTES_FOLDER = Flask_config.PASTES_FOLDER
|
||||||
|
r_cache = Flask_config.r_cache
|
||||||
|
|
||||||
def exist_item(item_id):
|
def exist_item(item_id):
|
||||||
if os.path.isfile(os.path.join(PASTES_FOLDER, item_id)):
|
if os.path.isfile(os.path.join(PASTES_FOLDER, item_id)):
|
||||||
|
@ -18,3 +21,74 @@ def exist_item(item_id):
|
||||||
def get_item_date(item_id):
|
def get_item_date(item_id):
|
||||||
l_directory = item_id.split('/')
|
l_directory = item_id.split('/')
|
||||||
return '{}{}{}'.format(l_directory[-4], l_directory[-3], l_directory[-2])
|
return '{}{}{}'.format(l_directory[-4], l_directory[-3], l_directory[-2])
|
||||||
|
|
||||||
|
def get_item_size(item_id):
|
||||||
|
return round(os.path.getsize(os.path.join(PASTES_FOLDER, item_id))/1024.0, 2)
|
||||||
|
|
||||||
|
def get_lines_info(item_id, item_content=None):
|
||||||
|
if not item_content:
|
||||||
|
item_content = get_item_content(item_id)
|
||||||
|
max_length = 0
|
||||||
|
line_id = 0
|
||||||
|
nb_line = 0
|
||||||
|
for line in item_content.splitlines():
|
||||||
|
length = len(line)
|
||||||
|
if length > max_length:
|
||||||
|
max_length = length
|
||||||
|
nb_line += 1
|
||||||
|
return {'nb': nb_line, 'max_length': max_length}
|
||||||
|
|
||||||
|
|
||||||
|
def get_item_content(item_id):
|
||||||
|
item_full_path = os.path.join(PASTES_FOLDER, item_id)
|
||||||
|
try:
|
||||||
|
item_content = r_cache.get(item_full_path)
|
||||||
|
except UnicodeDecodeError:
|
||||||
|
item_content = None
|
||||||
|
except Exception as e:
|
||||||
|
print("ERROR in: " + item_id)
|
||||||
|
print(e)
|
||||||
|
item_content = None
|
||||||
|
if item_content is None:
|
||||||
|
try:
|
||||||
|
with gzip.open(item_full_path, 'r') as f:
|
||||||
|
item_content = f.read()
|
||||||
|
r_cache.set(item_full_path, item_content)
|
||||||
|
r_cache.expire(item_full_path, 300)
|
||||||
|
except:
|
||||||
|
item_content = ''
|
||||||
|
return str(item_content)
|
||||||
|
|
||||||
|
# API
|
||||||
|
def get_item(request_dict):
|
||||||
|
if not request_dict:
|
||||||
|
return Response({'status': 'error', 'reason': 'Malformed JSON'}, 400)
|
||||||
|
|
||||||
|
item_id = request_dict.get('id', None)
|
||||||
|
if not item_id:
|
||||||
|
return ( {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400 )
|
||||||
|
if not exist_item(item_id):
|
||||||
|
return ( {'status': 'error', 'reason': 'Item not found'}, 404 )
|
||||||
|
|
||||||
|
dict_item = {}
|
||||||
|
dict_item['id'] = item_id
|
||||||
|
date = request_dict.get('date', True)
|
||||||
|
if date:
|
||||||
|
dict_item['date'] = get_item_date(item_id)
|
||||||
|
tags = request_dict.get('tags', True)
|
||||||
|
if tags:
|
||||||
|
dict_item['tags'] = Tag.get_item_tags(item_id)
|
||||||
|
|
||||||
|
size = request_dict.get('size', False)
|
||||||
|
if size:
|
||||||
|
dict_item['size'] = get_item_size(item_id)
|
||||||
|
|
||||||
|
content = request_dict.get('content', False)
|
||||||
|
if content:
|
||||||
|
dict_item['content'] = get_item_content(item_id)
|
||||||
|
|
||||||
|
lines_info = request_dict.get('lines', False)
|
||||||
|
if lines_info:
|
||||||
|
dict_item['lines'] = get_lines_info(item_id, dict_item.get('content', 'None'))
|
||||||
|
|
||||||
|
return (dict_item, 200)
|
||||||
|
|
130
doc/README.md
130
doc/README.md
|
@ -27,10 +27,10 @@ curl --header "Authorization: YOUR_API_KEY" --header "Content-Type: application/
|
||||||
|
|
||||||
## Item management
|
## Item management
|
||||||
|
|
||||||
### Get item: `api/get/item/basic/<path:item_id>`
|
### Get item: `api/get/item/default/<path:item_id>`
|
||||||
|
|
||||||
#### Description
|
#### Description
|
||||||
Get anitem basic information.
|
Get item default info.
|
||||||
|
|
||||||
**Method** : `GET`
|
**Method** : `GET`
|
||||||
|
|
||||||
|
@ -56,7 +56,7 @@ Get anitem basic information.
|
||||||
|
|
||||||
#### Example
|
#### Example
|
||||||
```
|
```
|
||||||
curl https://127.0.0.1:7000/api/get/item/info/submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json"
|
curl https://127.0.0.1:7000/api/get/item/default/submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json"
|
||||||
```
|
```
|
||||||
|
|
||||||
#### Expected Success Response
|
#### Expected Success Response
|
||||||
|
@ -81,7 +81,10 @@ curl https://127.0.0.1:7000/api/get/item/info/submitted/2019/07/26/3efb8a79-08e9
|
||||||
#### Expected Fail Response
|
#### Expected Fail Response
|
||||||
|
|
||||||
**HTTP Status Code** : `400`
|
**HTTP Status Code** : `400`
|
||||||
|
```json
|
||||||
|
{"status": "error", "reason": "Mandatory parameter(s) not provided"}
|
||||||
|
```
|
||||||
|
**HTTP Status Code** : `404`
|
||||||
```json
|
```json
|
||||||
{"status": "error", "reason": "Item not found"}
|
{"status": "error", "reason": "Item not found"}
|
||||||
```
|
```
|
||||||
|
@ -128,7 +131,10 @@ curl https://127.0.0.1:7000/api/get/item/content/submitted/2019/07/26/3efb8a79-0
|
||||||
#### Expected Fail Response
|
#### Expected Fail Response
|
||||||
|
|
||||||
**HTTP Status Code** : `400`
|
**HTTP Status Code** : `400`
|
||||||
|
```json
|
||||||
|
{"status": "error", "reason": "Mandatory parameter(s) not provided"}
|
||||||
|
```
|
||||||
|
**HTTP Status Code** : `404`
|
||||||
```json
|
```json
|
||||||
{"status": "error", "reason": "Item not found"}
|
{"status": "error", "reason": "Item not found"}
|
||||||
```
|
```
|
||||||
|
@ -181,13 +187,125 @@ curl https://127.0.0.1:7000/api/get/item/tag/submitted/2019/07/26/3efb8a79-08e9-
|
||||||
#### Expected Fail Response
|
#### Expected Fail Response
|
||||||
|
|
||||||
**HTTP Status Code** : `400`
|
**HTTP Status Code** : `400`
|
||||||
|
```json
|
||||||
|
{"status": "error", "reason": "Mandatory parameter(s) not provided"}
|
||||||
|
```
|
||||||
|
**HTTP Status Code** : `404`
|
||||||
```json
|
```json
|
||||||
{"status": "error", "reason": "Item not found"}
|
{"status": "error", "reason": "Item not found"}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
### Advanced Get item: `api/get/item`
|
||||||
|
|
||||||
|
#### Description
|
||||||
|
Get item. Filter requested field.
|
||||||
|
|
||||||
|
**Method** : `POST`
|
||||||
|
|
||||||
|
#### Parameters
|
||||||
|
- `id`
|
||||||
|
- item id
|
||||||
|
- *str - relative item path*
|
||||||
|
- mandatory
|
||||||
|
- `date`
|
||||||
|
- get item date
|
||||||
|
- *boolean*
|
||||||
|
- default: `true`
|
||||||
|
- `tags`
|
||||||
|
- get item tags
|
||||||
|
- *boolean*
|
||||||
|
- default: `true`
|
||||||
|
- `content`
|
||||||
|
- get item content
|
||||||
|
- *boolean*
|
||||||
|
- default: `false`
|
||||||
|
- `size`
|
||||||
|
- get item size
|
||||||
|
- *boolean*
|
||||||
|
- default: `false`
|
||||||
|
- `lines`
|
||||||
|
- get item lines info
|
||||||
|
- *boolean*
|
||||||
|
- default: `false`
|
||||||
|
|
||||||
|
#### JSON response
|
||||||
|
- `content`
|
||||||
|
- item content
|
||||||
|
- *str*
|
||||||
|
- `id`
|
||||||
|
- item id
|
||||||
|
- *str*
|
||||||
|
- `date`
|
||||||
|
- item date
|
||||||
|
- *str - YYMMDD*
|
||||||
|
- `tags`
|
||||||
|
- item tags list
|
||||||
|
- *list*
|
||||||
|
- `size`
|
||||||
|
- item size (Kb)
|
||||||
|
- *int*
|
||||||
|
- `lines`
|
||||||
|
- item lines info
|
||||||
|
- *{}*
|
||||||
|
- `max_length`
|
||||||
|
- line max length line
|
||||||
|
- *int*
|
||||||
|
- `nb`
|
||||||
|
- nb lines item
|
||||||
|
- *int*
|
||||||
|
|
||||||
|
|
||||||
|
#### Example
|
||||||
|
```
|
||||||
|
curl https://127.0.0.1:7000/api/get/item --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json" --data @input.json -X POST
|
||||||
|
```
|
||||||
|
|
||||||
|
#### input.json Example
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz",
|
||||||
|
"content": true,
|
||||||
|
"lines_info": true,
|
||||||
|
"tags": true,
|
||||||
|
"size": true
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Expected Success Response
|
||||||
|
**HTTP Status Code** : `200`
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"content": "b'dsvcdsvcdsc vvvv'",
|
||||||
|
"date": "20190726",
|
||||||
|
"id": "submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz",
|
||||||
|
"lines": {
|
||||||
|
"max_length": 19,
|
||||||
|
"nb": 1
|
||||||
|
},
|
||||||
|
"size": 0.03,
|
||||||
|
"tags": [
|
||||||
|
"misp-galaxy:stealer=\"Vidar\"",
|
||||||
|
"infoleak:submission=\"manual\""
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Expected Fail Response
|
||||||
|
**HTTP Status Code** : `400`
|
||||||
|
```json
|
||||||
|
{"status": "error", "reason": "Mandatory parameter(s) not provided"}
|
||||||
|
```
|
||||||
|
**HTTP Status Code** : `404`
|
||||||
|
```json
|
||||||
|
{"status": "error", "reason": "Item not found"}
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### add item tags: `api/add/item/tag`
|
### add item tags: `api/add/item/tag`
|
||||||
|
|
||||||
#### Description
|
#### Description
|
||||||
|
|
|
@ -24,7 +24,7 @@ import json
|
||||||
import Paste
|
import Paste
|
||||||
|
|
||||||
import Import_helper
|
import Import_helper
|
||||||
import Tags
|
import Tag
|
||||||
|
|
||||||
from pytaxonomies import Taxonomies
|
from pytaxonomies import Taxonomies
|
||||||
from pymispgalaxies import Galaxies, Clusters
|
from pymispgalaxies import Galaxies, Clusters
|
||||||
|
@ -224,8 +224,8 @@ def hive_create_case(hive_tlp, threat_level, hive_description, hive_case_title,
|
||||||
@login_analyst
|
@login_analyst
|
||||||
def PasteSubmit_page():
|
def PasteSubmit_page():
|
||||||
# Get all active tags/galaxy
|
# Get all active tags/galaxy
|
||||||
active_taxonomies = Tags.get_active_taxonomies()
|
active_taxonomies = Tag.get_active_taxonomies()
|
||||||
active_galaxies = Tags.get_active_galaxies()
|
active_galaxies = Tag.get_active_galaxies()
|
||||||
|
|
||||||
return render_template("submit_items.html",
|
return render_template("submit_items.html",
|
||||||
active_taxonomies = active_taxonomies,
|
active_taxonomies = active_taxonomies,
|
||||||
|
@ -253,9 +253,9 @@ def submit():
|
||||||
submitted_tag = 'infoleak:submission="manual"'
|
submitted_tag = 'infoleak:submission="manual"'
|
||||||
|
|
||||||
#active taxonomies
|
#active taxonomies
|
||||||
active_taxonomies = Tags.get_active_taxonomies()
|
active_taxonomies = Tag.get_active_taxonomies()
|
||||||
#active galaxies
|
#active galaxies
|
||||||
active_galaxies = Tags.get_active_galaxies()
|
active_galaxies = Tag.get_active_galaxies()
|
||||||
|
|
||||||
if ltags or ltagsgalaxies:
|
if ltags or ltagsgalaxies:
|
||||||
|
|
||||||
|
|
|
@ -139,11 +139,38 @@ def one():
|
||||||
# def api():
|
# def api():
|
||||||
# return 'api doc'
|
# return 'api doc'
|
||||||
|
|
||||||
@restApi.route("api/get/item/basic/<path:item_id>", methods=['GET'])
|
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||||
|
# POST
|
||||||
|
#
|
||||||
|
# {
|
||||||
|
# "id": item_id, mandatory
|
||||||
|
# "content": true,
|
||||||
|
# "tags": true,
|
||||||
|
#
|
||||||
|
#
|
||||||
|
# }
|
||||||
|
#
|
||||||
|
# response: {
|
||||||
|
# "id": "item_id",
|
||||||
|
# "tags": [],
|
||||||
|
# }
|
||||||
|
#
|
||||||
|
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||||
|
@restApi.route("api/get/item", methods=['GET', 'POST'])
|
||||||
@token_required('admin')
|
@token_required('admin')
|
||||||
def get_item_id(item_id):
|
def get_item_id():
|
||||||
|
if request.method == 'POST':
|
||||||
|
data = request.get_json()
|
||||||
|
res = Item.get_item(data)
|
||||||
|
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||||
|
else:
|
||||||
|
return 'description API endpoint'
|
||||||
|
|
||||||
|
@restApi.route("api/get/item/default/<path:item_id>", methods=['GET'])
|
||||||
|
@token_required('admin')
|
||||||
|
def get_item_id_basic(item_id):
|
||||||
"""
|
"""
|
||||||
**GET api/get/item/info/<item id>**
|
**POST api/get/item/default/<item_id>**
|
||||||
|
|
||||||
**Get item**
|
**Get item**
|
||||||
|
|
||||||
|
@ -155,7 +182,7 @@ def get_item_id(item_id):
|
||||||
|
|
||||||
- Example::
|
- Example::
|
||||||
|
|
||||||
curl -k https://127.0.0.1:7000/api/get/item/info/submitted/2019/07/26/3efb8a79-08e9-4776-94ab-615eb370b6d4.gz --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json"
|
curl -k https://127.0.0.1:7000/api/get/item/default --header "Authorization: iHc1_ChZxj1aXmiFiF1mkxxQkzawwriEaZpPqyTQj " -H "Content-Type: application/json --data @input.json -X POST"
|
||||||
|
|
||||||
- Expected Success Response::
|
- Expected Success Response::
|
||||||
|
|
||||||
|
@ -182,13 +209,10 @@ def get_item_id(item_id):
|
||||||
{'status': 'error', 'reason': 'Item not found'}
|
{'status': 'error', 'reason': 'Item not found'}
|
||||||
|
|
||||||
"""
|
"""
|
||||||
try:
|
|
||||||
item_object = Paste.Paste(item_id)
|
|
||||||
except FileNotFoundError:
|
|
||||||
return Response(json.dumps({'status': 'error', 'reason': 'Item not found'}, indent=2, sort_keys=True), mimetype='application/json'), 404
|
|
||||||
|
|
||||||
data = item_object.get_item_dict()
|
data = {'id': item_id, 'date': True, 'content': True, 'tags': True}
|
||||||
return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json')
|
res = Item.get_item(data)
|
||||||
|
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||||
|
|
||||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||||
# GET
|
# GET
|
||||||
|
@ -244,13 +268,9 @@ def get_item_tag(item_id):
|
||||||
{'status': 'error', 'reason': 'Item not found'}
|
{'status': 'error', 'reason': 'Item not found'}
|
||||||
|
|
||||||
"""
|
"""
|
||||||
if not Item.exist_item(item_id):
|
data = {'id': item_id, 'date': False, 'tags': True}
|
||||||
return Response(json.dumps({'status': 'error', 'reason': 'Item not found'}, indent=2, sort_keys=True), mimetype='application/json'), 404
|
res = Item.get_item(data)
|
||||||
tags = Tag.get_item_tags(item_id)
|
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||||
dict_tags = {}
|
|
||||||
dict_tags['id'] = item_id
|
|
||||||
dict_tags['tags'] = tags
|
|
||||||
return Response(json.dumps(dict_tags, indent=2, sort_keys=True), mimetype='application/json')
|
|
||||||
|
|
||||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||||
# POST
|
# POST
|
||||||
|
@ -461,15 +481,9 @@ def get_item_content(item_id):
|
||||||
{'status': 'error', 'reason': 'Item not found'}
|
{'status': 'error', 'reason': 'Item not found'}
|
||||||
|
|
||||||
"""
|
"""
|
||||||
try:
|
data = {'id': item_id, 'date': False, 'content': True, 'tags': False}
|
||||||
item_object = Paste.Paste(item_id)
|
res = Item.get_item(data)
|
||||||
except FileNotFoundError:
|
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||||
return Response(json.dumps({'status': 'error', 'reason': 'Item not found'}, indent=2, sort_keys=True), mimetype='application/json'), 404
|
|
||||||
item_object = Paste.Paste(item_id)
|
|
||||||
dict_content = {}
|
|
||||||
dict_content['id'] = item_id
|
|
||||||
dict_content['content'] = item_object.get_p_content()
|
|
||||||
return Response(json.dumps(dict_content, indent=2, sort_keys=True), mimetype='application/json')
|
|
||||||
|
|
||||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||||
#
|
#
|
||||||
|
|
Loading…
Reference in a new issue