Merge pull request #130 from TonyJabbour/master

New restAPIs
This commit is contained in:
Thirion Aurélien 2021-11-26 16:42:58 +01:00 committed by GitHub
commit 65eb084e41
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 194 additions and 67 deletions

View file

@ -9,6 +9,8 @@ import redis
import uuid
import yara
import datetime
import base64
from flask import escape
@ -159,6 +161,34 @@ def get_tracker_metadata(tracker_uuid, user_id=False, description=False, level=F
return dict_uuid
def get_tracker_metadata_api(request_dict):
tracker_uuid = request_dict.get('tracker_uuid', None)
if not request_dict:
return {'status': 'error', 'reason': 'Malformed JSON'}, 400
if not tracker_uuid:
return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400
if not is_valid_uuid_v4(tracker_uuid):
return {"status": "error", "reason": "Invalid Tracker UUID"}, 400
if not r_serv_tracker.exists(f'tracker:{tracker_uuid}'):
return {'status': 'error', 'reason': 'Tracker not found'}, 404
dict_tracker = {'status': 'success',
'uuid': tracker_uuid,
'user_id': get_tracker_user_id(tracker_uuid),
'tracker': get_tracker_by_uuid(tracker_uuid),
'type': get_tracker_type(tracker_uuid),
'date': get_tracker_date(tracker_uuid),
'first_seen': get_tracker_first_seen(tracker_uuid),
'last_seen': get_tracker_last_seen(tracker_uuid),
'level': get_tracker_level(tracker_uuid),
'mails': get_tracker_mails(tracker_uuid),
'tags': get_tracker_tags(tracker_uuid),
'description': get_tracker_description(tracker_uuid),
'webhook': get_tracker_webhook(tracker_uuid)
}
return dict_tracker, 200
# tracker sparkline
def get_tracker_sparkline(tracker_uuid, num_day=6):
date_range_sparkline = Date.get_date_range(num_day)
@ -278,13 +308,13 @@ def is_tracker_in_user_level(tracker, tracker_type, user_id):
def api_is_allowed_to_edit_tracker(tracker_uuid, user_id):
if not is_valid_uuid_v4(tracker_uuid):
return ({"status": "error", "reason": "Invalid uuid"}, 400)
return {"status": "error", "reason": "Invalid uuid"}, 400
tracker_creator = r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'user_id')
if not tracker_creator:
return ({"status": "error", "reason": "Unknown uuid"}, 404)
return {"status": "error", "reason": "Unknown uuid"}, 404
if not is_in_role(user_id, 'admin') and user_id != tracker_creator:
return ({"status": "error", "reason": "Access Denied"}, 403)
return ({"uuid": tracker_uuid}, 200)
return {"status": "error", "reason": "Access Denied"}, 403
return {"uuid": tracker_uuid}, 200
##-- ACL --##
@ -335,14 +365,14 @@ def fix_all_tracker_uuid_list():
def api_validate_tracker_to_add(tracker , tracker_type, nb_words=1):
if tracker_type=='regex':
if not is_valid_regex(tracker):
return ({"status": "error", "reason": "Invalid regex"}, 400)
return {"status": "error", "reason": "Invalid regex"}, 400
elif tracker_type=='word' or tracker_type=='set':
# force lowercase
tracker = tracker.lower()
word_set = set(tracker)
set_inter = word_set.intersection(special_characters)
if set_inter:
return ({"status": "error", "reason": f'special character(s) not allowed: {set_inter}', "message": "Please use a python regex or remove all special characters"}, 400)
return {"status": "error", "reason": f'special character(s) not allowed: {set_inter}', "message": "Please use a python regex or remove all special characters"}, 400
words = tracker.split()
# not a word
if tracker_type=='word' and len(words)>1:
@ -368,13 +398,13 @@ def api_validate_tracker_to_add(tracker , tracker_type, nb_words=1):
elif tracker_type=='yara_custom':
if not is_valid_yara_rule(tracker):
return ({"status": "error", "reason": "Invalid custom Yara Rule"}, 400)
return {"status": "error", "reason": "Invalid custom Yara Rule"}, 400
elif tracker_type=='yara_default':
if not is_valid_default_yara_rule(tracker):
return ({"status": "error", "reason": "The Yara Rule doesn't exist"}, 400)
return {"status": "error", "reason": "The Yara Rule doesn't exist"}, 400
else:
return ({"status": "error", "reason": "Incorrect type"}, 400)
return ({"status": "success", "tracker": tracker, "type": tracker_type}, 200)
return {"status": "error", "reason": "Incorrect type"}, 400
return {"status": "success", "tracker": tracker, "type": tracker_type}, 200
def create_tracker(tracker, tracker_type, user_id, level, tags, mails, description, webhook, dashboard=0, tracker_uuid=None, sources=[]):
# edit tracker
@ -683,17 +713,36 @@ def api_get_default_rule_content(default_yara_rule):
yara_dir = get_yara_rules_default_dir()
filename = os.path.join(yara_dir, default_yara_rule)
filename = os.path.realpath(filename)
# incorrect filename
if not os.path.commonprefix([filename, yara_dir]) == yara_dir:
return ({'status': 'error', 'reason': 'file transversal detected'}, 400)
return {'status': 'error', 'reason': 'file traversal detected'}, 400
if not os.path.isfile(filename):
return ({'status': 'error', 'reason': 'yara rule not found'}, 400)
return {'status': 'error', 'reason': 'yara rule not found'}, 400
with open(filename, 'r') as f:
rule_content = f.read()
return ({'rule_name': default_yara_rule, 'content': rule_content}, 200)
return {'rule_name': default_yara_rule, 'content': rule_content}, 200
def get_yara_rule_content_restapi(request_dict):
rule_name = request_dict.get('rule_name', None)
if not request_dict:
return {'status': 'error', 'reason': 'Malformed JSON'}, 400
if not rule_name:
return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400
yara_dir = get_yara_rules_dir()
filename = os.path.join(yara_dir, rule_name)
filename = os.path.realpath(filename)
if not os.path.commonprefix([filename, yara_dir]) == yara_dir:
return {'status': 'error', 'reason': 'File Path Traversal'}, 400
if not os.path.isfile(filename):
return {'status': 'error', 'reason': 'yara rule not found'}, 400
with open(filename, 'r') as f:
rule_content = f.read()
rule_content = base64.b64encode((rule_content.encode('utf-8'))).decode('UTF-8')
return {'status': 'success', 'content': rule_content}, 200
##-- YARA --##
@ -1086,9 +1135,9 @@ def get_retro_hunt_nb_item_by_day(l_task_uuid, date_from=None, date_to=None):
## API ##
def api_check_retro_hunt_task_uuid(task_uuid):
if not is_valid_uuid_v4(task_uuid):
return ({"status": "error", "reason": "Invalid uuid"}, 400)
return {"status": "error", "reason": "Invalid uuid"}, 400
if not r_serv_tracker.exists(f'tracker:retro_hunt:task:{task_uuid}'):
return ({"status": "error", "reason": "Unknown uuid"}, 404)
return {"status": "error", "reason": "Unknown uuid"}, 404
return None
def api_get_retro_hunt_items(dict_input):
@ -1116,7 +1165,7 @@ def api_get_retro_hunt_items(dict_input):
res_dict['date_from'] = date_from
res_dict['date_to'] = date_to
res_dict['items'] = all_items_id
return (res_dict, 200)
return res_dict, 200
def api_pause_retro_hunt_task(task_uuid):
res = api_check_retro_hunt_task_uuid(task_uuid)
@ -1124,9 +1173,9 @@ def api_pause_retro_hunt_task(task_uuid):
return res
task_state = get_retro_hunt_task_state(task_uuid)
if task_state not in ['pending', 'running']:
return ({"status": "error", "reason": f"Task {task_uuid} not paused, current state: {task_state}"}, 400)
return {"status": "error", "reason": f"Task {task_uuid} not paused, current state: {task_state}"}, 400
pause_retro_hunt_task(task_uuid)
return (task_uuid, 200)
return task_uuid, 200
def api_resume_retro_hunt_task(task_uuid):
res = api_check_retro_hunt_task_uuid(task_uuid)
@ -1134,9 +1183,9 @@ def api_resume_retro_hunt_task(task_uuid):
return res
task_state = get_retro_hunt_task_state(task_uuid)
if not r_serv_tracker.sismember('tracker:retro_hunt:task:paused', task_uuid):
return ({"status": "error", "reason": f"Task {task_uuid} not paused, current state: {get_retro_hunt_task_state(task_uuid)}"}, 400)
return {"status": "error", "reason": f"Task {task_uuid} not paused, current state: {get_retro_hunt_task_state(task_uuid)}"}, 400
resume_retro_hunt_task(task_uuid)
return (task_uuid, 200)
return task_uuid, 200
def api_validate_rule_to_add(rule, rule_type):
if rule_type=='yara_custom':

View file

@ -221,7 +221,7 @@ def verify_sources_list(sources):
all_sources = get_all_items_sources()
for source in sources:
if source not in all_sources:
return ({'status': 'error', 'reason': 'Invalid source', 'value': source}, 400)
return {'status': 'error', 'reason': 'Invalid source', 'value': source}, 400
return None
def get_all_items_metadata_dict(list_id):

View file

@ -60,6 +60,9 @@ def get_item_date(item_id, add_separator=False):
def get_source(item_id):
return item_basic.get_source(item_id)
def get_all_sources():
return item_basic.get_all_items_sources(r_list=True)
def get_item_basename(item_id):
return os.path.basename(item_id)
@ -88,14 +91,12 @@ def get_item_metadata(item_id, item_content=None):
# encoding
# language
# lines info
item_metadata = {}
item_metadata['date'] = get_item_date(item_id, add_separator=True)
item_metadata['source'] = get_source(item_id)
item_metadata['size'] = get_item_size(item_id)
item_metadata['encoding'] = get_item_encoding(item_id)
item_metadata['lines'] = get_lines_info(item_id, item_content=item_content)
item_metadata = {'date': get_item_date(item_id, add_separator=True),
'source': get_source(item_id),
'size': get_item_size(item_id),
'encoding': get_item_encoding(item_id),
'lines': get_lines_info(item_id, item_content=item_content)
}
return item_metadata
def get_item_parent(item_id):
@ -166,13 +167,13 @@ def get_item_languages(item_id, min_len=600, num_langs=3, min_proportion=0.2, mi
# API
def get_item(request_dict):
if not request_dict:
return Response({'status': 'error', 'reason': 'Malformed JSON'}, 400)
return {'status': 'error', 'reason': 'Malformed JSON'}, 400
item_id = request_dict.get('id', None)
if not item_id:
return ( {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400 )
return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400
if not exist_item(item_id):
return ( {'status': 'error', 'reason': 'Item not found'}, 404 )
return {'status': 'error', 'reason': 'Item not found'}, 404
dict_item = {}
dict_item['id'] = item_id
@ -217,9 +218,40 @@ def get_item(request_dict):
if request_dict['cryptocurrency'].get('bitcoin'):
dict_item['cryptocurrency']['bitcoin'] = get_item_bitcoin(item_id)
return (dict_item, 200)
return dict_item, 200
def get_item_content_encoded_text(request_dict):
item_id = request_dict.get('id', None)
if not request_dict:
return {'status': 'error', 'reason': 'Malformed JSON'}, 400
if not item_id:
return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400
if not exist_item(item_id):
return {'status': 'error', 'reason': 'Item not found'}, 404
item_content = get_item_content(item_id)
item_content = base64.b64encode((item_content.encode('utf-8'))).decode('UTF-8')
return {'status': 'success', 'content': item_content}, 200
def get_item_sources():
item_content = {'sources': get_all_sources()}
return item_content, 200
def check_item_source(request_dict):
source = request_dict.get('source', None)
if not request_dict:
return {'status': 'error', 'reason': 'Malformed JSON'}, 400
if not source:
return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400
all_sources = item_basic.get_all_items_sources()
if source not in all_sources:
return {'status': 'error', 'reason': 'Invalid source', 'provide': source}, 400
return {'status': 'success', 'reason': 'Valid source', 'provide': source}, 200
###
### correlation
###

View file

@ -57,14 +57,14 @@ def is_in_role(user_id, role):
def check_term_uuid_valid_access(term_uuid, user_id):
if not is_valid_uuid_v4(term_uuid):
return ({"status": "error", "reason": "Invalid uuid"}, 400)
return {"status": "error", "reason": "Invalid uuid"}, 400
level = r_serv_term.hget('tracker:{}'.format(term_uuid), 'level')
if not level:
return ({"status": "error", "reason": "Unknown uuid"}, 404)
return {"status": "error", "reason": "Unknown uuid"}, 404
if level == 0:
if r_serv_term.hget('tracker:{}'.format(term_uuid), 'user_id') != user_id:
if not is_in_role(user_id, 'admin'):
return ({"status": "error", "reason": "Unknown uuid"}, 404)
return {"status": "error", "reason": "Unknown uuid"}, 404
return None
@ -78,7 +78,7 @@ def is_valid_mail(email):
def verify_mail_list(mail_list):
for mail in mail_list:
if not is_valid_mail(mail):
return ({'status': 'error', 'reason': 'Invalid email', 'value': mail}, 400)
return {'status': 'error', 'reason': 'Invalid email', 'value': mail}, 400
return None
def is_valid_regex(term_regex):
@ -148,14 +148,19 @@ def is_term_tracked_in_user_level(term, term_type, user_id):
def parse_json_term_to_add(dict_input, user_id):
term = dict_input.get('term', None)
if not term:
return ({"status": "error", "reason": "Term not provided"}, 400)
return {"status": "error", "reason": "Term not provided"}, 400
term_type = dict_input.get('type', None)
if not term_type:
return ({"status": "error", "reason": "Term type not provided"}, 400)
return {"status": "error", "reason": "Term type not provided"}, 400
nb_words = dict_input.get('nb_words', 1)
description = dict_input.get('description', '')
description = escape(description)
webhook = dict_input.get('webhook', '')
webhook = escape(webhook)
res = parse_tracked_term_to_add(term , term_type, nb_words=nb_words)
if res[1]!=200:
return res
@ -168,7 +173,6 @@ def parse_json_term_to_add(dict_input, user_id):
if res:
return res
## TODO: add dashboard key
level = dict_input.get('level', 1)
try:
level = int(level)
@ -180,27 +184,27 @@ def parse_json_term_to_add(dict_input, user_id):
# check if term already tracked in global
if level==1:
if is_term_tracked_in_global_level(term, term_type):
return ({"status": "error", "reason": "Term already tracked"}, 409)
return {"status": "error", "reason": "Term already tracked"}, 409
else:
if is_term_tracked_in_user_level(term, term_type, user_id):
return ({"status": "error", "reason": "Term already tracked"}, 409)
return {"status": "error", "reason": "Term already tracked"}, 409
term_uuid = add_tracked_term(term , term_type, user_id, level, tags, mails, description)
term_uuid = add_tracked_term(term , term_type, user_id, level, tags, mails, description,webhook)
return ({'term': term, 'type': term_type, 'uuid': term_uuid}, 200)
return {'term': term, 'type': term_type, 'uuid': term_uuid}, 200
def parse_tracked_term_to_add(term , term_type, nb_words=1):
if term_type=='regex':
if not is_valid_regex(term):
return ({"status": "error", "reason": "Invalid regex"}, 400)
return {"status": "error", "reason": "Invalid regex"}, 400
elif term_type=='word' or term_type=='set':
# force lowercase
term = term.lower()
word_set = set(term)
set_inter = word_set.intersection(special_characters)
if set_inter:
return ({"status": "error", "reason": "special character not allowed", "message": "Please use a regex or remove all special characters"}, 400)
return {"status": "error", "reason": "special character not allowed", "message": "Please use a regex or remove all special characters"}, 400
words = term.split()
# not a word
if term_type=='word' and len(words)>1:
@ -226,15 +230,15 @@ def parse_tracked_term_to_add(term , term_type, nb_words=1):
elif term_type=='yara_custom':
if not Tracker.is_valid_yara_rule(term):
return ({"status": "error", "reason": "Invalid custom Yara Rule"}, 400)
return {"status": "error", "reason": "Invalid custom Yara Rule"}, 400
elif term_type=='yara_default':
if not Tracker.is_valid_default_yara_rule(term):
return ({"status": "error", "reason": "The Yara Rule doesn't exist"}, 400)
return {"status": "error", "reason": "The Yara Rule doesn't exist"}, 400
else:
return ({"status": "error", "reason": "Incorrect type"}, 400)
return ({"status": "success", "term": term, "type": term_type}, 200)
return {"status": "error", "reason": "Incorrect type"}, 400
return {"status": "success", "term": term, "type": term_type}, 200
def add_tracked_term(term , term_type, user_id, level, tags, mails, description, dashboard=0):
def add_tracked_term(term , term_type, user_id, level, tags, mails, description,webhook, dashboard=0):
term_uuid = str(uuid.uuid4())
@ -254,6 +258,9 @@ def add_tracked_term(term , term_type, user_id, level, tags, mails, description,
if description:
r_serv_term.hset('tracker:{}'.format(term_uuid), 'description', description)
if webhook:
r_serv_term.hset('tracker:{}'.format(term_uuid), 'webhook', webhook)
# create all term set
r_serv_term.sadd('all:tracker:{}'.format(term_type), term)
@ -281,14 +288,15 @@ def add_tracked_term(term , term_type, user_id, level, tags, mails, description,
return term_uuid
def parse_tracked_term_to_delete(dict_input, user_id):
term_uuid = dict_input.get("uuid", None)
res = check_term_uuid_valid_access(term_uuid, user_id)
if res:
return res
delete_term(term_uuid)
return ({"uuid": term_uuid}, 200)
return {"uuid": term_uuid}, 200
# # TODO: MOVE IN TRACKER
def delete_term(term_uuid):
@ -328,7 +336,6 @@ def delete_term(term_uuid):
r_serv_term.delete('tracker:sources:{}'.format(term_uuid))
# remove item set
#########################3
all_item_date = r_serv_term.zrange(f'tracker:stat:{term_uuid}', 0, -1, withscores=True)
if all_item_date:
all_item_date = dict(all_item_date)
@ -426,12 +433,8 @@ def parse_get_tracker_term_item(dict_input, user_id):
all_item_id = Tracker.get_tracker_items_by_daterange(term_uuid, date_from, date_to)
all_item_id = Item.get_item_list_desc(all_item_id)
res_dict = {}
res_dict['uuid'] = term_uuid
res_dict['date_from'] = date_from
res_dict['date_to'] = date_to
res_dict['items'] = all_item_id
return (res_dict, 200)
res_dict = {'uuid': term_uuid, 'date_from': date_from, 'date_to': date_to, 'items': all_item_id}
return res_dict, 200
def get_tracked_term_first_seen(term_uuid):
return Tracker.get_tracker_first_seen(term_uuid)
@ -444,7 +447,6 @@ def get_term_metedata(term_uuid, user_id=False, description=False, level=False,
dict_uuid['term'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'tracked')
dict_uuid['type'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'type')
dict_uuid['date'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'date')
dict_uuid['description'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'description')
dict_uuid['first_seen'] = get_tracked_term_first_seen(term_uuid)
dict_uuid['last_seen'] = get_tracked_term_last_seen(term_uuid)
if user_id:
@ -457,6 +459,8 @@ def get_term_metedata(term_uuid, user_id=False, description=False, level=False,
dict_uuid['tags'] = get_list_trackeed_term_tags(term_uuid)
if sparkline:
dict_uuid['sparkline'] = get_tracked_term_sparkline(term_uuid)
if description:
dict_uuid['description'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'description')
dict_uuid['uuid'] = term_uuid
return dict_uuid

View file

@ -112,11 +112,9 @@ def add_tracked_menu():
else:
tracker = yara_default_rule
tracker_type='yara_default'
# #
if level == 'on':
level = 1
if mails:
mails = mails.split()
if tags:

View file

@ -23,6 +23,7 @@ import Item
import Paste
import Tag
import Term
import Tracker
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'import'))
import importer
@ -305,7 +306,6 @@ def delete_item_tags():
@restApi.route("api/v1/get/item/content", methods=['POST'])
@token_required('read_only')
def get_item_content():
data = request.get_json()
item_id = data.get('id', None)
req_data = {'id': item_id, 'date': False, 'content': True, 'tags': False}
@ -313,6 +313,32 @@ def get_item_content():
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
@restApi.route("api/v1/get/item/content/encoded/text", methods=['POST'])
@token_required('read_only')
def get_item_content_encoded_text():
data = request.get_json()
item_id = data.get('id', None)
req_data = {'id': item_id}
res = Item.get_item_content_encoded_text(req_data)
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
@restApi.route("api/v1/get/item/sources", methods=['GET'])
@token_required('read_only')
def get_item_sources():
res = Item.get_item_sources()
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
@restApi.route("api/v1/get/item/source/check", methods=['POST'])
@token_required('read_only')
def get_check_item_source():
data = request.get_json()
source = data.get('source', None)
req_data = {'source': source}
res = Item.check_item_source(req_data)
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # TAGS # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@ -342,7 +368,7 @@ def add_tracker_term():
data = request.get_json()
user_token = get_auth_from_header()
user_id = get_user_from_token(user_token)
res = Term.parse_json_term_to_add(data, user_id)
res = Tracker.api_add_tracker(data, user_id)
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
@restApi.route("api/v1/delete/tracker", methods=['DELETE'])
@ -364,6 +390,24 @@ def get_tracker_term_item():
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
@restApi.route("api/v1/get/tracker/yara/content", methods=['POST'])
@token_required('read_only')
def get_default_yara_rule_content():
data = request.get_json()
rule_name = data.get('rule_name', None)
req_data = {'rule_name': rule_name}
res = Tracker.get_yara_rule_content_restapi(req_data)
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
@restApi.route("api/v1/get/tracker/metadata", methods=['POST'])
@token_required('read_only')
def get_tracker_metadata_api():
data = request.get_json()
tracker_uuid = data.get('tracker_uuid', None)
req_data = {'tracker_uuid': tracker_uuid}
res = Tracker.get_tracker_metadata_api(req_data)
return Response(json.dumps(res[0], indent=2, sort_keys=False), mimetype='application/json'), res[1]
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # CRYPTOCURRENCY # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #