From c2bf5aebdfe2af7f0a93692776d52248d51e54d0 Mon Sep 17 00:00:00 2001 From: TonyJabbour Date: Mon, 4 Oct 2021 15:01:14 +0200 Subject: [PATCH 01/12] Add new API endpoint that return only content encoded in base64 --- bin/packages/Item.py | 54 +++++++++++++++++++++--- var/www/modules/restApi/Flask_restApi.py | 11 +++++ 2 files changed, 60 insertions(+), 5 deletions(-) diff --git a/bin/packages/Item.py b/bin/packages/Item.py index 6ca7ba43..8331a33a 100755 --- a/bin/packages/Item.py +++ b/bin/packages/Item.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -*-coding:UTF-8 -* - +import base64 import os import re import sys @@ -165,13 +165,13 @@ def get_item_languages(item_id, min_len=600, num_langs=3, min_proportion=0.2, mi # API def get_item(request_dict): if not request_dict: - return Response({'status': 'error', 'reason': 'Malformed JSON'}, 400) + return {'status': 'error', 'reason': 'Malformed JSON'}, 400 item_id = request_dict.get('id', None) if not item_id: - return ( {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400 ) + return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400 if not exist_item(item_id): - return ( {'status': 'error', 'reason': 'Item not found'}, 404 ) + return {'status': 'error', 'reason': 'Item not found'}, 404 dict_item = {} dict_item['id'] = item_id @@ -216,7 +216,51 @@ def get_item(request_dict): if request_dict['cryptocurrency'].get('bitcoin'): dict_item['cryptocurrency']['bitcoin'] = get_item_bitcoin(item_id) - return (dict_item, 200) + return dict_item, 200 + + + + + + + +# API as txt +def get_item_as_txt(request_dict): + if not request_dict: + return {'status': 'error', 'reason': 'Malformed JSON'}, 400 + item_id = request_dict.get('id', None) + if not item_id: + return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400 + if not exist_item(item_id): + return {'status': 'error', 'reason': 'Item not found'}, 404 + + item_content = get_item_content(item_id) + encoded_item_content = item_content.encode("UTF-8") + base64_output = base64.b64encode(encoded_item_content) + dict_item = {'id': item_id, 'content': base64_output} + + return dict_item, 200 + + + + + + + + + + + + + + + + + + + + + ### diff --git a/var/www/modules/restApi/Flask_restApi.py b/var/www/modules/restApi/Flask_restApi.py index 8e96bf67..b5cc5457 100644 --- a/var/www/modules/restApi/Flask_restApi.py +++ b/var/www/modules/restApi/Flask_restApi.py @@ -313,6 +313,17 @@ def get_item_content(): return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] +@restApi.route("api/v1/get/item/content/text", methods=['POST']) +@token_required('read_only') +def get_item_content(): + + data = request.get_json() + item_id = data.get('id', None) + req_data = {'id': item_id} + res = Item.get_item_as_txt(req_data) + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='text/plain'), res[1] + + # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # TAGS # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # From 7669373002be69bf96b962dfd8dde745d67472c5 Mon Sep 17 00:00:00 2001 From: TonyJabbour Date: Mon, 4 Oct 2021 15:25:14 +0200 Subject: [PATCH 02/12] Error Fixed --- bin/packages/Item.py | 7 +++---- var/www/modules/restApi/Flask_restApi.py | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/bin/packages/Item.py b/bin/packages/Item.py index 8331a33a..b20e51d2 100755 --- a/bin/packages/Item.py +++ b/bin/packages/Item.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 # -*-coding:UTF-8 -* -import base64 import os import re import sys @@ -235,9 +234,9 @@ def get_item_as_txt(request_dict): return {'status': 'error', 'reason': 'Item not found'}, 404 item_content = get_item_content(item_id) - encoded_item_content = item_content.encode("UTF-8") - base64_output = base64.b64encode(encoded_item_content) - dict_item = {'id': item_id, 'content': base64_output} + #encoded_item_content = item_content.encode("UTF-8") + #base64_output = base64.b64encode(encoded_item_content) + dict_item = {'id': item_id, 'content': item_content} return dict_item, 200 diff --git a/var/www/modules/restApi/Flask_restApi.py b/var/www/modules/restApi/Flask_restApi.py index b5cc5457..4bed7193 100644 --- a/var/www/modules/restApi/Flask_restApi.py +++ b/var/www/modules/restApi/Flask_restApi.py @@ -321,7 +321,7 @@ def get_item_content(): item_id = data.get('id', None) req_data = {'id': item_id} res = Item.get_item_as_txt(req_data) - return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='text/plain'), res[1] + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # From e66b7aa19a31077f87a2bbf5e6401a7900c14286 Mon Sep 17 00:00:00 2001 From: TonyJabbour Date: Mon, 4 Oct 2021 15:31:18 +0200 Subject: [PATCH 03/12] Type fixed --- bin/packages/Item.py | 7 ++++--- var/www/modules/restApi/Flask_restApi.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/bin/packages/Item.py b/bin/packages/Item.py index b20e51d2..8331a33a 100755 --- a/bin/packages/Item.py +++ b/bin/packages/Item.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 # -*-coding:UTF-8 -* +import base64 import os import re import sys @@ -234,9 +235,9 @@ def get_item_as_txt(request_dict): return {'status': 'error', 'reason': 'Item not found'}, 404 item_content = get_item_content(item_id) - #encoded_item_content = item_content.encode("UTF-8") - #base64_output = base64.b64encode(encoded_item_content) - dict_item = {'id': item_id, 'content': item_content} + encoded_item_content = item_content.encode("UTF-8") + base64_output = base64.b64encode(encoded_item_content) + dict_item = {'id': item_id, 'content': base64_output} return dict_item, 200 diff --git a/var/www/modules/restApi/Flask_restApi.py b/var/www/modules/restApi/Flask_restApi.py index 4bed7193..6ba69f63 100644 --- a/var/www/modules/restApi/Flask_restApi.py +++ b/var/www/modules/restApi/Flask_restApi.py @@ -315,7 +315,7 @@ def get_item_content(): @restApi.route("api/v1/get/item/content/text", methods=['POST']) @token_required('read_only') -def get_item_content(): +def get_item_content_text(): data = request.get_json() item_id = data.get('id', None) From dd9cee83db1c41574de6c8316b94452f93055d89 Mon Sep 17 00:00:00 2001 From: TonyJabbour Date: Mon, 4 Oct 2021 15:39:32 +0200 Subject: [PATCH 04/12] Type fixed --- bin/packages/Item.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/packages/Item.py b/bin/packages/Item.py index 8331a33a..9c0302a6 100755 --- a/bin/packages/Item.py +++ b/bin/packages/Item.py @@ -226,17 +226,17 @@ def get_item(request_dict): # API as txt def get_item_as_txt(request_dict): + item_id = request_dict.get('id', None) + if not request_dict: return {'status': 'error', 'reason': 'Malformed JSON'}, 400 - item_id = request_dict.get('id', None) if not item_id: return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400 if not exist_item(item_id): return {'status': 'error', 'reason': 'Item not found'}, 404 item_content = get_item_content(item_id) - encoded_item_content = item_content.encode("UTF-8") - base64_output = base64.b64encode(encoded_item_content) + base64_output = base64.b64encode(item_content) dict_item = {'id': item_id, 'content': base64_output} return dict_item, 200 From bc8df8e6713d839f3c88ef9005481dc1d7d25075 Mon Sep 17 00:00:00 2001 From: TonyJabbour Date: Mon, 4 Oct 2021 15:44:41 +0200 Subject: [PATCH 05/12] Base64 Problem --- bin/packages/Item.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/packages/Item.py b/bin/packages/Item.py index 9c0302a6..e62688d0 100755 --- a/bin/packages/Item.py +++ b/bin/packages/Item.py @@ -236,7 +236,7 @@ def get_item_as_txt(request_dict): return {'status': 'error', 'reason': 'Item not found'}, 404 item_content = get_item_content(item_id) - base64_output = base64.b64encode(item_content) + base64_output = base64.b64encode(item_content.encode()) dict_item = {'id': item_id, 'content': base64_output} return dict_item, 200 From 7fb796172c714222b0f1c083825a5bcc5ef4078d Mon Sep 17 00:00:00 2001 From: TonyJabbour Date: Mon, 4 Oct 2021 15:54:57 +0200 Subject: [PATCH 06/12] Base64 Problem --- bin/packages/Item.py | 25 +----------------------- var/www/modules/restApi/Flask_restApi.py | 2 -- 2 files changed, 1 insertion(+), 26 deletions(-) diff --git a/bin/packages/Item.py b/bin/packages/Item.py index e62688d0..cfe4c37b 100755 --- a/bin/packages/Item.py +++ b/bin/packages/Item.py @@ -222,9 +222,6 @@ def get_item(request_dict): - - -# API as txt def get_item_as_txt(request_dict): item_id = request_dict.get('id', None) @@ -236,32 +233,12 @@ def get_item_as_txt(request_dict): return {'status': 'error', 'reason': 'Item not found'}, 404 item_content = get_item_content(item_id) - base64_output = base64.b64encode(item_content.encode()) + base64_output = base64.b64encode((item_content.encode('utf-8')).decode()) dict_item = {'id': item_id, 'content': base64_output} return dict_item, 200 - - - - - - - - - - - - - - - - - - - - ### ### correlation diff --git a/var/www/modules/restApi/Flask_restApi.py b/var/www/modules/restApi/Flask_restApi.py index 6ba69f63..7901f5e9 100644 --- a/var/www/modules/restApi/Flask_restApi.py +++ b/var/www/modules/restApi/Flask_restApi.py @@ -305,7 +305,6 @@ def delete_item_tags(): @restApi.route("api/v1/get/item/content", methods=['POST']) @token_required('read_only') def get_item_content(): - data = request.get_json() item_id = data.get('id', None) req_data = {'id': item_id, 'date': False, 'content': True, 'tags': False} @@ -316,7 +315,6 @@ def get_item_content(): @restApi.route("api/v1/get/item/content/text", methods=['POST']) @token_required('read_only') def get_item_content_text(): - data = request.get_json() item_id = data.get('id', None) req_data = {'id': item_id} From 4af54b7ff26204d31fe123dd0dc1e21a23b7b14d Mon Sep 17 00:00:00 2001 From: TonyJabbour Date: Mon, 4 Oct 2021 16:19:58 +0200 Subject: [PATCH 07/12] unnecessarily parenthesis removed --- bin/packages/Term.py | 47 ++++++++++++-------------- var/www/modules/hunter/Flask_hunter.py | 2 -- 2 files changed, 22 insertions(+), 27 deletions(-) diff --git a/bin/packages/Term.py b/bin/packages/Term.py index 5a495570..10093339 100755 --- a/bin/packages/Term.py +++ b/bin/packages/Term.py @@ -57,14 +57,14 @@ def is_in_role(user_id, role): def check_term_uuid_valid_access(term_uuid, user_id): if not is_valid_uuid_v4(term_uuid): - return ({"status": "error", "reason": "Invalid uuid"}, 400) + return {"status": "error", "reason": "Invalid uuid"}, 400 level = r_serv_term.hget('tracker:{}'.format(term_uuid), 'level') if not level: - return ({"status": "error", "reason": "Unknown uuid"}, 404) + return {"status": "error", "reason": "Unknown uuid"}, 404 if level == 0: if r_serv_term.hget('tracker:{}'.format(term_uuid), 'user_id') != user_id: if not is_in_role(user_id, 'admin'): - return ({"status": "error", "reason": "Unknown uuid"}, 404) + return {"status": "error", "reason": "Unknown uuid"}, 404 return None @@ -78,7 +78,7 @@ def is_valid_mail(email): def verify_mail_list(mail_list): for mail in mail_list: if not is_valid_mail(mail): - return ({'status': 'error', 'reason': 'Invalid email', 'value': mail}, 400) + return {'status': 'error', 'reason': 'Invalid email', 'value': mail}, 400 return None def is_valid_regex(term_regex): @@ -148,10 +148,10 @@ def is_term_tracked_in_user_level(term, term_type, user_id): def parse_json_term_to_add(dict_input, user_id): term = dict_input.get('term', None) if not term: - return ({"status": "error", "reason": "Term not provided"}, 400) + return {"status": "error", "reason": "Term not provided"}, 400 term_type = dict_input.get('type', None) if not term_type: - return ({"status": "error", "reason": "Term type not provided"}, 400) + return {"status": "error", "reason": "Term type not provided"}, 400 nb_words = dict_input.get('nb_words', 1) description = dict_input.get('description', '') description = escape(description) @@ -180,27 +180,27 @@ def parse_json_term_to_add(dict_input, user_id): # check if term already tracked in global if level==1: if is_term_tracked_in_global_level(term, term_type): - return ({"status": "error", "reason": "Term already tracked"}, 409) + return {"status": "error", "reason": "Term already tracked"}, 409 else: if is_term_tracked_in_user_level(term, term_type, user_id): - return ({"status": "error", "reason": "Term already tracked"}, 409) + return {"status": "error", "reason": "Term already tracked"}, 409 term_uuid = add_tracked_term(term , term_type, user_id, level, tags, mails, description) - return ({'term': term, 'type': term_type, 'uuid': term_uuid}, 200) + return {'term': term, 'type': term_type, 'uuid': term_uuid}, 200 def parse_tracked_term_to_add(term , term_type, nb_words=1): if term_type=='regex': if not is_valid_regex(term): - return ({"status": "error", "reason": "Invalid regex"}, 400) + return {"status": "error", "reason": "Invalid regex"}, 400 elif term_type=='word' or term_type=='set': # force lowercase term = term.lower() word_set = set(term) set_inter = word_set.intersection(special_characters) if set_inter: - return ({"status": "error", "reason": "special character not allowed", "message": "Please use a regex or remove all special characters"}, 400) + return {"status": "error", "reason": "special character not allowed", "message": "Please use a regex or remove all special characters"}, 400 words = term.split() # not a word if term_type=='word' and len(words)>1: @@ -226,13 +226,13 @@ def parse_tracked_term_to_add(term , term_type, nb_words=1): elif term_type=='yara_custom': if not Tracker.is_valid_yara_rule(term): - return ({"status": "error", "reason": "Invalid custom Yara Rule"}, 400) + return {"status": "error", "reason": "Invalid custom Yara Rule"}, 400 elif term_type=='yara_default': if not Tracker.is_valid_default_yara_rule(term): - return ({"status": "error", "reason": "The Yara Rule doesn't exist"}, 400) + return {"status": "error", "reason": "The Yara Rule doesn't exist"}, 400 else: - return ({"status": "error", "reason": "Incorrect type"}, 400) - return ({"status": "success", "term": term, "type": term_type}, 200) + return {"status": "error", "reason": "Incorrect type"}, 400 + return {"status": "success", "term": term, "type": term_type}, 200 def add_tracked_term(term , term_type, user_id, level, tags, mails, description, dashboard=0): @@ -281,14 +281,15 @@ def add_tracked_term(term , term_type, user_id, level, tags, mails, description, return term_uuid + def parse_tracked_term_to_delete(dict_input, user_id): term_uuid = dict_input.get("uuid", None) res = check_term_uuid_valid_access(term_uuid, user_id) if res: return res - delete_term(term_uuid) - return ({"uuid": term_uuid}, 200) + return {"uuid": term_uuid}, 200 + # # TODO: MOVE IN TRACKER def delete_term(term_uuid): @@ -328,7 +329,6 @@ def delete_term(term_uuid): r_serv_term.delete('tracker:sources:{}'.format(term_uuid)) # remove item set - #########################3 all_item_date = r_serv_term.zrange(f'tracker:stat:{term_uuid}', 0, -1, withscores=True) if all_item_date: all_item_date = dict(all_item_date) @@ -426,12 +426,8 @@ def parse_get_tracker_term_item(dict_input, user_id): all_item_id = Tracker.get_tracker_items_by_daterange(term_uuid, date_from, date_to) all_item_id = Item.get_item_list_desc(all_item_id) - res_dict = {} - res_dict['uuid'] = term_uuid - res_dict['date_from'] = date_from - res_dict['date_to'] = date_to - res_dict['items'] = all_item_id - return (res_dict, 200) + res_dict = {'uuid': term_uuid, 'date_from': date_from, 'date_to': date_to, 'items': all_item_id} + return res_dict, 200 def get_tracked_term_first_seen(term_uuid): return Tracker.get_tracker_first_seen(term_uuid) @@ -444,7 +440,6 @@ def get_term_metedata(term_uuid, user_id=False, description=False, level=False, dict_uuid['term'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'tracked') dict_uuid['type'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'type') dict_uuid['date'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'date') - dict_uuid['description'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'description') dict_uuid['first_seen'] = get_tracked_term_first_seen(term_uuid) dict_uuid['last_seen'] = get_tracked_term_last_seen(term_uuid) if user_id: @@ -457,6 +452,8 @@ def get_term_metedata(term_uuid, user_id=False, description=False, level=False, dict_uuid['tags'] = get_list_trackeed_term_tags(term_uuid) if sparkline: dict_uuid['sparkline'] = get_tracked_term_sparkline(term_uuid) + if description: + dict_uuid['description'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'description') dict_uuid['uuid'] = term_uuid return dict_uuid diff --git a/var/www/modules/hunter/Flask_hunter.py b/var/www/modules/hunter/Flask_hunter.py index 0853257f..1a52e31b 100644 --- a/var/www/modules/hunter/Flask_hunter.py +++ b/var/www/modules/hunter/Flask_hunter.py @@ -112,11 +112,9 @@ def add_tracked_menu(): else: tracker = yara_default_rule tracker_type='yara_default' - # # if level == 'on': level = 1 - if mails: mails = mails.split() if tags: From ddd2c81a978ccd22808eb6941bf5c56d0b3077d4 Mon Sep 17 00:00:00 2001 From: TonyJabbour Date: Tue, 5 Oct 2021 14:17:39 +0200 Subject: [PATCH 08/12] New API Endpoint: Return Item Content in base64 in non JSON format --- bin/packages/Item.py | 17 ++++++----------- var/www/modules/restApi/Flask_restApi.py | 11 +++++++---- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/bin/packages/Item.py b/bin/packages/Item.py index cfe4c37b..6e6e048e 100755 --- a/bin/packages/Item.py +++ b/bin/packages/Item.py @@ -220,24 +220,19 @@ def get_item(request_dict): - - -def get_item_as_txt(request_dict): +def get_item_content_encoded_text(request_dict): item_id = request_dict.get('id', None) - if not request_dict: - return {'status': 'error', 'reason': 'Malformed JSON'}, 400 + return {'status': 'error', 'reason': 'Malformed JSON'}, 400, 1 if not item_id: - return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400 + return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400, 1 if not exist_item(item_id): - return {'status': 'error', 'reason': 'Item not found'}, 404 + return {'status': 'error', 'reason': 'Item not found'}, 404, 1 item_content = get_item_content(item_id) - base64_output = base64.b64encode((item_content.encode('utf-8')).decode()) - dict_item = {'id': item_id, 'content': base64_output} - - return dict_item, 200 + base64_output = base64.b64encode((item_content.encode('utf-8'))) + return base64_output, 200, 0 ### diff --git a/var/www/modules/restApi/Flask_restApi.py b/var/www/modules/restApi/Flask_restApi.py index 7901f5e9..0169526a 100644 --- a/var/www/modules/restApi/Flask_restApi.py +++ b/var/www/modules/restApi/Flask_restApi.py @@ -312,14 +312,17 @@ def get_item_content(): return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] -@restApi.route("api/v1/get/item/content/text", methods=['POST']) +@restApi.route("api/v1/get/item/content/encoded/text", methods=['POST']) @token_required('read_only') -def get_item_content_text(): +def get_item_content_encoded_text(): data = request.get_json() item_id = data.get('id', None) req_data = {'id': item_id} - res = Item.get_item_as_txt(req_data) - return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + res = Item.get_item_content_encoded_text(req_data) + if res[2] == 1: + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + else: + return res[0], res[1] # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # From 3c24c636f84064504731e205fe2488fab209b039 Mon Sep 17 00:00:00 2001 From: TonyJabbour Date: Wed, 6 Oct 2021 11:11:10 +0200 Subject: [PATCH 09/12] Removed unnecessarily parentheses --- bin/lib/item_basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/lib/item_basic.py b/bin/lib/item_basic.py index f9a49260..fdc6587d 100755 --- a/bin/lib/item_basic.py +++ b/bin/lib/item_basic.py @@ -221,7 +221,7 @@ def verify_sources_list(sources): all_sources = get_all_items_sources() for source in sources: if source not in all_sources: - return ({'status': 'error', 'reason': 'Invalid source', 'value': source}, 400) + return {'status': 'error', 'reason': 'Invalid source', 'value': source}, 400 return None def get_all_items_metadata_dict(list_id): From fc817a5389e6ccd63f6685f10a72a27b4eea8dd4 Mon Sep 17 00:00:00 2001 From: TonyJabbour Date: Wed, 6 Oct 2021 11:12:43 +0200 Subject: [PATCH 10/12] New API Endpoint: Fixed get_item_content_encoded_text Added get_item_sources Added get_check_item_source Added get_default_yara_rule_content --- bin/lib/Tracker.py | 30 +++++++++++++--- bin/packages/Item.py | 44 ++++++++++++++++-------- var/www/modules/restApi/Flask_restApi.py | 30 +++++++++++++--- 3 files changed, 81 insertions(+), 23 deletions(-) diff --git a/bin/lib/Tracker.py b/bin/lib/Tracker.py index 5e929b8b..74543244 100755 --- a/bin/lib/Tracker.py +++ b/bin/lib/Tracker.py @@ -9,6 +9,7 @@ import redis import uuid import yara import datetime +import base64 from flask import escape @@ -683,17 +684,36 @@ def api_get_default_rule_content(default_yara_rule): yara_dir = get_yara_rules_default_dir() filename = os.path.join(yara_dir, default_yara_rule) filename = os.path.realpath(filename) - - # incorrect filename if not os.path.commonprefix([filename, yara_dir]) == yara_dir: - return ({'status': 'error', 'reason': 'file transversal detected'}, 400) + return {'status': 'error', 'reason': 'file traversal detected'}, 400 if not os.path.isfile(filename): - return ({'status': 'error', 'reason': 'yara rule not found'}, 400) + return {'status': 'error', 'reason': 'yara rule not found'}, 400 with open(filename, 'r') as f: rule_content = f.read() - return ({'rule_name': default_yara_rule, 'content': rule_content}, 200) + return {'rule_name': default_yara_rule, 'content': rule_content}, 200 + + +def get_yara_rule_content_restapi(request_dict): + rule_name = request_dict.get('rule_name', None) + if not request_dict: + return {'status': 'error', 'reason': 'Malformed JSON'}, 400 + if not rule_name: + return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400 + yara_dir = get_yara_rules_dir() + filename = os.path.join(yara_dir, rule_name) + filename = os.path.realpath(filename) + if not os.path.commonprefix([filename, yara_dir]) == yara_dir: + return {'status': 'error', 'reason': 'File Path Traversal'}, 400 + if not os.path.isfile(filename): + return {'status': 'error', 'reason': 'yara rule not found'}, 400 + with open(filename, 'r') as f: + rule_content = f.read() + rule_content = base64.b64encode((rule_content.encode('utf-8'))).decode('UTF-8') + return {'status': 'success', 'content': rule_content}, 200 + + ##-- YARA --## diff --git a/bin/packages/Item.py b/bin/packages/Item.py index 6e6e048e..b4d86a36 100755 --- a/bin/packages/Item.py +++ b/bin/packages/Item.py @@ -59,6 +59,9 @@ def get_item_date(item_id, add_separator=False): def get_source(item_id): return item_basic.get_source(item_id) +def get_all_sources(): + return item_basic.get_all_items_sources(r_list=True) + def get_item_basename(item_id): return os.path.basename(item_id) @@ -87,14 +90,12 @@ def get_item_metadata(item_id, item_content=None): # encoding # language # lines info - - item_metadata = {} - item_metadata['date'] = get_item_date(item_id, add_separator=True) - item_metadata['source'] = get_source(item_id) - item_metadata['size'] = get_item_size(item_id) - item_metadata['encoding'] = get_item_encoding(item_id) - item_metadata['lines'] = get_lines_info(item_id, item_content=item_content) - + item_metadata = {'date': get_item_date(item_id, add_separator=True), + 'source': get_source(item_id), + 'size': get_item_size(item_id), + 'encoding': get_item_encoding(item_id), + 'lines': get_lines_info(item_id, item_content=item_content) + } return item_metadata def get_item_parent(item_id): @@ -223,18 +224,33 @@ def get_item(request_dict): def get_item_content_encoded_text(request_dict): item_id = request_dict.get('id', None) if not request_dict: - return {'status': 'error', 'reason': 'Malformed JSON'}, 400, 1 + return {'status': 'error', 'reason': 'Malformed JSON'}, 400 if not item_id: - return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400, 1 + return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400 if not exist_item(item_id): - return {'status': 'error', 'reason': 'Item not found'}, 404, 1 + return {'status': 'error', 'reason': 'Item not found'}, 404 item_content = get_item_content(item_id) - base64_output = base64.b64encode((item_content.encode('utf-8'))) - - return base64_output, 200, 0 + item_content = base64.b64encode((item_content.encode('utf-8'))).decode('UTF-8') + return {'status': 'success', 'content': item_content}, 200 +def get_item_sources(): + item_content = {'sources': get_all_sources()} + return item_content, 200 + +def check_item_source(request_dict): + source = request_dict.get('source', None) + if not request_dict: + return {'status': 'error', 'reason': 'Malformed JSON'}, 400 + if not source: + return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400 + + all_sources = item_basic.get_all_items_sources() + + if source not in all_sources: + return {'status': 'error', 'reason': 'Invalid source', 'provide': source}, 400 + return {'status': 'success', 'reason': 'Valid source', 'provide': source}, 200 ### ### correlation ### diff --git a/var/www/modules/restApi/Flask_restApi.py b/var/www/modules/restApi/Flask_restApi.py index 0169526a..443e007f 100644 --- a/var/www/modules/restApi/Flask_restApi.py +++ b/var/www/modules/restApi/Flask_restApi.py @@ -23,6 +23,7 @@ import Item import Paste import Tag import Term +import Tracker sys.path.append(os.path.join(os.environ['AIL_BIN'], 'import')) import importer @@ -319,12 +320,25 @@ def get_item_content_encoded_text(): item_id = data.get('id', None) req_data = {'id': item_id} res = Item.get_item_content_encoded_text(req_data) - if res[2] == 1: - return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] - else: - return res[0], res[1] + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] +@restApi.route("api/v1/get/item/sources", methods=['GET']) +@token_required('read_only') +def get_item_sources(): + res = Item.get_item_sources() + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + + + +@restApi.route("api/v1/get/item/source/check", methods=['POST']) +@token_required('read_only') +def get_check_item_source(): + data = request.get_json() + source = data.get('source', None) + req_data = {'source': source} + res = Item.check_item_source(req_data) + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # TAGS # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # @@ -376,6 +390,14 @@ def get_tracker_term_item(): return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] +@restApi.route("api/v1/get/tracker/yara/content", methods=['POST']) +@token_required('read_only') +def get_default_yara_rule_content(): + data = request.get_json() + rule_name = data.get('rule_name', None) + req_data = {'rule_name': rule_name} + res = Tracker.get_yara_rule_content_restapi(req_data) + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # CRYPTOCURRENCY # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # From a0c978f2c7e4110abae7100ee09810b49dfe60d3 Mon Sep 17 00:00:00 2001 From: TonyJabbour Date: Wed, 6 Oct 2021 14:32:45 +0200 Subject: [PATCH 11/12] Added get_tracker_metadata_api Removed unnecessarily parentheses --- bin/lib/Tracker.py | 63 +++++++++++++++++------- var/www/modules/restApi/Flask_restApi.py | 10 ++++ 2 files changed, 56 insertions(+), 17 deletions(-) diff --git a/bin/lib/Tracker.py b/bin/lib/Tracker.py index 74543244..4abfd805 100755 --- a/bin/lib/Tracker.py +++ b/bin/lib/Tracker.py @@ -11,6 +11,7 @@ import yara import datetime import base64 + from flask import escape sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) @@ -160,6 +161,34 @@ def get_tracker_metadata(tracker_uuid, user_id=False, description=False, level=F return dict_uuid +def get_tracker_metadata_api(request_dict): + tracker_uuid = request_dict.get('tracker_uuid', None) + if not request_dict: + return {'status': 'error', 'reason': 'Malformed JSON'}, 400 + if not tracker_uuid: + return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400 + if not is_valid_uuid_v4(tracker_uuid): + return {"status": "error", "reason": "Invalid Tracker UUID"}, 400 + if not r_serv_tracker.exists(f'tracker:{tracker_uuid}'): + return {'status': 'error', 'reason': 'Tracker not found'}, 404 + + dict_tracker = {'status': 'success', + 'uuid': tracker_uuid, + 'user_id': get_tracker_user_id(tracker_uuid), + 'tracker': get_tracker_by_uuid(tracker_uuid), + 'type': get_tracker_type(tracker_uuid), + 'date': get_tracker_date(tracker_uuid), + 'first_seen': get_tracker_first_seen(tracker_uuid), + 'last_seen': get_tracker_last_seen(tracker_uuid), + 'level': get_tracker_level(tracker_uuid), + 'mails': get_tracker_mails(tracker_uuid), + 'tags': get_tracker_tags(tracker_uuid), + 'description': get_tracker_description(tracker_uuid), + 'webhook': get_tracker_webhook(tracker_uuid) + } + + return dict_tracker, 200 + # tracker sparkline def get_tracker_sparkline(tracker_uuid, num_day=6): date_range_sparkline = Date.get_date_range(num_day) @@ -279,13 +308,13 @@ def is_tracker_in_user_level(tracker, tracker_type, user_id): def api_is_allowed_to_edit_tracker(tracker_uuid, user_id): if not is_valid_uuid_v4(tracker_uuid): - return ({"status": "error", "reason": "Invalid uuid"}, 400) + return {"status": "error", "reason": "Invalid uuid"}, 400 tracker_creator = r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'user_id') if not tracker_creator: - return ({"status": "error", "reason": "Unknown uuid"}, 404) + return {"status": "error", "reason": "Unknown uuid"}, 404 if not is_in_role(user_id, 'admin') and user_id != tracker_creator: - return ({"status": "error", "reason": "Access Denied"}, 403) - return ({"uuid": tracker_uuid}, 200) + return {"status": "error", "reason": "Access Denied"}, 403 + return {"uuid": tracker_uuid}, 200 ##-- ACL --## @@ -336,14 +365,14 @@ def fix_all_tracker_uuid_list(): def api_validate_tracker_to_add(tracker , tracker_type, nb_words=1): if tracker_type=='regex': if not is_valid_regex(tracker): - return ({"status": "error", "reason": "Invalid regex"}, 400) + return {"status": "error", "reason": "Invalid regex"}, 400 elif tracker_type=='word' or tracker_type=='set': # force lowercase tracker = tracker.lower() word_set = set(tracker) set_inter = word_set.intersection(special_characters) if set_inter: - return ({"status": "error", "reason": f'special character(s) not allowed: {set_inter}', "message": "Please use a python regex or remove all special characters"}, 400) + return {"status": "error", "reason": f'special character(s) not allowed: {set_inter}', "message": "Please use a python regex or remove all special characters"}, 400 words = tracker.split() # not a word if tracker_type=='word' and len(words)>1: @@ -369,13 +398,13 @@ def api_validate_tracker_to_add(tracker , tracker_type, nb_words=1): elif tracker_type=='yara_custom': if not is_valid_yara_rule(tracker): - return ({"status": "error", "reason": "Invalid custom Yara Rule"}, 400) + return {"status": "error", "reason": "Invalid custom Yara Rule"}, 400 elif tracker_type=='yara_default': if not is_valid_default_yara_rule(tracker): - return ({"status": "error", "reason": "The Yara Rule doesn't exist"}, 400) + return {"status": "error", "reason": "The Yara Rule doesn't exist"}, 400 else: - return ({"status": "error", "reason": "Incorrect type"}, 400) - return ({"status": "success", "tracker": tracker, "type": tracker_type}, 200) + return {"status": "error", "reason": "Incorrect type"}, 400 + return {"status": "success", "tracker": tracker, "type": tracker_type}, 200 def create_tracker(tracker, tracker_type, user_id, level, tags, mails, description, webhook, dashboard=0, tracker_uuid=None, sources=[]): # edit tracker @@ -1106,9 +1135,9 @@ def get_retro_hunt_nb_item_by_day(l_task_uuid, date_from=None, date_to=None): ## API ## def api_check_retro_hunt_task_uuid(task_uuid): if not is_valid_uuid_v4(task_uuid): - return ({"status": "error", "reason": "Invalid uuid"}, 400) + return {"status": "error", "reason": "Invalid uuid"}, 400 if not r_serv_tracker.exists(f'tracker:retro_hunt:task:{task_uuid}'): - return ({"status": "error", "reason": "Unknown uuid"}, 404) + return {"status": "error", "reason": "Unknown uuid"}, 404 return None def api_get_retro_hunt_items(dict_input): @@ -1136,7 +1165,7 @@ def api_get_retro_hunt_items(dict_input): res_dict['date_from'] = date_from res_dict['date_to'] = date_to res_dict['items'] = all_items_id - return (res_dict, 200) + return res_dict, 200 def api_pause_retro_hunt_task(task_uuid): res = api_check_retro_hunt_task_uuid(task_uuid) @@ -1144,9 +1173,9 @@ def api_pause_retro_hunt_task(task_uuid): return res task_state = get_retro_hunt_task_state(task_uuid) if task_state not in ['pending', 'running']: - return ({"status": "error", "reason": f"Task {task_uuid} not paused, current state: {task_state}"}, 400) + return {"status": "error", "reason": f"Task {task_uuid} not paused, current state: {task_state}"}, 400 pause_retro_hunt_task(task_uuid) - return (task_uuid, 200) + return task_uuid, 200 def api_resume_retro_hunt_task(task_uuid): res = api_check_retro_hunt_task_uuid(task_uuid) @@ -1154,9 +1183,9 @@ def api_resume_retro_hunt_task(task_uuid): return res task_state = get_retro_hunt_task_state(task_uuid) if not r_serv_tracker.sismember('tracker:retro_hunt:task:paused', task_uuid): - return ({"status": "error", "reason": f"Task {task_uuid} not paused, current state: {get_retro_hunt_task_state(task_uuid)}"}, 400) + return {"status": "error", "reason": f"Task {task_uuid} not paused, current state: {get_retro_hunt_task_state(task_uuid)}"}, 400 resume_retro_hunt_task(task_uuid) - return (task_uuid, 200) + return task_uuid, 200 def api_validate_rule_to_add(rule, rule_type): if rule_type=='yara_custom': diff --git a/var/www/modules/restApi/Flask_restApi.py b/var/www/modules/restApi/Flask_restApi.py index 443e007f..c180bdb6 100644 --- a/var/www/modules/restApi/Flask_restApi.py +++ b/var/www/modules/restApi/Flask_restApi.py @@ -398,6 +398,16 @@ def get_default_yara_rule_content(): req_data = {'rule_name': rule_name} res = Tracker.get_yara_rule_content_restapi(req_data) return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + + +@restApi.route("api/v1/get/tracker/metadata", methods=['POST']) +@token_required('read_only') +def get_tracker_metadata_api(): + data = request.get_json() + tracker_uuid = data.get('tracker_uuid', None) + req_data = {'tracker_uuid': tracker_uuid} + res = Tracker.get_tracker_metadata_api(req_data) + return Response(json.dumps(res[0], indent=2, sort_keys=False), mimetype='application/json'), res[1] # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # CRYPTOCURRENCY # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # From de9799e7ff0397247d3163c8b0cc266fbebf55c7 Mon Sep 17 00:00:00 2001 From: TonyJabbour Date: Fri, 19 Nov 2021 10:55:47 +0100 Subject: [PATCH 12/12] add tracker fixed api function replaced it with internal function --- bin/packages/Term.py | 13 ++++++++++--- var/www/modules/restApi/Flask_restApi.py | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/bin/packages/Term.py b/bin/packages/Term.py index 10093339..f46ec4ea 100755 --- a/bin/packages/Term.py +++ b/bin/packages/Term.py @@ -152,10 +152,15 @@ def parse_json_term_to_add(dict_input, user_id): term_type = dict_input.get('type', None) if not term_type: return {"status": "error", "reason": "Term type not provided"}, 400 + nb_words = dict_input.get('nb_words', 1) + description = dict_input.get('description', '') description = escape(description) + webhook = dict_input.get('webhook', '') + webhook = escape(webhook) + res = parse_tracked_term_to_add(term , term_type, nb_words=nb_words) if res[1]!=200: return res @@ -168,7 +173,6 @@ def parse_json_term_to_add(dict_input, user_id): if res: return res - ## TODO: add dashboard key level = dict_input.get('level', 1) try: level = int(level) @@ -185,7 +189,7 @@ def parse_json_term_to_add(dict_input, user_id): if is_term_tracked_in_user_level(term, term_type, user_id): return {"status": "error", "reason": "Term already tracked"}, 409 - term_uuid = add_tracked_term(term , term_type, user_id, level, tags, mails, description) + term_uuid = add_tracked_term(term , term_type, user_id, level, tags, mails, description,webhook) return {'term': term, 'type': term_type, 'uuid': term_uuid}, 200 @@ -234,7 +238,7 @@ def parse_tracked_term_to_add(term , term_type, nb_words=1): return {"status": "error", "reason": "Incorrect type"}, 400 return {"status": "success", "term": term, "type": term_type}, 200 -def add_tracked_term(term , term_type, user_id, level, tags, mails, description, dashboard=0): +def add_tracked_term(term , term_type, user_id, level, tags, mails, description,webhook, dashboard=0): term_uuid = str(uuid.uuid4()) @@ -254,6 +258,9 @@ def add_tracked_term(term , term_type, user_id, level, tags, mails, description, if description: r_serv_term.hset('tracker:{}'.format(term_uuid), 'description', description) + if webhook: + r_serv_term.hset('tracker:{}'.format(term_uuid), 'webhook', webhook) + # create all term set r_serv_term.sadd('all:tracker:{}'.format(term_type), term) diff --git a/var/www/modules/restApi/Flask_restApi.py b/var/www/modules/restApi/Flask_restApi.py index c180bdb6..b48c6198 100644 --- a/var/www/modules/restApi/Flask_restApi.py +++ b/var/www/modules/restApi/Flask_restApi.py @@ -368,7 +368,7 @@ def add_tracker_term(): data = request.get_json() user_token = get_auth_from_header() user_id = get_user_from_token(user_token) - res = Term.parse_json_term_to_add(data, user_id) + res = Tracker.api_add_tracker(data, user_id) return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] @restApi.route("api/v1/delete/tracker", methods=['DELETE'])