diff --git a/bin/lib/objects/Items.py b/bin/lib/objects/Items.py index b72aada9..d28654ca 100755 --- a/bin/lib/objects/Items.py +++ b/bin/lib/objects/Items.py @@ -406,9 +406,43 @@ def _manual_set_items_date_first_last(): if last != 0: update_obj_date(last, 'item') +#### API #### + +def api_get_item(data): + item_id = data.get('id', None) + if not item_id: + return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400 + item = Item(item_id) + if not item.exists(): + return {'status': 'error', 'reason': 'Item not found'}, 404 + + options = set() + if data.get('content'): + options.add('content') + if data.get('crawler'): + options.add('crawler') + if data.get('duplicates'): + options.add('duplicates') + if data.get('lines'): + options.add('lines') + if data.get('mimetype'): + options.add('mimetype') + if data.get('parent'): + options.add('parent') + if data.get('size'): + options.add('size') + + # TODO correlation + + return item.get_meta(options=options), 200 + + +# -- API -- # + ################################################################################ ################################################################################ ################################################################################ + # TODO def exist_item(item_id): diff --git a/doc/README.md b/doc/README.md index 44987c94..366f4146 100644 --- a/doc/README.md +++ b/doc/README.md @@ -1239,64 +1239,36 @@ curl -k https://127.0.0.1:7000/api/v1/get/import/item --header "Authorization: i - - - - # FUTURE endpoints -### Text search by daterange -##### ``api/search/textIndexer/item`` POST +
+Endpoints -### Get tagged items by daterange -##### ``api/search/tag/item`` POST +### Submit a domain to crawl TODO +##### ``api/add/crawler/task`` POST -### Submit a domain to crawl -##### ``api/add/crawler/domain`` POST +### Create a term/set/regex/yara tracker +##### ``api/add/tracker/`` POST -### Create a term/set/regex tracker -##### ``api/add/termTracker/`` POST - -### Get tracker items list -##### ``api/get/termTracker/item`` POST +### Get tracker +##### ``api/get/tracker`` POST ----- -### Check if a tor/regular domain have been crawled -##### ``api/get/crawler/domain/`` POST -### Check if a tor/regular domain have been crawled -##### ``api/get/crawler/domain/metadata/ `` POST ### Get domain tags -##### ``api/get/crawler/domain/tag/ `` POST +##### ``api/get/domain/tags/`` POST ### Get domain history -##### ``api/get/crawler/domain/history/ `` POST - -### Get domain list of items -##### ``api/get/crawler/domain/item/ `` POST +##### ``api/get/domain/history/`` POST ----- -### Create auto-crawlers -##### ``api/add/crawler/autoCrawler/`` POST - ------ - -### get item by mime type/ decoded type -##### ``api/get/decoded`` POST - -### Check if a decoded item exists (via sha1) -##### ``api/get/decoded/exist/`` POST - ### Get decoded item metadata ### Check if a decoded item exists (via sha1) ##### ``api/get/decoded/metadata/`` POST -### Get decoded item correlation (1 depth) -##### ``api/get/decoded/metadata/`` POST - ----- @@ -1304,18 +1276,20 @@ curl -k https://127.0.0.1:7000/api/v1/get/import/item --header "Authorization: i ##### ``api/get/cryptocurrency`` POST ### Check if a cryptocurrency address (bitcoin, ..) exists -##### ``api/get/cryptocurrency/exist/`` POST +##### ``api/get/cryptocurrency/`` POST ### Get cryptocurrency address metadata -##### ``api/get/cryptocurrency/metadata/`` POST +##### ``api/get/cryptocurrency/metadata/`` POST ----- -### Item correlation (1 depth) -##### ``api/get/item/correlation/`` POST +### Object correlation (1 depth) +##### ``api/get/correlation/`` POST -### Create MISP event from item -##### ``api/export/item/misp`` POST +### Create MISP event from object +##### ``api/export/misp`` POST + +
+ +----- -### Create TheHive case from item -##### ``api/export/item/thehive`` POST diff --git a/var/www/modules/restApi/Flask_restApi.py b/var/www/modules/restApi/Flask_restApi.py index 049dab4d..3df13f00 100644 --- a/var/www/modules/restApi/Flask_restApi.py +++ b/var/www/modules/restApi/Flask_restApi.py @@ -10,38 +10,42 @@ import re import sys import uuid import json -import datetime sys.path.append(os.environ['AIL_BIN']) ################################## # Import Project packages ################################## +from lib.ConfigLoader import ConfigLoader from lib import crawlers from lib import Users -from lib.objects.Items import Item +from lib.objects import Items from lib import Tag from lib import Tracker -from packages import Term +from packages import Term # TODO REMOVE ME from packages import Import_helper from importer.FeederImporter import api_add_json_feeder_to_queue -from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, escape -from flask_login import login_required +from flask import jsonify, request, Blueprint, redirect, url_for, Response, escape from functools import wraps # ============ VARIABLES ============ +config_loader = ConfigLoader() +baseUrl = config_loader.get_config_str("Flask", "baseurl") +baseUrl = baseUrl.replace('/', '') +if baseUrl != '': + baseUrl = '/' + baseUrl + +r_cache = config_loader.get_redis_conn("Redis_Cache") + +config_loader = None + import Flask_config - - app = Flask_config.app -baseUrl = Flask_config.baseUrl -r_cache = Flask_config.r_cache - restApi = Blueprint('restApi', __name__, template_folder='templates') @@ -51,7 +55,7 @@ def check_token_format(token, search=re.compile(r'[^a-zA-Z0-9_-]').search): return not bool(search(token)) def verify_token(token): - if len(token) != 41: + if len(token) != 55: return False if not check_token_format(token): @@ -76,7 +80,7 @@ def token_required(user_role): def actual_decorator(funct): @wraps(funct) def api_token(*args, **kwargs): - data = authErrors(user_role) + data = auth_errors(user_role) if data: return Response(json.dumps(data[0], indent=2, sort_keys=True), mimetype='application/json'), data[1] else: @@ -85,10 +89,10 @@ def token_required(user_role): return actual_decorator def get_auth_from_header(): - token = request.headers.get('Authorization').replace(' ', '') # remove space + token = request.headers.get('Authorization').replace(' ', '') # remove space return token -def authErrors(user_role): +def auth_errors(user_role): # Check auth if not request.headers.get('Authorization'): return {'status': 'error', 'reason': 'Authentication needed'}, 401 @@ -98,16 +102,18 @@ def authErrors(user_role): # brute force protection current_ip = request.remote_addr - login_failed_ip = r_cache.get('failed_login_ip_api:{}'.format(current_ip)) + login_failed_ip = r_cache.get(f'failed_login_ip_api:{current_ip}') # brute force by ip if login_failed_ip: login_failed_ip = int(login_failed_ip) if login_failed_ip >= 5: - return {'status': 'error', 'reason': 'Max Connection Attempts reached, Please wait {}s'.format(r_cache.ttl('failed_login_ip_api:{}'.format(current_ip)))}, 401 + return {'status': 'error', + 'reason': 'Max Connection Attempts reached, Please wait {}s'.format(r_cache.ttl('failed_login_ip_api:{}'.format(current_ip))) + }, 401 try: authenticated = False - if verify_token(token): + if verify_token(token): # TODO Improve Returned error authenticated = True # check user role @@ -115,8 +121,8 @@ def authErrors(user_role): data = ({'status': 'error', 'reason': 'Access Forbidden'}, 403) if not authenticated: - r_cache.incr('failed_login_ip_api:{}'.format(current_ip)) - r_cache.expire('failed_login_ip_api:{}'.format(current_ip), 300) + r_cache.incr(f'failed_login_ip_api:{current_ip}') + r_cache.expire(f'failed_login_ip_api:{current_ip}', 300) data = ({'status': 'error', 'reason': 'Authentication failed'}, 401) except Exception as e: print(e) @@ -141,7 +147,7 @@ def get_mandatory_fields(json_data, required_fields): def is_valid_uuid_v4(header_uuid): try: - header_uuid=header_uuid.replace('-', '') + header_uuid = header_uuid.replace('-', '') uuid_test = uuid.UUID(hex=header_uuid, version=4) return uuid_test.hex == header_uuid except: @@ -160,7 +166,6 @@ def is_valid_uuid_v4(header_uuid): # { # "id": item_id, mandatory # "content": true, -# "tags": true, # # # } @@ -175,17 +180,7 @@ def is_valid_uuid_v4(header_uuid): @token_required('read_only') def get_item_id(): data = request.get_json() - res = Item.get_item(data) - return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] - -@restApi.route("api/v1/get/item/default", methods=['POST']) -@token_required('read_only') -def get_item_id_basic(): - - data = request.get_json() - item_id = data.get('id', None) - req_data = {'id': item_id, 'date': True, 'content': True, 'tags': True} - res = Item.get_item(req_data) + res = Items.api_get_item(data) return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # @@ -197,6 +192,7 @@ def get_item_id_basic(): # # response: { # "id": "item_id", +# "date": "date", # "tags": [], # } # @@ -204,7 +200,6 @@ def get_item_id_basic(): @restApi.route("api/v1/get/item/tag", methods=['POST']) @token_required('read_only') def get_item_tag(): - data = request.get_json() item_id = data.get('id', None) req_data = {'id': item_id, 'date': False, 'tags': True} @@ -309,17 +304,6 @@ def get_item_sources(): res = Item.api_get_items_sources() return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] - - -# @restApi.route("api/v1/get/item/source/check", methods=['POST']) -# @token_required('read_only') -# def get_check_item_source(): -# data = request.get_json() -# source = data.get('source', None) -# req_data = {'source': source} -# res = Item.check_item_source(req_data) -# return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] - # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # TAGS # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # @@ -390,6 +374,8 @@ def get_tracker_metadata_api(): res = Tracker.get_tracker_metadata_api(req_data) return Response(json.dumps(res[0], indent=2, sort_keys=False), mimetype='application/json'), res[1] +''' + # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # CRYPTOCURRENCY # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # @@ -476,8 +462,6 @@ def get_pgp_name_item(): res = 0 return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] -''' - @restApi.route("api/v1/get/item/cryptocurrency/key", methods=['POST']) @@ -511,25 +495,19 @@ def get_item_cryptocurrency_bitcoin(): # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # CRAWLER # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # -@restApi.route("api/v1/crawl/domain", methods=['POST']) +# # TODO: ADD RESULT JSON Response +@restApi.route("api/v1/add/crawler/task", methods=['POST']) @token_required('analyst') -def crawl_domain(): +def add_crawler_task(): data = request.get_json() + user_token = get_auth_from_header() + user_id = Users.get_token_user(user_token) + res = crawlers.api_add_crawler_task(data, user_id=user_id) + if res: + return create_json_response(res[0], res[1]) - url = data.get('url', None) - screenshot = data.get('screenshot', None) - har = data.get('har', None) - depth_limit = data.get('depth_limit', None) - max_pages = data.get('max_pages', None) - auto_crawler = data.get('auto_crawler', None) - crawler_delta = data.get('crawler_delta', None) - crawler_type = data.get('url', None) - cookiejar_uuid = data.get('url', None) - user_agent = data.get('url', None) - - res = crawlers.api_create_crawler_task(json_dict) - res[0]['domain'] = domain - return create_json_response(res[0], res[1]) + dict_res = {'url': data['url']} + return create_json_response(dict_res, 200) # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # @@ -574,27 +552,11 @@ def get_crawled_domain_list(): dict_res['domain_type'] = domain_type return create_json_response(dict_res, res[1]) -# # TODO: ADD RESULT JSON Response -@restApi.route("api/v1/add/crawler/task", methods=['POST']) -@token_required('analyst') -def add_crawler_task(): - data = request.get_json() - user_token = get_auth_from_header() - user_id = get_user_from_token(user_token) - res = crawlers.api_add_crawler_task(data, user_id=user_id) - if res: - return create_json_response(res[0], res[1]) - - dict_res = {'url': data['url']} - return create_json_response(dict_res, 200) - # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # IMPORT # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # - - # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # POST JSON FORMAT @@ -616,12 +578,12 @@ def import_item(): data = request.get_json() if not data: - return Response(json.dumps({'status': 'error', 'reason': 'Malformed JSON'}, indent=2, sort_keys=True), mimetype='application/json'), 400 + return create_json_response({'status': 'error', 'reason': 'Malformed JSON'}, 400) # unpack json text_to_import = data.get('text', None) if not text_to_import: - return Response(json.dumps({'status': 'error', 'reason': 'No text supplied'}, indent=2, sort_keys=True), mimetype='application/json'), 400 + return create_json_response({'status': 'error', 'reason': 'No text supplied'}, 400) tags = data.get('tags', []) if not type(tags) is list: @@ -631,19 +593,19 @@ def import_item(): galaxy = [] if not Tag.is_valid_tags_taxonomies_galaxy(tags, galaxy): - return Response(json.dumps({'status': 'error', 'reason': 'Tags or Galaxy not enabled'}, indent=2, sort_keys=True), mimetype='application/json'), 400 + return create_json_response({'status': 'error', 'reason': 'Tags or Galaxy not enabled'}, 400) default_tags = data.get('default_tags', True) if default_tags: tags.append('infoleak:submission="manual"') if sys.getsizeof(text_to_import) > 900000: - return Response(json.dumps({'status': 'error', 'reason': 'Size exceeds default'}, indent=2, sort_keys=True), mimetype='application/json'), 413 + return create_json_response({'status': 'error', 'reason': 'Size exceeds default'}, 413) - UUID = str(uuid.uuid4()) - Import_helper.create_import_queue(tags, galaxy, text_to_import, UUID) + import_uuid = str(uuid.uuid4()) + Import_helper.create_import_queue(tags, galaxy, text_to_import, import_uuid) - return Response(json.dumps({'uuid': UUID}, indent=2, sort_keys=True), mimetype='application/json') + return Response(json.dumps({'uuid': import_uuid}, indent=2, sort_keys=True), mimetype='application/json') # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # GET @@ -662,13 +624,13 @@ def import_item(): @token_required('analyst') def import_item_uuid(): data = request.get_json() - UUID = data.get('uuid', None) + import_uuid = data.get('uuid', None) # Verify uuid - if not is_valid_uuid_v4(UUID): + if not is_valid_uuid_v4(import_uuid): return Response(json.dumps({'status': 'error', 'reason': 'Invalid uuid'}), mimetype='application/json'), 400 - data = Import_helper.check_import_status(UUID) + data = Import_helper.check_import_status(import_uuid) if data: return Response(json.dumps(data[0]), mimetype='application/json'), data[1] @@ -700,5 +662,6 @@ def import_json_item(): def v1_ping(): return Response(json.dumps({'status': 'pong'}), mimetype='application/json'), 200 + # ========= REGISTRATION ========= app.register_blueprint(restApi, url_prefix=baseUrl)