From dfd1128daa3376570ae054a2a2649c5f04b5f83e Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 16 Aug 2019 17:59:44 +0200 Subject: [PATCH] chg: [api] add new endpoints: get bitcoin/pgp name/pgp keys/pgp mail metadata + items list --- bin/packages/Correlation.py | 64 +++++++++++++ bin/packages/Cryptocurrency.py | 48 ++++++++++ bin/packages/Item.py | 40 ++++++++ bin/packages/Pgp.py | 25 +++++ var/www/modules/restApi/Flask_restApi.py | 114 +++++++++++++++++++++++ 5 files changed, 291 insertions(+) create mode 100755 bin/packages/Correlation.py create mode 100755 bin/packages/Cryptocurrency.py create mode 100755 bin/packages/Pgp.py diff --git a/bin/packages/Correlation.py b/bin/packages/Correlation.py new file mode 100755 index 00000000..351d480a --- /dev/null +++ b/bin/packages/Correlation.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import redis + +import Flask_config + +r_serv_metadata = Flask_config.r_serv_metadata + + +class Correlation(object): + + def __init__(self, correlation_name): + self.correlation_name = correlation_name + + def _exist_corelation_field(self, correlation_type, field_name): + return r_serv_metadata.exists('set_{}_{}:{}'.format(self.correlation_name, correlation_type, field_name)) + + + def _get_items(self, correlation_type, field_name): + res = r_serv_metadata.smembers('set_{}_{}:{}'.format(self.correlation_name, correlation_type, field_name)) + if res: + return list(res) + else: + return {} + + + def _get_metadata(self, correlation_type, field_name): + meta_dict = {} + meta_dict['first_seen'] = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, correlation_type, field_name), 'first_seen') + meta_dict['last_seen'] = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, correlation_type, field_name), 'last_seen') + return meta_dict + + def _get_correlation_by_date(self, correlation_type, date): + return r_serv_metadata.hkeys('{}:{}:{}'.format(self.correlation_name, correlation_type, date)) + + def verify_correlation_field_request(self, request_dict, correlation_type): + if not request_dict: + return Response({'status': 'error', 'reason': 'Malformed JSON'}, 400) + + field_name = request_dict.get(correlation_type, None) + if not field_name: + return ( {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400 ) + if not self._exist_corelation_field(correlation_type, field_name): + return ( {'status': 'error', 'reason': 'Item not found'}, 404 ) + + def get_correlation(self, request_dict, correlation_type, field_name): + dict_resp = {} + + if request_dict.get('items'): + dict_resp['items'] = self._get_items(correlation_type, field_name) + + if request_dict.get('metadata'): + dict_resp['metadata'] = self._get_metadata(correlation_type, field_name) + + dict_resp[correlation_type] = field_name + + return (dict_resp, 200) + + + + +#cryptocurrency_all:cryptocurrency name cryptocurrency address nb seen diff --git a/bin/packages/Cryptocurrency.py b/bin/packages/Cryptocurrency.py new file mode 100755 index 00000000..1b4e4e3f --- /dev/null +++ b/bin/packages/Cryptocurrency.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import redis + +from hashlib import sha256 + +import Flask_config +from Correlation import Correlation + +r_serv_metadata = Flask_config.r_serv_metadata + +digits58 = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' + +cryptocurrency = Correlation('cryptocurrency') + +def decode_base58(bc, length): + n = 0 + for char in bc: + n = n * 58 + digits58.index(char) + return n.to_bytes(length, 'big') + +def check_bitcoin_address(bc): + try: + bcbytes = decode_base58(bc, 25) + return bcbytes[-4:] == sha256(sha256(bcbytes[:-4]).digest()).digest()[:4] + except Exception: + return False + +def verify_cryptocurrency_address(cryptocurrency_type, cryptocurrency_address): + if cryptocurrency_type == 'bitcoin': + return check_bitcoin_address(cryptocurrency_address) + else: + return True + + +def get_cryptocurrency(request_dict, cryptocurrency_type): + # basic verification + res = cryptocurrency.verify_correlation_field_request(request_dict, cryptocurrency_type) + if res: + return res + # cerify address + field_name = request_dict.get(cryptocurrency_type) + if not verify_cryptocurrency_address(cryptocurrency_type, field_name): + return ( {'status': 'error', 'reason': 'Invalid Cryptocurrency address'}, 400 ) + + return cryptocurrency.get_correlation(request_dict, cryptocurrency_type, field_name) diff --git a/bin/packages/Item.py b/bin/packages/Item.py index 2c10cb85..92c59a2f 100755 --- a/bin/packages/Item.py +++ b/bin/packages/Item.py @@ -11,6 +11,7 @@ import Tag PASTES_FOLDER = Flask_config.PASTES_FOLDER r_cache = Flask_config.r_cache +r_serv_metadata = Flask_config.r_serv_metadata def exist_item(item_id): if os.path.isfile(os.path.join(PASTES_FOLDER, item_id)): @@ -90,4 +91,43 @@ def get_item(request_dict): if lines_info: dict_item['lines'] = get_lines_info(item_id, dict_item.get('content', 'None')) + if request_dict.get('pgp'): + dict_item['pgp'] = {} + if request_dict['pgp'].get('key'): + dict_item['pgp']['key'] = get_item_pgp_key(item_id) + if request_dict['pgp'].get('mail'): + dict_item['pgp']['mail'] = get_item_pgp_mail(item_id) + if request_dict['pgp'].get('name'): + dict_item['pgp']['name'] = get_item_pgp_name(item_id) + + if request_dict.get('cryptocurrency'): + dict_item['cryptocurrency'] = {} + if request_dict['cryptocurrency'].get('bitcoin'): + dict_item['cryptocurrency']['bitcoin'] = get_item_bitcoin(item_id) + return (dict_item, 200) + + +### +### correlation +### + +def _get_item_correlation(correlation_name, correlation_type, item_id): + print('item_{}_{}:{}'.format(correlation_name, correlation_type, item_id)) + res = r_serv_metadata.smembers('item_{}_{}:{}'.format(correlation_name, correlation_type, item_id)) + if res: + return list(res) + else: + return [] + +def get_item_bitcoin(item_id): + return _get_item_correlation('cryptocurrency', 'bitcoin', item_id) + +def get_item_pgp_key(item_id): + return _get_item_correlation('pgpdump', 'key', item_id) + +def get_item_pgp_name(item_id): + return _get_item_correlation('pgpdump', 'name', item_id) + +def get_item_pgp_mail(item_id): + return _get_item_correlation('pgpdump', 'mail', item_id) diff --git a/bin/packages/Pgp.py b/bin/packages/Pgp.py new file mode 100755 index 00000000..9c7b0ec4 --- /dev/null +++ b/bin/packages/Pgp.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import redis + +from hashlib import sha256 + +import Flask_config +from Correlation import Correlation + +r_serv_metadata = Flask_config.r_serv_metadata + +pgpdump = Correlation('pgpdump') + + +def get_pgp(request_dict, pgp_type): + # basic verification + res = pgpdump.verify_correlation_field_request(request_dict, pgp_type) + if res: + return res + # cerify address + field_name = request_dict.get(pgp_type) + + return pgpdump.get_correlation(request_dict, pgp_type, field_name) diff --git a/var/www/modules/restApi/Flask_restApi.py b/var/www/modules/restApi/Flask_restApi.py index 6ea8dd69..f942e432 100644 --- a/var/www/modules/restApi/Flask_restApi.py +++ b/var/www/modules/restApi/Flask_restApi.py @@ -14,6 +14,8 @@ import redis import datetime import Import_helper +import Cryptocurrency +import Pgp import Item import Paste import Tag @@ -287,6 +289,7 @@ def get_item_content(): res = Item.get_item(req_data) return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # TAGS # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # @@ -307,6 +310,117 @@ def get_all_tags(): res = {'tags': Tag.get_all_tags()} return Response(json.dumps(res, indent=2, sort_keys=True), mimetype='application/json'), 200 +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# # # # # # # # # # # # CRYPTOCURRENCY # # # # # # # # # # # # # # +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +@restApi.route("api/v1/get/cryptocurrency/bitcoin/metadata", methods=['POST']) +@token_required('analyst') +def get_cryptocurrency_bitcoin_metadata(): + data = request.get_json() + crypto_address = data.get('bitcoin', None) + req_data = {'bitcoin': crypto_address, 'metadata': True} + res = Cryptocurrency.get_cryptocurrency(req_data, 'bitcoin') + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + +@restApi.route("api/v1/get/cryptocurrency/bitcoin/item", methods=['POST']) +@token_required('analyst') +def get_cryptocurrency_bitcoin_item(): + data = request.get_json() + bitcoin_address = data.get('bitcoin', None) + req_data = {'bitcoin': bitcoin_address, 'items': True} + res = Cryptocurrency.get_cryptocurrency(req_data, 'bitcoin') + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# # # # # # # # # # # # # # # PGP # # # # # # # # # # # # # # # # # +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +@restApi.route("api/v1/get/pgp/key/metadata", methods=['POST']) +@token_required('analyst') +def get_pgp_key_metadata(): + data = request.get_json() + pgp_field = data.get('key', None) + req_data = {'key': pgp_field, 'metadata': True} + res = Pgp.get_pgp(req_data, 'key') + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + +@restApi.route("api/v1/get/pgp/mail/metadata", methods=['POST']) +@token_required('analyst') +def get_pgp_mail_metadata(): + data = request.get_json() + pgp_field = data.get('mail', None) + req_data = {'mail': pgp_field, 'metadata': True} + res = Pgp.get_pgp(req_data, 'mail') + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + +@restApi.route("api/v1/get/pgp/name/metadata", methods=['POST']) +@token_required('analyst') +def get_pgp_name_metadata(): + data = request.get_json() + pgp_field = data.get('name', None) + req_data = {'name': pgp_field, 'metadata': True} + res = Pgp.get_pgp(req_data, 'name') + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + +@restApi.route("api/v1/get/pgp/key/item", methods=['POST']) +@token_required('analyst') +def get_pgp_key_item(): + data = request.get_json() + pgp_field = data.get('key', None) + req_data = {'key': pgp_field, 'items': True} + res = Pgp.get_pgp(req_data, 'key') + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + +@restApi.route("api/v1/get/pgp/mail/item", methods=['POST']) +@token_required('analyst') +def get_pgp_mail_item(): + data = request.get_json() + pgp_mail = data.get('mail', None) + req_data = {'mail': pgp_mail, 'items': True} + res = Pgp.get_pgp(req_data, 'mail') + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + +@restApi.route("api/v1/get/pgp/name/item", methods=['POST']) +@token_required('analyst') +def get_pgp_name_item(): + data = request.get_json() + pgp_name = data.get('name', None) + req_data = {'name': pgp_name, 'items': True} + res = Pgp.get_pgp(req_data, 'name') + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + +''' + + + +@restApi.route("api/v1/get/item/cryptocurrency/key", methods=['POST']) +@token_required('analyst') +def get_item_cryptocurrency_bitcoin(): + data = request.get_json() + item_id = data.get('id', None) + req_data = {'id': item_id, 'date': False, 'tags': False, 'pgp': {'key': True}} + res = Item.get_item(req_data) + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + +@restApi.route("api/v1/get/item/pgp/mail", methods=['POST']) +@token_required('analyst') +def get_item_cryptocurrency_bitcoin(): + data = request.get_json() + item_id = data.get('id', None) + req_data = {'id': item_id, 'date': False, 'tags': False, 'pgp': {'mail': True}} + res = Item.get_item(req_data) + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + +@restApi.route("api/v1/get/item/pgp/name", methods=['POST']) +@token_required('analyst') +def get_item_cryptocurrency_bitcoin(): + data = request.get_json() + item_id = data.get('id', None) + req_data = {'id': item_id, 'date': False, 'tags': False, 'pgp': {'name': True}} + res = Item.get_item(req_data) + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] +''' + + # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # IMPORT # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #