chg: [api] add new endpoints: get bitcoin/pgp name/pgp keys/pgp mail metadata + items list

This commit is contained in:
Terrtia 2019-08-16 17:59:44 +02:00
parent ab45ac0fef
commit dfd1128daa
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
5 changed files with 291 additions and 0 deletions

64
bin/packages/Correlation.py Executable file
View file

@ -0,0 +1,64 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import redis
import Flask_config
r_serv_metadata = Flask_config.r_serv_metadata
class Correlation(object):
def __init__(self, correlation_name):
self.correlation_name = correlation_name
def _exist_corelation_field(self, correlation_type, field_name):
return r_serv_metadata.exists('set_{}_{}:{}'.format(self.correlation_name, correlation_type, field_name))
def _get_items(self, correlation_type, field_name):
res = r_serv_metadata.smembers('set_{}_{}:{}'.format(self.correlation_name, correlation_type, field_name))
if res:
return list(res)
else:
return {}
def _get_metadata(self, correlation_type, field_name):
meta_dict = {}
meta_dict['first_seen'] = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, correlation_type, field_name), 'first_seen')
meta_dict['last_seen'] = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, correlation_type, field_name), 'last_seen')
return meta_dict
def _get_correlation_by_date(self, correlation_type, date):
return r_serv_metadata.hkeys('{}:{}:{}'.format(self.correlation_name, correlation_type, date))
def verify_correlation_field_request(self, request_dict, correlation_type):
if not request_dict:
return Response({'status': 'error', 'reason': 'Malformed JSON'}, 400)
field_name = request_dict.get(correlation_type, None)
if not field_name:
return ( {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400 )
if not self._exist_corelation_field(correlation_type, field_name):
return ( {'status': 'error', 'reason': 'Item not found'}, 404 )
def get_correlation(self, request_dict, correlation_type, field_name):
dict_resp = {}
if request_dict.get('items'):
dict_resp['items'] = self._get_items(correlation_type, field_name)
if request_dict.get('metadata'):
dict_resp['metadata'] = self._get_metadata(correlation_type, field_name)
dict_resp[correlation_type] = field_name
return (dict_resp, 200)
#cryptocurrency_all:cryptocurrency name cryptocurrency address nb seen

48
bin/packages/Cryptocurrency.py Executable file
View file

@ -0,0 +1,48 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import redis
from hashlib import sha256
import Flask_config
from Correlation import Correlation
r_serv_metadata = Flask_config.r_serv_metadata
digits58 = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
cryptocurrency = Correlation('cryptocurrency')
def decode_base58(bc, length):
n = 0
for char in bc:
n = n * 58 + digits58.index(char)
return n.to_bytes(length, 'big')
def check_bitcoin_address(bc):
try:
bcbytes = decode_base58(bc, 25)
return bcbytes[-4:] == sha256(sha256(bcbytes[:-4]).digest()).digest()[:4]
except Exception:
return False
def verify_cryptocurrency_address(cryptocurrency_type, cryptocurrency_address):
if cryptocurrency_type == 'bitcoin':
return check_bitcoin_address(cryptocurrency_address)
else:
return True
def get_cryptocurrency(request_dict, cryptocurrency_type):
# basic verification
res = cryptocurrency.verify_correlation_field_request(request_dict, cryptocurrency_type)
if res:
return res
# cerify address
field_name = request_dict.get(cryptocurrency_type)
if not verify_cryptocurrency_address(cryptocurrency_type, field_name):
return ( {'status': 'error', 'reason': 'Invalid Cryptocurrency address'}, 400 )
return cryptocurrency.get_correlation(request_dict, cryptocurrency_type, field_name)

View file

@ -11,6 +11,7 @@ import Tag
PASTES_FOLDER = Flask_config.PASTES_FOLDER PASTES_FOLDER = Flask_config.PASTES_FOLDER
r_cache = Flask_config.r_cache r_cache = Flask_config.r_cache
r_serv_metadata = Flask_config.r_serv_metadata
def exist_item(item_id): def exist_item(item_id):
if os.path.isfile(os.path.join(PASTES_FOLDER, item_id)): if os.path.isfile(os.path.join(PASTES_FOLDER, item_id)):
@ -90,4 +91,43 @@ def get_item(request_dict):
if lines_info: if lines_info:
dict_item['lines'] = get_lines_info(item_id, dict_item.get('content', 'None')) dict_item['lines'] = get_lines_info(item_id, dict_item.get('content', 'None'))
if request_dict.get('pgp'):
dict_item['pgp'] = {}
if request_dict['pgp'].get('key'):
dict_item['pgp']['key'] = get_item_pgp_key(item_id)
if request_dict['pgp'].get('mail'):
dict_item['pgp']['mail'] = get_item_pgp_mail(item_id)
if request_dict['pgp'].get('name'):
dict_item['pgp']['name'] = get_item_pgp_name(item_id)
if request_dict.get('cryptocurrency'):
dict_item['cryptocurrency'] = {}
if request_dict['cryptocurrency'].get('bitcoin'):
dict_item['cryptocurrency']['bitcoin'] = get_item_bitcoin(item_id)
return (dict_item, 200) return (dict_item, 200)
###
### correlation
###
def _get_item_correlation(correlation_name, correlation_type, item_id):
print('item_{}_{}:{}'.format(correlation_name, correlation_type, item_id))
res = r_serv_metadata.smembers('item_{}_{}:{}'.format(correlation_name, correlation_type, item_id))
if res:
return list(res)
else:
return []
def get_item_bitcoin(item_id):
return _get_item_correlation('cryptocurrency', 'bitcoin', item_id)
def get_item_pgp_key(item_id):
return _get_item_correlation('pgpdump', 'key', item_id)
def get_item_pgp_name(item_id):
return _get_item_correlation('pgpdump', 'name', item_id)
def get_item_pgp_mail(item_id):
return _get_item_correlation('pgpdump', 'mail', item_id)

25
bin/packages/Pgp.py Executable file
View file

@ -0,0 +1,25 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import redis
from hashlib import sha256
import Flask_config
from Correlation import Correlation
r_serv_metadata = Flask_config.r_serv_metadata
pgpdump = Correlation('pgpdump')
def get_pgp(request_dict, pgp_type):
# basic verification
res = pgpdump.verify_correlation_field_request(request_dict, pgp_type)
if res:
return res
# cerify address
field_name = request_dict.get(pgp_type)
return pgpdump.get_correlation(request_dict, pgp_type, field_name)

View file

@ -14,6 +14,8 @@ import redis
import datetime import datetime
import Import_helper import Import_helper
import Cryptocurrency
import Pgp
import Item import Item
import Paste import Paste
import Tag import Tag
@ -287,6 +289,7 @@ def get_item_content():
res = Item.get_item(req_data) res = Item.get_item(req_data)
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # TAGS # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # TAGS # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@ -307,6 +310,117 @@ def get_all_tags():
res = {'tags': Tag.get_all_tags()} res = {'tags': Tag.get_all_tags()}
return Response(json.dumps(res, indent=2, sort_keys=True), mimetype='application/json'), 200 return Response(json.dumps(res, indent=2, sort_keys=True), mimetype='application/json'), 200
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # CRYPTOCURRENCY # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@restApi.route("api/v1/get/cryptocurrency/bitcoin/metadata", methods=['POST'])
@token_required('analyst')
def get_cryptocurrency_bitcoin_metadata():
data = request.get_json()
crypto_address = data.get('bitcoin', None)
req_data = {'bitcoin': crypto_address, 'metadata': True}
res = Cryptocurrency.get_cryptocurrency(req_data, 'bitcoin')
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
@restApi.route("api/v1/get/cryptocurrency/bitcoin/item", methods=['POST'])
@token_required('analyst')
def get_cryptocurrency_bitcoin_item():
data = request.get_json()
bitcoin_address = data.get('bitcoin', None)
req_data = {'bitcoin': bitcoin_address, 'items': True}
res = Cryptocurrency.get_cryptocurrency(req_data, 'bitcoin')
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # PGP # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@restApi.route("api/v1/get/pgp/key/metadata", methods=['POST'])
@token_required('analyst')
def get_pgp_key_metadata():
data = request.get_json()
pgp_field = data.get('key', None)
req_data = {'key': pgp_field, 'metadata': True}
res = Pgp.get_pgp(req_data, 'key')
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
@restApi.route("api/v1/get/pgp/mail/metadata", methods=['POST'])
@token_required('analyst')
def get_pgp_mail_metadata():
data = request.get_json()
pgp_field = data.get('mail', None)
req_data = {'mail': pgp_field, 'metadata': True}
res = Pgp.get_pgp(req_data, 'mail')
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
@restApi.route("api/v1/get/pgp/name/metadata", methods=['POST'])
@token_required('analyst')
def get_pgp_name_metadata():
data = request.get_json()
pgp_field = data.get('name', None)
req_data = {'name': pgp_field, 'metadata': True}
res = Pgp.get_pgp(req_data, 'name')
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
@restApi.route("api/v1/get/pgp/key/item", methods=['POST'])
@token_required('analyst')
def get_pgp_key_item():
data = request.get_json()
pgp_field = data.get('key', None)
req_data = {'key': pgp_field, 'items': True}
res = Pgp.get_pgp(req_data, 'key')
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
@restApi.route("api/v1/get/pgp/mail/item", methods=['POST'])
@token_required('analyst')
def get_pgp_mail_item():
data = request.get_json()
pgp_mail = data.get('mail', None)
req_data = {'mail': pgp_mail, 'items': True}
res = Pgp.get_pgp(req_data, 'mail')
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
@restApi.route("api/v1/get/pgp/name/item", methods=['POST'])
@token_required('analyst')
def get_pgp_name_item():
data = request.get_json()
pgp_name = data.get('name', None)
req_data = {'name': pgp_name, 'items': True}
res = Pgp.get_pgp(req_data, 'name')
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
'''
@restApi.route("api/v1/get/item/cryptocurrency/key", methods=['POST'])
@token_required('analyst')
def get_item_cryptocurrency_bitcoin():
data = request.get_json()
item_id = data.get('id', None)
req_data = {'id': item_id, 'date': False, 'tags': False, 'pgp': {'key': True}}
res = Item.get_item(req_data)
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
@restApi.route("api/v1/get/item/pgp/mail", methods=['POST'])
@token_required('analyst')
def get_item_cryptocurrency_bitcoin():
data = request.get_json()
item_id = data.get('id', None)
req_data = {'id': item_id, 'date': False, 'tags': False, 'pgp': {'mail': True}}
res = Item.get_item(req_data)
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
@restApi.route("api/v1/get/item/pgp/name", methods=['POST'])
@token_required('analyst')
def get_item_cryptocurrency_bitcoin():
data = request.get_json()
item_id = data.get('id', None)
req_data = {'id': item_id, 'date': False, 'tags': False, 'pgp': {'name': True}}
res = Item.get_item(req_data)
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
'''
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # IMPORT # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # IMPORT # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #