mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 08:38:28 +00:00
chg: [api] add new endpoints: get bitcoin/pgp name/pgp keys/pgp mail metadata + items list
This commit is contained in:
parent
ab45ac0fef
commit
dfd1128daa
5 changed files with 291 additions and 0 deletions
64
bin/packages/Correlation.py
Executable file
64
bin/packages/Correlation.py
Executable file
|
@ -0,0 +1,64 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*-coding:UTF-8 -*
|
||||||
|
|
||||||
|
import os
|
||||||
|
import redis
|
||||||
|
|
||||||
|
import Flask_config
|
||||||
|
|
||||||
|
r_serv_metadata = Flask_config.r_serv_metadata
|
||||||
|
|
||||||
|
|
||||||
|
class Correlation(object):
|
||||||
|
|
||||||
|
def __init__(self, correlation_name):
|
||||||
|
self.correlation_name = correlation_name
|
||||||
|
|
||||||
|
def _exist_corelation_field(self, correlation_type, field_name):
|
||||||
|
return r_serv_metadata.exists('set_{}_{}:{}'.format(self.correlation_name, correlation_type, field_name))
|
||||||
|
|
||||||
|
|
||||||
|
def _get_items(self, correlation_type, field_name):
|
||||||
|
res = r_serv_metadata.smembers('set_{}_{}:{}'.format(self.correlation_name, correlation_type, field_name))
|
||||||
|
if res:
|
||||||
|
return list(res)
|
||||||
|
else:
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
def _get_metadata(self, correlation_type, field_name):
|
||||||
|
meta_dict = {}
|
||||||
|
meta_dict['first_seen'] = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, correlation_type, field_name), 'first_seen')
|
||||||
|
meta_dict['last_seen'] = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, correlation_type, field_name), 'last_seen')
|
||||||
|
return meta_dict
|
||||||
|
|
||||||
|
def _get_correlation_by_date(self, correlation_type, date):
|
||||||
|
return r_serv_metadata.hkeys('{}:{}:{}'.format(self.correlation_name, correlation_type, date))
|
||||||
|
|
||||||
|
def verify_correlation_field_request(self, request_dict, correlation_type):
|
||||||
|
if not request_dict:
|
||||||
|
return Response({'status': 'error', 'reason': 'Malformed JSON'}, 400)
|
||||||
|
|
||||||
|
field_name = request_dict.get(correlation_type, None)
|
||||||
|
if not field_name:
|
||||||
|
return ( {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400 )
|
||||||
|
if not self._exist_corelation_field(correlation_type, field_name):
|
||||||
|
return ( {'status': 'error', 'reason': 'Item not found'}, 404 )
|
||||||
|
|
||||||
|
def get_correlation(self, request_dict, correlation_type, field_name):
|
||||||
|
dict_resp = {}
|
||||||
|
|
||||||
|
if request_dict.get('items'):
|
||||||
|
dict_resp['items'] = self._get_items(correlation_type, field_name)
|
||||||
|
|
||||||
|
if request_dict.get('metadata'):
|
||||||
|
dict_resp['metadata'] = self._get_metadata(correlation_type, field_name)
|
||||||
|
|
||||||
|
dict_resp[correlation_type] = field_name
|
||||||
|
|
||||||
|
return (dict_resp, 200)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#cryptocurrency_all:cryptocurrency name cryptocurrency address nb seen
|
48
bin/packages/Cryptocurrency.py
Executable file
48
bin/packages/Cryptocurrency.py
Executable file
|
@ -0,0 +1,48 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*-coding:UTF-8 -*
|
||||||
|
|
||||||
|
import os
|
||||||
|
import redis
|
||||||
|
|
||||||
|
from hashlib import sha256
|
||||||
|
|
||||||
|
import Flask_config
|
||||||
|
from Correlation import Correlation
|
||||||
|
|
||||||
|
r_serv_metadata = Flask_config.r_serv_metadata
|
||||||
|
|
||||||
|
digits58 = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
|
||||||
|
|
||||||
|
cryptocurrency = Correlation('cryptocurrency')
|
||||||
|
|
||||||
|
def decode_base58(bc, length):
|
||||||
|
n = 0
|
||||||
|
for char in bc:
|
||||||
|
n = n * 58 + digits58.index(char)
|
||||||
|
return n.to_bytes(length, 'big')
|
||||||
|
|
||||||
|
def check_bitcoin_address(bc):
|
||||||
|
try:
|
||||||
|
bcbytes = decode_base58(bc, 25)
|
||||||
|
return bcbytes[-4:] == sha256(sha256(bcbytes[:-4]).digest()).digest()[:4]
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
|
||||||
|
def verify_cryptocurrency_address(cryptocurrency_type, cryptocurrency_address):
|
||||||
|
if cryptocurrency_type == 'bitcoin':
|
||||||
|
return check_bitcoin_address(cryptocurrency_address)
|
||||||
|
else:
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def get_cryptocurrency(request_dict, cryptocurrency_type):
|
||||||
|
# basic verification
|
||||||
|
res = cryptocurrency.verify_correlation_field_request(request_dict, cryptocurrency_type)
|
||||||
|
if res:
|
||||||
|
return res
|
||||||
|
# cerify address
|
||||||
|
field_name = request_dict.get(cryptocurrency_type)
|
||||||
|
if not verify_cryptocurrency_address(cryptocurrency_type, field_name):
|
||||||
|
return ( {'status': 'error', 'reason': 'Invalid Cryptocurrency address'}, 400 )
|
||||||
|
|
||||||
|
return cryptocurrency.get_correlation(request_dict, cryptocurrency_type, field_name)
|
|
@ -11,6 +11,7 @@ import Tag
|
||||||
|
|
||||||
PASTES_FOLDER = Flask_config.PASTES_FOLDER
|
PASTES_FOLDER = Flask_config.PASTES_FOLDER
|
||||||
r_cache = Flask_config.r_cache
|
r_cache = Flask_config.r_cache
|
||||||
|
r_serv_metadata = Flask_config.r_serv_metadata
|
||||||
|
|
||||||
def exist_item(item_id):
|
def exist_item(item_id):
|
||||||
if os.path.isfile(os.path.join(PASTES_FOLDER, item_id)):
|
if os.path.isfile(os.path.join(PASTES_FOLDER, item_id)):
|
||||||
|
@ -90,4 +91,43 @@ def get_item(request_dict):
|
||||||
if lines_info:
|
if lines_info:
|
||||||
dict_item['lines'] = get_lines_info(item_id, dict_item.get('content', 'None'))
|
dict_item['lines'] = get_lines_info(item_id, dict_item.get('content', 'None'))
|
||||||
|
|
||||||
|
if request_dict.get('pgp'):
|
||||||
|
dict_item['pgp'] = {}
|
||||||
|
if request_dict['pgp'].get('key'):
|
||||||
|
dict_item['pgp']['key'] = get_item_pgp_key(item_id)
|
||||||
|
if request_dict['pgp'].get('mail'):
|
||||||
|
dict_item['pgp']['mail'] = get_item_pgp_mail(item_id)
|
||||||
|
if request_dict['pgp'].get('name'):
|
||||||
|
dict_item['pgp']['name'] = get_item_pgp_name(item_id)
|
||||||
|
|
||||||
|
if request_dict.get('cryptocurrency'):
|
||||||
|
dict_item['cryptocurrency'] = {}
|
||||||
|
if request_dict['cryptocurrency'].get('bitcoin'):
|
||||||
|
dict_item['cryptocurrency']['bitcoin'] = get_item_bitcoin(item_id)
|
||||||
|
|
||||||
return (dict_item, 200)
|
return (dict_item, 200)
|
||||||
|
|
||||||
|
|
||||||
|
###
|
||||||
|
### correlation
|
||||||
|
###
|
||||||
|
|
||||||
|
def _get_item_correlation(correlation_name, correlation_type, item_id):
|
||||||
|
print('item_{}_{}:{}'.format(correlation_name, correlation_type, item_id))
|
||||||
|
res = r_serv_metadata.smembers('item_{}_{}:{}'.format(correlation_name, correlation_type, item_id))
|
||||||
|
if res:
|
||||||
|
return list(res)
|
||||||
|
else:
|
||||||
|
return []
|
||||||
|
|
||||||
|
def get_item_bitcoin(item_id):
|
||||||
|
return _get_item_correlation('cryptocurrency', 'bitcoin', item_id)
|
||||||
|
|
||||||
|
def get_item_pgp_key(item_id):
|
||||||
|
return _get_item_correlation('pgpdump', 'key', item_id)
|
||||||
|
|
||||||
|
def get_item_pgp_name(item_id):
|
||||||
|
return _get_item_correlation('pgpdump', 'name', item_id)
|
||||||
|
|
||||||
|
def get_item_pgp_mail(item_id):
|
||||||
|
return _get_item_correlation('pgpdump', 'mail', item_id)
|
||||||
|
|
25
bin/packages/Pgp.py
Executable file
25
bin/packages/Pgp.py
Executable file
|
@ -0,0 +1,25 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*-coding:UTF-8 -*
|
||||||
|
|
||||||
|
import os
|
||||||
|
import redis
|
||||||
|
|
||||||
|
from hashlib import sha256
|
||||||
|
|
||||||
|
import Flask_config
|
||||||
|
from Correlation import Correlation
|
||||||
|
|
||||||
|
r_serv_metadata = Flask_config.r_serv_metadata
|
||||||
|
|
||||||
|
pgpdump = Correlation('pgpdump')
|
||||||
|
|
||||||
|
|
||||||
|
def get_pgp(request_dict, pgp_type):
|
||||||
|
# basic verification
|
||||||
|
res = pgpdump.verify_correlation_field_request(request_dict, pgp_type)
|
||||||
|
if res:
|
||||||
|
return res
|
||||||
|
# cerify address
|
||||||
|
field_name = request_dict.get(pgp_type)
|
||||||
|
|
||||||
|
return pgpdump.get_correlation(request_dict, pgp_type, field_name)
|
|
@ -14,6 +14,8 @@ import redis
|
||||||
import datetime
|
import datetime
|
||||||
|
|
||||||
import Import_helper
|
import Import_helper
|
||||||
|
import Cryptocurrency
|
||||||
|
import Pgp
|
||||||
import Item
|
import Item
|
||||||
import Paste
|
import Paste
|
||||||
import Tag
|
import Tag
|
||||||
|
@ -287,6 +289,7 @@ def get_item_content():
|
||||||
res = Item.get_item(req_data)
|
res = Item.get_item(req_data)
|
||||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||||
|
|
||||||
|
|
||||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||||
# # # # # # # # # # # # # # TAGS # # # # # # # # # # # # # # # # #
|
# # # # # # # # # # # # # # TAGS # # # # # # # # # # # # # # # # #
|
||||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||||
|
@ -307,6 +310,117 @@ def get_all_tags():
|
||||||
res = {'tags': Tag.get_all_tags()}
|
res = {'tags': Tag.get_all_tags()}
|
||||||
return Response(json.dumps(res, indent=2, sort_keys=True), mimetype='application/json'), 200
|
return Response(json.dumps(res, indent=2, sort_keys=True), mimetype='application/json'), 200
|
||||||
|
|
||||||
|
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||||
|
# # # # # # # # # # # # CRYPTOCURRENCY # # # # # # # # # # # # # #
|
||||||
|
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||||
|
@restApi.route("api/v1/get/cryptocurrency/bitcoin/metadata", methods=['POST'])
|
||||||
|
@token_required('analyst')
|
||||||
|
def get_cryptocurrency_bitcoin_metadata():
|
||||||
|
data = request.get_json()
|
||||||
|
crypto_address = data.get('bitcoin', None)
|
||||||
|
req_data = {'bitcoin': crypto_address, 'metadata': True}
|
||||||
|
res = Cryptocurrency.get_cryptocurrency(req_data, 'bitcoin')
|
||||||
|
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||||
|
|
||||||
|
@restApi.route("api/v1/get/cryptocurrency/bitcoin/item", methods=['POST'])
|
||||||
|
@token_required('analyst')
|
||||||
|
def get_cryptocurrency_bitcoin_item():
|
||||||
|
data = request.get_json()
|
||||||
|
bitcoin_address = data.get('bitcoin', None)
|
||||||
|
req_data = {'bitcoin': bitcoin_address, 'items': True}
|
||||||
|
res = Cryptocurrency.get_cryptocurrency(req_data, 'bitcoin')
|
||||||
|
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||||
|
|
||||||
|
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||||
|
# # # # # # # # # # # # # # # PGP # # # # # # # # # # # # # # # # #
|
||||||
|
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||||
|
@restApi.route("api/v1/get/pgp/key/metadata", methods=['POST'])
|
||||||
|
@token_required('analyst')
|
||||||
|
def get_pgp_key_metadata():
|
||||||
|
data = request.get_json()
|
||||||
|
pgp_field = data.get('key', None)
|
||||||
|
req_data = {'key': pgp_field, 'metadata': True}
|
||||||
|
res = Pgp.get_pgp(req_data, 'key')
|
||||||
|
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||||
|
|
||||||
|
@restApi.route("api/v1/get/pgp/mail/metadata", methods=['POST'])
|
||||||
|
@token_required('analyst')
|
||||||
|
def get_pgp_mail_metadata():
|
||||||
|
data = request.get_json()
|
||||||
|
pgp_field = data.get('mail', None)
|
||||||
|
req_data = {'mail': pgp_field, 'metadata': True}
|
||||||
|
res = Pgp.get_pgp(req_data, 'mail')
|
||||||
|
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||||
|
|
||||||
|
@restApi.route("api/v1/get/pgp/name/metadata", methods=['POST'])
|
||||||
|
@token_required('analyst')
|
||||||
|
def get_pgp_name_metadata():
|
||||||
|
data = request.get_json()
|
||||||
|
pgp_field = data.get('name', None)
|
||||||
|
req_data = {'name': pgp_field, 'metadata': True}
|
||||||
|
res = Pgp.get_pgp(req_data, 'name')
|
||||||
|
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||||
|
|
||||||
|
@restApi.route("api/v1/get/pgp/key/item", methods=['POST'])
|
||||||
|
@token_required('analyst')
|
||||||
|
def get_pgp_key_item():
|
||||||
|
data = request.get_json()
|
||||||
|
pgp_field = data.get('key', None)
|
||||||
|
req_data = {'key': pgp_field, 'items': True}
|
||||||
|
res = Pgp.get_pgp(req_data, 'key')
|
||||||
|
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||||
|
|
||||||
|
@restApi.route("api/v1/get/pgp/mail/item", methods=['POST'])
|
||||||
|
@token_required('analyst')
|
||||||
|
def get_pgp_mail_item():
|
||||||
|
data = request.get_json()
|
||||||
|
pgp_mail = data.get('mail', None)
|
||||||
|
req_data = {'mail': pgp_mail, 'items': True}
|
||||||
|
res = Pgp.get_pgp(req_data, 'mail')
|
||||||
|
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||||
|
|
||||||
|
@restApi.route("api/v1/get/pgp/name/item", methods=['POST'])
|
||||||
|
@token_required('analyst')
|
||||||
|
def get_pgp_name_item():
|
||||||
|
data = request.get_json()
|
||||||
|
pgp_name = data.get('name', None)
|
||||||
|
req_data = {'name': pgp_name, 'items': True}
|
||||||
|
res = Pgp.get_pgp(req_data, 'name')
|
||||||
|
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||||
|
|
||||||
|
'''
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
@restApi.route("api/v1/get/item/cryptocurrency/key", methods=['POST'])
|
||||||
|
@token_required('analyst')
|
||||||
|
def get_item_cryptocurrency_bitcoin():
|
||||||
|
data = request.get_json()
|
||||||
|
item_id = data.get('id', None)
|
||||||
|
req_data = {'id': item_id, 'date': False, 'tags': False, 'pgp': {'key': True}}
|
||||||
|
res = Item.get_item(req_data)
|
||||||
|
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||||
|
|
||||||
|
@restApi.route("api/v1/get/item/pgp/mail", methods=['POST'])
|
||||||
|
@token_required('analyst')
|
||||||
|
def get_item_cryptocurrency_bitcoin():
|
||||||
|
data = request.get_json()
|
||||||
|
item_id = data.get('id', None)
|
||||||
|
req_data = {'id': item_id, 'date': False, 'tags': False, 'pgp': {'mail': True}}
|
||||||
|
res = Item.get_item(req_data)
|
||||||
|
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||||
|
|
||||||
|
@restApi.route("api/v1/get/item/pgp/name", methods=['POST'])
|
||||||
|
@token_required('analyst')
|
||||||
|
def get_item_cryptocurrency_bitcoin():
|
||||||
|
data = request.get_json()
|
||||||
|
item_id = data.get('id', None)
|
||||||
|
req_data = {'id': item_id, 'date': False, 'tags': False, 'pgp': {'name': True}}
|
||||||
|
res = Item.get_item(req_data)
|
||||||
|
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||||
|
'''
|
||||||
|
|
||||||
|
|
||||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||||
# # # # # # # # # # # # # IMPORT # # # # # # # # # # # # # # # # # #
|
# # # # # # # # # # # # # IMPORT # # # # # # # # # # # # # # # # # #
|
||||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||||
|
|
Loading…
Reference in a new issue