mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 08:38:28 +00:00
chg: [api] add new endpoints: get bitcoin/pgp name/pgp keys/pgp mail metadata + items list
This commit is contained in:
parent
ab45ac0fef
commit
dfd1128daa
5 changed files with 291 additions and 0 deletions
64
bin/packages/Correlation.py
Executable file
64
bin/packages/Correlation.py
Executable file
|
@ -0,0 +1,64 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import redis
|
||||
|
||||
import Flask_config
|
||||
|
||||
r_serv_metadata = Flask_config.r_serv_metadata
|
||||
|
||||
|
||||
class Correlation(object):
|
||||
|
||||
def __init__(self, correlation_name):
|
||||
self.correlation_name = correlation_name
|
||||
|
||||
def _exist_corelation_field(self, correlation_type, field_name):
|
||||
return r_serv_metadata.exists('set_{}_{}:{}'.format(self.correlation_name, correlation_type, field_name))
|
||||
|
||||
|
||||
def _get_items(self, correlation_type, field_name):
|
||||
res = r_serv_metadata.smembers('set_{}_{}:{}'.format(self.correlation_name, correlation_type, field_name))
|
||||
if res:
|
||||
return list(res)
|
||||
else:
|
||||
return {}
|
||||
|
||||
|
||||
def _get_metadata(self, correlation_type, field_name):
|
||||
meta_dict = {}
|
||||
meta_dict['first_seen'] = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, correlation_type, field_name), 'first_seen')
|
||||
meta_dict['last_seen'] = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, correlation_type, field_name), 'last_seen')
|
||||
return meta_dict
|
||||
|
||||
def _get_correlation_by_date(self, correlation_type, date):
|
||||
return r_serv_metadata.hkeys('{}:{}:{}'.format(self.correlation_name, correlation_type, date))
|
||||
|
||||
def verify_correlation_field_request(self, request_dict, correlation_type):
|
||||
if not request_dict:
|
||||
return Response({'status': 'error', 'reason': 'Malformed JSON'}, 400)
|
||||
|
||||
field_name = request_dict.get(correlation_type, None)
|
||||
if not field_name:
|
||||
return ( {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400 )
|
||||
if not self._exist_corelation_field(correlation_type, field_name):
|
||||
return ( {'status': 'error', 'reason': 'Item not found'}, 404 )
|
||||
|
||||
def get_correlation(self, request_dict, correlation_type, field_name):
|
||||
dict_resp = {}
|
||||
|
||||
if request_dict.get('items'):
|
||||
dict_resp['items'] = self._get_items(correlation_type, field_name)
|
||||
|
||||
if request_dict.get('metadata'):
|
||||
dict_resp['metadata'] = self._get_metadata(correlation_type, field_name)
|
||||
|
||||
dict_resp[correlation_type] = field_name
|
||||
|
||||
return (dict_resp, 200)
|
||||
|
||||
|
||||
|
||||
|
||||
#cryptocurrency_all:cryptocurrency name cryptocurrency address nb seen
|
48
bin/packages/Cryptocurrency.py
Executable file
48
bin/packages/Cryptocurrency.py
Executable file
|
@ -0,0 +1,48 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import redis
|
||||
|
||||
from hashlib import sha256
|
||||
|
||||
import Flask_config
|
||||
from Correlation import Correlation
|
||||
|
||||
r_serv_metadata = Flask_config.r_serv_metadata
|
||||
|
||||
digits58 = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
|
||||
|
||||
cryptocurrency = Correlation('cryptocurrency')
|
||||
|
||||
def decode_base58(bc, length):
|
||||
n = 0
|
||||
for char in bc:
|
||||
n = n * 58 + digits58.index(char)
|
||||
return n.to_bytes(length, 'big')
|
||||
|
||||
def check_bitcoin_address(bc):
|
||||
try:
|
||||
bcbytes = decode_base58(bc, 25)
|
||||
return bcbytes[-4:] == sha256(sha256(bcbytes[:-4]).digest()).digest()[:4]
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def verify_cryptocurrency_address(cryptocurrency_type, cryptocurrency_address):
|
||||
if cryptocurrency_type == 'bitcoin':
|
||||
return check_bitcoin_address(cryptocurrency_address)
|
||||
else:
|
||||
return True
|
||||
|
||||
|
||||
def get_cryptocurrency(request_dict, cryptocurrency_type):
|
||||
# basic verification
|
||||
res = cryptocurrency.verify_correlation_field_request(request_dict, cryptocurrency_type)
|
||||
if res:
|
||||
return res
|
||||
# cerify address
|
||||
field_name = request_dict.get(cryptocurrency_type)
|
||||
if not verify_cryptocurrency_address(cryptocurrency_type, field_name):
|
||||
return ( {'status': 'error', 'reason': 'Invalid Cryptocurrency address'}, 400 )
|
||||
|
||||
return cryptocurrency.get_correlation(request_dict, cryptocurrency_type, field_name)
|
|
@ -11,6 +11,7 @@ import Tag
|
|||
|
||||
PASTES_FOLDER = Flask_config.PASTES_FOLDER
|
||||
r_cache = Flask_config.r_cache
|
||||
r_serv_metadata = Flask_config.r_serv_metadata
|
||||
|
||||
def exist_item(item_id):
|
||||
if os.path.isfile(os.path.join(PASTES_FOLDER, item_id)):
|
||||
|
@ -90,4 +91,43 @@ def get_item(request_dict):
|
|||
if lines_info:
|
||||
dict_item['lines'] = get_lines_info(item_id, dict_item.get('content', 'None'))
|
||||
|
||||
if request_dict.get('pgp'):
|
||||
dict_item['pgp'] = {}
|
||||
if request_dict['pgp'].get('key'):
|
||||
dict_item['pgp']['key'] = get_item_pgp_key(item_id)
|
||||
if request_dict['pgp'].get('mail'):
|
||||
dict_item['pgp']['mail'] = get_item_pgp_mail(item_id)
|
||||
if request_dict['pgp'].get('name'):
|
||||
dict_item['pgp']['name'] = get_item_pgp_name(item_id)
|
||||
|
||||
if request_dict.get('cryptocurrency'):
|
||||
dict_item['cryptocurrency'] = {}
|
||||
if request_dict['cryptocurrency'].get('bitcoin'):
|
||||
dict_item['cryptocurrency']['bitcoin'] = get_item_bitcoin(item_id)
|
||||
|
||||
return (dict_item, 200)
|
||||
|
||||
|
||||
###
|
||||
### correlation
|
||||
###
|
||||
|
||||
def _get_item_correlation(correlation_name, correlation_type, item_id):
|
||||
print('item_{}_{}:{}'.format(correlation_name, correlation_type, item_id))
|
||||
res = r_serv_metadata.smembers('item_{}_{}:{}'.format(correlation_name, correlation_type, item_id))
|
||||
if res:
|
||||
return list(res)
|
||||
else:
|
||||
return []
|
||||
|
||||
def get_item_bitcoin(item_id):
|
||||
return _get_item_correlation('cryptocurrency', 'bitcoin', item_id)
|
||||
|
||||
def get_item_pgp_key(item_id):
|
||||
return _get_item_correlation('pgpdump', 'key', item_id)
|
||||
|
||||
def get_item_pgp_name(item_id):
|
||||
return _get_item_correlation('pgpdump', 'name', item_id)
|
||||
|
||||
def get_item_pgp_mail(item_id):
|
||||
return _get_item_correlation('pgpdump', 'mail', item_id)
|
||||
|
|
25
bin/packages/Pgp.py
Executable file
25
bin/packages/Pgp.py
Executable file
|
@ -0,0 +1,25 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import redis
|
||||
|
||||
from hashlib import sha256
|
||||
|
||||
import Flask_config
|
||||
from Correlation import Correlation
|
||||
|
||||
r_serv_metadata = Flask_config.r_serv_metadata
|
||||
|
||||
pgpdump = Correlation('pgpdump')
|
||||
|
||||
|
||||
def get_pgp(request_dict, pgp_type):
|
||||
# basic verification
|
||||
res = pgpdump.verify_correlation_field_request(request_dict, pgp_type)
|
||||
if res:
|
||||
return res
|
||||
# cerify address
|
||||
field_name = request_dict.get(pgp_type)
|
||||
|
||||
return pgpdump.get_correlation(request_dict, pgp_type, field_name)
|
|
@ -14,6 +14,8 @@ import redis
|
|||
import datetime
|
||||
|
||||
import Import_helper
|
||||
import Cryptocurrency
|
||||
import Pgp
|
||||
import Item
|
||||
import Paste
|
||||
import Tag
|
||||
|
@ -287,6 +289,7 @@ def get_item_content():
|
|||
res = Item.get_item(req_data)
|
||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
|
||||
|
||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||
# # # # # # # # # # # # # # TAGS # # # # # # # # # # # # # # # # #
|
||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||
|
@ -307,6 +310,117 @@ def get_all_tags():
|
|||
res = {'tags': Tag.get_all_tags()}
|
||||
return Response(json.dumps(res, indent=2, sort_keys=True), mimetype='application/json'), 200
|
||||
|
||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||
# # # # # # # # # # # # CRYPTOCURRENCY # # # # # # # # # # # # # #
|
||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||
@restApi.route("api/v1/get/cryptocurrency/bitcoin/metadata", methods=['POST'])
|
||||
@token_required('analyst')
|
||||
def get_cryptocurrency_bitcoin_metadata():
|
||||
data = request.get_json()
|
||||
crypto_address = data.get('bitcoin', None)
|
||||
req_data = {'bitcoin': crypto_address, 'metadata': True}
|
||||
res = Cryptocurrency.get_cryptocurrency(req_data, 'bitcoin')
|
||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
|
||||
@restApi.route("api/v1/get/cryptocurrency/bitcoin/item", methods=['POST'])
|
||||
@token_required('analyst')
|
||||
def get_cryptocurrency_bitcoin_item():
|
||||
data = request.get_json()
|
||||
bitcoin_address = data.get('bitcoin', None)
|
||||
req_data = {'bitcoin': bitcoin_address, 'items': True}
|
||||
res = Cryptocurrency.get_cryptocurrency(req_data, 'bitcoin')
|
||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
|
||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||
# # # # # # # # # # # # # # # PGP # # # # # # # # # # # # # # # # #
|
||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||
@restApi.route("api/v1/get/pgp/key/metadata", methods=['POST'])
|
||||
@token_required('analyst')
|
||||
def get_pgp_key_metadata():
|
||||
data = request.get_json()
|
||||
pgp_field = data.get('key', None)
|
||||
req_data = {'key': pgp_field, 'metadata': True}
|
||||
res = Pgp.get_pgp(req_data, 'key')
|
||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
|
||||
@restApi.route("api/v1/get/pgp/mail/metadata", methods=['POST'])
|
||||
@token_required('analyst')
|
||||
def get_pgp_mail_metadata():
|
||||
data = request.get_json()
|
||||
pgp_field = data.get('mail', None)
|
||||
req_data = {'mail': pgp_field, 'metadata': True}
|
||||
res = Pgp.get_pgp(req_data, 'mail')
|
||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
|
||||
@restApi.route("api/v1/get/pgp/name/metadata", methods=['POST'])
|
||||
@token_required('analyst')
|
||||
def get_pgp_name_metadata():
|
||||
data = request.get_json()
|
||||
pgp_field = data.get('name', None)
|
||||
req_data = {'name': pgp_field, 'metadata': True}
|
||||
res = Pgp.get_pgp(req_data, 'name')
|
||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
|
||||
@restApi.route("api/v1/get/pgp/key/item", methods=['POST'])
|
||||
@token_required('analyst')
|
||||
def get_pgp_key_item():
|
||||
data = request.get_json()
|
||||
pgp_field = data.get('key', None)
|
||||
req_data = {'key': pgp_field, 'items': True}
|
||||
res = Pgp.get_pgp(req_data, 'key')
|
||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
|
||||
@restApi.route("api/v1/get/pgp/mail/item", methods=['POST'])
|
||||
@token_required('analyst')
|
||||
def get_pgp_mail_item():
|
||||
data = request.get_json()
|
||||
pgp_mail = data.get('mail', None)
|
||||
req_data = {'mail': pgp_mail, 'items': True}
|
||||
res = Pgp.get_pgp(req_data, 'mail')
|
||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
|
||||
@restApi.route("api/v1/get/pgp/name/item", methods=['POST'])
|
||||
@token_required('analyst')
|
||||
def get_pgp_name_item():
|
||||
data = request.get_json()
|
||||
pgp_name = data.get('name', None)
|
||||
req_data = {'name': pgp_name, 'items': True}
|
||||
res = Pgp.get_pgp(req_data, 'name')
|
||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
|
||||
'''
|
||||
|
||||
|
||||
|
||||
@restApi.route("api/v1/get/item/cryptocurrency/key", methods=['POST'])
|
||||
@token_required('analyst')
|
||||
def get_item_cryptocurrency_bitcoin():
|
||||
data = request.get_json()
|
||||
item_id = data.get('id', None)
|
||||
req_data = {'id': item_id, 'date': False, 'tags': False, 'pgp': {'key': True}}
|
||||
res = Item.get_item(req_data)
|
||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
|
||||
@restApi.route("api/v1/get/item/pgp/mail", methods=['POST'])
|
||||
@token_required('analyst')
|
||||
def get_item_cryptocurrency_bitcoin():
|
||||
data = request.get_json()
|
||||
item_id = data.get('id', None)
|
||||
req_data = {'id': item_id, 'date': False, 'tags': False, 'pgp': {'mail': True}}
|
||||
res = Item.get_item(req_data)
|
||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
|
||||
@restApi.route("api/v1/get/item/pgp/name", methods=['POST'])
|
||||
@token_required('analyst')
|
||||
def get_item_cryptocurrency_bitcoin():
|
||||
data = request.get_json()
|
||||
item_id = data.get('id', None)
|
||||
req_data = {'id': item_id, 'date': False, 'tags': False, 'pgp': {'name': True}}
|
||||
res = Item.get_item(req_data)
|
||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
'''
|
||||
|
||||
|
||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||
# # # # # # # # # # # # # IMPORT # # # # # # # # # # # # # # # # # #
|
||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||
|
|
Loading…
Reference in a new issue