chg: [api] add new endpoints: get bitcoin/pgp name/pgp keys/pgp mail metadata + items list

bin/packages/Correlation.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import redis
+
+import Flask_config
+
+r_serv_metadata = Flask_config.r_serv_metadata
+
+
+class Correlation(object):
+
+    def __init__(self, correlation_name):
+        self.correlation_name = correlation_name
+
+    def _exist_corelation_field(self, correlation_type, field_name):
+        return r_serv_metadata.exists('set_{}_{}:{}'.format(self.correlation_name, correlation_type, field_name))
+
+
+    def _get_items(self, correlation_type, field_name):
+        res =  r_serv_metadata.smembers('set_{}_{}:{}'.format(self.correlation_name, correlation_type, field_name))
+        if res:
+            return list(res)
+        else:
+            return {}
+
+
+    def _get_metadata(self, correlation_type, field_name):
+        meta_dict = {}
+        meta_dict['first_seen'] = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, correlation_type, field_name), 'first_seen')
+        meta_dict['last_seen'] = r_serv_metadata.hget('{}_metadata_{}:{}'.format(self.correlation_name, correlation_type, field_name), 'last_seen')
+        return meta_dict
+
+    def _get_correlation_by_date(self, correlation_type, date):
+        return r_serv_metadata.hkeys('{}:{}:{}'.format(self.correlation_name, correlation_type, date))
+
+    def verify_correlation_field_request(self, request_dict, correlation_type):
+        if not request_dict:
+            return Response({'status': 'error', 'reason': 'Malformed JSON'}, 400)
+
+        field_name = request_dict.get(correlation_type, None)
+        if not field_name:
+            return ( {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400 )
+        if not self._exist_corelation_field(correlation_type, field_name):
+            return ( {'status': 'error', 'reason': 'Item not found'}, 404 )
+
+    def get_correlation(self, request_dict, correlation_type, field_name):
+        dict_resp = {}
+
+        if request_dict.get('items'):
+            dict_resp['items'] = self._get_items(correlation_type, field_name)
+
+        if request_dict.get('metadata'):
+            dict_resp['metadata'] = self._get_metadata(correlation_type, field_name)
+
+        dict_resp[correlation_type] = field_name
+
+        return (dict_resp, 200)
+
+
+
+
+#cryptocurrency_all:cryptocurrency name	cryptocurrency address	nb seen

bin/packages/Cryptocurrency.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import redis
+
+from hashlib import sha256
+
+import Flask_config
+from Correlation import Correlation
+
+r_serv_metadata = Flask_config.r_serv_metadata
+
+digits58 = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
+
+cryptocurrency = Correlation('cryptocurrency')
+
+def decode_base58(bc, length):
+    n = 0
+    for char in bc:
+        n = n * 58 + digits58.index(char)
+    return n.to_bytes(length, 'big')
+
+def check_bitcoin_address(bc):
+    try:
+        bcbytes = decode_base58(bc, 25)
+        return bcbytes[-4:] == sha256(sha256(bcbytes[:-4]).digest()).digest()[:4]
+    except Exception:
+        return False
+
+def verify_cryptocurrency_address(cryptocurrency_type, cryptocurrency_address):
+    if cryptocurrency_type == 'bitcoin':
+        return check_bitcoin_address(cryptocurrency_address)
+    else:
+        return True
+
+
+def get_cryptocurrency(request_dict, cryptocurrency_type):
+    # basic verification
+    res = cryptocurrency.verify_correlation_field_request(request_dict, cryptocurrency_type)
+    if res:
+        return res
+    # cerify address
+    field_name = request_dict.get(cryptocurrency_type)
+    if not verify_cryptocurrency_address(cryptocurrency_type, field_name):
+        return ( {'status': 'error', 'reason': 'Invalid Cryptocurrency address'}, 400 )
+
+    return cryptocurrency.get_correlation(request_dict, cryptocurrency_type, field_name)

bin/packages/Item.py

@@ -11,6 +11,7 @@ import Tag
 
 PASTES_FOLDER = Flask_config.PASTES_FOLDER
 r_cache = Flask_config.r_cache
+r_serv_metadata = Flask_config.r_serv_metadata
 
 def exist_item(item_id):
     if os.path.isfile(os.path.join(PASTES_FOLDER, item_id)):
@@ -90,4 +91,43 @@ def get_item(request_dict):
     if lines_info:
         dict_item['lines'] = get_lines_info(item_id, dict_item.get('content', 'None'))
 
+    if request_dict.get('pgp'):
+        dict_item['pgp'] = {}
+        if request_dict['pgp'].get('key'):
+            dict_item['pgp']['key'] = get_item_pgp_key(item_id)
+        if request_dict['pgp'].get('mail'):
+            dict_item['pgp']['mail'] = get_item_pgp_mail(item_id)
+        if request_dict['pgp'].get('name'):
+            dict_item['pgp']['name'] = get_item_pgp_name(item_id)
+
+    if request_dict.get('cryptocurrency'):
+        dict_item['cryptocurrency'] = {}
+        if request_dict['cryptocurrency'].get('bitcoin'):
+            dict_item['cryptocurrency']['bitcoin'] = get_item_bitcoin(item_id)
+
     return (dict_item, 200)
+
+
+###
+### correlation
+###
+
+def _get_item_correlation(correlation_name, correlation_type, item_id):
+    print('item_{}_{}:{}'.format(correlation_name, correlation_type, item_id))
+    res = r_serv_metadata.smembers('item_{}_{}:{}'.format(correlation_name, correlation_type, item_id))
+    if res:
+        return list(res)
+    else:
+        return []
+
+def get_item_bitcoin(item_id):
+    return _get_item_correlation('cryptocurrency', 'bitcoin', item_id)
+
+def get_item_pgp_key(item_id):
+    return _get_item_correlation('pgpdump', 'key', item_id)
+
+def get_item_pgp_name(item_id):
+    return _get_item_correlation('pgpdump', 'name', item_id)
+
+def get_item_pgp_mail(item_id):
+    return _get_item_correlation('pgpdump', 'mail', item_id)

bin/packages/Pgp.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import redis
+
+from hashlib import sha256
+
+import Flask_config
+from Correlation import Correlation
+
+r_serv_metadata = Flask_config.r_serv_metadata
+
+pgpdump = Correlation('pgpdump')
+
+
+def get_pgp(request_dict, pgp_type):
+    # basic verification
+    res = pgpdump.verify_correlation_field_request(request_dict, pgp_type)
+    if res:
+        return res
+    # cerify address
+    field_name = request_dict.get(pgp_type)
+
+    return pgpdump.get_correlation(request_dict, pgp_type, field_name)

var/www/modules/restApi/Flask_restApi.py

@@ -14,6 +14,8 @@ import redis
 import datetime
 
 import Import_helper
+import Cryptocurrency
+import Pgp
 import Item
 import Paste
 import Tag
@@ -287,6 +289,7 @@ def get_item_content():
     res = Item.get_item(req_data)
     return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
 
+
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 # # # # # # # # # # # # # #        TAGS       # # # # # # # # # # # # # # # # #
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@@ -307,6 +310,117 @@ def get_all_tags():
     res = {'tags': Tag.get_all_tags()}
     return Response(json.dumps(res, indent=2, sort_keys=True), mimetype='application/json'), 200
 
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+# # # # # # # # # # # #        CRYPTOCURRENCY       # # # # # # # # # # # # # #
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+@restApi.route("api/v1/get/cryptocurrency/bitcoin/metadata", methods=['POST'])
+@token_required('analyst')
+def get_cryptocurrency_bitcoin_metadata():
+    data = request.get_json()
+    crypto_address = data.get('bitcoin', None)
+    req_data = {'bitcoin': crypto_address, 'metadata': True}
+    res = Cryptocurrency.get_cryptocurrency(req_data, 'bitcoin')
+    return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
+
+@restApi.route("api/v1/get/cryptocurrency/bitcoin/item", methods=['POST'])
+@token_required('analyst')
+def get_cryptocurrency_bitcoin_item():
+    data = request.get_json()
+    bitcoin_address = data.get('bitcoin', None)
+    req_data = {'bitcoin': bitcoin_address, 'items': True}
+    res = Cryptocurrency.get_cryptocurrency(req_data, 'bitcoin')
+    return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+# # # # # # # # # # # # # # #       PGP       # # # # # # # # # # # # # # # # #
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+@restApi.route("api/v1/get/pgp/key/metadata", methods=['POST'])
+@token_required('analyst')
+def get_pgp_key_metadata():
+    data = request.get_json()
+    pgp_field = data.get('key', None)
+    req_data = {'key': pgp_field, 'metadata': True}
+    res = Pgp.get_pgp(req_data, 'key')
+    return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
+
+@restApi.route("api/v1/get/pgp/mail/metadata", methods=['POST'])
+@token_required('analyst')
+def get_pgp_mail_metadata():
+    data = request.get_json()
+    pgp_field = data.get('mail', None)
+    req_data = {'mail': pgp_field, 'metadata': True}
+    res = Pgp.get_pgp(req_data, 'mail')
+    return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
+
+@restApi.route("api/v1/get/pgp/name/metadata", methods=['POST'])
+@token_required('analyst')
+def get_pgp_name_metadata():
+    data = request.get_json()
+    pgp_field = data.get('name', None)
+    req_data = {'name': pgp_field, 'metadata': True}
+    res = Pgp.get_pgp(req_data, 'name')
+    return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
+
+@restApi.route("api/v1/get/pgp/key/item", methods=['POST'])
+@token_required('analyst')
+def get_pgp_key_item():
+    data = request.get_json()
+    pgp_field = data.get('key', None)
+    req_data = {'key': pgp_field, 'items': True}
+    res = Pgp.get_pgp(req_data, 'key')
+    return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
+
+@restApi.route("api/v1/get/pgp/mail/item", methods=['POST'])
+@token_required('analyst')
+def get_pgp_mail_item():
+    data = request.get_json()
+    pgp_mail = data.get('mail', None)
+    req_data = {'mail': pgp_mail, 'items': True}
+    res = Pgp.get_pgp(req_data, 'mail')
+    return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
+
+@restApi.route("api/v1/get/pgp/name/item", methods=['POST'])
+@token_required('analyst')
+def get_pgp_name_item():
+    data = request.get_json()
+    pgp_name = data.get('name', None)
+    req_data = {'name': pgp_name, 'items': True}
+    res = Pgp.get_pgp(req_data, 'name')
+    return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
+
+'''
+
+
+
+@restApi.route("api/v1/get/item/cryptocurrency/key", methods=['POST'])
+@token_required('analyst')
+def get_item_cryptocurrency_bitcoin():
+    data = request.get_json()
+    item_id = data.get('id', None)
+    req_data = {'id': item_id, 'date': False, 'tags': False, 'pgp': {'key': True}}
+    res = Item.get_item(req_data)
+    return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
+
+@restApi.route("api/v1/get/item/pgp/mail", methods=['POST'])
+@token_required('analyst')
+def get_item_cryptocurrency_bitcoin():
+    data = request.get_json()
+    item_id = data.get('id', None)
+    req_data = {'id': item_id, 'date': False, 'tags': False, 'pgp': {'mail': True}}
+    res = Item.get_item(req_data)
+    return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
+
+@restApi.route("api/v1/get/item/pgp/name", methods=['POST'])
+@token_required('analyst')
+def get_item_cryptocurrency_bitcoin():
+    data = request.get_json()
+    item_id = data.get('id', None)
+    req_data = {'id': item_id, 'date': False, 'tags': False, 'pgp': {'name': True}}
+    res = Item.get_item(req_data)
+    return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
+'''
+
+
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 # # # # # # # # # # # # #        IMPORT     # # # # # # # # # # # # # # # # # #
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #