chg: [Bitcoin] map cryptocurrency: bitcoin (DB pivot)

This commit is contained in:
Terrtia 2019-05-21 16:14:09 +02:00
parent ff3d6e22ef
commit ff23a2bce2
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
3 changed files with 119 additions and 1 deletions

View file

@ -202,6 +202,38 @@ Redis and ARDB overview
| | |
| item_pgp_mail:*item_path* | *mail* |
#### Cryptocurrency
Supported cryptocurrency:
- bitcoin
##### Hset:
| Key | Field | Value |
| ------ | ------ | ------ |
| cryptocurrency_metadata_**cryptocurrency name**:**cryptocurrency address** | first_seen | **date** |
| | last_seen | **date** |
##### set:
| Key | Value |
| ------ | ------ |
| set_cryptocurrency_**cryptocurrency name**:**cryptocurrency address** | **item_path** |
##### Hset date:
| Key | Field | Value |
| ------ | ------ |
| cryptocurrency_**cryptocurrency name**:**date** | **cryptocurrency address** | **nb seen** |
##### zset:
| Key | Field | Value |
| ------ | ------ | ------ |
| cryptocurrency_all:**cryptocurrency name** | **cryptocurrency address** | **nb seen** |
##### set:
| Key | Value |
| ------ | ------ |
| item_cryptocurrency_**cryptocurrency name**:**item_path** | **cryptocurrency address** |
## DB9 - Crawler:
##### Hset:

View file

@ -44,6 +44,7 @@ def check_bc(bc):
def search_key(content, message, paste):
bitcoin_address = re.findall(regex_bitcoin_public_address, content)
bitcoin_private_key = re.findall(regex_bitcoin_private_key, content)
date = str(paste._get_p_date())
validate_address = False
key = False
if(len(bitcoin_address) >0):
@ -56,6 +57,8 @@ def search_key(content, message, paste):
for private_key in bitcoin_private_key:
print('Bitcoin private key found : {}'.format(private_key))
key = True
# build bitcoin correlation
save_bitcoin_data(date, message, address):
if(validate_address):
p.populate_set_out(message, 'Duplicate')
@ -75,6 +78,31 @@ def search_key(content, message, paste):
publisher.warning('{}Detected {} Bitcoin private key;{}'.format(
to_print, len(bitcoin_private_key),paste.p_rel_path))
def save_bitcoin_data(cryptocurrency_name, date, item_path, cryptocurrency_address):
# create basic medata
if not serv_metadata.exists('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address)):
serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'first_seen', date)
serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen', date)
else:
last_seen = serv_metadata.hget('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen')
if not last_seen:
serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen', date)
else:
if int(last_seen) < int(date):
serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen', date)
# global set
serv_metadata.sadd('set_cryptocurrency_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), item_path)
# daily
serv_metadata.hincrby('cryptocurrency_{}:{}'.format(cryptocurrency_name, date), cryptocurrency_address, 1)
# all type
serv_metadata.zincrby('cryptocurrency_all:{}'.format(cryptocurrency_name), cryptocurrency_address, 1)
# item_metadata
serv_metadata.sadd('item_cryptocurrency_{}:{}'.format(cryptocurrency_name, item_path), cryptocurrency_address)
if __name__ == "__main__":
publisher.port = 6380
publisher.channel = "Script"

View file

@ -13,6 +13,8 @@ from Date import Date
from io import BytesIO
import zipfile
from hashlib import sha256
import requests
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, send_file
@ -158,6 +160,38 @@ def get_all_pgp_from_item(item_path):
def one():
return 1
def decode_base58(bc, length):
n = 0
for char in bc:
n = n * 58 + digits58.index(char)
return n.to_bytes(length, 'big')
def check_bc(bc):
try:
bcbytes = decode_base58(bc, 25)
return bcbytes[-4:] == sha256(sha256(bcbytes[:-4]).digest()).digest()[:4]
except Exception:
return False
def get_bitcoin_address_metadata(bitcoin_address):
address_metadata = {}
if r_serv_metadata.exists('bitcoin_metadata:{}'.format(bitcoin_address)):
address_metadata['first_seen'] = r_serv_metadata.hget('bitcoin_metadata:{}'.format(bitcoin_address), 'first_seen')
address_metadata['first_seen'] = '{}/{}/{}'.format(address_metadata['first_seen'][0:4], address_metadata['first_seen'][4:6], address_metadata['first_seen'][6:8])
address_metadata['last_seen'] = r_serv_metadata.hget('bitcoin_metadata:{}'.format(bitcoin_address), 'last_seen')
address_metadata['last_seen'] = '{}/{}/{}'.format(address_metadata['last_seen'][0:4], address_metadata['last_seen'][4:6], address_metadata['last_seen'][6:8])
address_metadata['nb_seen'] = r_serv_metadata.scard('bitcoin:{}'.format(bitcoin_address))
return address_metadata
def list_sparkline_bitcoin_values(date_range_sparkline, bitcoin_address):
sparklines_value = []
for date_day in date_range_sparkline:
nb_seen_this_day = r_serv_metadata.hget('bitcoin:{}'.format(bitcoin_address), bitcoin_address)
if nb_seen_this_day is None:
nb_seen_this_day = 0
sparklines_value.append(int(nb_seen_this_day))
return sparklines_value
# ============= ROUTES ==============
@hashDecoded.route("/hashDecoded/all_hash_search", methods=['POST'])
def all_hash_search():
@ -777,7 +811,7 @@ def update_vt_result():
# TODO FIXME make json response
return jsonify()
## PGPDump ##
############################ PGPDump ############################
@hashDecoded.route("/decoded/pgpdump", methods=['GET'])
def pgpdump_page():
@ -1069,5 +1103,29 @@ def pgp_by_type_json():
else:
return jsonify()
############################ Bitcoin ############################
'''
@hashDecoded.route('/correlation/show_bitcoin_address')
def show_bitcoin_address():
bitcoin_address = request.args.get('bitcoin_address')
# validate user input
if check_bc(bitcoin_address):
bitcoin_address_metadata = get_bitcoin_address_metadata(bitcoin_address)
if bitcoin_address_metadata:
num_day_sparkline = 6
date_range_sparkline = get_date_range(num_day_sparkline)
sparkline_values = list_sparkline_bitcoin_values(date_range_sparkline, bitcoin_address)
return render_template('showPgpDump.html', bitcoin_address=bitcoin_address,
key_id_metadata=bitcoin_address_metadata,
sparkline_values=sparkline_values)
else:
return '404'
else:
return 'error'
'''
# ========= REGISTRATION =========
app.register_blueprint(hashDecoded, url_prefix=baseUrl)