chg: [Cryptocurrency] add new Cryptocurrency module

This commit is contained in:
Terrtia 2019-09-19 16:31:15 +02:00
parent 8612d89275
commit 3a9d0157e4
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
2 changed files with 168 additions and 0 deletions

139
bin/Cryptocurrencies.py Executable file
View file

@ -0,0 +1,139 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
"""
The Cryptocurrency Module
============================
It trying to extract Bitcoin address and secret key from paste
..seealso:: Paste method (get_regex)
Requirements
------------
*Need running Redis instances. (Redis).
"""
from Helper import Process
from pubsublogger import publisher
import os
import re
import sys
import time
import redis
import signal
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
import Cryptocurrency
import Item
class TimeoutException(Exception):
pass
def timeout_handler(signum, frame):
raise TimeoutException
signal.signal(signal.SIGALRM, timeout_handler)
def search_crytocurrency(item_id, item_content):
# bitcoin_private_key = re.findall(regex_bitcoin_private_key, content)
is_cryptocurrency_found = False
for crypto_name in cryptocurrency_dict:
crypto_dict = cryptocurrency_dict[crypto_name]
signal.alarm(crypto_dict['max_execution_time'])
try:
crypto_addr = re.findall(crypto_dict['regex'], item_content)
except TimeoutException:
crypto_addr = []
p.incr_module_timeout_statistic() # add encoder type
print ("{0} processing timeout".format(item_id))
continue
else:
signal.alarm(0)
if crypto_addr:
is_valid_crypto_addr = False
# validate cryptocurrency address
for address in crypto_addr:
if(Cryptocurrency.verify_cryptocurrency_address(crypto_name, address)):
is_valid_crypto_addr = True
print('{} address found : {}'.format(crypto_name, address))
# build bitcoin correlation
Cryptocurrency.save_cryptocurrency_data(crypto_name, Item.get_item_date(item_id), item_id, address)
# # TODO: add private key validation
#if(len(bitcoin_private_key) > 0):
# for private_key in bitcoin_private_key:
# print('Bitcoin private key found : {}'.format(private_key))
# to_print = 'Bitcoin found: {} address and {} private Keys'.format(len(bitcoin_address), len(bitcoin_private_key))
# print(to_print)
# publisher.warning(to_print)
# msg = 'infoleak:automatic-detection="bitcoin-private-key";{}'.format(message)
# p.populate_set_out(msg, 'Tags')
# to_print = 'Bitcoin;{};{};{};'.format(paste.p_source, paste.p_date,
# paste.p_name)
# publisher.warning('{}Detected {} Bitcoin private key;{}'.format(
# to_print, len(bitcoin_private_key),paste.p_rel_path))
if(is_valid_crypto_addr):
# valid cryptocurrency found in this item
is_cryptocurrency_found = True
# Tag Item
msg = '{};{}'.format(crypto_dict['tag'], item_id)
p.populate_set_out(msg, 'Tags')
if is_cryptocurrency_found:
# send to duplicate module
p.populate_set_out(item_id, 'Duplicate')
default_max_execution_time = 30
regex_bitcoin_public_address = r'(?<![a-km-zA-HJ-NP-Z0-9])[13][a-km-zA-HJ-NP-Z0-9]{26,33}(?![a-km-zA-HJ-NP-Z0-9])'
cryptocurrency_dict = {'bitcoin': {
'name': 'bitcoin',
'regex': regex_bitcoin_public_address,
'max_execution_time': default_max_execution_time,
'tag': 'infoleak:automatic-detection="bitcoin-address"',
}
}
if __name__ == "__main__":
publisher.port = 6380
publisher.channel = "Script"
config_section = 'Bitcoin'
# Setup the I/O queues
p = Process(config_section)
# Sent to the logging a description of the module
publisher.info("Run Cryptocurrency module ")
regex_bitcoin_private_key = re.compile(r'[5KL][1-9A-HJ-NP-Za-km-z]{50,51}')
# Endless loop getting messages from the input queue
while True:
# Get one message from the input queue
item_id = p.get_from_set()
if item_id is None:
publisher.debug("{} queue is empty, waiting".format(config_section))
time.sleep(1)
continue
# Do something with the message from the queue
item_content = Item.get_item_content(item_id)
search_crytocurrency(item_id, item_content)

View file

@ -2,10 +2,12 @@
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
import os import os
import sys
import redis import redis
from hashlib import sha256 from hashlib import sha256
sys.path.append(os.path.join(os.environ['AIL_FLASK'], 'modules'))
import Flask_config import Flask_config
from Correlation import Correlation from Correlation import Correlation
@ -15,12 +17,14 @@ digits58 = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
cryptocurrency = Correlation('cryptocurrency') cryptocurrency = Correlation('cryptocurrency')
# http://rosettacode.org/wiki/Bitcoin/address_validation#Python
def decode_base58(bc, length): def decode_base58(bc, length):
n = 0 n = 0
for char in bc: for char in bc:
n = n * 58 + digits58.index(char) n = n * 58 + digits58.index(char)
return n.to_bytes(length, 'big') return n.to_bytes(length, 'big')
# http://rosettacode.org/wiki/Bitcoin/address_validation#Python
def check_bitcoin_address(bc): def check_bitcoin_address(bc):
try: try:
bcbytes = decode_base58(bc, 25) bcbytes = decode_base58(bc, 25)
@ -46,3 +50,28 @@ def get_cryptocurrency(request_dict, cryptocurrency_type):
return ( {'status': 'error', 'reason': 'Invalid Cryptocurrency address'}, 400 ) return ( {'status': 'error', 'reason': 'Invalid Cryptocurrency address'}, 400 )
return cryptocurrency.get_correlation(request_dict, cryptocurrency_type, field_name) return cryptocurrency.get_correlation(request_dict, cryptocurrency_type, field_name)
def save_cryptocurrency_data(cryptocurrency_name, date, item_path, cryptocurrency_address):
# create basic medata
if not r_serv_metadata.exists('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address)):
r_serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'first_seen', date)
r_serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen', date)
else:
last_seen = r_serv_metadata.hget('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen')
if not last_seen:
r_serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen', date)
else:
if int(last_seen) < int(date):
r_serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen', date)
# global set
r_serv_metadata.sadd('set_cryptocurrency_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), item_path)
# daily
r_serv_metadata.hincrby('cryptocurrency:{}:{}'.format(cryptocurrency_name, date), cryptocurrency_address, 1)
# all type
r_serv_metadata.zincrby('cryptocurrency_all:{}'.format(cryptocurrency_name), cryptocurrency_address, 1)
# item_metadata
r_serv_metadata.sadd('item_cryptocurrency_{}:{}'.format(cryptocurrency_name, item_path), cryptocurrency_address)