mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-27 00:07:16 +00:00
chg: [Cryptocurrency] add new Cryptocurrency module
This commit is contained in:
Terrtia
parent
8612d89275
commit
3a9d0157e4
2 changed files with 168 additions and 0 deletions
139
bin/Cryptocurrencies.py
Executable file
139
bin/Cryptocurrencies.py
Executable file
|
|
@ -0,0 +1,139 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
The Cryptocurrency Module
|
||||
============================
|
||||
|
||||
It trying to extract Bitcoin address and secret key from paste
|
||||
|
||||
..seealso:: Paste method (get_regex)
|
||||
|
||||
Requirements
|
||||
------------
|
||||
|
||||
*Need running Redis instances. (Redis).
|
||||
|
||||
"""
|
||||
|
||||
from Helper import Process
|
||||
from pubsublogger import publisher
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import signal
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
|
||||
import Cryptocurrency
|
||||
import Item
|
||||
|
||||
|
||||
class TimeoutException(Exception):
|
||||
pass
|
||||
|
||||
def timeout_handler(signum, frame):
|
||||
raise TimeoutException
|
||||
|
||||
signal.signal(signal.SIGALRM, timeout_handler)
|
||||
|
||||
|
||||
def search_crytocurrency(item_id, item_content):
|
||||
|
||||
# bitcoin_private_key = re.findall(regex_bitcoin_private_key, content)
|
||||
|
||||
is_cryptocurrency_found = False
|
||||
|
||||
for crypto_name in cryptocurrency_dict:
|
||||
crypto_dict = cryptocurrency_dict[crypto_name]
|
||||
|
||||
signal.alarm(crypto_dict['max_execution_time'])
|
||||
try:
|
||||
crypto_addr = re.findall(crypto_dict['regex'], item_content)
|
||||
except TimeoutException:
|
||||
crypto_addr = []
|
||||
p.incr_module_timeout_statistic() # add encoder type
|
||||
print ("{0} processing timeout".format(item_id))
|
||||
continue
|
||||
else:
|
||||
signal.alarm(0)
|
||||
|
||||
if crypto_addr:
|
||||
is_valid_crypto_addr = False
|
||||
# validate cryptocurrency address
|
||||
for address in crypto_addr:
|
||||
if(Cryptocurrency.verify_cryptocurrency_address(crypto_name, address)):
|
||||
is_valid_crypto_addr = True
|
||||
print('{} address found : {}'.format(crypto_name, address))
|
||||
# build bitcoin correlation
|
||||
Cryptocurrency.save_cryptocurrency_data(crypto_name, Item.get_item_date(item_id), item_id, address)
|
||||
|
||||
# # TODO: add private key validation
|
||||
#if(len(bitcoin_private_key) > 0):
|
||||
# for private_key in bitcoin_private_key:
|
||||
# print('Bitcoin private key found : {}'.format(private_key))
|
||||
# to_print = 'Bitcoin found: {} address and {} private Keys'.format(len(bitcoin_address), len(bitcoin_private_key))
|
||||
# print(to_print)
|
||||
# publisher.warning(to_print)
|
||||
# msg = 'infoleak:automatic-detection="bitcoin-private-key";{}'.format(message)
|
||||
# p.populate_set_out(msg, 'Tags')
|
||||
# to_print = 'Bitcoin;{};{};{};'.format(paste.p_source, paste.p_date,
|
||||
# paste.p_name)
|
||||
# publisher.warning('{}Detected {} Bitcoin private key;{}'.format(
|
||||
# to_print, len(bitcoin_private_key),paste.p_rel_path))
|
||||
|
||||
if(is_valid_crypto_addr):
|
||||
# valid cryptocurrency found in this item
|
||||
is_cryptocurrency_found = True
|
||||
|
||||
# Tag Item
|
||||
msg = '{};{}'.format(crypto_dict['tag'], item_id)
|
||||
p.populate_set_out(msg, 'Tags')
|
||||
|
||||
if is_cryptocurrency_found:
|
||||
# send to duplicate module
|
||||
p.populate_set_out(item_id, 'Duplicate')
|
||||
|
||||
|
||||
|
||||
|
||||
default_max_execution_time = 30
|
||||
regex_bitcoin_public_address = r'(?<![a-km-zA-HJ-NP-Z0-9])[13][a-km-zA-HJ-NP-Z0-9]{26,33}(?![a-km-zA-HJ-NP-Z0-9])'
|
||||
|
||||
cryptocurrency_dict = {'bitcoin': {
|
||||
'name': 'bitcoin',
|
||||
'regex': regex_bitcoin_public_address,
|
||||
'max_execution_time': default_max_execution_time,
|
||||
'tag': 'infoleak:automatic-detection="bitcoin-address"',
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
publisher.port = 6380
|
||||
publisher.channel = "Script"
|
||||
|
||||
config_section = 'Bitcoin'
|
||||
|
||||
# Setup the I/O queues
|
||||
p = Process(config_section)
|
||||
|
||||
# Sent to the logging a description of the module
|
||||
publisher.info("Run Cryptocurrency module ")
|
||||
|
||||
|
||||
regex_bitcoin_private_key = re.compile(r'[5KL][1-9A-HJ-NP-Za-km-z]{50,51}')
|
||||
|
||||
# Endless loop getting messages from the input queue
|
||||
while True:
|
||||
# Get one message from the input queue
|
||||
item_id = p.get_from_set()
|
||||
if item_id is None:
|
||||
publisher.debug("{} queue is empty, waiting".format(config_section))
|
||||
time.sleep(1)
|
||||
continue
|
||||
|
||||
# Do something with the message from the queue
|
||||
item_content = Item.get_item_content(item_id)
|
||||
search_crytocurrency(item_id, item_content)
|
||||
|
|
@ -2,10 +2,12 @@
|
|||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
import redis
|
||||
|
||||
from hashlib import sha256
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_FLASK'], 'modules'))
|
||||
import Flask_config
|
||||
from Correlation import Correlation
|
||||
|
||||
|
|
@ -15,12 +17,14 @@ digits58 = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
|
|||
|
||||
cryptocurrency = Correlation('cryptocurrency')
|
||||
|
||||
# http://rosettacode.org/wiki/Bitcoin/address_validation#Python
|
||||
def decode_base58(bc, length):
|
||||
n = 0
|
||||
for char in bc:
|
||||
n = n * 58 + digits58.index(char)
|
||||
return n.to_bytes(length, 'big')
|
||||
|
||||
# http://rosettacode.org/wiki/Bitcoin/address_validation#Python
|
||||
def check_bitcoin_address(bc):
|
||||
try:
|
||||
bcbytes = decode_base58(bc, 25)
|
||||
|
|
@ -46,3 +50,28 @@ def get_cryptocurrency(request_dict, cryptocurrency_type):
|
|||
return ( {'status': 'error', 'reason': 'Invalid Cryptocurrency address'}, 400 )
|
||||
|
||||
return cryptocurrency.get_correlation(request_dict, cryptocurrency_type, field_name)
|
||||
|
||||
def save_cryptocurrency_data(cryptocurrency_name, date, item_path, cryptocurrency_address):
|
||||
# create basic medata
|
||||
if not r_serv_metadata.exists('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address)):
|
||||
r_serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'first_seen', date)
|
||||
r_serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen', date)
|
||||
else:
|
||||
last_seen = r_serv_metadata.hget('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen')
|
||||
if not last_seen:
|
||||
r_serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen', date)
|
||||
else:
|
||||
if int(last_seen) < int(date):
|
||||
r_serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen', date)
|
||||
|
||||
# global set
|
||||
r_serv_metadata.sadd('set_cryptocurrency_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), item_path)
|
||||
|
||||
# daily
|
||||
r_serv_metadata.hincrby('cryptocurrency:{}:{}'.format(cryptocurrency_name, date), cryptocurrency_address, 1)
|
||||
|
||||
# all type
|
||||
r_serv_metadata.zincrby('cryptocurrency_all:{}'.format(cryptocurrency_name), cryptocurrency_address, 1)
|
||||
|
||||
# item_metadata
|
||||
r_serv_metadata.sadd('item_cryptocurrency_{}:{}'.format(cryptocurrency_name, item_path), cryptocurrency_address)
|
||||
|
|
|
|||
Loading…
Reference in a new issue