chg: [Cryptocurrency] add new Cryptocurrency module

bin/Cryptocurrencies.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+"""
+The Cryptocurrency Module
+============================
+
+It trying to extract Bitcoin address and secret key from paste
+
+    ..seealso:: Paste method (get_regex)
+
+Requirements
+------------
+
+*Need running Redis instances. (Redis).
+
+"""
+
+from Helper import Process
+from pubsublogger import publisher
+
+import os
+import re
+import sys
+import time
+import redis
+import signal
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
+import Cryptocurrency
+import Item
+
+
+class TimeoutException(Exception):
+    pass
+
+def timeout_handler(signum, frame):
+    raise TimeoutException
+
+signal.signal(signal.SIGALRM, timeout_handler)
+
+
+def search_crytocurrency(item_id, item_content):
+
+    # bitcoin_private_key = re.findall(regex_bitcoin_private_key, content)
+
+    is_cryptocurrency_found = False
+
+    for crypto_name in cryptocurrency_dict:
+        crypto_dict = cryptocurrency_dict[crypto_name]
+
+        signal.alarm(crypto_dict['max_execution_time'])
+        try:
+            crypto_addr = re.findall(crypto_dict['regex'], item_content)
+        except TimeoutException:
+            crypto_addr = []
+            p.incr_module_timeout_statistic() # add encoder type
+            print ("{0} processing timeout".format(item_id))
+            continue
+        else:
+            signal.alarm(0)
+
+        if crypto_addr:
+            is_valid_crypto_addr = False
+            # validate cryptocurrency address
+            for address in crypto_addr:
+                if(Cryptocurrency.verify_cryptocurrency_address(crypto_name, address)):
+                    is_valid_crypto_addr = True
+                    print('{} address found : {}'.format(crypto_name, address))
+                    # build bitcoin correlation
+                    Cryptocurrency.save_cryptocurrency_data(crypto_name, Item.get_item_date(item_id), item_id, address)
+
+            # # TODO: add private key validation
+            #if(len(bitcoin_private_key) > 0):
+            #    for private_key in bitcoin_private_key:
+            #        print('Bitcoin private key found : {}'.format(private_key))
+            #        to_print = 'Bitcoin found: {} address and {} private Keys'.format(len(bitcoin_address), len(bitcoin_private_key))
+            #        print(to_print)
+            #        publisher.warning(to_print)
+            #        msg = 'infoleak:automatic-detection="bitcoin-private-key";{}'.format(message)
+            #        p.populate_set_out(msg, 'Tags')
+            #            to_print = 'Bitcoin;{};{};{};'.format(paste.p_source, paste.p_date,
+            #                                            paste.p_name)
+            #        publisher.warning('{}Detected {} Bitcoin private key;{}'.format(
+            #            to_print, len(bitcoin_private_key),paste.p_rel_path))
+
+            if(is_valid_crypto_addr):
+                # valid cryptocurrency found in this item
+                is_cryptocurrency_found = True
+
+                # Tag Item
+                msg = '{};{}'.format(crypto_dict['tag'], item_id)
+                p.populate_set_out(msg, 'Tags')
+
+    if is_cryptocurrency_found:
+        # send to duplicate module
+        p.populate_set_out(item_id, 'Duplicate')
+
+
+
+
+default_max_execution_time = 30
+regex_bitcoin_public_address = r'(?<![a-km-zA-HJ-NP-Z0-9])[13][a-km-zA-HJ-NP-Z0-9]{26,33}(?![a-km-zA-HJ-NP-Z0-9])'
+
+cryptocurrency_dict = {'bitcoin': {
+                            'name': 'bitcoin',
+                            'regex': regex_bitcoin_public_address,
+                            'max_execution_time': default_max_execution_time,
+                            'tag': 'infoleak:automatic-detection="bitcoin-address"',
+                            }
+                        }
+
+
+if __name__ == "__main__":
+    publisher.port = 6380
+    publisher.channel = "Script"
+
+    config_section = 'Bitcoin'
+
+    # Setup the I/O queues
+    p = Process(config_section)
+
+    # Sent to the logging a description of the module
+    publisher.info("Run Cryptocurrency module ")
+
+
+    regex_bitcoin_private_key = re.compile(r'[5KL][1-9A-HJ-NP-Za-km-z]{50,51}')
+
+    # Endless loop getting messages from the input queue
+    while True:
+        # Get one message from the input queue
+        item_id = p.get_from_set()
+        if item_id is None:
+            publisher.debug("{} queue is empty, waiting".format(config_section))
+            time.sleep(1)
+            continue
+
+        # Do something with the message from the queue
+        item_content = Item.get_item_content(item_id)
+        search_crytocurrency(item_id, item_content)

bin/packages/Cryptocurrency.py

@@ -2,10 +2,12 @@
 # -*-coding:UTF-8 -*
 
 import os
+import sys
 import redis
 
 from hashlib import sha256
 
+sys.path.append(os.path.join(os.environ['AIL_FLASK'], 'modules'))
 import Flask_config
 from Correlation import Correlation
 
@@ -15,12 +17,14 @@ digits58 = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
 
 cryptocurrency = Correlation('cryptocurrency')
 
+# http://rosettacode.org/wiki/Bitcoin/address_validation#Python
 def decode_base58(bc, length):
     n = 0
     for char in bc:
         n = n * 58 + digits58.index(char)
     return n.to_bytes(length, 'big')
 
+# http://rosettacode.org/wiki/Bitcoin/address_validation#Python
 def check_bitcoin_address(bc):
     try:
         bcbytes = decode_base58(bc, 25)
@@ -46,3 +50,28 @@ def get_cryptocurrency(request_dict, cryptocurrency_type):
         return ( {'status': 'error', 'reason': 'Invalid Cryptocurrency address'}, 400 )
 
     return cryptocurrency.get_correlation(request_dict, cryptocurrency_type, field_name)
+
+def save_cryptocurrency_data(cryptocurrency_name, date, item_path, cryptocurrency_address):
+    # create basic medata
+    if not r_serv_metadata.exists('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address)):
+        r_serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'first_seen', date)
+        r_serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen', date)
+    else:
+        last_seen = r_serv_metadata.hget('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen')
+        if not last_seen:
+            r_serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen', date)
+        else:
+            if int(last_seen) < int(date):
+                r_serv_metadata.hset('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), 'last_seen', date)
+
+    # global set
+    r_serv_metadata.sadd('set_cryptocurrency_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), item_path)
+
+    # daily
+    r_serv_metadata.hincrby('cryptocurrency:{}:{}'.format(cryptocurrency_name, date), cryptocurrency_address, 1)
+
+    # all type
+    r_serv_metadata.zincrby('cryptocurrency_all:{}'.format(cryptocurrency_name), cryptocurrency_address, 1)
+
+    # item_metadata
+    r_serv_metadata.sadd('item_cryptocurrency_{}:{}'.format(cryptocurrency_name, item_path), cryptocurrency_address)