mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-22 22:27:17 +00:00
Merge branch 'master' into onion_crawler
This commit is contained in:
Terrtia
commit
823c33174a
31 changed files with 2732 additions and 62 deletions
2
HOWTO.md
2
HOWTO.md
|
|
@ -6,7 +6,7 @@ How to feed the AIL framework
|
|||
|
||||
For the moment, there are three different ways to feed AIL with data:
|
||||
|
||||
1. Be a collaborator of CIRCL and ask to access our feed. It will be sent to the static IP your are using for AIL.
|
||||
1. Be a collaborator of CIRCL and ask to access our feed. It will be sent to the static IP you are using for AIL.
|
||||
|
||||
2. You can setup [pystemon](https://github.com/CIRCL/pystemon) and use the custom feeder provided by AIL (see below).
|
||||
|
||||
|
|
|
|||
44
OVERVIEW.md
44
OVERVIEW.md
|
|
@ -20,4 +20,46 @@ Redis and ARDB overview
|
|||
- DB 0 - Lines duplicate
|
||||
- DB 1 - Hashes
|
||||
|
||||
To be updated
|
||||
|
||||
ARDB overview
|
||||
---------------------------
|
||||
|
||||
* DB 7 - Metadata:
|
||||
----------------------------------------- BASE64 ----------------------------------------
|
||||
|
||||
HSET - 'metadata_hash:'+hash 'saved_path' saved_path
|
||||
'size' size
|
||||
'first_seen' first_seen
|
||||
'last_seen' last_seen
|
||||
'estimated_type' estimated_type
|
||||
'vt_link' vt_link
|
||||
'vt_report' vt_report
|
||||
'nb_seen_in_all_pastes' nb_seen_in_all_pastes
|
||||
'base64_decoder' nb_encoded
|
||||
'binary_decoder' nb_encoded
|
||||
|
||||
SET - 'all_decoder' decoder*
|
||||
|
||||
SET - 'hash_all_type' hash_type *
|
||||
SET - 'hash_base64_all_type' hash_type *
|
||||
SET - 'hash_binary_all_type' hash_type *
|
||||
|
||||
SET - 'hash_paste:'+paste hash *
|
||||
SET - 'base64_paste:'+paste hash *
|
||||
SET - 'binary_paste:'+paste hash *
|
||||
|
||||
ZADD - 'hash_date:'+20180622 hash * nb_seen_this_day
|
||||
ZADD - 'base64_date:'+20180622 hash * nb_seen_this_day
|
||||
ZADD - 'binary_date:'+20180622 hash * nb_seen_this_day
|
||||
|
||||
ZADD - 'nb_seen_hash:'+hash paste * nb_seen_in_paste
|
||||
ZADD - 'base64_hash:'+hash paste * nb_seen_in_paste
|
||||
ZADD - 'binary_hash:'+hash paste * nb_seen_in_paste
|
||||
|
||||
ZADD - 'hash_type:'+type date nb_seen
|
||||
ZADD - 'base64_type:'+type date nb_seen
|
||||
ZADD - 'binary_type:'+type date nb_seen
|
||||
|
||||
GET - 'base64_decoded:'+date nd_decoded
|
||||
GET - 'binary_decoded:'+date nd_decoded
|
||||
|
||||
|
|
|
|||
31
README.md
31
README.md
|
|
@ -1,5 +1,3 @@
|
|||
[](https://travis-ci.org/CIRCL/AIL-framework)
|
||||
|
||||
AIL
|
||||
===
|
||||
|
||||
|
|
@ -11,6 +9,22 @@ AIL is a modular framework to analyse potential information leaks from unstructu
|
|||
|
||||
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<td>Latest Release</td>
|
||||
<td><a href="https://badge.fury.io/gh/CIRCL%2FAIL-Framework"><img src="https://badge.fury.io/gh/CIRCL%2FAIL-Framework.svg" alt="GitHub version" height="18"></a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Contributors</td>
|
||||
<td><img src="https://img.shields.io/github/contributors/CIRCL/AIL-Framework.svg" /></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>License</td>
|
||||
<td><img src="https://img.shields.io/github/license/CIRCL/AIL-Framework.svg" /></td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
Features
|
||||
--------
|
||||
|
||||
|
|
@ -31,14 +45,17 @@ Features
|
|||
* Terms, Set of terms and Regex tracking and occurrence
|
||||
* Many more modules for extracting phone numbers, credentials and others
|
||||
* Alerting to [MISP](https://github.com/MISP/MISP) to share found leaks within a threat intelligence platform using [MISP standard](https://www.misp-project.org/objects.html#_ail_leak)
|
||||
* Detect and decode Base64 and store files
|
||||
* Detect and decode encoded file (Base64, hex encoded or your own decoding scheme) and store files
|
||||
* Detect Amazon AWS and Google API keys
|
||||
* Detect Bitcoin address and Bitcoin private keys
|
||||
* Detect private keys and certificate
|
||||
* Detect private keys, certificate, keys (including SSH, OpenVPN)
|
||||
* Detect IBAN bank accounts
|
||||
* Tagging system with [MISP Galaxy](https://github.com/MISP/misp-galaxy) and [MISP Taxonomies](https://github.com/MISP/misp-taxonomies) tags
|
||||
* UI paste submission
|
||||
* Create events on [MISP](https://github.com/MISP/MISP) and cases on [The Hive](https://github.com/TheHive-Project/TheHive)
|
||||
* Automatic paste export at detection on [MISP](https://github.com/MISP/MISP) (events) and [The Hive](https://github.com/TheHive-Project/TheHive) (alerts) on selected tags
|
||||
* Extracted and decoded files can be searched by date range, type of file (mime-type) and encoding discovered
|
||||
* Graph relationships between decoded file (hashes)
|
||||
|
||||
Installation
|
||||
------------
|
||||
|
|
@ -152,6 +169,12 @@ Trending charts
|
|||
|
||||
|
||||
|
||||
Extracted encoded files from pastes
|
||||
-----------------------------------
|
||||
|
||||
|
||||
|
||||
|
||||
Browsing
|
||||
--------
|
||||
|
||||
|
|
|
|||
|
|
@ -86,8 +86,7 @@ if __name__ == "__main__":
|
|||
|
||||
if message is not None:
|
||||
|
||||
search_api_key(message)
|
||||
|
||||
search_api_key(message)
|
||||
|
||||
else:
|
||||
publisher.debug("Script ApiKey is Idling 10s")
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
import time
|
||||
import os
|
||||
import datetime
|
||||
import redis
|
||||
|
||||
from pubsublogger import publisher
|
||||
|
||||
|
|
@ -31,7 +32,7 @@ def timeout_handler(signum, frame):
|
|||
signal.signal(signal.SIGALRM, timeout_handler)
|
||||
|
||||
|
||||
def search_base64(content, message):
|
||||
def search_base64(content, message, date):
|
||||
find = False
|
||||
base64_list = re.findall(regex_base64, content)
|
||||
if(len(base64_list) > 0):
|
||||
|
|
@ -39,6 +40,7 @@ def search_base64(content, message):
|
|||
for b64 in base64_list:
|
||||
if len(b64) >= 40 :
|
||||
decode = base64.b64decode(b64)
|
||||
print(decode)
|
||||
|
||||
type = magic.from_buffer(decode, mime=True)
|
||||
#print(type)
|
||||
|
|
@ -46,6 +48,8 @@ def search_base64(content, message):
|
|||
|
||||
find = True
|
||||
hash = sha1(decode).hexdigest()
|
||||
print(message)
|
||||
print(hash)
|
||||
|
||||
data = {}
|
||||
data['name'] = hash
|
||||
|
|
@ -54,8 +58,37 @@ def search_base64(content, message):
|
|||
data['estimated type'] = type
|
||||
json_data = json.dumps(data)
|
||||
|
||||
save_base64_as_file(decode, type, hash, json_data)
|
||||
print('found {} '.format(type))
|
||||
date_paste = '{}/{}/{}'.format(date[0:4], date[4:6], date[6:8])
|
||||
date_key = date[0:4] + date[4:6] + date[6:8]
|
||||
|
||||
serv_metadata.zincrby('base64_date:'+date_key, hash, 1)
|
||||
|
||||
# first time we see this hash
|
||||
if not serv_metadata.hexists('metadata_hash:'+hash, 'estimated_type'):
|
||||
serv_metadata.hset('metadata_hash:'+hash, 'first_seen', date_paste)
|
||||
serv_metadata.hset('metadata_hash:'+hash, 'last_seen', date_paste)
|
||||
else:
|
||||
serv_metadata.hset('metadata_hash:'+hash, 'last_seen', date_paste)
|
||||
|
||||
# first time we see this file on this paste
|
||||
if serv_metadata.zscore('base64_hash:'+hash, message) is None:
|
||||
print('first')
|
||||
serv_metadata.hincrby('metadata_hash:'+hash, 'nb_seen_in_all_pastes', 1)
|
||||
|
||||
serv_metadata.sadd('base64_paste:'+message, hash) # paste - hash map
|
||||
serv_metadata.zincrby('base64_hash:'+hash, message, 1)# hash - paste map
|
||||
|
||||
# create hash metadata
|
||||
serv_metadata.hset('metadata_hash:'+hash, 'estimated_type', type)
|
||||
serv_metadata.sadd('hash_all_type', type)
|
||||
serv_metadata.sadd('hash_base64_all_type', type)
|
||||
serv_metadata.zincrby('base64_type:'+type, date_key, 1)
|
||||
|
||||
save_base64_as_file(decode, type, hash, json_data, id)
|
||||
print('found {} '.format(type))
|
||||
# duplicate
|
||||
else:
|
||||
serv_metadata.zincrby('base64_hash:'+hash, message, 1) # number of b64 on this paste
|
||||
|
||||
if(find):
|
||||
publisher.warning('base64 decoded')
|
||||
|
|
@ -68,10 +101,10 @@ def search_base64(content, message):
|
|||
msg = 'infoleak:automatic-detection="base64";{}'.format(message)
|
||||
p.populate_set_out(msg, 'Tags')
|
||||
|
||||
def save_base64_as_file(decode, type, hash, json_data):
|
||||
def save_base64_as_file(decode, type, hash, json_data, id):
|
||||
|
||||
filename_b64 = os.path.join(os.environ['AIL_HOME'],
|
||||
p.config.get("Directories", "base64"), type, hash[:2], hash)
|
||||
local_filename_b64 = os.path.join(p.config.get("Directories", "base64"), type, hash[:2], hash)
|
||||
filename_b64 = os.path.join(os.environ['AIL_HOME'], local_filename_b64)
|
||||
|
||||
filename_json = os.path.join(os.environ['AIL_HOME'],
|
||||
p.config.get("Directories", "base64"), type, hash[:2], hash + '.json')
|
||||
|
|
@ -83,6 +116,10 @@ def save_base64_as_file(decode, type, hash, json_data):
|
|||
with open(filename_b64, 'wb') as f:
|
||||
f.write(decode)
|
||||
|
||||
# create hash metadata
|
||||
serv_metadata.hset('metadata_hash:'+hash, 'saved_path', local_filename_b64)
|
||||
serv_metadata.hset('metadata_hash:'+hash, 'size', os.path.getsize(filename_b64))
|
||||
|
||||
with open(filename_json, 'w') as f:
|
||||
f.write(json_data)
|
||||
|
||||
|
|
@ -103,6 +140,12 @@ if __name__ == '__main__':
|
|||
p = Process(config_section)
|
||||
max_execution_time = p.config.getint("Base64", "max_execution_time")
|
||||
|
||||
serv_metadata = redis.StrictRedis(
|
||||
host=p.config.get("ARDB_Metadata", "host"),
|
||||
port=p.config.getint("ARDB_Metadata", "port"),
|
||||
db=p.config.getint("ARDB_Metadata", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
# Sent to the logging a description of the module
|
||||
publisher.info("Base64 started")
|
||||
|
||||
|
|
@ -127,13 +170,12 @@ if __name__ == '__main__':
|
|||
# Do something with the message from the queue
|
||||
#print(filename)
|
||||
content = paste.get_p_content()
|
||||
search_base64(content,message)
|
||||
|
||||
# (Optional) Send that thing to the next queue
|
||||
#p.populate_set_out(something_has_been_done)
|
||||
date = str(paste._get_p_date())
|
||||
search_base64(content,message, date)
|
||||
|
||||
except TimeoutException:
|
||||
print ("{0} processing timeout".format(paste.p_path))
|
||||
continue
|
||||
p.incr_module_timeout_statistic()
|
||||
print ("{0} processing timeout".format(paste.p_path))
|
||||
continue
|
||||
else:
|
||||
signal.alarm(0)
|
||||
|
|
|
|||
184
bin/Binary.py
Executable file
184
bin/Binary.py
Executable file
|
|
@ -0,0 +1,184 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
Binary module
|
||||
|
||||
Dectect Binary and decode it
|
||||
"""
|
||||
import time
|
||||
import os
|
||||
import datetime
|
||||
import redis
|
||||
|
||||
from pubsublogger import publisher
|
||||
|
||||
from Helper import Process
|
||||
from packages import Paste
|
||||
|
||||
import re
|
||||
from hashlib import sha1
|
||||
import magic
|
||||
import json
|
||||
|
||||
import signal
|
||||
|
||||
class TimeoutException(Exception):
|
||||
pass
|
||||
|
||||
def timeout_handler(signum, frame):
|
||||
raise TimeoutException
|
||||
|
||||
signal.signal(signal.SIGALRM, timeout_handler)
|
||||
|
||||
def decode_binary_string(binary_string):
|
||||
return ''.join(chr(int(s[i*8:i*8+8],2)) for i in range(len(s)//8))
|
||||
|
||||
def decode_binary(binary_string):
|
||||
return bytes(bytearray([int(binary_string[i:i+8], 2) for i in range(0, len(binary_string), 8)]))
|
||||
|
||||
|
||||
def search_binary(content, message, date):
|
||||
find = False
|
||||
binary_list = re.findall(regex_binary, content)
|
||||
if(len(binary_list) > 0):
|
||||
|
||||
for binary in binary_list:
|
||||
if len(binary) >= 40 :
|
||||
decode = decode_binary(binary)
|
||||
print(message)
|
||||
|
||||
type = magic.from_buffer(decode, mime=True)
|
||||
print(type)
|
||||
|
||||
find = True
|
||||
hash = sha1(decode).hexdigest()
|
||||
print(hash)
|
||||
|
||||
data = {}
|
||||
data['name'] = hash
|
||||
data['date'] = datetime.datetime.now().strftime("%d/%m/%y")
|
||||
data['origin'] = message
|
||||
data['estimated type'] = type
|
||||
json_data = json.dumps(data)
|
||||
|
||||
date_paste = '{}/{}/{}'.format(date[0:4], date[4:6], date[6:8])
|
||||
date_key = date[0:4] + date[4:6] + date[6:8]
|
||||
|
||||
serv_metadata.zincrby('binary_date:'+date_key, hash, 1)
|
||||
|
||||
# first time we see this hash
|
||||
if not serv_metadata.hexists('metadata_hash:'+hash, 'estimated_type'):
|
||||
serv_metadata.hset('metadata_hash:'+hash, 'first_seen', date_paste)
|
||||
serv_metadata.hset('metadata_hash:'+hash, 'last_seen', date_paste)
|
||||
else:
|
||||
serv_metadata.hset('metadata_hash:'+hash, 'last_seen', date_paste)
|
||||
|
||||
# first time we see this file encoding on this paste
|
||||
if serv_metadata.zscore('binary_hash:'+hash, message) is None:
|
||||
print('first binary')
|
||||
serv_metadata.hincrby('metadata_hash:'+hash, 'nb_seen_in_all_pastes', 1)
|
||||
|
||||
serv_metadata.sadd('binary_paste:'+message, hash) # paste - hash map
|
||||
serv_metadata.zincrby('binary_hash:'+hash, message, 1)# hash - paste map
|
||||
|
||||
# create hash metadata
|
||||
serv_metadata.hset('metadata_hash:'+hash, 'estimated_type', type)
|
||||
serv_metadata.sadd('hash_all_type', type)
|
||||
serv_metadata.sadd('hash_binary_all_type', type)
|
||||
serv_metadata.zincrby('binary_type:'+type, date_key, 1)
|
||||
|
||||
save_binary_as_file(decode, type, hash, json_data, id)
|
||||
print('found {} '.format(type))
|
||||
# duplicate
|
||||
else:
|
||||
serv_metadata.zincrby('binary_hash:'+hash, message, 1) # number of b64 on this paste
|
||||
|
||||
if(find):
|
||||
publisher.warning('binary decoded')
|
||||
#Send to duplicate
|
||||
p.populate_set_out(message, 'Duplicate')
|
||||
#send to Browse_warning_paste
|
||||
msg = ('binary;{}'.format(message))
|
||||
p.populate_set_out( msg, 'alertHandler')
|
||||
|
||||
msg = 'infoleak:automatic-detection="binary";{}'.format(message)
|
||||
p.populate_set_out(msg, 'Tags')
|
||||
|
||||
def save_binary_as_file(decode, type, hash, json_data, id):
|
||||
|
||||
local_filename_b64 = os.path.join(p.config.get("Directories", "base64"), type, hash[:2], hash)
|
||||
filename_b64 = os.path.join(os.environ['AIL_HOME'], local_filename_b64)
|
||||
|
||||
filename_json = os.path.join(os.environ['AIL_HOME'],
|
||||
p.config.get("Directories", "base64"), type, hash[:2], hash + '.json')
|
||||
|
||||
dirname = os.path.dirname(filename_b64)
|
||||
if not os.path.exists(dirname):
|
||||
os.makedirs(dirname)
|
||||
|
||||
with open(filename_b64, 'wb') as f:
|
||||
f.write(decode)
|
||||
|
||||
# create hash metadata
|
||||
serv_metadata.hset('metadata_hash:'+hash, 'saved_path', local_filename_b64)
|
||||
serv_metadata.hset('metadata_hash:'+hash, 'size', os.path.getsize(filename_b64))
|
||||
|
||||
with open(filename_json, 'w') as f:
|
||||
f.write(json_data)
|
||||
|
||||
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
|
||||
# Port of the redis instance used by pubsublogger
|
||||
publisher.port = 6380
|
||||
# Script is the default channel used for the modules.
|
||||
publisher.channel = 'Script'
|
||||
|
||||
# Section name in bin/packages/modules.cfg
|
||||
config_section = 'Binary'
|
||||
|
||||
# Setup the I/O queues
|
||||
p = Process(config_section)
|
||||
max_execution_time = p.config.getint("Binary", "max_execution_time")
|
||||
|
||||
serv_metadata = redis.StrictRedis(
|
||||
host=p.config.get("ARDB_Metadata", "host"),
|
||||
port=p.config.getint("ARDB_Metadata", "port"),
|
||||
db=p.config.getint("ARDB_Metadata", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
# Sent to the logging a description of the module
|
||||
publisher.info("Binary started")
|
||||
|
||||
regex_binary = '[0-1]{40,}'
|
||||
re.compile(regex_binary)
|
||||
|
||||
# Endless loop getting messages from the input queue
|
||||
while True:
|
||||
# Get one message from the input queue
|
||||
message = p.get_from_set()
|
||||
if message is None:
|
||||
|
||||
publisher.debug("{} queue is empty, waiting".format(config_section))
|
||||
time.sleep(1)
|
||||
continue
|
||||
|
||||
filename = message
|
||||
paste = Paste.Paste(filename)
|
||||
|
||||
signal.alarm(max_execution_time)
|
||||
try:
|
||||
# Do something with the message from the queue
|
||||
#print(filename)
|
||||
content = paste.get_p_content()
|
||||
date = str(paste._get_p_date())
|
||||
search_binary(content,message, date)
|
||||
|
||||
except TimeoutException:
|
||||
p.incr_module_timeout_statistic()
|
||||
print ("{0} processing timeout".format(paste.p_path))
|
||||
continue
|
||||
else:
|
||||
signal.alarm(0)
|
||||
238
bin/Decoder.py
Executable file
238
bin/Decoder.py
Executable file
|
|
@ -0,0 +1,238 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
Decoder module
|
||||
|
||||
Dectect Binary and decode it
|
||||
"""
|
||||
import time
|
||||
import os
|
||||
import redis
|
||||
import base64
|
||||
from hashlib import sha1
|
||||
import magic
|
||||
import json
|
||||
import datetime
|
||||
|
||||
from pubsublogger import publisher
|
||||
|
||||
from Helper import Process
|
||||
from packages import Paste
|
||||
|
||||
import re
|
||||
import signal
|
||||
|
||||
class TimeoutException(Exception):
|
||||
pass
|
||||
|
||||
def timeout_handler(signum, frame):
|
||||
raise TimeoutException
|
||||
|
||||
signal.signal(signal.SIGALRM, timeout_handler)
|
||||
|
||||
def hex_decoder(hexStr):
|
||||
#hexStr = ''.join( hex_string.split(" ") )
|
||||
return bytes(bytearray([int(hexStr[i:i+2], 16) for i in range(0, len(hexStr), 2)]))
|
||||
|
||||
def binary_decoder(binary_string):
|
||||
return bytes(bytearray([int(binary_string[i:i+8], 2) for i in range(0, len(binary_string), 8)]))
|
||||
|
||||
def base64_decoder(base64_string):
|
||||
return base64.b64decode(base64_string)
|
||||
|
||||
def decode_string(content, message, date, encoded_list, decoder_name, encoded_min_size):
|
||||
find = False
|
||||
for encoded in encoded_list:
|
||||
if len(encoded) >= encoded_min_size:
|
||||
decode = decoder_function[decoder_name](encoded)
|
||||
find = True
|
||||
|
||||
save_hash(decoder_name, message, date, decode)
|
||||
|
||||
#remove encoded from paste content
|
||||
content = content.replace(encoded, '', 1)
|
||||
|
||||
if(find):
|
||||
set_out_paste(decoder_name, message)
|
||||
|
||||
return content
|
||||
|
||||
# # TODO: FIXME check db
|
||||
def save_hash(decoder_name, message, date, decoded):
|
||||
print(decoder_name)
|
||||
type = magic.from_buffer(decoded, mime=True)
|
||||
hash = sha1(decoded).hexdigest()
|
||||
print(hash)
|
||||
|
||||
data = {}
|
||||
data['name'] = hash
|
||||
data['date'] = datetime.datetime.now().strftime("%d/%m/%y")
|
||||
data['origin'] = message
|
||||
data['estimated type'] = type
|
||||
json_data = json.dumps(data)
|
||||
|
||||
date_paste = '{}/{}/{}'.format(date[0:4], date[4:6], date[6:8])
|
||||
date_key = date[0:4] + date[4:6] + date[6:8]
|
||||
|
||||
serv_metadata.incrby(decoder_name+'_decoded:'+date_key, 1)
|
||||
serv_metadata.zincrby('hash_date:'+date_key, hash, 1)
|
||||
serv_metadata.zincrby(decoder_name+'_date:'+date_key, hash, 1)
|
||||
|
||||
# first time we see this hash
|
||||
if not serv_metadata.hexists('metadata_hash:'+hash, 'estimated_type'):
|
||||
serv_metadata.hset('metadata_hash:'+hash, 'first_seen', date_paste)
|
||||
serv_metadata.hset('metadata_hash:'+hash, 'last_seen', date_paste)
|
||||
else:
|
||||
serv_metadata.hset('metadata_hash:'+hash, 'last_seen', date_paste)
|
||||
|
||||
# first time we see this hash (all encoding) on this paste
|
||||
if serv_metadata.zscore('nb_seen_hash:'+hash, message) is None:
|
||||
serv_metadata.hincrby('metadata_hash:'+hash, 'nb_seen_in_all_pastes', 1)
|
||||
serv_metadata.sadd('hash_paste:'+message, hash) # paste - hash map
|
||||
# create hash metadata
|
||||
serv_metadata.hset('metadata_hash:'+hash, 'estimated_type', type)
|
||||
serv_metadata.sadd('hash_all_type', type)
|
||||
|
||||
# first time we see this hash encoding on this paste
|
||||
if serv_metadata.zscore(decoder_name+'_hash:'+hash, message) is None:
|
||||
print('first '+decoder_name)
|
||||
|
||||
serv_metadata.sadd(decoder_name+'_paste:'+message, hash) # paste - hash map
|
||||
|
||||
# create hash metadata
|
||||
serv_metadata.sadd('hash_'+ decoder_name +'_all_type', type)
|
||||
|
||||
# first time we see this hash today
|
||||
if serv_metadata.zscore('hash_date:'+date_key, hash) is None:
|
||||
serv_metadata.zincrby('hash_type:'+type, date_key, 1)
|
||||
|
||||
# first time we see this hash encoding today
|
||||
if serv_metadata.zscore(decoder_name+'_date:'+date_key, hash) is None:
|
||||
serv_metadata.zincrby(decoder_name+'_type:'+type, date_key, 1)
|
||||
|
||||
save_hash_on_disk(decoded, type, hash, json_data)
|
||||
print('found {} '.format(type))
|
||||
|
||||
serv_metadata.hincrby('metadata_hash:'+hash, decoder_name+'_decoder', 1)
|
||||
|
||||
serv_metadata.zincrby(decoder_name+'_type:'+type, date_key, 1)
|
||||
|
||||
serv_metadata.zincrby('nb_seen_hash:'+hash, message, 1)# hash - paste map
|
||||
serv_metadata.zincrby(decoder_name+'_hash:'+hash, message, 1) # number of b64 on this paste
|
||||
|
||||
|
||||
def save_hash_on_disk(decode, type, hash, json_data):
|
||||
|
||||
local_filename_hash = os.path.join(p.config.get("Directories", "hash"), type, hash[:2], hash)
|
||||
filename_hash = os.path.join(os.environ['AIL_HOME'], local_filename_hash)
|
||||
|
||||
filename_json = os.path.join(os.environ['AIL_HOME'],
|
||||
p.config.get("Directories", "hash"), type, hash[:2], hash + '.json')
|
||||
|
||||
dirname = os.path.dirname(filename_hash)
|
||||
if not os.path.exists(dirname):
|
||||
os.makedirs(dirname)
|
||||
|
||||
with open(filename_hash, 'wb') as f:
|
||||
f.write(decode)
|
||||
|
||||
# create hash metadata
|
||||
serv_metadata.hset('metadata_hash:'+hash, 'saved_path', local_filename_hash)
|
||||
serv_metadata.hset('metadata_hash:'+hash, 'size', os.path.getsize(filename_hash))
|
||||
|
||||
with open(filename_json, 'w') as f:
|
||||
f.write(json_data)
|
||||
|
||||
def set_out_paste(decoder_name, message):
|
||||
publisher.warning(decoder_name+' decoded')
|
||||
#Send to duplicate
|
||||
p.populate_set_out(message, 'Duplicate')
|
||||
#send to Browse_warning_paste
|
||||
msg = (decoder_name+';{}'.format(message))
|
||||
p.populate_set_out( msg, 'alertHandler')
|
||||
|
||||
msg = 'infoleak:automatic-detection="'+decoder_name+'";{}'.format(message)
|
||||
p.populate_set_out(msg, 'Tags')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
|
||||
# Port of the redis instance used by pubsublogger
|
||||
publisher.port = 6380
|
||||
# Script is the default channel used for the modules.
|
||||
publisher.channel = 'Script'
|
||||
|
||||
# Section name in bin/packages/modules.cfg
|
||||
config_section = 'Decoder'
|
||||
|
||||
# Setup the I/O queues
|
||||
p = Process(config_section)
|
||||
|
||||
serv_metadata = redis.StrictRedis(
|
||||
host=p.config.get("ARDB_Metadata", "host"),
|
||||
port=p.config.getint("ARDB_Metadata", "port"),
|
||||
db=p.config.getint("ARDB_Metadata", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
# Sent to the logging a description of the module
|
||||
publisher.info("Decoder started")
|
||||
|
||||
regex_binary = '[0-1]{40,}'
|
||||
#regex_hex = '(0[xX])?[A-Fa-f0-9]{40,}'
|
||||
regex_hex = '[A-Fa-f0-9]{40,}'
|
||||
regex_base64 = '(?:[A-Za-z0-9+/]{4}){2,}(?:[A-Za-z0-9+/]{2}[AEIMQUYcgkosw048]=|[A-Za-z0-9+/][AQgw]==)'
|
||||
|
||||
re.compile(regex_binary)
|
||||
re.compile(regex_hex)
|
||||
re.compile(regex_base64)
|
||||
|
||||
# map decoder function
|
||||
decoder_function = {'binary':binary_decoder,'hexadecimal':hex_decoder, 'base64':base64_decoder}
|
||||
|
||||
hex_max_execution_time = p.config.getint("Hex", "max_execution_time")
|
||||
binary_max_execution_time = p.config.getint("Binary", "max_execution_time")
|
||||
base64_max_execution_time = p.config.getint("Base64", "max_execution_time")
|
||||
|
||||
# list all decoder yith regex,
|
||||
decoder_binary = {'name': 'binary', 'regex': regex_binary, 'encoded_min_size': 300, 'max_execution_time': binary_max_execution_time}
|
||||
decoder_hexadecimal = {'name': 'hexadecimal', 'regex': regex_hex, 'encoded_min_size': 300, 'max_execution_time': hex_max_execution_time}
|
||||
decoder_base64 = {'name': 'base64', 'regex': regex_base64, 'encoded_min_size': 40, 'max_execution_time': base64_max_execution_time}
|
||||
|
||||
decoder_order = [ decoder_base64, decoder_binary, decoder_hexadecimal, decoder_base64]
|
||||
|
||||
for decoder in decoder_order:
|
||||
serv_metadata.sadd('all_decoder', decoder['name'])
|
||||
|
||||
# Endless loop getting messages from the input queue
|
||||
while True:
|
||||
# Get one message from the input queue
|
||||
message = p.get_from_set()
|
||||
if message is None:
|
||||
|
||||
publisher.debug("{} queue is empty, waiting".format(config_section))
|
||||
time.sleep(1)
|
||||
continue
|
||||
|
||||
filename = message
|
||||
paste = Paste.Paste(filename)
|
||||
|
||||
# Do something with the message from the queue
|
||||
content = paste.get_p_content()
|
||||
date = str(paste._get_p_date())
|
||||
|
||||
for decoder in decoder_order: # add threshold and size limit
|
||||
|
||||
# max execution time on regex
|
||||
signal.alarm(decoder['max_execution_time'])
|
||||
try:
|
||||
encoded_list = re.findall(decoder['regex'], content)
|
||||
except TimeoutException:
|
||||
encoded_list = []
|
||||
p.incr_module_timeout_statistic() # add encoder type
|
||||
print ("{0} processing timeout".format(paste.p_path))
|
||||
continue
|
||||
else:
|
||||
signal.alarm(0)
|
||||
|
||||
if(len(encoded_list) > 0):
|
||||
content = decode_string(content, message, date, encoded_list, decoder['name'], decoder['encoded_min_size'])
|
||||
|
|
@ -135,6 +135,12 @@ class Process(object):
|
|||
db=self.config.get('RedisPubSub', 'db'),
|
||||
decode_responses=True)
|
||||
|
||||
self.serv_statistics = redis.StrictRedis(
|
||||
host=self.config.get('ARDB_Statistics', 'host'),
|
||||
port=self.config.get('ARDB_Statistics', 'port'),
|
||||
db=self.config.get('ARDB_Statistics', 'db'),
|
||||
decode_responses=True)
|
||||
|
||||
self.moduleNum = os.getpid()
|
||||
|
||||
def populate_set_in(self):
|
||||
|
|
@ -164,36 +170,39 @@ class Process(object):
|
|||
return None
|
||||
|
||||
else:
|
||||
#try:
|
||||
if '.gz' in message:
|
||||
path = message.split(".")[-2].split("/")[-1]
|
||||
#find start of path with AIL_HOME
|
||||
index_s = message.find(os.environ['AIL_HOME'])
|
||||
#Stop when .gz
|
||||
index_e = message.find(".gz")+3
|
||||
if(index_s == -1):
|
||||
complete_path = message[0:index_e]
|
||||
try:
|
||||
if '.gz' in message:
|
||||
path = message.split(".")[-2].split("/")[-1]
|
||||
#find start of path with AIL_HOME
|
||||
index_s = message.find(os.environ['AIL_HOME'])
|
||||
#Stop when .gz
|
||||
index_e = message.find(".gz")+3
|
||||
if(index_s == -1):
|
||||
complete_path = message[0:index_e]
|
||||
else:
|
||||
complete_path = message[index_s:index_e]
|
||||
|
||||
else:
|
||||
complete_path = message[index_s:index_e]
|
||||
path = "-"
|
||||
complete_path = "?"
|
||||
|
||||
else:
|
||||
path = "-"
|
||||
complete_path = "?"
|
||||
value = str(timestamp) + ", " + path
|
||||
self.r_temp.set("MODULE_"+self.subscriber_name + "_" + str(self.moduleNum), value)
|
||||
self.r_temp.set("MODULE_"+self.subscriber_name + "_" + str(self.moduleNum) + "_PATH", complete_path)
|
||||
self.r_temp.sadd("MODULE_TYPE_"+self.subscriber_name, str(self.moduleNum))
|
||||
|
||||
value = str(timestamp) + ", " + path
|
||||
self.r_temp.set("MODULE_"+self.subscriber_name + "_" + str(self.moduleNum), value)
|
||||
self.r_temp.set("MODULE_"+self.subscriber_name + "_" + str(self.moduleNum) + "_PATH", complete_path)
|
||||
self.r_temp.sadd("MODULE_TYPE_"+self.subscriber_name, str(self.moduleNum))
|
||||
return message
|
||||
curr_date = datetime.date.today()
|
||||
self.serv_statistics.hincrby(curr_date.strftime("%Y%m%d"),'paste_by_modules_in:'+self.subscriber_name, 1)
|
||||
return message
|
||||
|
||||
#except:
|
||||
#print('except')
|
||||
#path = "?"
|
||||
#value = str(timestamp) + ", " + path
|
||||
#self.r_temp.set("MODULE_"+self.subscriber_name + "_" + str(self.moduleNum), value)
|
||||
#self.r_temp.set("MODULE_"+self.subscriber_name + "_" + str(self.moduleNum) + "_PATH", "?")
|
||||
#self.r_temp.sadd("MODULE_TYPE_"+self.subscriber_name, str(self.moduleNum))
|
||||
#return message
|
||||
except:
|
||||
print('except')
|
||||
path = "?"
|
||||
value = str(timestamp) + ", " + path
|
||||
self.r_temp.set("MODULE_"+self.subscriber_name + "_" + str(self.moduleNum), value)
|
||||
self.r_temp.set("MODULE_"+self.subscriber_name + "_" + str(self.moduleNum) + "_PATH", "?")
|
||||
self.r_temp.sadd("MODULE_TYPE_"+self.subscriber_name, str(self.moduleNum))
|
||||
return message
|
||||
|
||||
def populate_set_out(self, msg, channel=None):
|
||||
# multiproc
|
||||
|
|
@ -220,3 +229,7 @@ class Process(object):
|
|||
time.sleep(1)
|
||||
continue
|
||||
self.pubsub.publish(message)
|
||||
|
||||
def incr_module_timeout_statistic(self):
|
||||
curr_date = datetime.date.today()
|
||||
self.serv_statistics.hincrby(curr_date.strftime("%Y%m%d"),'paste_by_modules_timeout:'+self.subscriber_name, 1)
|
||||
|
|
|
|||
|
|
@ -168,7 +168,7 @@ function launching_scripts {
|
|||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Keys" bash -c 'cd '${AIL_BIN}'; ./Keys.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Base64" bash -c 'cd '${AIL_BIN}'; ./Base64.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "Decoder" bash -c 'cd '${AIL_BIN}'; ./Decoder.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c 'cd '${AIL_BIN}'; ./Bitcoin.py; read x'
|
||||
sleep 0.1
|
||||
|
|
|
|||
|
|
@ -150,16 +150,18 @@ if __name__ == "__main__":
|
|||
if flag_the_hive:
|
||||
try:
|
||||
HiveApi = TheHiveApi(the_hive_url, the_hive_key, cert = the_hive_verifycert)
|
||||
r_serv_db.set('ail:thehive', True)
|
||||
except:
|
||||
HiveApi = False
|
||||
flag_the_hive = False
|
||||
r_serv_db.set('ail:thehive', False)
|
||||
print('Not connected to The HIVE')
|
||||
else:
|
||||
HiveApi = False
|
||||
|
||||
if HiveApi != False and flag_the_hive:
|
||||
try:
|
||||
HiveApi.get_alert(0)
|
||||
r_serv_db.set('ail:thehive', True)
|
||||
print('Connected to The HIVE:', the_hive_url)
|
||||
except thehive4py.exceptions.AlertException:
|
||||
HiveApi = False
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ if __name__ == "__main__":
|
|||
|
||||
regex = '|'.join(regexs)
|
||||
while True:
|
||||
signal.alarm(max_execution_time)
|
||||
filepath = p.get_from_set()
|
||||
if filepath is None:
|
||||
publisher.debug("Script Release is Idling 10s")
|
||||
|
|
@ -47,7 +48,7 @@ if __name__ == "__main__":
|
|||
paste = Paste.Paste(filepath)
|
||||
content = paste.get_p_content()
|
||||
|
||||
signal.alarm(max_execution_time)
|
||||
#signal.alarm(max_execution_time)
|
||||
try:
|
||||
releases = set(re.findall(regex, content))
|
||||
if len(releases) == 0:
|
||||
|
|
@ -61,7 +62,8 @@ if __name__ == "__main__":
|
|||
publisher.info(to_print)
|
||||
|
||||
except TimeoutException:
|
||||
print ("{0} processing timeout".format(paste.p_path))
|
||||
continue
|
||||
p.incr_module_timeout_statistic()
|
||||
print ("{0} processing timeout".format(paste.p_path))
|
||||
continue
|
||||
else:
|
||||
signal.alarm(0)
|
||||
|
|
|
|||
|
|
@ -167,6 +167,7 @@ if __name__ == '__main__':
|
|||
try:
|
||||
Analyse(message, server)
|
||||
except TimeoutException:
|
||||
p.incr_module_timeout_statistic()
|
||||
print ("{0} processing timeout".format(message))
|
||||
continue
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ This module create tags.
|
|||
import redis
|
||||
|
||||
import time
|
||||
import datetime
|
||||
|
||||
from pubsublogger import publisher
|
||||
from Helper import Process
|
||||
|
|
@ -41,6 +42,12 @@ if __name__ == '__main__':
|
|||
db=p.config.get("ARDB_Metadata", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
serv_statistics = redis.StrictRedis(
|
||||
host=p.config.get('ARDB_Statistics', 'host'),
|
||||
port=p.config.get('ARDB_Statistics', 'port'),
|
||||
db=p.config.get('ARDB_Statistics', 'db'),
|
||||
decode_responses=True)
|
||||
|
||||
# Sent to the logging a description of the module
|
||||
publisher.info("Tags module started")
|
||||
|
||||
|
|
@ -67,4 +74,6 @@ if __name__ == '__main__':
|
|||
print(" tagged: {}".format(tag))
|
||||
server_metadata.sadd('tag:'+path, tag)
|
||||
|
||||
curr_date = datetime.date.today()
|
||||
serv_statistics.hincrby(curr_date.strftime("%Y%m%d"),'paste_tagged:'+tag, 1)
|
||||
p.populate_set_out(message, 'MISP_The_Hive_feeder')
|
||||
|
|
|
|||
|
|
@ -60,8 +60,9 @@ if __name__ == "__main__":
|
|||
msg = '{} {} {}'.format(paste.p_path, word, score)
|
||||
p.populate_set_out(msg)
|
||||
except TimeoutException:
|
||||
print ("{0} processing timeout".format(paste.p_path))
|
||||
continue
|
||||
p.incr_module_timeout_statistic()
|
||||
print ("{0} processing timeout".format(paste.p_path))
|
||||
continue
|
||||
else:
|
||||
signal.alarm(0)
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -124,6 +124,7 @@ if __name__ == "__main__":
|
|||
except ipaddress.AddressValueError:
|
||||
continue
|
||||
cc = getattr(l, 'cc')
|
||||
asn = ''
|
||||
if getattr(l, 'asn') is not None:
|
||||
asn = getattr(l, 'asn')[2:] #remobe b'
|
||||
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@ class Paste(object):
|
|||
|
||||
:Example:
|
||||
|
||||
PST = Paste("/home/2013/ZEeGaez5.gz")
|
||||
PST = Paste("/home/2013/01/12/ZEeGaez5.gz")
|
||||
|
||||
"""
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
bloomfilters = Blooms
|
||||
dicofilters = Dicos
|
||||
pastes = PASTES
|
||||
hash = HASHS
|
||||
base64 = BASE64
|
||||
crawled = crawled
|
||||
crawled_screenshot = CRAWLED_SCREENSHOT
|
||||
|
|
@ -69,6 +70,14 @@ max_execution_time = 90
|
|||
path = Base64/
|
||||
max_execution_time = 60
|
||||
|
||||
[Binary]
|
||||
path = Base64/
|
||||
max_execution_time = 60
|
||||
|
||||
[Hex]
|
||||
path = Base64/
|
||||
max_execution_time = 60
|
||||
|
||||
[Modules_Duplicates]
|
||||
#Number of month to look back
|
||||
maximum_month_range = 3
|
||||
|
|
|
|||
|
|
@ -125,7 +125,7 @@ publish = Redis_Duplicate,Redis_alertHandler,Redis_Tags
|
|||
subscribe = Redis_ApiKey
|
||||
publish = Redis_Duplicate,Redis_alertHandler,Redis_Tags
|
||||
|
||||
[Base64]
|
||||
[Decoder]
|
||||
subscribe = Redis_Global
|
||||
publish = Redis_Duplicate,Redis_alertHandler,Redis_Tags
|
||||
|
||||
|
|
|
|||
|
|
@ -55,6 +55,9 @@ def create_paste(uuid, paste_content, ltags, ltagsgalaxies, name):
|
|||
print(' {} send to Global'.format(save_path))
|
||||
r_serv_log_submit.sadd(uuid + ':paste_submit_link', full_path)
|
||||
|
||||
curr_date = datetime.date.today()
|
||||
serv_statistics.hincrby(curr_date.strftime("%Y%m%d"),'submit_paste', 1)
|
||||
|
||||
return 0
|
||||
|
||||
def addError(uuid, errorMessage):
|
||||
|
|
@ -67,6 +70,8 @@ def addError(uuid, errorMessage):
|
|||
def abord_file_submission(uuid, errorMessage):
|
||||
addError(uuid, errorMessage)
|
||||
r_serv_log_submit.set(uuid + ':end', 1)
|
||||
curr_date = datetime.date.today()
|
||||
serv_statistics.hincrby(curr_date.strftime("%Y%m%d"),'submit_abord', 1)
|
||||
remove_submit_uuid(uuid)
|
||||
|
||||
|
||||
|
|
@ -161,6 +166,12 @@ if __name__ == "__main__":
|
|||
db=cfg.getint("ARDB_Metadata", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
serv_statistics = redis.StrictRedis(
|
||||
host=cfg.get('ARDB_Statistics', 'host'),
|
||||
port=cfg.getint('ARDB_Statistics', 'port'),
|
||||
db=cfg.getint('ARDB_Statistics', 'db'),
|
||||
decode_responses=True)
|
||||
|
||||
expire_time = 120
|
||||
MAX_FILE_SIZE = 1000000000
|
||||
ALLOWED_EXTENSIONS = ['txt', 'sh', 'pdf']
|
||||
|
|
|
|||
BIN
doc/screenshots/ail-hashedfiles.png
Normal file
BIN
doc/screenshots/ail-hashedfiles.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 138 KiB |
BIN
doc/screenshots/hashedfile-graph.png
Normal file
BIN
doc/screenshots/hashedfile-graph.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 562 KiB |
|
|
@ -153,3 +153,18 @@ UPLOAD_FOLDER = os.path.join(os.environ['AIL_FLASK'], 'submitted')
|
|||
SCREENSHOT_FOLDER = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "crawled_screenshot"))
|
||||
|
||||
max_dashboard_logs = int(cfg.get("Flask", "max_dashboard_logs"))
|
||||
|
||||
# VT
|
||||
try:
|
||||
from virusTotalKEYS import vt_key
|
||||
if vt_key != '':
|
||||
vt_auth = vt_key
|
||||
vt_enabled = True
|
||||
print('VT submission is enabled')
|
||||
else:
|
||||
vt_enabled = False
|
||||
print('VT submission is disabled')
|
||||
except:
|
||||
vt_auth = {'apikey': cfg.get("Flask", "max_preview_char")}
|
||||
vt_enabled = False
|
||||
print('VT submission is disabled')
|
||||
|
|
|
|||
|
|
@ -32,6 +32,8 @@
|
|||
<div class="row">
|
||||
<!-- /.col-lg-12 -->
|
||||
|
||||
|
||||
|
||||
<form action="/PasteSubmit/submit" id="pasteSubmitForm" method="post" enctype=multipart/form-data onsubmit="submitPaste()">
|
||||
|
||||
<input type="hidden" id="tags_taxonomies" name="tags_taxonomies" value="test">
|
||||
|
|
|
|||
629
var/www/modules/hashDecoded/Flask_hashDecoded.py
Normal file
629
var/www/modules/hashDecoded/Flask_hashDecoded.py
Normal file
|
|
@ -0,0 +1,629 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
'''
|
||||
Flask functions and routes for the trending modules page
|
||||
'''
|
||||
import redis
|
||||
import os
|
||||
import datetime
|
||||
import json
|
||||
from Date import Date
|
||||
|
||||
from io import BytesIO
|
||||
import zipfile
|
||||
|
||||
import requests
|
||||
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, send_file
|
||||
|
||||
# ============ VARIABLES ============
|
||||
import Flask_config
|
||||
|
||||
app = Flask_config.app
|
||||
cfg = Flask_config.cfg
|
||||
r_serv_metadata = Flask_config.r_serv_metadata
|
||||
vt_enabled = Flask_config.vt_enabled
|
||||
vt_auth = Flask_config.vt_auth
|
||||
|
||||
hashDecoded = Blueprint('hashDecoded', __name__, template_folder='templates')
|
||||
|
||||
# ============ FUNCTIONS ============
|
||||
|
||||
def get_date_range(num_day):
|
||||
curr_date = datetime.date.today()
|
||||
date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2))
|
||||
date_list = []
|
||||
|
||||
for i in range(0, num_day+1):
|
||||
date_list.append(date.substract_day(i))
|
||||
|
||||
return list(reversed(date_list))
|
||||
|
||||
def substract_date(date_from, date_to):
|
||||
date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8]))
|
||||
date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8]))
|
||||
delta = date_to - date_from # timedelta
|
||||
l_date = []
|
||||
for i in range(delta.days + 1):
|
||||
date = date_from + datetime.timedelta(i)
|
||||
l_date.append( date.strftime('%Y%m%d') )
|
||||
return l_date
|
||||
|
||||
def list_sparkline_values(date_range_sparkline, hash):
|
||||
sparklines_value = []
|
||||
for date_day in date_range_sparkline:
|
||||
nb_seen_this_day = r_serv_metadata.zscore('hash_date:'+date_day, hash)
|
||||
if nb_seen_this_day is None:
|
||||
nb_seen_this_day = 0
|
||||
sparklines_value.append(int(nb_seen_this_day))
|
||||
return sparklines_value
|
||||
|
||||
def get_file_icon(estimated_type):
|
||||
file_type = estimated_type.split('/')[0]
|
||||
# set file icon
|
||||
if file_type == 'application':
|
||||
file_icon = 'fa-file '
|
||||
elif file_type == 'audio':
|
||||
file_icon = 'fa-file-video-o '
|
||||
elif file_type == 'image':
|
||||
file_icon = 'fa-file-image-o'
|
||||
elif file_type == 'text':
|
||||
file_icon = 'fa-file-text-o'
|
||||
else:
|
||||
file_icon = 'fa-file-o'
|
||||
|
||||
return file_icon
|
||||
|
||||
def get_file_icon_text(estimated_type):
|
||||
file_type = estimated_type.split('/')[0]
|
||||
# set file icon
|
||||
if file_type == 'application':
|
||||
file_icon_text = '\uf15b'
|
||||
elif file_type == 'audio':
|
||||
file_icon_text = '\uf1c7'
|
||||
elif file_type == 'image':
|
||||
file_icon_text = '\uf1c5'
|
||||
elif file_type == 'text':
|
||||
file_icon_text = '\uf15c'
|
||||
else:
|
||||
file_icon_text = '\uf15b'
|
||||
|
||||
return file_icon_text
|
||||
|
||||
def one():
|
||||
return 1
|
||||
|
||||
# ============= ROUTES ==============
|
||||
@hashDecoded.route("/hashDecoded/all_hash_search", methods=['POST'])
|
||||
def all_hash_search():
|
||||
date_from = request.form.get('date_from')
|
||||
date_to = request.form.get('date_to')
|
||||
type = request.form.get('type')
|
||||
encoding = request.form.get('encoding')
|
||||
return redirect(url_for('hashDecoded.hashDecoded_page', date_from=date_from, date_to=date_to, type=type, encoding=encoding))
|
||||
|
||||
@hashDecoded.route("/hashDecoded/", methods=['GET'])
|
||||
def hashDecoded_page():
|
||||
date_from = request.args.get('date_from')
|
||||
date_to = request.args.get('date_to')
|
||||
type = request.args.get('type')
|
||||
encoding = request.args.get('encoding')
|
||||
|
||||
if type == 'All types':
|
||||
type = None
|
||||
|
||||
if encoding == 'All encoding':
|
||||
encoding = None
|
||||
|
||||
#date_from = '20180628' or date_from = '2018-06-28'
|
||||
#date_to = '20180628' or date_to = '2018-06-28'
|
||||
|
||||
# verify file type input
|
||||
if type is not None:
|
||||
#retrieve + char
|
||||
type = type.replace(' ', '+')
|
||||
if type not in r_serv_metadata.smembers('hash_all_type'):
|
||||
type = None
|
||||
|
||||
all_encoding = r_serv_metadata.smembers('all_decoder')
|
||||
# verify encoding input
|
||||
if encoding is not None:
|
||||
if encoding not in all_encoding:
|
||||
encoding = None
|
||||
|
||||
date_range = []
|
||||
if date_from is not None and date_to is not None:
|
||||
#change format
|
||||
try:
|
||||
if len(date_from) != 8:
|
||||
date_from = date_from[0:4] + date_from[5:7] + date_from[8:10]
|
||||
date_to = date_to[0:4] + date_to[5:7] + date_to[8:10]
|
||||
date_range = substract_date(date_from, date_to)
|
||||
except:
|
||||
pass
|
||||
|
||||
if not date_range:
|
||||
date_range.append(datetime.date.today().strftime("%Y%m%d"))
|
||||
date_from = date_range[0][0:4] + '-' + date_range[0][4:6] + '-' + date_range[0][6:8]
|
||||
date_to = date_from
|
||||
|
||||
else:
|
||||
date_from = date_from[0:4] + '-' + date_from[4:6] + '-' + date_from[6:8]
|
||||
date_to = date_to[0:4] + '-' + date_to[4:6] + '-' + date_to[6:8]
|
||||
|
||||
# display day type bar chart
|
||||
if len(date_range) == 1 and type is None:
|
||||
daily_type_chart = True
|
||||
daily_date = date_range[0]
|
||||
else:
|
||||
daily_type_chart = False
|
||||
daily_date = None
|
||||
|
||||
l_64 = set()
|
||||
for date in date_range:
|
||||
if encoding is None:
|
||||
l_hash = r_serv_metadata.zrange('hash_date:' +date, 0, -1)
|
||||
else:
|
||||
l_hash = r_serv_metadata.zrange(encoding+'_date:' +date, 0, -1)
|
||||
if l_hash:
|
||||
for hash in l_hash:
|
||||
l_64.add(hash)
|
||||
|
||||
num_day_sparkline = 6
|
||||
date_range_sparkline = get_date_range(num_day_sparkline)
|
||||
|
||||
b64_metadata = []
|
||||
l_64 = list(l_64)
|
||||
for hash in l_64:
|
||||
# select requested base 64 type
|
||||
estimated_type = r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type')
|
||||
if type is not None:
|
||||
if estimated_type is not None:
|
||||
if estimated_type != type:
|
||||
continue
|
||||
|
||||
first_seen = r_serv_metadata.hget('metadata_hash:'+hash, 'first_seen')
|
||||
last_seen = r_serv_metadata.hget('metadata_hash:'+hash, 'last_seen')
|
||||
nb_seen_in_paste = r_serv_metadata.hget('metadata_hash:'+hash, 'nb_seen_in_all_pastes')
|
||||
size = r_serv_metadata.hget('metadata_hash:'+hash, 'size')
|
||||
|
||||
if hash is not None and first_seen is not None and \
|
||||
last_seen is not None and \
|
||||
nb_seen_in_paste is not None and \
|
||||
size is not None:
|
||||
|
||||
file_icon = get_file_icon(estimated_type)
|
||||
|
||||
if r_serv_metadata.hexists('metadata_hash:'+hash, 'vt_link'):
|
||||
b64_vt = True
|
||||
b64_vt_link = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_link')
|
||||
b64_vt_report = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_report')
|
||||
else:
|
||||
b64_vt = False
|
||||
b64_vt_link = ''
|
||||
b64_vt_report = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_report')
|
||||
# hash never refreshed
|
||||
if b64_vt_report is None:
|
||||
b64_vt_report = ''
|
||||
|
||||
sparklines_value = list_sparkline_values(date_range_sparkline, hash)
|
||||
|
||||
b64_metadata.append( (file_icon, estimated_type, hash, nb_seen_in_paste, size, first_seen, last_seen, b64_vt, b64_vt_link, b64_vt_report, sparklines_value) )
|
||||
|
||||
l_type = r_serv_metadata.smembers('hash_all_type')
|
||||
|
||||
return render_template("hashDecoded.html", l_64=b64_metadata, vt_enabled=vt_enabled, l_type=l_type, type=type, daily_type_chart=daily_type_chart, daily_date=daily_date,
|
||||
encoding=encoding, all_encoding=all_encoding, date_from=date_from, date_to=date_to)
|
||||
|
||||
@hashDecoded.route('/hashDecoded/hash_by_type')
|
||||
def hash_by_type():
|
||||
type = request.args.get('type')
|
||||
type = 'text/plain'
|
||||
return render_template('hash_type.html',type = type)
|
||||
|
||||
@hashDecoded.route('/hashDecoded/hash_hash')
|
||||
def hash_hash():
|
||||
hash = request.args.get('hash')
|
||||
return render_template('hash_hash.html')
|
||||
|
||||
@hashDecoded.route('/hashDecoded/showHash')
|
||||
def showHash():
|
||||
hash = request.args.get('hash')
|
||||
#hash = 'e02055d3efaad5d656345f6a8b1b6be4fe8cb5ea'
|
||||
|
||||
# TODO FIXME show error
|
||||
if hash is None:
|
||||
return hashDecoded_page()
|
||||
|
||||
estimated_type = r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type')
|
||||
# hash not found
|
||||
# TODO FIXME show error
|
||||
if estimated_type is None:
|
||||
return hashDecoded_page()
|
||||
|
||||
else:
|
||||
file_icon = get_file_icon(estimated_type)
|
||||
size = r_serv_metadata.hget('metadata_hash:'+hash, 'size')
|
||||
first_seen = r_serv_metadata.hget('metadata_hash:'+hash, 'first_seen')
|
||||
last_seen = r_serv_metadata.hget('metadata_hash:'+hash, 'last_seen')
|
||||
nb_seen_in_all_pastes = r_serv_metadata.hget('metadata_hash:'+hash, 'nb_seen_in_all_pastes')
|
||||
|
||||
# get all encoding for this hash
|
||||
list_hash_decoder = []
|
||||
list_decoder = r_serv_metadata.smembers('all_decoder')
|
||||
for decoder in list_decoder:
|
||||
encoding = r_serv_metadata.hget('metadata_hash:'+hash, decoder+'_decoder')
|
||||
if encoding is not None:
|
||||
list_hash_decoder.append({'encoding': decoder, 'nb_seen': encoding})
|
||||
|
||||
num_day_type = 6
|
||||
date_range_sparkline = get_date_range(num_day_type)
|
||||
sparkline_values = list_sparkline_values(date_range_sparkline, hash)
|
||||
|
||||
if r_serv_metadata.hexists('metadata_hash:'+hash, 'vt_link'):
|
||||
b64_vt = True
|
||||
b64_vt_link = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_link')
|
||||
b64_vt_report = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_report')
|
||||
else:
|
||||
b64_vt = False
|
||||
b64_vt_link = ''
|
||||
b64_vt_report = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_report')
|
||||
# hash never refreshed
|
||||
if b64_vt_report is None:
|
||||
b64_vt_report = ''
|
||||
|
||||
return render_template('showHash.html', hash=hash, vt_enabled=vt_enabled, b64_vt=b64_vt, b64_vt_link=b64_vt_link,
|
||||
b64_vt_report=b64_vt_report,
|
||||
size=size, estimated_type=estimated_type, file_icon=file_icon,
|
||||
first_seen=first_seen, list_hash_decoder=list_hash_decoder,
|
||||
last_seen=last_seen, nb_seen_in_all_pastes=nb_seen_in_all_pastes, sparkline_values=sparkline_values)
|
||||
|
||||
@app.route('/hashDecoded/downloadHash')
|
||||
def downloadHash():
|
||||
hash = request.args.get('hash')
|
||||
# sanitize hash
|
||||
hash = hash.split('/')[0]
|
||||
|
||||
# hash exist
|
||||
if r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type') is not None:
|
||||
|
||||
b64_path = r_serv_metadata.hget('metadata_hash:'+hash, 'saved_path')
|
||||
b64_full_path = os.path.join(os.environ['AIL_HOME'], b64_path)
|
||||
hash_content = ''
|
||||
try:
|
||||
with open(b64_full_path, 'rb') as f:
|
||||
hash_content = f.read()
|
||||
|
||||
# zip buffer
|
||||
result = BytesIO()
|
||||
temp = BytesIO()
|
||||
temp.write(hash_content)
|
||||
|
||||
with zipfile.ZipFile(result, "w") as zf:
|
||||
#zf.setpassword(b"infected")
|
||||
zf.writestr( hash, temp.getvalue())
|
||||
|
||||
filename = hash + '.zip'
|
||||
result.seek(0)
|
||||
|
||||
return send_file(result, attachment_filename=filename, as_attachment=True)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
return 'Server Error'
|
||||
else:
|
||||
return 'hash: ' + hash + " don't exist"
|
||||
|
||||
@hashDecoded.route('/hashDecoded/hash_by_type_json')
|
||||
def hash_by_type_json():
|
||||
type = request.args.get('type')
|
||||
|
||||
#retrieve + char
|
||||
type = type.replace(' ', '+')
|
||||
|
||||
num_day_type = 30
|
||||
date_range = get_date_range(num_day_type)
|
||||
|
||||
#verify input
|
||||
if type in r_serv_metadata.smembers('hash_all_type'):
|
||||
type_value = []
|
||||
all_decoder = r_serv_metadata.smembers('all_decoder')
|
||||
|
||||
range_decoder = []
|
||||
for date in date_range:
|
||||
day_decoder = {}
|
||||
day_decoder['date']= date[0:4] + '-' + date[4:6] + '-' + date[6:8]
|
||||
for decoder in all_decoder:
|
||||
num_day_decoder = r_serv_metadata.zscore(decoder+'_type:'+type, date)
|
||||
if num_day_decoder is None:
|
||||
num_day_decoder = 0
|
||||
day_decoder[decoder]= num_day_decoder
|
||||
range_decoder.append(day_decoder)
|
||||
|
||||
|
||||
|
||||
return jsonify(range_decoder)
|
||||
else:
|
||||
return jsonify()
|
||||
|
||||
@hashDecoded.route('/hashDecoded/decoder_type_json')
|
||||
def decoder_type_json():
|
||||
date_from = request.args.get('date_from')
|
||||
date_to = request.args.get('date_to')
|
||||
|
||||
typ = request.args.get('type')
|
||||
|
||||
if typ == 'All types':
|
||||
typ = None
|
||||
|
||||
# verify file type input
|
||||
if typ is not None:
|
||||
#retrieve + char
|
||||
typ = typ.replace(' ', '+')
|
||||
if typ not in r_serv_metadata.smembers('hash_all_type'):
|
||||
typ = None
|
||||
|
||||
all_decoder = r_serv_metadata.smembers('all_decoder')
|
||||
# sort DESC decoder for color
|
||||
all_decoder = sorted(all_decoder)
|
||||
|
||||
date_range = []
|
||||
if date_from is not None and date_to is not None:
|
||||
#change format
|
||||
try:
|
||||
if len(date_from) != 8:
|
||||
date_from = date_from[0:4] + date_from[5:7] + date_from[8:10]
|
||||
date_to = date_to[0:4] + date_to[5:7] + date_to[8:10]
|
||||
date_range = substract_date(date_from, date_to)
|
||||
except:
|
||||
pass
|
||||
|
||||
if not date_range:
|
||||
date_range.append(datetime.date.today().strftime("%Y%m%d"))
|
||||
|
||||
nb_decoded = {}
|
||||
for decoder in all_decoder:
|
||||
nb_decoded[decoder] = 0
|
||||
|
||||
for date in date_range:
|
||||
for decoder in all_decoder:
|
||||
if typ is None:
|
||||
nb_decod = r_serv_metadata.get(decoder+'_decoded:'+date)
|
||||
else:
|
||||
nb_decod = r_serv_metadata.zscore(decoder+'_type:'+typ, date)
|
||||
|
||||
if nb_decod is not None:
|
||||
nb_decoded[decoder] = nb_decoded[decoder] + int(nb_decod)
|
||||
|
||||
to_json = []
|
||||
for decoder in all_decoder:
|
||||
to_json.append({'name': decoder, 'value': nb_decoded[decoder]})
|
||||
return jsonify(to_json)
|
||||
|
||||
|
||||
@hashDecoded.route('/hashDecoded/daily_type_json')
|
||||
def daily_type_json():
|
||||
date = request.args.get('date')
|
||||
|
||||
daily_type = set()
|
||||
l_b64 = r_serv_metadata.zrange('hash_date:' +date, 0, -1)
|
||||
for hash in l_b64:
|
||||
estimated_type = r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type')
|
||||
if estimated_type is not None:
|
||||
daily_type.add(estimated_type)
|
||||
|
||||
type_value = []
|
||||
for day_type in daily_type:
|
||||
num_day_type = r_serv_metadata.zscore('hash_type:'+day_type, date)
|
||||
type_value.append({ 'date' : day_type, 'value' : int( num_day_type )})
|
||||
|
||||
return jsonify(type_value)
|
||||
|
||||
@hashDecoded.route('/hashDecoded/range_type_json')
|
||||
def range_type_json():
|
||||
date_from = request.args.get('date_from')
|
||||
date_to = request.args.get('date_to')
|
||||
|
||||
date_range = []
|
||||
if date_from is not None and date_to is not None:
|
||||
#change format
|
||||
if len(date_from) != 8:
|
||||
date_from = date_from[0:4] + date_from[5:7] + date_from[8:10]
|
||||
date_to = date_to[0:4] + date_to[5:7] + date_to[8:10]
|
||||
date_range = substract_date(date_from, date_to)
|
||||
|
||||
if not date_range:
|
||||
date_range.append(datetime.date.today().strftime("%Y%m%d"))
|
||||
|
||||
all_type = set()
|
||||
for date in date_range:
|
||||
l_hash = r_serv_metadata.zrange('hash_date:' +date, 0, -1)
|
||||
if l_hash:
|
||||
for hash in l_hash:
|
||||
estimated_type = r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type')
|
||||
all_type.add(estimated_type)
|
||||
|
||||
range_type = []
|
||||
|
||||
for date in date_range:
|
||||
if len(date_range) == 1:
|
||||
if date==date_from and date==date_to:
|
||||
for type in all_type:
|
||||
day_type = {}
|
||||
day_type['date']= type
|
||||
list_decoder = r_serv_metadata.smembers('all_decoder')
|
||||
for decoder in list_decoder:
|
||||
num_day_decoder = r_serv_metadata.zscore(decoder+'_type:'+type, date)
|
||||
if num_day_decoder is None:
|
||||
num_day_decoder = 0
|
||||
day_type[decoder]= num_day_decoder
|
||||
range_type.append(day_type)
|
||||
else:
|
||||
range_type = ''
|
||||
else:
|
||||
day_type = {}
|
||||
day_type['date']= date[0:4] + '-' + date[4:6] + '-' + date[6:8]
|
||||
for type in all_type:
|
||||
num_day_type = r_serv_metadata.zscore('hash_type:'+type, date)
|
||||
if num_day_type is None:
|
||||
num_day_type = 0
|
||||
day_type[type]= num_day_type
|
||||
range_type.append(day_type)
|
||||
|
||||
return jsonify(range_type)
|
||||
|
||||
@hashDecoded.route('/hashDecoded/hash_graph_line_json')
|
||||
def hash_graph_line_json():
|
||||
hash = request.args.get('hash')
|
||||
date_from = request.args.get('date_from')
|
||||
date_to = request.args.get('date_to')
|
||||
|
||||
#hash = '9c748d28d78a64aef99e7ba866a433eb635c6d7a'
|
||||
|
||||
if date_from is None or date_to is None:
|
||||
nb_days_seen_in_pastes = 30
|
||||
else:
|
||||
# # TODO: # FIXME:
|
||||
nb_days_seen_in_pastes = 30
|
||||
|
||||
date_range_seen_in_pastes = get_date_range(nb_days_seen_in_pastes)
|
||||
|
||||
#verify input
|
||||
if r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type') is not None:
|
||||
json_seen_in_paste = []
|
||||
for date in date_range_seen_in_pastes:
|
||||
nb_seen_this_day = r_serv_metadata.zscore('hash_date:'+date, hash)
|
||||
if nb_seen_this_day is None:
|
||||
nb_seen_this_day = 0
|
||||
date = date[0:4] + '-' + date[4:6] + '-' + date[6:8]
|
||||
json_seen_in_paste.append({ 'date' : date, 'value' : int( nb_seen_this_day )})
|
||||
|
||||
return jsonify(json_seen_in_paste)
|
||||
else:
|
||||
return jsonify()
|
||||
|
||||
|
||||
@hashDecoded.route('/hashDecoded/hash_graph_node_json')
|
||||
def hash_graph_node_json():
|
||||
hash = request.args.get('hash')
|
||||
|
||||
estimated_type = r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type')
|
||||
|
||||
if hash is not None and estimated_type is not None:
|
||||
|
||||
nodes_set_hash = set()
|
||||
nodes_set_paste = set()
|
||||
links_set = set()
|
||||
|
||||
url = hash
|
||||
first_seen = r_serv_metadata.hget('metadata_hash:'+hash, 'first_seen')
|
||||
last_seen = r_serv_metadata.hget('metadata_hash:'+hash, 'last_seen')
|
||||
nb_seen_in_paste = r_serv_metadata.hget('metadata_hash:'+hash, 'nb_seen_in_all_pastes')
|
||||
size = r_serv_metadata.hget('metadata_hash:'+hash, 'size')
|
||||
|
||||
nodes_set_hash.add((hash, 1, first_seen, last_seen, estimated_type, nb_seen_in_paste, size, url))
|
||||
|
||||
#get related paste
|
||||
l_pastes = r_serv_metadata.zrange('nb_seen_hash:'+hash, 0, -1)
|
||||
for paste in l_pastes:
|
||||
url = paste
|
||||
#nb_seen_in_this_paste = nb_in_file = int(r_serv_metadata.zscore('nb_seen_hash:'+hash, paste))
|
||||
nb_hash_in_paste = r_serv_metadata.scard('hash_paste:'+paste)
|
||||
|
||||
nodes_set_paste.add((paste, 2,nb_hash_in_paste,url))
|
||||
links_set.add((hash, paste))
|
||||
|
||||
l_hash = r_serv_metadata.smembers('hash_paste:'+paste)
|
||||
for child_hash in l_hash:
|
||||
if child_hash != hash:
|
||||
url = child_hash
|
||||
first_seen = r_serv_metadata.hget('metadata_hash:'+child_hash, 'first_seen')
|
||||
last_seen = r_serv_metadata.hget('metadata_hash:'+child_hash, 'last_seen')
|
||||
nb_seen_in_paste = r_serv_metadata.hget('metadata_hash:'+child_hash, 'nb_seen_in_all_pastes')
|
||||
size = r_serv_metadata.hget('metadata_hash:'+child_hash, 'size')
|
||||
estimated_type = r_serv_metadata.hget('metadata_hash:'+child_hash, 'estimated_type')
|
||||
|
||||
nodes_set_hash.add((child_hash, 3, first_seen, last_seen, estimated_type, nb_seen_in_paste, size, url))
|
||||
links_set.add((child_hash, paste))
|
||||
|
||||
#l_pastes_child = r_serv_metadata.zrange('nb_seen_hash:'+child_hash, 0, -1)
|
||||
#for child_paste in l_pastes_child:
|
||||
|
||||
nodes = []
|
||||
for node in nodes_set_hash:
|
||||
nodes.append({"id": node[0], "group": node[1], "first_seen": node[2], "last_seen": node[3], 'estimated_type': node[4], "nb_seen_in_paste": node[5], "size": node[6], 'icon': get_file_icon_text(node[4]),"url": url_for('hashDecoded.showHash', hash=node[7]), 'hash': True})
|
||||
for node in nodes_set_paste:
|
||||
nodes.append({"id": node[0], "group": node[1], "nb_seen_in_paste": node[2],"url": url_for('showsavedpastes.showsavedpaste', paste=node[3]), 'hash': False})
|
||||
links = []
|
||||
for link in links_set:
|
||||
links.append({"source": link[0], "target": link[1]})
|
||||
json = {"nodes": nodes, "links": links}
|
||||
return jsonify(json)
|
||||
|
||||
else:
|
||||
return jsonify({})
|
||||
|
||||
@hashDecoded.route('/hashDecoded/hash_types')
|
||||
def hash_types():
|
||||
date_from = 20180701
|
||||
date_to = 20180706
|
||||
return render_template('hash_types.html', date_from=date_from, date_to=date_to)
|
||||
|
||||
@hashDecoded.route('/hashDecoded/send_file_to_vt_js')
|
||||
def send_file_to_vt_js():
|
||||
hash = request.args.get('hash')
|
||||
|
||||
b64_path = r_serv_metadata.hget('metadata_hash:'+hash, 'saved_path')
|
||||
b64_full_path = os.path.join(os.environ['AIL_HOME'], b64_path)
|
||||
b64_content = ''
|
||||
with open(b64_full_path, 'rb') as f:
|
||||
b64_content = f.read()
|
||||
|
||||
files = {'file': (hash, b64_content)}
|
||||
response = requests.post('https://www.virustotal.com/vtapi/v2/file/scan', files=files, params= {'apikey': vt_auth})
|
||||
json_response = response.json()
|
||||
#print(json_response)
|
||||
|
||||
vt_link = json_response['permalink'].split('analysis')[0] + 'analysis/'
|
||||
r_serv_metadata.hset('metadata_hash:'+hash, 'vt_link', vt_link)
|
||||
vt_report = 'Please Refresh'
|
||||
r_serv_metadata.hset('metadata_hash:'+hash, 'vt_report', vt_report)
|
||||
|
||||
return jsonify({'vt_link': vt_link, 'vt_report': vt_report})
|
||||
|
||||
|
||||
@hashDecoded.route('/hashDecoded/update_vt_result')
|
||||
def update_vt_result():
|
||||
hash = request.args.get('hash')
|
||||
|
||||
params = {'apikey': vt_auth, 'resource': hash}
|
||||
response = requests.get('https://www.virustotal.com/vtapi/v2/file/report',params=params)
|
||||
if response.status_code == 200:
|
||||
json_response = response.json()
|
||||
response_code = json_response['response_code']
|
||||
# report exist
|
||||
if response_code == 1:
|
||||
total = json_response['total']
|
||||
positive = json_response['positives']
|
||||
|
||||
b64_vt_report = 'Detection {}/{}'.format(positive,total)
|
||||
# no report found
|
||||
elif response_code == 0:
|
||||
b64_vt_report = 'No report found'
|
||||
pass
|
||||
# file in queue
|
||||
elif response_code == -2:
|
||||
b64_vt_report = 'File in queue'
|
||||
pass
|
||||
|
||||
r_serv_metadata.hset('metadata_hash:'+hash, 'vt_report', b64_vt_report)
|
||||
return jsonify(hash=hash, report_vt=b64_vt_report)
|
||||
elif response.status_code == 403:
|
||||
Flask_config.vt_enabled = False
|
||||
print('VT is disabled')
|
||||
return jsonify()
|
||||
else:
|
||||
# TODO FIXME make json response
|
||||
return jsonify()
|
||||
|
||||
# ========= REGISTRATION =========
|
||||
app.register_blueprint(hashDecoded)
|
||||
680
var/www/modules/hashDecoded/templates/hashDecoded.html
Normal file
680
var/www/modules/hashDecoded/templates/hashDecoded.html
Normal file
|
|
@ -0,0 +1,680 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
|
||||
<title>HashesDecoded - AIL</title>
|
||||
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
|
||||
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet" type="text/css" />
|
||||
<!-- JS -->
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
|
||||
<style>
|
||||
.input-group .form-control {
|
||||
position: unset;
|
||||
}
|
||||
.red_table thead{
|
||||
background: #d91f2d;
|
||||
color: #fff;
|
||||
}
|
||||
.line {
|
||||
fill: none;
|
||||
stroke: #000;
|
||||
stroke-width: 2.0px;
|
||||
}
|
||||
.bar {
|
||||
fill: steelblue;
|
||||
}
|
||||
.bar:hover{
|
||||
fill: brown;
|
||||
cursor: pointer;
|
||||
}
|
||||
.bar_stack:hover{
|
||||
cursor: pointer;
|
||||
}
|
||||
.pie_path:hover{
|
||||
cursor: pointer;
|
||||
}
|
||||
.svgText {
|
||||
pointer-events: none;
|
||||
}
|
||||
div.tooltip {
|
||||
position: absolute;
|
||||
text-align: center;
|
||||
padding: 2px;
|
||||
font: 12px sans-serif;
|
||||
background: #ebf4fb;
|
||||
border: 2px solid #b7ddf2;
|
||||
border-radius: 8px;
|
||||
pointer-events: none;
|
||||
color: #000000;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
{% include 'navbar.html' %}
|
||||
|
||||
<div id="page-wrapper">
|
||||
<div class="row">
|
||||
<div class="col-lg-12">
|
||||
<h1 class="page-header" data-page="page-termsfrequency" >Hashed Files</h1>
|
||||
<div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
<!-- /.col-lg-12 -->
|
||||
|
||||
<div class="row">
|
||||
<div class="col-md-10">
|
||||
<div id="barchart_type">
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="col-md-2">
|
||||
<div class="panel panel-info" style="text-align:center;">
|
||||
<div class="panel-heading">
|
||||
Select a date range :
|
||||
<form action="/hashDecoded/all_hash_search" id="hash_selector_form" method='post'>
|
||||
<div class="input-group">
|
||||
<span class="input-group-addon"><i class="fa fa-calendar fa" aria-hidden="true"></i></span>
|
||||
<input class="form-control" id="date-range-from" placeholder="yyyy-mm-dd" value="{{ date_from }}" name="date_from">
|
||||
</div>
|
||||
<div class="input-group">
|
||||
<span class="input-group-addon"><i class="fa fa-calendar fa" aria-hidden="true"></i></span>
|
||||
<input class="form-control" id="date-range-to" placeholder="yyyy-mm-dd" value="{{ date_to }}" name="date_to">
|
||||
</div>
|
||||
Encoding :
|
||||
<select class="form-control" name="encoding" style="width=100%;">
|
||||
<option>All encoding</option>
|
||||
{% for encod in all_encoding %}
|
||||
{% if encoding|string() == encod|string() %}
|
||||
<option selected>{{ encod }}</option>
|
||||
{% else %}
|
||||
<option>{{ encod }}</option>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</select>
|
||||
File Type :
|
||||
<select class="form-control" name="type" style="width=100%;">
|
||||
<option>All types</option>
|
||||
{% for typ in l_type %}
|
||||
{% if type|string() == typ|string() %}
|
||||
<option selected>{{ typ }}</option>
|
||||
{% else %}
|
||||
<option>{{ typ }}</option>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
</select>
|
||||
<br>
|
||||
<button class="btn btn-primary" style="text-align:center;">
|
||||
<i class="fa fa-files-o"></i> Search
|
||||
</button>
|
||||
<form>
|
||||
</div>
|
||||
</div>
|
||||
<div id="pie_chart_encoded">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- /#page-wrapper -->
|
||||
{% if l_64|length != 0 %}
|
||||
{% if date_from|string == date_to|string %}
|
||||
<h3> {{ date_from }} Hashed files: </h3>
|
||||
{% else %}
|
||||
<h3> {{ date_from }} to {{ date_to }} Hashed files: </h3>
|
||||
{% endif %}
|
||||
<table id="tableb64" class="red_table table table-striped table-bordered">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>estimated type</th>
|
||||
<th>hash</th>
|
||||
<th>first seen</th>
|
||||
<th>last seen</th>
|
||||
<th>nb paste</th>
|
||||
<th>size</th>
|
||||
<th>Virus Total</th>
|
||||
<th>Sparkline</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for b64 in l_64 %}
|
||||
<tr>
|
||||
<td><i class="fa {{ b64[0] }}"></i> {{ b64[1] }}</td>
|
||||
<td><a target="_blank" href="{{ url_for('hashDecoded.showHash') }}?hash={{ b64[2] }}">{{ b64[2] }}</a></td>
|
||||
<td>{{ b64[5] }}</td>
|
||||
<td>{{ b64[6] }}</td>
|
||||
<td>{{ b64[3] }}</td>
|
||||
<td>{{ b64[4] }}</td>
|
||||
<td style="text-align:center;max-width:150px;">
|
||||
{% if vt_enabled %}
|
||||
{% if not b64[7] %}
|
||||
<darkbutton_{{ b64[2] }}>
|
||||
<button id="submit_vt_{{ b64[2] }}" class="btn btn-primary" onclick="sendFileToVT('{{ b64[2] }}')">
|
||||
<i class="fa fa-paper-plane"></i> Send this file to VT
|
||||
</button>
|
||||
</darkbutton_{{ b64[2] }}>
|
||||
{% else %}
|
||||
<a class="btn btn-primary" target="_blank" href="{{ b64[8] }}"><i class="fa fa-link"> VT Report</i></a>
|
||||
{% endif %}
|
||||
<button class="btn btn-default" onclick="updateVTReport('{{ b64[2] }}')">
|
||||
<div id="report_vt_{{ b64[2] }}"><span class="glyphicon glyphicon-refresh"></span> {{ b64[9] }}</div>
|
||||
</button>
|
||||
{% else %}
|
||||
Virus Total submission is disabled
|
||||
{% endif %}
|
||||
|
||||
</td>
|
||||
<td id="sparklines_{{ b64[2] }}" style="text-align:center;">
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
{% else %}
|
||||
{% if date_from|string == date_to|string %}
|
||||
<h3> {{ date_from }}, No Hashes</h3>
|
||||
{% else %}
|
||||
<h3> {{ date_from }} to {{ date_to }}, No Hashes</h3>
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
</div>
|
||||
<!-- /.row -->
|
||||
|
||||
<script>
|
||||
var chart = {};
|
||||
$(document).ready(function(){
|
||||
activePage = "page-hashDecoded"
|
||||
$("#"+activePage).addClass("active");
|
||||
|
||||
$('#date-range-from').dateRangePicker({
|
||||
separator : ' to ',
|
||||
getValue: function()
|
||||
{
|
||||
if ($('#date-range-from').val() && $('#date-range-to').val() )
|
||||
return $('#date-range-from').val() + ' to ' + $('#date-range-to').val();
|
||||
else
|
||||
return '';
|
||||
},
|
||||
setValue: function(s,s1,s2)
|
||||
{
|
||||
$('#date-range-from').val(s1);
|
||||
$('#date-range-to').val(s2);
|
||||
}
|
||||
});
|
||||
$('#date-range-to').dateRangePicker({
|
||||
separator : ' to ',
|
||||
getValue: function()
|
||||
{
|
||||
if ($('#date-range-from').val() && $('#date-range-to').val() )
|
||||
return $('#date-range-from').val() + ' to ' + $('#date-range-to').val();
|
||||
else
|
||||
return '';
|
||||
},
|
||||
setValue: function(s,s1,s2)
|
||||
{
|
||||
$('#date-range-from').val(s1);
|
||||
$('#date-range-to').val(s2);
|
||||
}
|
||||
});
|
||||
|
||||
$('#tableb64').DataTable({
|
||||
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
|
||||
"iDisplayLength": 10,
|
||||
"order": [[ 3, "desc" ]]
|
||||
});
|
||||
|
||||
{% if type %}
|
||||
chart.stackBarChart =barchart_type_stack('/hashDecoded/hash_by_type_json?type={{type}}', 'id');
|
||||
{% elif daily_type_chart %}
|
||||
chart.stackBarChart =barchart_type_stack('/hashDecoded/range_type_json?date_from={{daily_date}}&date_to={{daily_date}}', 'id');
|
||||
{% else %}
|
||||
chart.stackBarChart = barchart_type_stack("/hashDecoded/range_type_json?date_from={{date_from}}&date_to={{date_to}}", 'id')
|
||||
{% endif %}
|
||||
|
||||
chart.onResize();
|
||||
$(window).on("resize", function() {
|
||||
chart.onResize();
|
||||
});
|
||||
});
|
||||
</script>
|
||||
<script>
|
||||
function updateVTReport(hash) {
|
||||
//updateReport
|
||||
$.getJSON('/hashDecoded/update_vt_result?hash='+hash,
|
||||
function(data) {
|
||||
content = '<span class="glyphicon glyphicon-refresh"></span> ' +data['report_vt']
|
||||
$( "#report_vt_"+hash ).html(content);
|
||||
});
|
||||
}
|
||||
|
||||
function sendFileToVT(hash) {
|
||||
//send file to vt
|
||||
$.getJSON('/hashDecoded/send_file_to_vt_js?hash='+hash,
|
||||
function(data) {
|
||||
var content = '<a id="submit_vt_'+hash+'" class="btn btn-primary" target="_blank" href="'+ data['vt_link'] +'"><i class="fa fa-link"> '+ ' VT Report' +'</i></a>';
|
||||
$('#submit_vt_'+hash).remove();
|
||||
$('darkbutton_'+hash).append(content);
|
||||
});
|
||||
}
|
||||
</script>
|
||||
<script>
|
||||
//var data = [6,3,3,2,5,3,9];
|
||||
|
||||
// a sparklines plot
|
||||
function sparklines(id, points) {
|
||||
var width = 100, height = 60;
|
||||
|
||||
var data = []
|
||||
for (i = 0; i < points.length; i++) {
|
||||
data[i] = {
|
||||
'x': i,
|
||||
'y': +points[i]
|
||||
}
|
||||
}
|
||||
|
||||
var x = d3.scaleLinear()
|
||||
.range([0, width - 10])
|
||||
.domain([0,5]);
|
||||
|
||||
var y = d3.scaleLinear()
|
||||
.range([height, 0])
|
||||
.domain([0,10]);
|
||||
|
||||
var line = d3.line()
|
||||
.x(function(d) {return x(d.x)})
|
||||
.y(function(d) {return y(d.y)});
|
||||
|
||||
d3.select("#"+id).append('svg')
|
||||
.attr('width', width)
|
||||
.attr('height', height)
|
||||
.append('path')
|
||||
.attr('class','line')
|
||||
.datum(data)
|
||||
.attr('d', line);
|
||||
|
||||
}
|
||||
</script>
|
||||
<script>
|
||||
{% for b64 in l_64 %}
|
||||
sparklines("sparklines_{{ b64[2] }}", {{ b64[10] }})
|
||||
{% endfor %}
|
||||
</script>
|
||||
|
||||
<script>
|
||||
var margin = {top: 20, right: 100, bottom: 55, left: 45},
|
||||
width = 1000 - margin.left - margin.right,
|
||||
height = 500 - margin.top - margin.bottom;
|
||||
var x = d3.scaleBand().rangeRound([0, width]).padding(0.1);
|
||||
|
||||
var y = d3.scaleLinear().rangeRound([height, 0]);
|
||||
|
||||
var xAxis = d3.axisBottom(x);
|
||||
|
||||
var yAxis = d3.axisLeft(y);
|
||||
|
||||
var color = d3.scaleOrdinal(d3.schemeSet3);
|
||||
|
||||
var svg = d3.select("#barchart_type").append("svg")
|
||||
.attr("id", "thesvg")
|
||||
.attr("viewBox", "0 0 1000 500")
|
||||
.attr("width", width + margin.left + margin.right)
|
||||
.attr("height", height + margin.top + margin.bottom)
|
||||
.append("g")
|
||||
.attr("transform", "translate(" + margin.left + "," + margin.top + ")");
|
||||
|
||||
function barchart_type_stack(url, id) {
|
||||
|
||||
d3.json(url)
|
||||
.then(function(data){
|
||||
|
||||
var labelVar = 'date'; //A
|
||||
var varNames = d3.keys(data[0])
|
||||
.filter(function (key) { return key !== labelVar;}); //B
|
||||
|
||||
data.forEach(function (d) { //D
|
||||
var y0 = 0;
|
||||
d.mapping = varNames.map(function (name) {
|
||||
return {
|
||||
name: name,
|
||||
label: d[labelVar],
|
||||
y0: y0,
|
||||
y1: y0 += +d[name]
|
||||
};
|
||||
});
|
||||
d.total = d.mapping[d.mapping.length - 1].y1;
|
||||
});
|
||||
|
||||
x.domain(data.map(function (d) { return (d.date); })); //E
|
||||
y.domain([0, d3.max(data, function (d) { return d.total; })]);
|
||||
|
||||
svg.append("g")
|
||||
.attr("class", "x axis")
|
||||
.attr("transform", "translate(0," + height + ")")
|
||||
.call(xAxis)
|
||||
.selectAll("text")
|
||||
.attr("class", "bar")
|
||||
{% if date_from|string == date_to|string and type is none %}
|
||||
.on("click", function (d) { window.location.href = '/hashDecoded/?date_from={{date_from}}&date_to={{date_to}}&type='+d })
|
||||
.attr("transform", "rotate(-18)" )
|
||||
{% elif date_from|string == date_to|string and type is not none %}
|
||||
.on("click", function (d) { window.location.href = '/hashDecoded/?date_from='+d+'&date_to='+d })
|
||||
.attr("transform", "rotate(-18)" )
|
||||
{% else %}
|
||||
.on("click", function (d) { window.location.href = '/hashDecoded/?date_from='+d+'&date_to='+d })
|
||||
.attr("transform", "rotate(-40)" )
|
||||
{% endif %}
|
||||
.style("text-anchor", "end");
|
||||
|
||||
svg.append("g")
|
||||
.attr("class", "y axis")
|
||||
.call(yAxis)
|
||||
.append("text")
|
||||
.attr("transform", "rotate(-90)")
|
||||
.attr("y", 6)
|
||||
.attr("dy", ".71em")
|
||||
.style("text-anchor", "end");
|
||||
|
||||
var selection = svg.selectAll(".series")
|
||||
.data(data)
|
||||
.enter().append("g")
|
||||
.attr("class", "series")
|
||||
.attr("transform", function (d) { return "translate(" + x((d.date)) + ",0)"; });
|
||||
|
||||
selection.selectAll("rect")
|
||||
.data(function (d) { return d.mapping; })
|
||||
.enter().append("rect")
|
||||
.attr("class", "bar_stack")
|
||||
.attr("width", x.bandwidth())
|
||||
.attr("y", function (d) { return y(d.y1); })
|
||||
.attr("height", function (d) { return y(d.y0) - y(d.y1); })
|
||||
.style("fill", function (d) { return color(d.name); })
|
||||
.style("stroke", "grey")
|
||||
.on("mouseover", function (d) { showPopover.call(this, d); })
|
||||
.on("mouseout", function (d) { removePopovers(); })
|
||||
{% if date_from|string == date_to|string and type is none %}
|
||||
.on("click", function(d){ window.location.href = "/hashDecoded/" +'?date_from={{date_from}}&date_to={{date_to}}&type='+d.label+'&encoding='+d.name; });
|
||||
{% elif date_from|string == date_to|string and type is not none %}
|
||||
.on("click", function(d){ window.location.href = "/hashDecoded/" +'?type={{type}}&date_from='+d.label+'&date_to='+d.label+'&encoding='+d.name; });
|
||||
{% else %}
|
||||
.on("click", function(d){ window.location.href = "/hashDecoded/" +'?type='+ d.name +'&date_from='+d.label+'&date_to='+d.label; });
|
||||
{% endif %}
|
||||
|
||||
data.forEach(function(d) {
|
||||
if(d.total != 0){
|
||||
svg.append("text")
|
||||
.attr("class", "bar")
|
||||
.attr("dy", "-.35em")
|
||||
//.on("click", (window.location.href = "/hashDecoded/"+'?date_from='+d.date) )
|
||||
.attr('x', x(d.date) + x.bandwidth()/2)
|
||||
.attr('y', y(d.total))
|
||||
{% if date_from|string == date_to|string and type is none %}
|
||||
.on("click", function () {window.location.href = "/hashDecoded/"+'?date_from={{date_from}}&date_to={{date_to}}&type='+d.date })
|
||||
{% elif date_from|string == date_to|string and type is not none %}
|
||||
.on("click", function () {window.location.href = '/hashDecoded/?type={{type}}&date_from='+d.date+'&date_to='+d.date })
|
||||
{% else %}
|
||||
.on("click", function () {window.location.href = "/hashDecoded/"+'?date_from='+d.date+'&date_to='+d.date })
|
||||
{% endif %}
|
||||
.style("text-anchor", "middle")
|
||||
.text(d.total);
|
||||
}
|
||||
});
|
||||
|
||||
drawLegend(varNames);
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
function drawLegend (varNames) {
|
||||
var legend = svg.selectAll(".legend")
|
||||
.data(varNames.slice().reverse())
|
||||
.enter().append("g")
|
||||
.attr("class", "legend")
|
||||
.attr("transform", function (d, i) { return "translate(0," + i * 20 + ")"; });
|
||||
|
||||
legend.append("rect")
|
||||
.attr("x", 943)
|
||||
.attr("width", 10)
|
||||
.attr("height", 10)
|
||||
.style("fill", color)
|
||||
.style("stroke", "grey");
|
||||
|
||||
legend.append("text")
|
||||
.attr("class", "svgText")
|
||||
.attr("x", 941)
|
||||
.attr("y", 6)
|
||||
.attr("dy", ".35em")
|
||||
.style("text-anchor", "end")
|
||||
.text(function (d) { return d; });
|
||||
}
|
||||
|
||||
function removePopovers () {
|
||||
$('.popover').each(function() {
|
||||
$(this).remove();
|
||||
});
|
||||
}
|
||||
|
||||
function showPopover (d) {
|
||||
$(this).popover({
|
||||
title: d.name,
|
||||
placement: 'auto top',
|
||||
container: 'body',
|
||||
trigger: 'manual',
|
||||
html : true,
|
||||
content: function() {
|
||||
return d.label +
|
||||
"<br/>num: " + d3.format(",")(d.value ? d.value: d.y1 - d.y0); }
|
||||
});
|
||||
$(this).popover('show')
|
||||
}
|
||||
|
||||
chart.onResize = function () {
|
||||
var aspect = 1000 / 500, chart = $("#thesvg");
|
||||
var targetWidth = chart.parent().width();
|
||||
chart.attr("width", targetWidth);
|
||||
chart.attr("height", targetWidth / aspect);
|
||||
}
|
||||
|
||||
window.chart = chart;
|
||||
|
||||
</script>
|
||||
|
||||
<script>
|
||||
|
||||
var width_pie = 200;
|
||||
var height_pie = 200;
|
||||
var padding_pie = 10;
|
||||
var opacity_pie = .8;
|
||||
|
||||
var radius_pie = Math.min(width_pie - padding_pie, height_pie - padding_pie) / 2;
|
||||
//var color_pie = d3.scaleOrdinal(d3.schemeCategory10);
|
||||
var color_pie = d3.scaleOrdinal(d3.schemeSet3);
|
||||
|
||||
var div_pie = d3.select("body").append("div")
|
||||
.attr("class", "tooltip")
|
||||
.style("opacity", 0);
|
||||
|
||||
var svg_pie = d3.select("#pie_chart_encoded")
|
||||
.append('svg')
|
||||
.attr("width", '100%')
|
||||
.attr("height", '100%')
|
||||
.attr('viewBox','0 0 '+Math.min(width_pie,height_pie) +' '+Math.min(width_pie,height_pie) )
|
||||
.attr('preserveAspectRatio','xMinYMin')
|
||||
|
||||
|
||||
var g_pie = svg_pie.append('g')
|
||||
.attr('transform', 'translate(' + (width_pie/2) + ',' + (height_pie/2) + ')');
|
||||
|
||||
var arc_pie = d3.arc()
|
||||
.innerRadius(0)
|
||||
.outerRadius(radius_pie);
|
||||
|
||||
d3.json("/hashDecoded/decoder_type_json?date_from={{date_from}}&date_to={{date_to}}&type={{type}}")
|
||||
.then(function(data){
|
||||
|
||||
var pie_pie = d3.pie()
|
||||
.value(function(d) { return d.value; })
|
||||
.sort(null);
|
||||
|
||||
var path_pie = g_pie.selectAll('path')
|
||||
.data(pie_pie(data))
|
||||
.enter()
|
||||
.append("g")
|
||||
.append('path')
|
||||
.attr('d', arc_pie)
|
||||
.attr('fill', (d,i) => color_pie(i))
|
||||
.attr('class', 'pie_path')
|
||||
.on("mouseover", mouseovered_pie)
|
||||
.on("mouseout", mouseouted_pie)
|
||||
.on("click", function (d) {window.location.href = '/hashDecoded/?date_from={{date_from}}&date_to={{date_to}}&type={{type}}&encoding='+d.data.name })
|
||||
.style('opacity', opacity_pie)
|
||||
.style('stroke', 'white');
|
||||
});
|
||||
|
||||
function mouseovered_pie(d) {
|
||||
|
||||
// tooltip
|
||||
var content;
|
||||
|
||||
content = "<b>"+d.data.name+"</b>"+"<br/>"+
|
||||
"<br/>"+
|
||||
"<i>Decoded</i>: "+d.data.value+"<br/>"
|
||||
|
||||
div_pie.transition()
|
||||
.duration(200)
|
||||
.style("opacity", .9);
|
||||
div_pie.html(content)
|
||||
.style("left", (d3.event.pageX) + "px")
|
||||
.style("top", (d3.event.pageY - 28) + "px");
|
||||
}
|
||||
|
||||
function mouseouted_pie() {
|
||||
div_pie.transition()
|
||||
.duration(500)
|
||||
.style("opacity", 0);
|
||||
}
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
<script>
|
||||
function barchart_type(url, id) {
|
||||
|
||||
|
||||
var margin = {top: 20, right: 20, bottom: 70, left: 40};
|
||||
|
||||
var width = 960 - margin.left - margin.right;
|
||||
var height = 500 - margin.top - margin.bottom;
|
||||
|
||||
var x = d3.scaleBand().rangeRound([0, width]).padding(0.1);
|
||||
var y = d3.scaleLinear().rangeRound([height, 0]);
|
||||
|
||||
var xAxis = d3.axisBottom(x)
|
||||
//.tickFormat(d3.time.format("%Y-%m"));
|
||||
|
||||
var yAxis = d3.axisLeft(y)
|
||||
.ticks(10);
|
||||
|
||||
/*var svg = d3.select(id).append("svg")
|
||||
.attr("width", width + margin.left + margin.right)
|
||||
.attr("height", height + margin.top + margin.bottom)
|
||||
.attr("id", "thesvg")
|
||||
.append("g")
|
||||
.attr("transform",
|
||||
"translate(" + margin.left + "," + margin.top + ")");*/
|
||||
|
||||
|
||||
d3.json(url)
|
||||
.then(function(data){
|
||||
|
||||
data.forEach(function(d) {
|
||||
d.value = +d.value;
|
||||
});
|
||||
|
||||
x.domain(data.map(function(d) { return d.date; }));
|
||||
y.domain([0, d3.max(data, function(d) { return d.value; })]);
|
||||
|
||||
var label = svg.append("g")
|
||||
.attr("class", "x axis")
|
||||
.attr("transform", "translate(0," + height + ")")
|
||||
.call(xAxis)
|
||||
.selectAll("text")
|
||||
.style("text-anchor", "end")
|
||||
.attr("dx", "-.8em")
|
||||
.attr("dy", "-.55em")
|
||||
{% if daily_type_chart %}
|
||||
.attr("transform", "rotate(-20)" );
|
||||
{% else %}
|
||||
.attr("transform", "rotate(-70)" )
|
||||
.attr("class", "bar")
|
||||
.on("click", function (d) { window.location.href = "/hashDecoded/"+'?date_from='+d+'&date_to='+d });
|
||||
{% endif %}
|
||||
|
||||
svg.append("g")
|
||||
.attr("class", "y axis")
|
||||
.call(yAxis)
|
||||
.append("text")
|
||||
.attr("transform", "rotate(-90)")
|
||||
.attr("y", 6)
|
||||
.attr("dy", ".71em")
|
||||
.style("text-anchor", "end")
|
||||
.text("Value ($)");
|
||||
|
||||
var bar = svg.selectAll("bar")
|
||||
.data(data)
|
||||
.enter().append("rect")
|
||||
.attr("class", "bar")
|
||||
//.style("fill", "steelblue")
|
||||
.attr("x", function(d) { return x(d.date); })
|
||||
.attr("width", x.bandwidth())
|
||||
.attr("y", function(d) { return y(d.value); })
|
||||
.attr("height", function(d) { return height - y(d.value); })
|
||||
{% if type %}
|
||||
.on("click", function(d){ window.location.href = "/hashDecoded/" +'?type={{type}}&date_from='+ d.date +'&date_to='+ d.date; });
|
||||
{% endif %}
|
||||
{% if daily_type_chart %}
|
||||
.on("click", function(d){ window.location.href = "/hashDecoded/" +'?type='+d.date+'&date_from={{ daily_date }}&date_to={{ daily_date }}'; });
|
||||
{% endif %}
|
||||
|
||||
|
||||
data.forEach(function(d) {
|
||||
if(d.value != 0){
|
||||
svg.append("text")
|
||||
.attr("class", "bar")
|
||||
.attr("dy", "-.35em")
|
||||
//.text(function(d) { return d.value; });
|
||||
.text(d.value)
|
||||
.style("text-anchor", "middle")
|
||||
.attr('x', x(d.date) + x.bandwidth()/2)
|
||||
.attr('y', y(d.value));
|
||||
}
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
}
|
||||
</script>
|
||||
|
||||
|
||||
</body>
|
||||
|
||||
</html>
|
||||
|
|
@ -0,0 +1 @@
|
|||
<li id='page-hashDecoded'><a href="{{ url_for('hashDecoded.hashDecoded_page') }}"><i class="fa fa-files-o"></i> hashesDecoded </a></li>
|
||||
611
var/www/modules/hashDecoded/templates/showHash.html
Normal file
611
var/www/modules/hashDecoded/templates/showHash.html
Normal file
|
|
@ -0,0 +1,611 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
|
||||
<title>Hash Information - AIL</title>
|
||||
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
|
||||
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet" type="text/css" />
|
||||
<!-- JS -->
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
|
||||
<style>
|
||||
line.link {
|
||||
stroke: #666;
|
||||
}
|
||||
line.link:hover{
|
||||
stroke: red;
|
||||
stroke-width: 2px
|
||||
}
|
||||
.line_sparkline {
|
||||
fill: none;
|
||||
stroke: #000;
|
||||
stroke-width: 2.0px;
|
||||
}
|
||||
.node {
|
||||
pointer-events: all;
|
||||
}
|
||||
|
||||
circle {
|
||||
stroke: none;
|
||||
}
|
||||
|
||||
.graph_text_node {
|
||||
font: 8px sans-serif;
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
.graph_node_icon {
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
.node text {
|
||||
font: 8px sans-serif;
|
||||
pointer-events: auto;
|
||||
}
|
||||
|
||||
div.tooltip {
|
||||
position: absolute;
|
||||
text-align: center;
|
||||
padding: 2px;
|
||||
font: 12px sans-serif;
|
||||
background: #ebf4fb;
|
||||
border: 2px solid #b7ddf2;
|
||||
border-radius: 8px;
|
||||
pointer-events: none;
|
||||
color: #000000;
|
||||
}
|
||||
|
||||
.graph_panel {
|
||||
padding: unset;
|
||||
}
|
||||
|
||||
.line_graph {
|
||||
fill: none;
|
||||
stroke: steelblue;
|
||||
stroke-width: 2px;
|
||||
stroke-linejoin: round;
|
||||
stroke-linecap: round;
|
||||
stroke-width: 1.5;
|
||||
/*attr('stroke', '#bcbd22').*/
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
{% include 'navbar.html' %}
|
||||
|
||||
<div id="page-wrapper">
|
||||
<div class="row">
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<!-- /#page-wrapper -->
|
||||
<div class="panel panel-info">
|
||||
<div class="panel-heading panelText">
|
||||
<h3>{{ hash }} :</h3>
|
||||
<span class="pull-right"> </span>
|
||||
<span class="badge pull-right">6 / 26</span>
|
||||
<ul class="list-group"><li class="list-group-item">
|
||||
|
||||
<div class="row">
|
||||
<div class="col-md-10">
|
||||
|
||||
<table class="table table-condensed">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Estimated type</th>
|
||||
<th>First_seen</th>
|
||||
<th>Last_seen</th>
|
||||
<th>Size (Kb)</th>
|
||||
<th>Nb seen</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td class="panelText"><i class="fa {{ file_icon }}"></i> {{ estimated_type }}</td>
|
||||
<td class="panelText">{{ first_seen }}</td>
|
||||
<td class="panelText">{{ last_seen }}</td>
|
||||
<td class="panelText">{{ size }}</td>
|
||||
<td class="panelText">{{ nb_seen_in_all_pastes }}</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<div class="col-md-1">
|
||||
<div id="sparkline"></div>
|
||||
</div>
|
||||
</div>
|
||||
</li></ul>
|
||||
|
||||
{% if vt_enabled %}
|
||||
{% if not b64_vt %}
|
||||
<darkbutton>
|
||||
<button id="submit_vt_b" class="btn btn-primary" onclick="sendFileToVT('{{ hash }}')">
|
||||
<i class="fa fa-paper-plane"></i> Send this file to VT
|
||||
</button>
|
||||
</darkbutton>
|
||||
{% else %}
|
||||
<a class="btn btn-primary" target="_blank" href="{{ b64_vt_link }}"><i class="fa fa-link"> VT Report</i></a>
|
||||
{% endif %}
|
||||
<button class="btn btn-default" onclick="updateVTReport('{{ hash }}')">
|
||||
<div id="report_vt_b"><span class="glyphicon glyphicon-refresh"></span> {{ b64_vt_report }}</div>
|
||||
</button>
|
||||
{% else %}
|
||||
Virus Total submission is disabled
|
||||
{% endif %}
|
||||
|
||||
<a href="/hashDecoded/downloadHash?hash={{hash}}" target="blank">
|
||||
<button class='btn btn-info pull-right'><i id="flash-tld" class="glyphicon glyphicon-download-alt " flash-tld=""></i> Download Hashed file
|
||||
</button>
|
||||
</a>
|
||||
</div></div>
|
||||
|
||||
<div class="row">
|
||||
<div class="col-md-10">
|
||||
|
||||
<div class="panel panel-default">
|
||||
<div class="panel-heading">
|
||||
<i id="flash-tld" class="glyphicon glyphicon-flash " flash-tld=""></i> Graph
|
||||
</div>
|
||||
<div class="panel-body graph_panel">
|
||||
<div id="graph">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="col-md-2">
|
||||
|
||||
<div class="panel panel-info">
|
||||
<div class="panel-heading">
|
||||
<i class="fa fa-unlock-alt" aria-hidden="true"></i> Encoding
|
||||
</div>
|
||||
<div class="panel-body" style="text-align:center;">
|
||||
{% for encoding in list_hash_decoder %}
|
||||
<button id="" class="btn btn-default">
|
||||
{{encoding['encoding']}} <span class="badge">{{encoding['nb_seen']}}</span>
|
||||
</button>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="panel panel-default">
|
||||
<div class="panel-heading">
|
||||
<i id="flash-tld" class="glyphicon glyphicon-flash " flash-tld=""></i> Graph
|
||||
</div>
|
||||
<div class="panel-body" style="text-align:center;">
|
||||
<button class="btn btn-primary" onclick="resize_graph();">
|
||||
<span class="glyphicon glyphicon-refresh"></span> Resize Graph</div>
|
||||
</button>
|
||||
|
||||
|
||||
<ul class="list-group">
|
||||
<li class="list-group-item list-group-item-info" style="text-align:center;"><i class="fa fa-info-circle fa-2x"></i></li>
|
||||
<li class="list-group-item">
|
||||
<p>Double click on a node to open Hash/Paste<br><br>
|
||||
<svg height="12" width="12"><g class="nodes"><circle cx="6" cy="6" r="6" fill="orange"></circle></g></svg>
|
||||
Current Hash<br>
|
||||
<svg height="12" width="12"><g class="nodes"><circle cx="6" cy="6" r="6" fill="rgb(141, 211, 199)"></circle></g></svg>
|
||||
Hashes<br>
|
||||
<svg height="12" width="12"><g class="nodes"><circle cx="6" cy="6" r="6" fill="#1f77b4"></circle></g></svg>
|
||||
Pastes
|
||||
</p>
|
||||
</li>
|
||||
<li class="list-group-item list-group-item-info">
|
||||
Hash Types:
|
||||
</li>
|
||||
<li class="list-group-item">
|
||||
<i class="fa fa-file"></i> Application<br>
|
||||
<i class="fa fa-file-video-o"></i> Audio<br>
|
||||
<i class="fa fa-file-image-o"></i> Image<br>
|
||||
<i class="fa fa-file-text-o"></i> Text<br>
|
||||
<i class="fa fa-file-o"></i> Other
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="panel panel-default">
|
||||
<div class="panel-heading">
|
||||
<i id="flash-tld" class="glyphicon glyphicon-stats" flash-tld=""></i> Graph
|
||||
</div>
|
||||
<div class="panel-body ">
|
||||
<div id="graph_line">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
</div>
|
||||
<!-- /.row -->
|
||||
|
||||
<script>
|
||||
var all_graph = {};
|
||||
$(document).ready(function(){
|
||||
sparklines("sparkline", {{ sparkline_values }})
|
||||
|
||||
all_graph.node_graph = create_graph('/hashDecoded/hash_graph_node_json?hash={{hash}}');
|
||||
all_graph.line_chart = create_line_chart('graph_line', '/hashDecoded/hash_graph_line_json?hash={{hash}}');
|
||||
all_graph.onResize();
|
||||
});
|
||||
|
||||
$(window).on("resize", function() {
|
||||
all_graph.onResize();
|
||||
});
|
||||
</script>
|
||||
<script>
|
||||
function sendFileToVT(hash) {
|
||||
//send file to vt
|
||||
$.getJSON('/hashDecoded/send_file_to_vt_js?hash='+hash,
|
||||
function(data) {
|
||||
var content = '<a id="submit_vt_b" class="btn btn-primary" target="_blank" href="'+ data['vt_link'] +'"><i class="fa fa-link"> '+ ' VT Report' +'</i></a>';
|
||||
$('#submit_vt_b').remove();
|
||||
$('darkbutton').append(content);
|
||||
});
|
||||
}
|
||||
|
||||
function updateVTReport(hash) {
|
||||
//updateReport
|
||||
$.getJSON('/hashDecoded/update_vt_result?hash='+hash,
|
||||
function(data) {
|
||||
var content = '<span class="glyphicon glyphicon-refresh"></span> ' +data['report_vt'];
|
||||
$( "#report_vt_b" ).html(content);
|
||||
});
|
||||
}
|
||||
|
||||
</script>
|
||||
<script>
|
||||
function resize_graph() {
|
||||
zoom.translateTo(svg_node, 200, 200);
|
||||
zoom.scaleTo(svg_node, 2);
|
||||
}
|
||||
|
||||
</script>
|
||||
<script>
|
||||
//var data = [6,3,3,2,5,3,9];
|
||||
|
||||
// a sparklines plot
|
||||
function sparklines(id, points) {
|
||||
var width_spark = 100, height_spark = 60;
|
||||
|
||||
var data = []
|
||||
for (i = 0; i < points.length; i++) {
|
||||
data[i] = {
|
||||
'x': i,
|
||||
'y': +points[i]
|
||||
}
|
||||
}
|
||||
|
||||
var x = d3.scaleLinear()
|
||||
.range([0, width_spark - 10])
|
||||
.domain([0,5]);
|
||||
|
||||
var y = d3.scaleLinear()
|
||||
.range([height_spark, 0])
|
||||
.domain([0,10]);
|
||||
|
||||
var line = d3.line()
|
||||
.x(function(d) {return x(d.x)})
|
||||
.y(function(d) {return y(d.y)});
|
||||
|
||||
d3.select("#"+id).append('svg')
|
||||
.attr('width', width_spark)
|
||||
.attr('height', height_spark)
|
||||
.append('path')
|
||||
.attr('class','line_sparkline')
|
||||
.datum(data)
|
||||
.attr('d', line);
|
||||
|
||||
}
|
||||
</script>
|
||||
|
||||
<script>
|
||||
var width = 400,
|
||||
height = 400;
|
||||
|
||||
var link;
|
||||
|
||||
var zoom = d3.zoom()
|
||||
.scaleExtent([.2, 10])
|
||||
.on("zoom", zoomed);
|
||||
|
||||
//var transform = d3.zoomIdentity;
|
||||
|
||||
var color = d3.scaleOrdinal(d3.schemeCategory10);
|
||||
|
||||
var div = d3.select("body").append("div")
|
||||
.attr("class", "tooltip")
|
||||
.style("opacity", 0);
|
||||
|
||||
var simulation = d3.forceSimulation()
|
||||
.force("link", d3.forceLink().id(function(d) { return d.id; }))
|
||||
.force("charge", d3.forceManyBody())
|
||||
.force("center", d3.forceCenter(width / 2, height / 2));
|
||||
//.on("tick", ticked);
|
||||
|
||||
var svg_node = d3.select("#graph").append("svg")
|
||||
.attr("id", "graph_div")
|
||||
.attr("width", width)
|
||||
.attr("height", height)
|
||||
.call(d3.zoom().scaleExtent([1, 8]).on("zoom", zoomed))
|
||||
.on("dblclick.zoom", null)
|
||||
|
||||
var container_graph = svg_node.append("g");
|
||||
//.attr("transform", "translate(40,0)")
|
||||
//.attr("transform", "scale(2)");
|
||||
|
||||
function create_graph(url){
|
||||
|
||||
d3.json(url)
|
||||
.then(function(data){
|
||||
|
||||
link = container_graph.append("g")
|
||||
.selectAll("line")
|
||||
.data(data.links)
|
||||
.enter().append("line")
|
||||
.attr("class", "link");
|
||||
//.attr("stroke-width", function(d) { return Math.sqrt(d.value); })
|
||||
|
||||
var node = container_graph.selectAll(".node")
|
||||
.data(data.nodes)
|
||||
.enter().append("g")
|
||||
.attr("class", "nodes")
|
||||
.on("dblclick", doubleclick)
|
||||
.on("click", click)
|
||||
.on("mouseover", mouseovered)
|
||||
.on("mouseout", mouseouted)
|
||||
.call(d3.drag()
|
||||
.on("start", drag_start)
|
||||
.on("drag", dragged)
|
||||
.on("end", drag_end));
|
||||
|
||||
|
||||
node.append("circle")
|
||||
.attr("r", function(d) {
|
||||
return (d.hash) ? 6 : 5; })
|
||||
.attr("fill", function(d) {
|
||||
if(!d.hash){ return color(d.group);}
|
||||
if(d.group == 1){ return "orange";}
|
||||
return "rgb(141, 211, 199)"; });
|
||||
|
||||
node.append('text')
|
||||
.attr('text-anchor', 'middle')
|
||||
.attr('dominant-baseline', 'central')
|
||||
.attr("class", "graph_node_icon")
|
||||
.attr('font-family', 'FontAwesome')
|
||||
.attr('font-size', '8px' )
|
||||
.attr('pointer-events', 'none')
|
||||
.text(function(d) {
|
||||
if(d.hash){
|
||||
return d.icon
|
||||
} });
|
||||
|
||||
zoom.translateTo(svg_node, 200, 200);
|
||||
zoom.scaleTo(svg_node, 2);
|
||||
|
||||
/* node.append("text")
|
||||
.attr("dy", 3)
|
||||
.attr("dx", 7)
|
||||
.attr("class", "graph_text_node")
|
||||
//.style("text-anchor", function(d) { return d.children ? "end" : "start"; })
|
||||
.text(function(d) { return d.id; });*/
|
||||
|
||||
simulation
|
||||
.nodes(data.nodes)
|
||||
.on("tick", ticked);
|
||||
|
||||
simulation.force("link")
|
||||
.links(data.links);
|
||||
|
||||
function ticked() {
|
||||
link
|
||||
.attr("x1", function(d) { return d.source.x; })
|
||||
.attr("y1", function(d) { return d.source.y; })
|
||||
.attr("x2", function(d) { return d.target.x; })
|
||||
.attr("y2", function(d) { return d.target.y; });
|
||||
|
||||
/*node
|
||||
.attr("cx", function(d) { return d.x; })
|
||||
.attr("cy", function(d) { return d.y; });*/
|
||||
node.attr("transform", function(d) { return "translate(" + d.x + "," + d.y + ")"; });
|
||||
}
|
||||
|
||||
});
|
||||
}
|
||||
|
||||
function zoomed() {
|
||||
container_graph.attr("transform", d3.event.transform);
|
||||
}
|
||||
|
||||
function doubleclick (d) {
|
||||
window.open(d.url, '_blank');
|
||||
}
|
||||
|
||||
function click (d) {
|
||||
console.log('clicked')
|
||||
}
|
||||
|
||||
function drag_start(d) {
|
||||
if (!d3.event.active) simulation.alphaTarget(0.3).restart();
|
||||
d.fx = d.x;
|
||||
d.fy = d.y;
|
||||
}
|
||||
|
||||
function dragged(d) {
|
||||
d.fx = d3.event.x;
|
||||
d.fy = d3.event.y;
|
||||
}
|
||||
|
||||
function drag_end(d) {
|
||||
if (!d3.event.active) simulation.alphaTarget(0);
|
||||
d.fx = d.x;
|
||||
d.fy = d.y;
|
||||
}
|
||||
|
||||
function mouseovered(d) {
|
||||
|
||||
// tooltip
|
||||
var content;
|
||||
|
||||
if(d.hash == true){
|
||||
content = "<b>"+d.id+"</b>"+"<br/>"+
|
||||
"<br/>"+
|
||||
"<i>First seen</i>: "+d.first_seen+"<br/>"+
|
||||
"<i>Last seen</i>: "+d.last_seen+"<br/>"+
|
||||
"<i>nb_seen_in_paste</i>: "+d.nb_seen_in_paste+"<br/>"+
|
||||
"<i>Size (kb)</i>: "+d.size+"<br/>"+
|
||||
"<br/>"+
|
||||
"<i>Estimated type</i>: "+d.estimated_type;
|
||||
} else {
|
||||
content = "<b>"+d.id+"</b>"+"<br/>";
|
||||
}
|
||||
|
||||
div.transition()
|
||||
.duration(200)
|
||||
.style("opacity", .9);
|
||||
div.html(content)
|
||||
.style("left", (d3.event.pageX) + "px")
|
||||
.style("top", (d3.event.pageY - 28) + "px");
|
||||
|
||||
//links
|
||||
/*link.style("stroke-opacity", function(o) {
|
||||
return o.source === d || o.target === d ? 1 : opacity;
|
||||
});*/
|
||||
link.style("stroke", function(o){
|
||||
return o.source === d || o.target === d ? "#666" : "#ddd";
|
||||
});
|
||||
}
|
||||
|
||||
function mouseouted() {
|
||||
div.transition()
|
||||
.duration(500)
|
||||
.style("opacity", 0);
|
||||
|
||||
link.style("stroke", "#666");
|
||||
}
|
||||
|
||||
all_graph.onResize = function () {
|
||||
var aspect = 1000 / 500, all_graph = $("#graph_div");
|
||||
var targetWidth = all_graph.parent().width();
|
||||
all_graph.attr("width", targetWidth);
|
||||
all_graph.attr("height", targetWidth / aspect);
|
||||
}
|
||||
|
||||
window.all_graph = all_graph;
|
||||
</script>
|
||||
|
||||
<script>
|
||||
function create_line_chart(id, url){
|
||||
|
||||
var width = 900;
|
||||
var height = Math.round(width / 4);
|
||||
|
||||
var margin = {top: 20, right: 55, bottom: 50, left: 40};
|
||||
|
||||
var x = d3.scaleTime().range([0, width]);
|
||||
var y = d3.scaleLinear().rangeRound([height, 0]);
|
||||
|
||||
var xAxis = d3.axisBottom(x);
|
||||
var yAxis = d3.axisLeft(y);
|
||||
|
||||
var parseTime = d3.timeParse("%Y-%m-%d");
|
||||
|
||||
var line = d3.line()
|
||||
.x(function(d) {
|
||||
return x(d.date);
|
||||
}).y(function(d) {
|
||||
return y(d.value);
|
||||
});
|
||||
|
||||
var svg_line = d3.select('#'+id).append('svg')
|
||||
.attr("id", "graph_div")
|
||||
.attr("width", width + margin.left + margin.right)
|
||||
.attr("height", height + margin.top + margin.bottom)
|
||||
.append('g')
|
||||
.attr('transform', "translate("+ margin.left +","+ margin.top +")");
|
||||
|
||||
var div = d3.select('body').append('div')
|
||||
.attr('class', 'tooltip')
|
||||
.style('opacity', 0);
|
||||
|
||||
//add div tooltip
|
||||
|
||||
d3.json(url)
|
||||
.then(function(data){
|
||||
|
||||
data.forEach(function(d) {
|
||||
d.date_label = d.date;
|
||||
d.date = parseTime(d.date);
|
||||
d.value = +d.value;
|
||||
});
|
||||
|
||||
// fit the data
|
||||
x.domain(d3.extent(data, function(d) { return d.date; }));
|
||||
//x.domain(data.map(function (d) { return d.date; })); //E
|
||||
y.domain([0, d3.max(data, function(d){ return d.value ; })]);
|
||||
|
||||
//line
|
||||
svg_line.append("path")
|
||||
.data([data])
|
||||
.attr("class", "line_graph")
|
||||
.attr("d", line);
|
||||
|
||||
// add X axis
|
||||
svg_line.append("g")
|
||||
.attr("transform", "translate(0," + height + ")")
|
||||
.call(d3.axisBottom(x))
|
||||
.selectAll("text")
|
||||
.style("text-anchor", "end")
|
||||
.attr("transform", "rotate(-45)" );
|
||||
|
||||
// Add the Y Axis
|
||||
svg_line.append("g")
|
||||
.call(d3.axisLeft(y));
|
||||
|
||||
//add a dot circle
|
||||
svg_line.selectAll('dot')
|
||||
.data(data).enter()
|
||||
.append('circle')
|
||||
.attr('r', 2)
|
||||
.attr('cx', function(d) { return x(d.date); })
|
||||
.attr('cy', function(d) { return y(d.value); })
|
||||
|
||||
.on('mouseover', function(d) {
|
||||
div.transition().style('opacity', .9);
|
||||
div.html('' + d.date_label+ '<br/>' + d.value).style('left', (d3.event.pageX) + 'px')
|
||||
.style("left", (d3.event.pageX) + "px")
|
||||
.style("top", (d3.event.pageY - 28) + "px");
|
||||
})
|
||||
.on('mouseout', function(d)
|
||||
{
|
||||
div.transition().style('opacity', 0);
|
||||
});
|
||||
|
||||
});
|
||||
}
|
||||
</script>
|
||||
|
||||
</body>
|
||||
|
||||
</html>
|
||||
|
|
@ -7,12 +7,18 @@
|
|||
import redis
|
||||
import os
|
||||
import json
|
||||
import os
|
||||
import flask
|
||||
<<<<<<< HEAD
|
||||
from flask import Flask, render_template, jsonify, request, Blueprint, make_response, Response, send_from_directory
|
||||
=======
|
||||
from flask import Flask, render_template, jsonify, request, Blueprint, make_response, redirect, url_for, Response, send_from_directory
|
||||
>>>>>>> master
|
||||
import difflib
|
||||
import ssdeep
|
||||
|
||||
import Paste
|
||||
import requests
|
||||
|
||||
# ============ VARIABLES ============
|
||||
import Flask_config
|
||||
|
|
@ -30,14 +36,16 @@ DiffMaxLineLength = Flask_config.DiffMaxLineLength
|
|||
bootstrap_label = Flask_config.bootstrap_label
|
||||
misp_event_url = Flask_config.misp_event_url
|
||||
hive_case_url = Flask_config.hive_case_url
|
||||
vt_enabled = Flask_config.vt_enabled
|
||||
SCREENSHOT_FOLDER = Flask_config.SCREENSHOT_FOLDER
|
||||
|
||||
showsavedpastes = Blueprint('showsavedpastes', __name__, template_folder='templates')
|
||||
|
||||
# ============ FUNCTIONS ============
|
||||
|
||||
def showpaste(content_range):
|
||||
requested_path = request.args.get('paste', '')
|
||||
def showpaste(content_range, requested_path):
|
||||
vt_enabled = Flask_config.vt_enabled
|
||||
|
||||
paste = Paste.Paste(requested_path)
|
||||
p_date = str(paste._get_p_date())
|
||||
p_date = p_date[6:]+'/'+p_date[4:6]+'/'+p_date[0:4]
|
||||
|
|
@ -121,7 +129,6 @@ def showpaste(content_range):
|
|||
else:
|
||||
automatic = False
|
||||
|
||||
tag_hash = ssdeep.hash(tag)
|
||||
if r_serv_statistics.sismember('tp:'+tag, requested_path):
|
||||
tag_status_tp = True
|
||||
else:
|
||||
|
|
@ -133,6 +140,40 @@ def showpaste(content_range):
|
|||
|
||||
list_tags.append( (tag, automatic, tag_status_tp, tag_status_fp) )
|
||||
|
||||
l_64 = []
|
||||
# load hash files
|
||||
if r_serv_metadata.scard('hash_paste:'+requested_path) > 0:
|
||||
set_b64 = r_serv_metadata.smembers('hash_paste:'+requested_path)
|
||||
for hash in set_b64:
|
||||
nb_in_file = int(r_serv_metadata.zscore('nb_seen_hash:'+hash, requested_path))
|
||||
estimated_type = r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type')
|
||||
file_type = estimated_type.split('/')[0]
|
||||
# set file icon
|
||||
if file_type == 'application':
|
||||
file_icon = 'fa-file-o '
|
||||
elif file_type == 'audio':
|
||||
file_icon = 'fa-file-video-o '
|
||||
elif file_type == 'image':
|
||||
file_icon = 'fa-file-image-o'
|
||||
elif file_type == 'text':
|
||||
file_icon = 'fa-file-text-o'
|
||||
else:
|
||||
file_icon = 'fa-file'
|
||||
saved_path = r_serv_metadata.hget('metadata_hash:'+hash, 'saved_path')
|
||||
if r_serv_metadata.hexists('metadata_hash:'+hash, 'vt_link'):
|
||||
b64_vt = True
|
||||
b64_vt_link = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_link')
|
||||
b64_vt_report = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_report')
|
||||
else:
|
||||
b64_vt = False
|
||||
b64_vt_link = ''
|
||||
b64_vt_report = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_report')
|
||||
# hash never refreshed
|
||||
if b64_vt_report is None:
|
||||
b64_vt_report = ''
|
||||
|
||||
l_64.append( (file_icon, estimated_type, hash, saved_path, nb_in_file, b64_vt, b64_vt_link, b64_vt_report) )
|
||||
|
||||
crawler_metadata = {}
|
||||
if 'infoleak:submission="crawler"' in l_tags:
|
||||
crawler_metadata['get_metadata'] = True
|
||||
|
|
@ -171,13 +212,15 @@ def showpaste(content_range):
|
|||
|
||||
return render_template("show_saved_paste.html", date=p_date, bootstrap_label=bootstrap_label, active_taxonomies=active_taxonomies, active_galaxies=active_galaxies, list_tags=list_tags, source=p_source, encoding=p_encoding, language=p_language, size=p_size, mime=p_mime, lineinfo=p_lineinfo, content=p_content, initsize=len(p_content), duplicate_list = p_duplicate_list, simil_list = p_simil_list, hashtype_list = p_hashtype_list, date_list=p_date_list,
|
||||
crawler_metadata=crawler_metadata,
|
||||
misp=misp, hive=hive, misp_eventid=misp_eventid, misp_url=misp_url, hive_caseid=hive_caseid, hive_url=hive_url)
|
||||
l_64=l_64, vt_enabled=vt_enabled, misp=misp, hive=hive, misp_eventid=misp_eventid, misp_url=misp_url, hive_caseid=hive_caseid, hive_url=hive_url)
|
||||
|
||||
# ============ ROUTES ============
|
||||
|
||||
@showsavedpastes.route("/showsavedpaste/") #completely shows the paste in a new tab
|
||||
def showsavedpaste():
|
||||
return showpaste(0)
|
||||
requested_path = request.args.get('paste', '')
|
||||
print(requested_path)
|
||||
return showpaste(0, requested_path)
|
||||
|
||||
@showsavedpastes.route("/showsavedrawpaste/") #shows raw
|
||||
def showsavedrawpaste():
|
||||
|
|
@ -189,7 +232,8 @@ def showsavedrawpaste():
|
|||
@showsavedpastes.route("/showpreviewpaste/")
|
||||
def showpreviewpaste():
|
||||
num = request.args.get('num', '')
|
||||
return "|num|"+num+"|num|"+showpaste(max_preview_modal)
|
||||
requested_path = request.args.get('paste', '')
|
||||
return "|num|"+num+"|num|"+showpaste(max_preview_modal, requested_path)
|
||||
|
||||
|
||||
@showsavedpastes.route("/getmoredata/")
|
||||
|
|
@ -220,5 +264,26 @@ def showDiff():
|
|||
def screenshot(filename):
|
||||
return send_from_directory(SCREENSHOT_FOLDER, filename+'.png', as_attachment=True)
|
||||
|
||||
@showsavedpastes.route('/send_file_to_vt/', methods=['POST'])
|
||||
def send_file_to_vt():
|
||||
b64_path = request.form['b64_path']
|
||||
paste = request.form['paste']
|
||||
hash = request.form['hash']
|
||||
|
||||
b64_full_path = os.path.join(os.environ['AIL_HOME'], b64_path)
|
||||
b64_content = ''
|
||||
with open(b64_full_path, 'rb') as f:
|
||||
b64_content = f.read()
|
||||
|
||||
files = {'file': (hash, b64_content)}
|
||||
response = requests.post('https://www.virustotal.com/vtapi/v2/file/scan', files=files, params=vt_auth)
|
||||
json_response = response.json()
|
||||
print(json_response)
|
||||
|
||||
vt_b64_link = json_response['permalink'].split('analysis')[0] + 'analysis/'
|
||||
r_serv_metadata.hset('metadata_hash:'+hash, 'vt_link', vt_b64_link)
|
||||
|
||||
return redirect(url_for('showsavedpastes.showsavedpaste', paste=paste))
|
||||
|
||||
# ========= REGISTRATION =========
|
||||
app.register_blueprint(showsavedpastes)
|
||||
|
|
|
|||
|
|
@ -28,6 +28,11 @@
|
|||
overflow-x: hidden;
|
||||
width:100%;
|
||||
}
|
||||
|
||||
.red_table thead{
|
||||
background: #d91f2d;
|
||||
color: #fff;
|
||||
}
|
||||
</style>
|
||||
|
||||
</head>
|
||||
|
|
@ -374,6 +379,47 @@
|
|||
</table>
|
||||
{% endif %}
|
||||
|
||||
{% if l_64|length != 0 %}
|
||||
<h3> Hash files: </h3>
|
||||
<table id="tableb64" class="red_table table table-striped table-bordered">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>estimated type</th>
|
||||
<th>hash</th>
|
||||
<th>saved_path</th>
|
||||
<th>Virus Total</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for b64 in l_64 %}
|
||||
<tr>
|
||||
<td><i class="fa {{ b64[0] }}"></i> {{ b64[1] }}</td>
|
||||
<td><a target="_blank" href="{{ url_for('hashDecoded.showHash') }}?hash={{ b64[2] }}">{{ b64[2] }}</a> ({{ b64[4] }})</td>
|
||||
<td>{{ b64[3] }}</td>
|
||||
<td style="text-align:center;">
|
||||
{% if vt_enabled %}
|
||||
{% if not b64[5] %}
|
||||
<darkbutton_{{ b64[2] }}>
|
||||
<button id="submit_vt_{{ b64[2] }}" class="btn btn-primary" onclick="sendFileToVT('{{ b64[2] }}')">
|
||||
<i class="fa fa-paper-plane"></i> Send this file to VT
|
||||
</button>
|
||||
</darkbutton_{{ b64[2] }}>
|
||||
{% else %}
|
||||
<a class="btn btn-primary" target="_blank" href="{{ b64[6] }}"><i class="fa fa-link"> VT Report</i></a>
|
||||
{% endif %}
|
||||
<button class="btn btn-default" onclick="updateVTReport('{{ b64[2] }}')">
|
||||
<div id="report_vt_{{ b64[2] }}"><span class="glyphicon glyphicon-refresh"></span> {{ b64[7] }}</div>
|
||||
</button>
|
||||
{% else %}
|
||||
Virus Total submission is disabled
|
||||
{% endif %}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
{% endif %}
|
||||
|
||||
{% if crawler_metadata['get_metadata'] %}
|
||||
<div class="row">
|
||||
<div class="col-md-7">
|
||||
|
|
@ -443,9 +489,36 @@
|
|||
});
|
||||
|
||||
$('#tableDup').DataTable();
|
||||
$('#tableb64').DataTable({
|
||||
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
|
||||
"iDisplayLength": 5,
|
||||
"order": [[ 1, "asc" ]]
|
||||
});
|
||||
});
|
||||
</script>
|
||||
|
||||
<script>
|
||||
function updateVTReport(hash) {
|
||||
//updateReport
|
||||
$.getJSON('/hashDecoded/update_vt_result?hash='+hash,
|
||||
function(data) {
|
||||
content = '<span class="glyphicon glyphicon-refresh"></span> ' +data['report_vt']
|
||||
$( "#report_vt_"+hash ).html(content);
|
||||
});
|
||||
}
|
||||
|
||||
function sendFileToVT(hash) {
|
||||
//send file to vt
|
||||
$.getJSON('/hashDecoded/send_file_to_vt_js?hash='+hash,
|
||||
function(data) {
|
||||
var content = '<a id="submit_vt_'+hash+'" class="btn btn-primary" target="_blank" href="'+ data['vt_link'] +'"><i class="fa fa-link"> '+ ' VT Report' +'</i></a>';
|
||||
$('#submit_vt_'+hash).remove();
|
||||
$('darkbutton_'+hash).append(content);
|
||||
});
|
||||
}
|
||||
|
||||
</script>
|
||||
|
||||
<script>
|
||||
jQuery("#all-tags-taxonomies").click(function(e){
|
||||
//change input tags list
|
||||
|
|
|
|||
|
|
@ -47,7 +47,6 @@
|
|||
-webkit-transition: none;
|
||||
outline: none;
|
||||
display: block;
|
||||
padding: 4px 6px;
|
||||
line-height: normal;
|
||||
overflow: hidden;
|
||||
height: auto;
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ wget http://dygraphs.com/dygraph-combined.js -O ./static/js/dygraph-combined.js
|
|||
|
||||
SBADMIN_VERSION='3.3.7'
|
||||
FONT_AWESOME_VERSION='4.7.0'
|
||||
D3_JS_VERSION='5.5.0'
|
||||
|
||||
rm -rf temp
|
||||
mkdir temp
|
||||
|
|
@ -13,9 +14,21 @@ mkdir temp
|
|||
wget https://github.com/BlackrockDigital/startbootstrap-sb-admin/archive/v${SBADMIN_VERSION}.zip -O temp/${SBADMIN_VERSION}.zip
|
||||
wget https://github.com/BlackrockDigital/startbootstrap-sb-admin-2/archive/v${SBADMIN_VERSION}.zip -O temp/${SBADMIN_VERSION}-2.zip
|
||||
wget https://github.com/FortAwesome/Font-Awesome/archive/v${FONT_AWESOME_VERSION}.zip -O temp/FONT_AWESOME_${FONT_AWESOME_VERSION}.zip
|
||||
wget https://github.com/d3/d3/releases/download/v${D3_JS_VERSION}/d3.zip -O temp/d3_${D3_JS_VERSION}.zip
|
||||
|
||||
# dateRangePicker
|
||||
wget https://github.com/moment/moment/archive/2.22.2.zip -O temp/moment_2.22.2.zip
|
||||
wget https://github.com/longbill/jquery-date-range-picker/archive/v0.18.0.zip -O temp/daterangepicker_v0.18.0.zip
|
||||
|
||||
|
||||
unzip temp/${SBADMIN_VERSION}.zip -d temp/
|
||||
unzip temp/${SBADMIN_VERSION}-2.zip -d temp/
|
||||
unzip temp/FONT_AWESOME_${FONT_AWESOME_VERSION}.zip -d temp/
|
||||
unzip temp/d3_${D3_JS_VERSION}.zip -d temp/
|
||||
|
||||
unzip temp/moment_2.22.2.zip -d temp/
|
||||
unzip temp/daterangepicker_v0.18.0.zip -d temp/
|
||||
|
||||
mv temp/startbootstrap-sb-admin-${SBADMIN_VERSION} temp/sb-admin
|
||||
mv temp/startbootstrap-sb-admin-2-${SBADMIN_VERSION} temp/sb-admin-2
|
||||
mv temp/Font-Awesome-${FONT_AWESOME_VERSION} temp/font-awesome
|
||||
|
|
@ -30,6 +43,11 @@ mv temp/font-awesome/ ./static/
|
|||
rm -rf ./static/css/plugins/
|
||||
mv temp/sb-admin/css/* ./static/css/
|
||||
mv temp/sb-admin-2/dist/css/* ./static/css/
|
||||
mv temp/jquery-date-range-picker-0.18.0/dist/daterangepicker.min.css ./static/css/
|
||||
|
||||
mv temp/d3.min.js ./static/js/
|
||||
mv temp/moment-2.22.2/min/moment.min.js ./static/js/
|
||||
mv temp/jquery-date-range-picker-0.18.0/dist/jquery.daterangepicker.min.js ./static/js/
|
||||
|
||||
rm -rf temp
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue