AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-10 08:38:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

chg: [Hash] add reprocess regex order

Browse source
This commit is contained in:
Terrtia 2018-07-25 10:47:36 +02:00
parent 8c97c70a3f
commit 03a37af5cf
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
3 changed files with 35 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
bin/Decoder.py
View file

@ -85,29 +85,33 @@ def save_hash(decoder_name, message, date, decoded):
else:
serv_metadata.hset('metadata_hash:'+hash, 'last_seen', date_paste)
# first time we see this file encoding on this paste
if serv_metadata.zscore(decoder_name+'_hash:'+hash, message) is None:
print('first '+decoder_name)
# first time we see this hash (all encoding) on this paste
if serv_metadata.zscore('nb_seen_hash:'+hash, message) is None:
serv_metadata.hincrby('metadata_hash:'+hash, 'nb_seen_in_all_pastes', 1)
serv_metadata.hincrby('metadata_hash:'+hash, decoder_name+'_decoder', 1)
serv_metadata.sadd('hash_paste:'+message, hash) # paste - hash map
serv_metadata.sadd(decoder_name+'_paste:'+message, hash) # paste - hash map
serv_metadata.zincrby('nb_seen_hash:'+hash, message, 1)# hash - paste map
serv_metadata.zincrby(decoder_name+'_hash:'+hash, message, 1)# hash - paste map
# create hash metadata
serv_metadata.hset('metadata_hash:'+hash, 'estimated_type', type)
serv_metadata.sadd('hash_all_type', type)
# first time we see this hash encoding on this paste
if serv_metadata.zscore(decoder_name+'_hash:'+hash, message) is None:
print('first '+decoder_name)
serv_metadata.sadd(decoder_name+'_paste:'+message, hash) # paste - hash map
# create hash metadata
serv_metadata.sadd('hash_'+ decoder_name +'_all_type', type)
serv_metadata.zincrby('hash_type:'+type, date_key, 1)
serv_metadata.zincrby(decoder_name+'_type:'+type, date_key, 1)
save_hash_on_disk(decoded, type, hash, json_data)
print('found {} '.format(type))
# duplicate
else:
serv_metadata.zincrby(decoder_name+'_hash:'+hash, message, 1) # number of b64 on this paste
serv_metadata.hincrby('metadata_hash:'+hash, decoder_name+'_decoder', 1)
serv_metadata.zincrby('hash_type:'+type, date_key, 1)
serv_metadata.zincrby(decoder_name+'_type:'+type, date_key, 1)
serv_metadata.zincrby('nb_seen_hash:'+hash, message, 1)# hash - paste map
serv_metadata.zincrby(decoder_name+'_hash:'+hash, message, 1) # number of b64 on this paste
def save_hash_on_disk(decode, type, hash, json_data):
@ -182,12 +186,14 @@ if __name__ == '__main__':
binary_max_execution_time = p.config.getint("Binary", "max_execution_time")
base64_max_execution_time = p.config.getint("Base64", "max_execution_time")
# list all decoder yith regex, the order is use to search content by order
all_decoder = [ {'name': 'binary', 'regex': regex_binary, 'encoded_min_size': 300, 'max_execution_time': binary_max_execution_time},
{'name': 'hexadecimal', 'regex': regex_hex, 'encoded_min_size': 300, 'max_execution_time': hex_max_execution_time},
{'name': 'base64', 'regex': regex_base64, 'encoded_min_size': 40, 'max_execution_time': base64_max_execution_time}]
# list all decoder yith regex,
decoder_binary = {'name': 'binary', 'regex': regex_binary, 'encoded_min_size': 300, 'max_execution_time': binary_max_execution_time}
decoder_hexadecimal = {'name': 'hexadecimal', 'regex': regex_hex, 'encoded_min_size': 300, 'max_execution_time': hex_max_execution_time}
decoder_base64 = {'name': 'base64', 'regex': regex_base64, 'encoded_min_size': 40, 'max_execution_time': base64_max_execution_time}
for decoder in all_decoder:
decoder_order = [ decoder_base64, decoder_binary, decoder_hexadecimal, decoder_base64]
for decoder in decoder_order:
serv_metadata.sadd('all_decoder', decoder['name'])
# Endless loop getting messages from the input queue
@ -207,7 +213,7 @@ if __name__ == '__main__':
content = paste.get_p_content()
date = str(paste._get_p_date())
for decoder in all_decoder: # add threshold and size limit
for decoder in decoder_order: # add threshold and size limit
# max execution time on regex
signal.alarm(decoder['max_execution_time'])

4
var/www/modules/hashDecoded/Flask_hashDecoded.py
View file

@ -364,9 +364,7 @@ def decoder_type_json():
all_decoder = r_serv_metadata.smembers('all_decoder')
# sort DESC decoder for color
all_decoder = sorted(all_decoder, reverse=True)
print(all_decoder)
print(type(all_decoder))
all_decoder = sorted(all_decoder)
date_range = []
if date_from is not None and date_to is not None:

11
var/www/modules/hashDecoded/templates/hashDecoded.html
View file

@ -47,6 +47,9 @@
.bar_stack:hover{
cursor: pointer;
}
.pie_path:hover{
cursor: pointer;
}
.svgText {
pointer-events: none;
}
@ -318,7 +321,7 @@ function sparklines(id, points) {
</script>
<script>
var margin = {top: 20, right: 55, bottom: 55, left: 40},
var margin = {top: 20, right: 100, bottom: 55, left: 45},
width = 1000 - margin.left - margin.right,
height = 500 - margin.top - margin.bottom;
var x = d3.scaleBand().rangeRound([0, width]).padding(0.1);
@ -449,7 +452,7 @@ function drawLegend (varNames) {
.attr("transform", function (d, i) { return "translate(0," + i * 20 + ")"; });
legend.append("rect")
.attr("x", 152)
.attr("x", 943)
.attr("width", 10)
.attr("height", 10)
.style("fill", color)
@ -457,7 +460,7 @@ function drawLegend (varNames) {
legend.append("text")
.attr("class", "svgText")
.attr("x", 150)
.attr("x", 941)
.attr("y", 6)
.attr("dy", ".35em")
.style("text-anchor", "end")
@ -539,8 +542,10 @@ d3.json("/hashDecoded/decoder_type_json?date_from={{date_from}}&date_to={{date_t
.append('path')
.attr('d', arc_pie)
.attr('fill', (d,i) => color_pie(i))
.attr('class', 'pie_path')
.on("mouseover", mouseovered_pie)
.on("mouseout", mouseouted_pie)
.on("click", function (d) {window.location.href = '/hashDecoded/?date_from={{date_from}}&date_to={{date_to}}&type={{type}}&encoding='+d.data.name })
.style('opacity', opacity_pie)
.style('stroke', 'white');
});