Bug fix related with redis:

Fixed typo key in redis for module creditcard and sqlinjection
Modified Curve redisLvlDb server
Modified Url.py so that it forwards name of protocol from saved protocolsfile
Added Cache control in Flask
Modified key-tab name into keys-tab
This commit is contained in:
Mokaddem 2016-08-09 11:59:36 +02:00
parent 60512a30a2
commit a80f6ca0c2
11 changed files with 47 additions and 17 deletions

View file

@ -68,7 +68,7 @@ if __name__ == "__main__":
#Send to duplicate
p.populate_set_out(filepath, 'Redis_Duplicate')
#send to Browse_warning_paste
p.populate_set_out('creditCard;{}'.format(filename), 'BrowseWarningPaste')
p.populate_set_out('creditcard;{}'.format(filename), 'BrowseWarningPaste')
else:
publisher.info('{}CreditCard related'.format(to_print))
else:

View file

@ -40,9 +40,9 @@ if __name__ == "__main__":
# REDIS #
r_serv1 = redis.StrictRedis(
host=p.config.get("Redis_Level_DB", "host"),
port=p.config.get("Redis_Level_DB", "port"),
db=p.config.get("Redis_Level_DB", "db"))
host=p.config.get("Redis_Level_DB_Curve", "host"),
port=p.config.get("Redis_Level_DB_Curve", "port"),
db=p.config.get("Redis_Level_DB_Curve", "db"))
# FUNCTIONS #
publisher.info("Script Curve started")

View file

@ -67,6 +67,7 @@ function launching_lvldb {
db1_y='2013'
db2_y='2014'
db3_y='2016'
db4_y='3016'
nb_db=13
screen -dmS "LevelDB"
@ -78,6 +79,10 @@ function launching_lvldb {
screen -S "LevelDB" -X screen -t "2014" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2014/ -P '$db2_y' -M '$nb_db'; read x'
sleep 0.1
screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db3_y' -M '$nb_db'; read x'
# For Curve
sleep 0.1
screen -S "LevelDB" -X screen -t "3016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'3016/ -P '$db4_y' -M '$nb_db'; read x'
}
function launching_logs {

View file

@ -77,7 +77,7 @@ def analyse(url, path):
#Send to duplicate
p.populate_set_out(path, 'Duplicate')
#send to Browse_warning_paste
p.populate_set_out('sqlInjectionDetection;{}'.format(path), 'BrowseWarningPaste')
p.populate_set_out('sqlinjection;{}'.format(path), 'BrowseWarningPaste')
else:
print "Potential SQL injection:"
print urllib2.unquote(url)

View file

@ -3,6 +3,7 @@
import redis
import pprint
import time
import os
import dns.exception
from packages import Paste
from packages import lib_refine
@ -39,6 +40,10 @@ if __name__ == "__main__":
port=p.config.getint("Redis_Cache", "port"),
db=p.config.getint("Redis_Cache", "db"))
# Protocol file path
protocolsfile_path = os.path.join(os.environ['AIL_HOME'],
p.config.get("Directories", "protocolsfile"))
# Country to log as critical
cc_critical = p.config.get("Url", "cc_critical")
@ -52,7 +57,14 @@ if __name__ == "__main__":
prec_filename = None
faup = Faup()
url_regex = "(http|https|ftp)\://([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.(com|edu|gov|int|mil|net|org|biz|arpa|info|name|pro|aero|coop|museum|[a-zA-Z]{2}))(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*"
# Get all uri from protocolsfile (Used for Curve)
uri_scheme = ""
with open(protocolsfile_path, 'r') as scheme_file:
for scheme in scheme_file:
uri_scheme += scheme[:-1]+"|"
uri_scheme = uri_scheme[:-1]
url_regex = "("+uri_scheme+")\://([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.(com|edu|gov|int|mil|net|org|biz|arpa|info|name|pro|aero|coop|museum|[a-zA-Z]{2}))(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*"
while True:
if message is not None:

View file

@ -26,6 +26,7 @@ def analyse(server, field_name, date, url_parsed):
prev_score = server.hget(field, date)
if prev_score is not None:
server.hset(field, date, int(prev_score) + 1)
else:
server.hset(field, date, 1)
if field_name == "domain": #save domain in a set for the monthly plot
@ -67,7 +68,6 @@ def compute_progression(server, field_name, num_day, url_parsed):
divisor = value_list_reversed[i-1] if value_list_reversed[i-1] != 0 else 1
keyword_increase += value_list_reversed[i] / divisor
# filter
if (keyword_total_sum > threshold_total_sum) and (keyword_increase > threshold_increase):

View file

@ -164,7 +164,6 @@ def showpaste(content_range):
def getPastebyType(server, module_name):
all_path = []
for path in server.smembers('WARNING_'+module_name):
#all_path.append("/home/mokaddem/AIL-framework/PASTES/archive/paste.debian.net/2016/06/30/771058.gz")
all_path.append(path)
return all_path
@ -198,7 +197,16 @@ def get_top_relevant_data(server, module_name):
member_set.insert(0, ("passed_days", days))
return member_set
# ========= CACHE CONTROL ========
@app.after_request
def add_header(response):
"""
Add headers to both force latest IE rendering engine or Chrome Frame,
and also to cache the rendered page for 10 minutes.
"""
response.headers['X-UA-Compatible'] = 'IE=Edge,chrome=1'
response.headers['Cache-Control'] = 'public, max-age=0'
return response
# ============ ROUTES ============

View file

@ -112,19 +112,21 @@ function Graph(id_pannel, path, header_size){
// display the top headers
function setVis(max_display){
headings = this.graph.getLabels();
var headings = this.graph.getLabels();
headings.splice(0,1);
var sorted_list = new Array();
today = new Date().getDate()-1; // Take the top from yesterday so that we can see the current evolution
var today = new Date().getDate();
//today = new Date().getDate()-1; // Could take the top from yesterday so that we can see the current evolution
for( i=0; i<headings.length; i++){
the_heading = headings[i];
//console.log('heading='+the_heading+' tab['+(today-1)+']['+(parseInt(i)+1)+']='+g.getValue(today-1, parseInt(i)+1));
var the_heading = headings[i];
//console.log('heading='+the_heading+' tab['+(today)+']['+(parseInt(i)+1)+']='+this.graph.getValue(today-1, parseInt(i)+1));
sorted_list.push({dom: the_heading, val: this.graph.getValue(today-1, parseInt(i)+1), index: parseInt(i)});
}
sorted_list.sort(function(a,b) {
return b.val - a.val;
});
var display_list = sorted_list.slice(0, max_display);
for( i=0; i<display_list.length; i++){
this.graph.setVisibility(display_list[i].index, true);

View file

@ -5,6 +5,10 @@
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta http-equiv="Cache-Control" content="no-cache, no-store, must-revalidate" />
<meta http-equiv="Pragma" content="no-cache" />
<meta http-equiv="Expires" content="0" />
<title>Analysis Information Leak framework Dashboard</title>
<!-- Core CSS -->

View file

@ -99,7 +99,7 @@
<li name='nav-pan'><a data-toggle="tab" href="#creditcard-tab" data-attribute-name="creditcard" data-panel="creditcard-panel">Credit cards</a></li>
<li name='nav-pan'><a data-toggle="tab" href="#sqlinjection-tab" data-attribute-name="sqlinjection" data-panel="sqlinjection-panel">SQL injections</a></li>
<li name='nav-pan'><a data-toggle="tab" href="#cve-tab" data-attribute-name="cve" data-panel="cve-panel">CVEs</a></li>
<li name='nav-pan'><a data-toggle="tab" href="#key-tab" data-attribute-name="key" data-panel="key-panel">Keys</a></li>
<li name='nav-pan'><a data-toggle="tab" href="#keys-tab" data-attribute-name="keys" data-panel="keys-panel">Keys</a></li>
<li name='nav-pan'><a data-toggle="tab" href="#mail-tab" data-attribute-name="mail" data-panel="mail-panel">Mails</a></li>
<li name='nav-pan'><a data-toggle="tab" href="#phone-tab" data-attribute-name="phone" data-panel="phone-panel">Phones</a></li>
</ul>
@ -119,7 +119,7 @@
<div class="col-lg-12 tab-pane fade" id="cve-tab">
<img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" style="margin: 4px;">
</div>
<div class="col-lg-12 tab-pane fade" id="key-tab">
<div class="col-lg-12 tab-pane fade" id="keys-tab">
<img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" style="margin: 4px;">
</div>
<div class="col-lg-12 tab-pane fade" id="mail-tab">
@ -150,7 +150,6 @@ var previous_tab = $('[data-attribute-name="credential');
var loading_gif = "<img id='loading-gif-modal' class='img-center' src=\"{{url_for('static', filename='image/loading.gif') }}\" height='26' width='26' style='margin: 4px;'>";
$('.nav-tabs a').on('shown.bs.tab', function(event){
console.log(event.target);
var dataPath = $(event.target).attr('data-attribute-name');
$.get("{{ url_for('importantPasteByModule') }}"+"?moduleName="+dataPath, function(data, status){
var currentTab = $('[name].active').children();