Merge branch 'master' into statistic

This commit is contained in:
Terrtia 2018-08-01 09:29:51 +02:00
commit 940b96734a
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
9 changed files with 302 additions and 18 deletions

115
bin/BankAccount.py Executable file
View file

@ -0,0 +1,115 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
"""
The BankAccount Module
======================
It apply IBAN regexes on paste content and warn if above a threshold.
"""
import redis
import time
import re
import string
from itertools import chain
from packages import Paste
from pubsublogger import publisher
from Helper import Process
import signal
class TimeoutException(Exception):
pass
def timeout_handler(signum, frame):
raise TimeoutException
signal.signal(signal.SIGALRM, timeout_handler)
_LETTERS_IBAN = chain(enumerate(string.digits + string.ascii_uppercase),
enumerate(string.ascii_lowercase, 10))
LETTERS_IBAN = {ord(d): str(i) for i, d in _LETTERS_IBAN}
def iban_number(iban):
return (iban[4:] + iban[:4]).translate(LETTERS_IBAN)
def is_valid_iban(iban):
iban_numb = iban_number(iban)
iban_numb_check = iban_number(iban[:2] + '00' + iban[4:])
check_digit = '{:0>2}'.format(98 - (int(iban_numb_check) % 97))
if check_digit == iban[2:4] and int(iban_numb) % 97 == 1:
# valid iban
print('valid iban')
return True
return False
def check_all_iban(l_iban, paste, filename):
nb_valid_iban = 0
for iban in l_iban:
iban = iban[0]+iban[1]+iban[2]
iban = ''.join(e for e in iban if e.isalnum())
#iban = iban.upper()
res = iban_regex_verify.findall(iban)
if res:
print('checking '+iban)
if is_valid_iban(iban):
print('------')
nb_valid_iban = nb_valid_iban + 1
if(nb_valid_iban > 0):
to_print = 'Iban;{};{};{};'.format(paste.p_source, paste.p_date, paste.p_name)
publisher.warning('{}Checked found {} IBAN;{}'.format(
to_print, nb_valid_iban, paste.p_path))
msg = 'infoleak:automatic-detection="iban";{}'.format(filename)
p.populate_set_out(msg, 'Tags')
#Send to duplicate
p.populate_set_out(filename, 'Duplicate')
if __name__ == "__main__":
publisher.port = 6380
publisher.channel = "Script"
config_section = 'BankAccount'
p = Process(config_section)
max_execution_time = p.config.getint("BankAccount", "max_execution_time")
publisher.info("BankAccount started")
message = p.get_from_set()
#iban_regex = re.compile(r'\b[A-Za-z]{2}[0-9]{2}(?:[ ]?[0-9]{4}){4}(?:[ ]?[0-9]{1,2})?\b')
iban_regex = re.compile(r'\b([A-Za-z]{2}[ \-]?[0-9]{2})(?=(?:[ \-]?[A-Za-z0-9]){9,30})((?:[ \-]?[A-Za-z0-9]{3,5}){2,6})([ \-]?[A-Za-z0-9]{1,3})\b')
iban_regex_verify = re.compile(r'^([A-Z]{2})([0-9]{2})([A-Z0-9]{9,30})$')
while True:
message = p.get_from_set()
if message is not None:
filename = message
paste = Paste.Paste(filename)
content = paste.get_p_content()
signal.alarm(max_execution_time)
try:
l_iban = iban_regex.findall(content)
except TimeoutException:
print ("{0} processing timeout".format(paste.p_path))
continue
else:
signal.alarm(0)
if(len(l_iban) > 0):
check_all_iban(l_iban, paste, filename)
else:
publisher.debug("Script BankAccount is Idling 10s")
time.sleep(10)

View file

@ -87,8 +87,6 @@ if __name__ == "__main__":
content = paste.get_p_content()
creds = set(re.findall(regex_cred, content))
publisher.warning('to_print')
if len(creds) == 0:
continue

View file

@ -144,6 +144,8 @@ function launching_scripts {
sleep 0.1
screen -S "Script_AIL" -X screen -t "CreditCards" bash -c 'cd '${AIL_BIN}'; ./CreditCards.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "BankAccount" bash -c 'cd '${AIL_BIN}'; ./BankAccount.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "Onion" bash -c 'cd '${AIL_BIN}'; ./Onion.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "Mail" bash -c 'cd '${AIL_BIN}'; ./Mail.py; read x'

View file

@ -43,6 +43,9 @@ minute_processed_paste = 10
DiffMaxLineLength = 10000
#### Modules ####
[BankAccount]
max_execution_time = 60
[Categ]
#Minimum number of match between the paste and the category file
matchingThreshold=1

View file

@ -51,6 +51,10 @@ publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Web,Redis_Credential,Re
subscribe = Redis_CreditCards
publish = Redis_Duplicate,Redis_ModuleStats,Redis_alertHandler,Redis_Tags
[BankAccount]
subscribe = Redis_Global
publish = Redis_Duplicate,Redis_Tags
[Mail]
subscribe = Redis_Mail
publish = Redis_Duplicate,Redis_ModuleStats,Redis_alertHandler,Redis_Tags

View file

@ -143,3 +143,5 @@ DiffMaxLineLength = int(cfg.get("Flask", "DiffMaxLineLength"))#Use to display t
bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
UPLOAD_FOLDER = os.path.join(os.environ['AIL_FLASK'], 'submitted')
max_dashboard_logs = int(cfg.get("Flask", "max_dashboard_logs"))

View file

@ -5,10 +5,14 @@
Flask functions and routes for the dashboard page
'''
import json
import os
import datetime
import time
import flask
from flask import Flask, render_template, jsonify, request, Blueprint
from Date import Date
from flask import Flask, render_template, jsonify, request, Blueprint, url_for
# ============ VARIABLES ============
import Flask_config
@ -18,6 +22,8 @@ cfg = Flask_config.cfg
r_serv = Flask_config.r_serv
r_serv_log = Flask_config.r_serv_log
max_dashboard_logs = Flask_config.max_dashboard_logs
dashboard = Blueprint('dashboard', __name__, template_folder='templates')
# ============ FUNCTIONS ============
@ -62,12 +68,87 @@ def get_queues(r):
return newData
def get_date_range(date_from, num_day):
date = Date(str(date_from[0:4])+str(date_from[4:6]).zfill(2)+str(date_from[6:8]).zfill(2))
date_list = []
for i in range(0, num_day+1):
new_date = date.substract_day(i)
date_list.append(new_date[0:4] +'-'+ new_date[4:6] +'-'+ new_date[6:8])
return date_list
def dashboard_alert(log):
# check if we need to display this log
if len(log)>50:
date = log[1:5]+log[6:8]+log[9:11]
utc_str = log[1:20]
log = log[46:].split(';')
if len(log) == 6:
time = datetime_from_utc_to_local(utc_str)
path = url_for('showsavedpastes.showsavedpaste',paste=log[5])
res = {'date': date, 'time': time, 'script': log[0], 'domain': log[1], 'date_paste': log[2],
'paste': log[3], 'message': log[4], 'path': path}
return res
else:
return False
else:
return False
def datetime_from_utc_to_local(utc_str):
utc_datetime = datetime.datetime.strptime(utc_str, '%Y-%m-%d %H:%M:%S')
now_timestamp = time.time()
offset = datetime.datetime.fromtimestamp(now_timestamp) - datetime.datetime.utcfromtimestamp(now_timestamp)
local_time_str = (utc_datetime + offset).strftime('%H:%M:%S')
return local_time_str
# ============ ROUTES ============
@dashboard.route("/_logs")
def logs():
return flask.Response(event_stream(), mimetype="text/event-stream")
@dashboard.route("/_get_last_logs_json")
def get_last_logs_json():
date = datetime.datetime.now().strftime("%Y%m%d")
max_day_search = 6
day_search = 0
warning_found = 0
warning_to_found = max_dashboard_logs
last_logs = []
date_range = get_date_range(date, max_day_search)
while max_day_search != day_search and warning_found != warning_to_found:
filename_warning_log = 'logs/Script_warn-'+ date_range[day_search] +'.log'
filename_log = os.path.join(os.environ['AIL_HOME'], filename_warning_log)
try:
with open(filename_log, 'r') as f:
lines = f.read().splitlines()
curr_index = -1
while warning_found != warning_to_found:
try:
# get lasts warning logs
log_warn = dashboard_alert(lines[curr_index])
if log_warn != False:
last_logs.append(log_warn)
warning_found = warning_found + 1
curr_index = curr_index - 1
except IndexError:
# check previous warning log file
day_search = day_search + 1
break
except FileNotFoundError:
# check previous warning log file
day_search = day_search + 1
return jsonify(list(reversed(last_logs)))
@dashboard.route("/_stuff", methods=['GET'])
def stuff():
@ -78,7 +159,12 @@ def stuff():
def index():
default_minute = cfg.get("Flask", "minute_processed_paste")
threshold_stucked_module = cfg.getint("Module_ModuleInformation", "threshold_stucked_module")
return render_template("index.html", default_minute = default_minute, threshold_stucked_module=threshold_stucked_module)
log_select = {10, 25, 50, 100}
log_select.add(max_dashboard_logs)
log_select = list(log_select)
log_select.sort()
return render_template("index.html", default_minute = default_minute, threshold_stucked_module=threshold_stucked_module,
log_select=log_select, selected=max_dashboard_logs)
# ========= REGISTRATION =========
app.register_blueprint(dashboard)

View file

@ -31,6 +31,14 @@
};
update_values();
</script>
<style>
.tableQueue tbody tr td,
.tableQueue tbody tr th,
.tableQueue thead tr td,
.tableQueue thead tr th{
padding: 1px;
}
</style>
</head>
<body>
@ -136,10 +144,13 @@
<div class="pull-right">
<label style="padding-bottom:2px;">
<select class="form-control input-sm" id="log_select">
<option value="10">10</option>
<option value="25">25</option>
<option value="50">50</option>
<option value="100">100</option>
{% for log_selection in log_select %}
{% if log_selection == selected %}
<option value="{{ log_selection }}" selected>{{ log_selection }}</option>
{% else %}
<option value="{{ log_selection }}">{{ log_selection }}</option>
{% endif %}
{% endfor %}
</select>
</label>
<input id="checkbox_log_info" type="checkbox" value="info"> INFO
@ -182,6 +193,66 @@
<script>
activePage = "page-index";
$("#"+activePage).addClass("active");
var tableBody = document.getElementById('tab_body')
$.getJSON('/_get_last_logs_json', function(data) {
data.forEach(function (d) {
var tr = document.createElement('TR')
var time = document.createElement('TD')
var chan = document.createElement('TD')
var level = document.createElement('TD')
var scrpt = document.createElement('TD')
var src = document.createElement('TD')
var pdate = document.createElement('TD')
var nam = document.createElement('TD')
var msage = document.createElement('TD')
var inspect = document.createElement('TD')
tr.className = "warning";
time.appendChild(document.createTextNode(d.time))
chan.appendChild(document.createTextNode('Script'))
level.appendChild(document.createTextNode('WARNING'))
scrpt.appendChild(document.createTextNode(d.script))
src.appendChild(document.createTextNode(d.domain))
pdate.appendChild(document.createTextNode(d.date_paste))
nam.appendChild(document.createTextNode(d.paste))
var iconspan = document.createElement('SPAN');
var message = d.message.split(" ")
if (message[0] == "Detected"){
iconspan.className = "glyphicon glyphicon-eye-open";
}
else if (message[0] == "Checked"){
iconspan.className = "glyphicon glyphicon-thumbs-up";
}
iconspan.innerHTML = "&nbsp;";
msage.appendChild(iconspan);
msage.appendChild(document.createTextNode(message.join(" ")));
var action_icon_a = document.createElement("A");
action_icon_a.setAttribute("TARGET", "_blank");
action_icon_a.setAttribute("HREF", d.path);
var action_icon_span = document.createElement('SPAN');
action_icon_span.className = "fa fa-search-plus";
action_icon_a.appendChild(action_icon_span);
inspect.appendChild(action_icon_a)
inspect.setAttribute("style", "text-align:center;");
tr.appendChild(time)
tr.appendChild(chan);
tr.appendChild(level);
tr.appendChild(scrpt);
tr.appendChild(src);
tr.appendChild(pdate);
tr.appendChild(nam);
tr.appendChild(msage);
tr.appendChild(inspect);
tableBody.appendChild(tr);
});
});
</script>
<script type="text/javascript">

View file

@ -248,6 +248,7 @@ function create_log_table(obj_json) {
action_icon_a.appendChild(action_icon_span);
inspect.appendChild(action_icon_a);
inspect.setAttribute("style", "text-align:center;");
tr.appendChild(time)
tr.appendChild(chan);
@ -281,7 +282,7 @@ function create_queue_table() {
document.getElementById("queueing").innerHTML = "";
var Tablediv = document.getElementById("queueing")
var table = document.createElement('TABLE')
table.className = "table table-bordered table-hover table-striped";
table.className = "table table-bordered table-hover table-striped tableQueue";
var tableHead = document.createElement('THEAD')
var tableBody = document.createElement('TBODY')
@ -295,7 +296,11 @@ function create_queue_table() {
for (i = 0; i < heading.length; i++) {
var th = document.createElement('TH')
if (heading[i] == "Amount") {
th.width = '50';
} else {
th.width = '100';
}
th.appendChild(document.createTextNode(heading[i]));
tr.appendChild(th);
}
@ -494,5 +499,3 @@ function manage_undefined() {
$(document).ready(function () {
manage_undefined();
});