mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-23 06:37:15 +00:00
Draft module + web link
This commit is contained in:
Mokaddem
parent
9b8570e40b
commit
882ab88b00
5 changed files with 168 additions and 28 deletions
|
|
@ -9,6 +9,17 @@ This module is consuming the Redis-list created by the Categ module.
|
|||
|
||||
It apply credential regexes on paste content and warn if above a threshold.
|
||||
|
||||
It also split the username and store it into redis for searching purposes.
|
||||
|
||||
Redis organization:
|
||||
uniqNumForUsername: unique number attached to unique username
|
||||
uniqNumForPath: unique number attached to unique path
|
||||
AllCredentials: hashed set where keys are username and value are their uniq number
|
||||
AllCredentialsRev: the opposite of AllCredentials, uniqNum -> username
|
||||
AllPath: hashed set where keys are path and value are their uniq number
|
||||
AllPathRev: the opposite of AllPath, uniqNum -> path
|
||||
splitedCred -> uniq_num (set)
|
||||
|
||||
"""
|
||||
|
||||
import time
|
||||
|
|
@ -17,8 +28,19 @@ from packages import Paste
|
|||
from pubsublogger import publisher
|
||||
from Helper import Process
|
||||
import re
|
||||
import redis
|
||||
from pyfaup.faup import Faup
|
||||
|
||||
#split username with spec. char or with upper case, distinguish start with upper
|
||||
REGEX_CRED = "[a-z]+|[A-Z]{3,}|[A-Z]{1,2}[a-z]+|[0-9]+"
|
||||
REDIS_KEY_NUM_USERNAME = 'uniqNumForUsername'
|
||||
REDIS_KEY_NUM_PATH = 'uniqNumForUsername'
|
||||
REDIS_KEY_ALL_CRED_SET = 'AllCredentials'
|
||||
REDIS_KEY_ALL_CRED_SET_REV = 'AllCredentialsRev'
|
||||
REDIS_KEY_ALL_PATH_SET = 'AllPath'
|
||||
REDIS_KEY_ALL_PATH_SET_REV = 'AllPathRev'
|
||||
REDIS_KEY_MAP_CRED_TO_PATH = 'CredToPathMapping'
|
||||
|
||||
if __name__ == "__main__":
|
||||
publisher.port = 6380
|
||||
publisher.channel = "Script"
|
||||
|
|
@ -27,6 +49,10 @@ if __name__ == "__main__":
|
|||
publisher.info("Find credentials")
|
||||
|
||||
faup = Faup()
|
||||
server_cred = redis.StrictRedis(
|
||||
host=p.config.get("Redis_Level_DB_TermCred", "host"),
|
||||
port=p.config.get("Redis_Level_DB_TermCred", "port"),
|
||||
db=p.config.get("Redis_Level_DB_TermCred", "db"))
|
||||
|
||||
critical = 8
|
||||
|
||||
|
|
@ -37,6 +63,7 @@ if __name__ == "__main__":
|
|||
message = p.get_from_set()
|
||||
if message is None:
|
||||
publisher.debug("Script Credential is Idling 10s")
|
||||
print('sleeping 10s')
|
||||
time.sleep(10)
|
||||
continue
|
||||
|
||||
|
|
@ -44,6 +71,7 @@ if __name__ == "__main__":
|
|||
|
||||
if count < 5:
|
||||
# Less than 5 matches from the top password list, false positive.
|
||||
print("false positive:", count)
|
||||
continue
|
||||
|
||||
paste = Paste.Paste(filepath)
|
||||
|
|
@ -63,6 +91,7 @@ if __name__ == "__main__":
|
|||
|
||||
print('\n '.join(creds))
|
||||
|
||||
#num of creds above tresh, publish an alert
|
||||
if len(creds) > critical:
|
||||
print("========> Found more than 10 credentials in this file : {}".format(filepath))
|
||||
publisher.warning(to_print)
|
||||
|
|
@ -97,3 +126,31 @@ if __name__ == "__main__":
|
|||
print("=======> Probably on : {}".format(', '.join(sites_set)))
|
||||
else:
|
||||
publisher.info(to_print)
|
||||
print('found {} credentials'.format(len(creds)))
|
||||
|
||||
|
||||
#for searching credential in cred seeker
|
||||
for cred in creds:
|
||||
cred = cred.split('@')[0]
|
||||
|
||||
#unique number attached to unique path
|
||||
uniq_num_path = server_cred.incr(REDIS_KEY_ALL_PATH_SET)
|
||||
print(REDIS_KEY_ALL_PATH_SET, {filepath: uniq_num_path})
|
||||
server_cred.hmset(REDIS_KEY_ALL_PATH_SET, {filepath: uniq_num_path})
|
||||
server_cred.hmset(REDIS_KEY_ALL_PATH_SET_REV, {uniq_num_path: filepath})
|
||||
|
||||
#unique number attached to unique username
|
||||
uniq_num_cred = server_cred.hget(REDIS_KEY_ALL_CRED_SET, cred)
|
||||
if uniq_num_cred is None: #cred do not exist, create new entries
|
||||
uniq_num_cred = server_cred.incr(REDIS_KEY_NUM_USERNAME)
|
||||
server_cred.hmset(REDIS_KEY_ALL_CRED_SET, {cred: uniq_num_cred})
|
||||
server_cred.hmset(REDIS_KEY_ALL_CRED_SET_REV, {uniq_num_cred: cred})
|
||||
|
||||
server_cred.hmset(REDIS_KEY_MAP_CRED_TO_PATH, {uniq_num_cred: uniq_num_path})
|
||||
|
||||
splitedCred = re.findall(REGEX_CRED, cred)
|
||||
print(splitedCred)
|
||||
for partCred in splitedCred:
|
||||
server_cred.sadd(partCred, uniq_num_cred)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -97,6 +97,11 @@ host = localhost
|
|||
port = 6382
|
||||
db = 2
|
||||
|
||||
[Redis_Level_DB_TermCred]
|
||||
host = localhost
|
||||
port = 6382
|
||||
db = 5
|
||||
|
||||
[Redis_Level_DB]
|
||||
host = localhost
|
||||
port = 2016
|
||||
|
|
|
|||
|
|
@ -53,6 +53,11 @@ r_serv_term = redis.StrictRedis(
|
|||
port=cfg.getint("Redis_Level_DB_TermFreq", "port"),
|
||||
db=cfg.getint("Redis_Level_DB_TermFreq", "db"))
|
||||
|
||||
r_serv_cred = redis.StrictRedis(
|
||||
host=cfg.get("Redis_Level_DB_TermCred", "host"),
|
||||
port=cfg.getint("Redis_Level_DB_TermCred", "port"),
|
||||
db=cfg.getint("Redis_Level_DB_TermCred", "db"))
|
||||
|
||||
r_serv_pasteName = redis.StrictRedis(
|
||||
host=cfg.get("Redis_Paste_Name", "host"),
|
||||
port=cfg.getint("Redis_Paste_Name", "port"),
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import flask
|
|||
from flask import Flask, render_template, jsonify, request, Blueprint
|
||||
import re
|
||||
import Paste
|
||||
from pprint import pprint
|
||||
|
||||
# ============ VARIABLES ============
|
||||
import Flask_config
|
||||
|
|
@ -18,6 +19,7 @@ import Flask_config
|
|||
app = Flask_config.app
|
||||
cfg = Flask_config.cfg
|
||||
r_serv_term = Flask_config.r_serv_term
|
||||
r_serv_cred = Flask_config.r_serv_cred
|
||||
|
||||
terms = Blueprint('terms', __name__, template_folder='templates')
|
||||
|
||||
|
|
@ -357,11 +359,80 @@ def credentials_management_query_paste():
|
|||
cred = request.args.get('cred')
|
||||
return 1
|
||||
|
||||
def mixUserName(supplied):
|
||||
#e.g.: John Smith
|
||||
terms = supplied.split()[:2]
|
||||
usernames = []
|
||||
if len(terms) == 1:
|
||||
terms.append(' ')
|
||||
|
||||
#john, smith, John, Smith, JOHN, SMITH
|
||||
usernames += [terms[0].lower()]
|
||||
usernames += [terms[1].lower()]
|
||||
usernames += [terms[0][0].upper() + terms[0][1:].lower()]
|
||||
usernames += [terms[1][0].upper() + terms[1][1:].lower()]
|
||||
usernames += [terms[0].upper()]
|
||||
usernames += [terms[1].upper()]
|
||||
|
||||
#johnsmith, smithjohn, JOHNsmith, johnSMITH, SMITHjohn, smithJOHN
|
||||
usernames += [(terms[0].lower() + terms[1].lower()).strip()]
|
||||
usernames += [(terms[1].lower() + terms[0].lower()).strip()]
|
||||
usernames += [(terms[0].upper() + terms[1].lower()).strip()]
|
||||
usernames += [(terms[0].lower() + terms[1].upper()).strip()]
|
||||
usernames += [(terms[1].upper() + terms[0].lower()).strip()]
|
||||
usernames += [(terms[1].lower() + terms[0].upper()).strip()]
|
||||
#Jsmith, JSmith, jsmith, jSmith, johnS, Js, JohnSmith, Johnsmith, johnSmith
|
||||
usernames += [(terms[0][0].upper() + terms[1][0].lower() + terms[1][1:].lower()).strip()]
|
||||
usernames += [(terms[0][0].upper() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
|
||||
usernames += [(terms[0][0].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()]
|
||||
usernames += [(terms[0][0].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
|
||||
usernames += [(terms[0].lower() + terms[1][0].upper()).strip()]
|
||||
usernames += [(terms[0].upper() + terms[1][0].lower()).strip()]
|
||||
usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
|
||||
usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()]
|
||||
usernames += [(terms[0][0].lower() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
|
||||
|
||||
return usernames
|
||||
|
||||
|
||||
|
||||
@terms.route("/credentials_management_action/", methods=['GET'])
|
||||
def cred_management_action():
|
||||
cred = request.args.get('cred')
|
||||
REGEX_CRED = '[a-z]+|[A-Z]{3,}|[A-Z]{1,2}[a-z]+|[0-9]+'
|
||||
REDIS_KEY_NUM_USERNAME = 'uniqNumForUsername'
|
||||
REDIS_KEY_NUM_PATH = 'uniqNumForUsername'
|
||||
REDIS_KEY_ALL_CRED_SET = 'AllCredentials'
|
||||
REDIS_KEY_ALL_CRED_SET_REV = 'AllCredentialsRev'
|
||||
REDIS_KEY_ALL_PATH_SET = 'AllPath'
|
||||
REDIS_KEY_ALL_PATH_SET_REV = 'AllPath'
|
||||
REDIS_KEY_MAP_CRED_TO_PATH = 'CredToPathMapping'
|
||||
|
||||
supplied = request.args.get('term')
|
||||
action = request.args.get('action')
|
||||
return 1
|
||||
section = request.args.get('section')
|
||||
|
||||
#splitedCred = re.findall(REGEX_CRED, cred)
|
||||
uniq_num_set = set()
|
||||
if action == "seek":
|
||||
possibilities = mixUserName(supplied)
|
||||
for poss in possibilities:
|
||||
for num in r_serv_cred.smembers(poss):
|
||||
uniq_num_set.add(num)
|
||||
|
||||
data = {'usr': [], 'path': []}
|
||||
for Unum in uniq_num_set:
|
||||
data['usr'].append(r_serv_cred.hget(REDIS_KEY_ALL_CRED_SET_REV, Unum))
|
||||
data['path'].append(r_serv_cred.hget(REDIS_KEY_MAP_CRED_TO_PATH, Unum))
|
||||
|
||||
pprint(data)
|
||||
to_return = {}
|
||||
to_return["section"] = section
|
||||
to_return["action"] = action
|
||||
to_return["term"] = supplied
|
||||
to_return["data"] = data
|
||||
|
||||
return jsonify(to_return)
|
||||
|
||||
|
||||
@terms.route("/credentials_management_query/")
|
||||
def cred_management_query():
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@
|
|||
<div id="page-wrapper">
|
||||
<div class="row">
|
||||
<div class="col-lg-12">
|
||||
<h1 class="page-header" data-page="page-termsfrequency" >Credentials tracking</h1>
|
||||
<h1 class="page-header" data-page="page-termsfrequency" >Credential seeker</h1>
|
||||
</div>
|
||||
<!-- /.col-lg-12 -->
|
||||
<!-- Panel OPTIONS -->
|
||||
|
|
@ -43,24 +43,23 @@
|
|||
<div class="col-lg-12">
|
||||
<div id="panel-today" class="panel panel-success">
|
||||
<div class="panel-heading">
|
||||
<strong>Manage tracked credentials</strong>
|
||||
<strong>Credential seeker</strong>
|
||||
</div>
|
||||
<div class="panel-body">
|
||||
|
||||
<div class="form-group input-group" style="margin-bottom: 30px;">
|
||||
<span class="input-group-addon"><span class="fa fa-eye"></span></span>
|
||||
<input id="followCredInput" class="form-control" placeholder="Credential to track." type="text" style="max-width: 400px;">
|
||||
<button id="followTermBtn" class="btn btn-success btn-interaction" style="margin-left: 10px;" data-section="followTerm" data-action="add"> Track</button>
|
||||
<input id="seekInput" class="form-control" placeholder="Credential to seek." type="text" style="max-width: 400px;">
|
||||
<button id="followTermBtn" class="btn btn-success btn-interaction" style="margin-left: 10px;" data-section="seek" data-action="seek"> Seek</button>
|
||||
<span id="nodata" class="alert alert-danger" style="margin-left: 10px; display: none; padding: 6px;">No data</span>
|
||||
</div>
|
||||
|
||||
<table class="table table-striped table-bordered table-hover" id="myTable">
|
||||
<thead>
|
||||
<tr>
|
||||
<th style="max-width: 800px;">Credential</th>
|
||||
<th>Added date</th>
|
||||
<th>Day occurence</th>
|
||||
<th>Week occurence</th>
|
||||
<th>Month occurence</th>
|
||||
<th>Date</th>
|
||||
<th># line in the paste</th>
|
||||
<th># tracked paste</th>
|
||||
<th>Action</th>
|
||||
</tr>
|
||||
|
|
@ -70,14 +69,12 @@
|
|||
{% for set in trackSet_list %}
|
||||
<tr style="background-color: #cdffca;">
|
||||
<td>{{ set }}</td>
|
||||
<td>{{ trackSet_list_values[loop.index0][3] }}</td>
|
||||
<td>{{ trackSet_list_values[loop.index0][0] }}</td>
|
||||
<td>{{ trackSet_list_values[loop.index0][1] }}</td>
|
||||
<td>{{ trackSet_list_values[loop.index0][2] }}</td>
|
||||
<td>{{ trackSet_list_num_of_paste[loop.index0] }}</td>
|
||||
<td><p style="margin: 0px;">
|
||||
<span data-toggle="modal" data-target="#mymodal" data-term="{{ set }}" ><button class="btn-link" data-toggle="tooltip" data-placement="right" title="Show concerned paste(s)"><span class="glyphicon glyphicon-info-sign"></span></button></span>
|
||||
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ set }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
|
||||
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Track this credentials" data-content="{{ set }}" data-section="followTerm" data-action="add"><span class="fa fa-eye"></span></button>
|
||||
</p></td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
|
|
@ -184,32 +181,37 @@
|
|||
var curr_action = $(this).attr('data-action');
|
||||
if (curr_action == "add") {
|
||||
var curr_term = $('#'+curr_section+'Input').val();
|
||||
} else if (curr_action == "seek") {
|
||||
var curr_term = $('#'+curr_section+'Input').val();
|
||||
} else {
|
||||
var curr_term = $(this).attr('data-content');
|
||||
}
|
||||
var data_to_send = { section: curr_section, action:curr_action, term: curr_term};
|
||||
|
||||
if (curr_term != "") {
|
||||
console.log(data_to_send);
|
||||
//console.log(data_to_send);
|
||||
$.get("{{ url_for('terms.cred_management_action') }}", data_to_send, function(data, status){
|
||||
if(status == "success") {
|
||||
var json = data;
|
||||
console.log(json)
|
||||
|
||||
if(json.section == "followTerm") {
|
||||
if(json.action == "add") {
|
||||
// query data
|
||||
$.get("{{ url_for('terms.cred_management_query') }}", { term: json.term, section: json.section }, function(data2, status){
|
||||
|
||||
} else if (json.action == "seek") {
|
||||
var rep = json.data
|
||||
var action_button = "<button class=\"btn-link btn-interaction\" data-toggle=\"tooltip\" data-placement=\"left\" title=\"Remove this term\" data-content=\"" + json.term + "\" data-section=\"followTerm\" data-action=\"delete\"><span class=\"glyphicon glyphicon-trash\"></span></button>"
|
||||
table_track.row.add( [ json.term, data2[3], data2[0], data2[1], data2[2], 0, action_button ] ).draw( false );
|
||||
perform_binding();
|
||||
if (rep.usr.length == 0) {
|
||||
console.log('noData');
|
||||
$( "#nodata" ).fadeIn( "fast", function() {
|
||||
setTimeout(function() {
|
||||
$( "#nodata" ).fadeOut("fast");
|
||||
}, 2000);
|
||||
});
|
||||
} else if (json.action == "delete") {
|
||||
// Find indexes of row which have the term in the first column
|
||||
var index = table_track.rows().eq( 0 ).filter( function (rowIdx) {
|
||||
console.log(table_track.cell( rowIdx, 0 ).data())
|
||||
return table_track.cell( rowIdx, 0 ).data() === json.term;
|
||||
} );
|
||||
table_track.rows(index).remove().draw( false );
|
||||
} else {
|
||||
for(i=0; i < rep.length; i++) {
|
||||
table_track.row.add( [ json.term, "xx/xx/xxxx", "xxxx", "xxxx", action_button ] ).draw( false );
|
||||
}
|
||||
//perform_binding();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue