mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-23 06:37:15 +00:00
Added similarity checks + started link webInterface
This commit is contained in:
Mokaddem
parent
882ab88b00
commit
ef741f85c0
6 changed files with 146 additions and 112 deletions
|
|
@ -40,6 +40,7 @@ REDIS_KEY_ALL_CRED_SET_REV = 'AllCredentialsRev'
|
|||
REDIS_KEY_ALL_PATH_SET = 'AllPath'
|
||||
REDIS_KEY_ALL_PATH_SET_REV = 'AllPathRev'
|
||||
REDIS_KEY_MAP_CRED_TO_PATH = 'CredToPathMapping'
|
||||
MINIMUMSIZETHRESHOLD = 3
|
||||
|
||||
if __name__ == "__main__":
|
||||
publisher.port = 6380
|
||||
|
|
@ -53,7 +54,7 @@ if __name__ == "__main__":
|
|||
host=p.config.get("Redis_Level_DB_TermCred", "host"),
|
||||
port=p.config.get("Redis_Level_DB_TermCred", "port"),
|
||||
db=p.config.get("Redis_Level_DB_TermCred", "db"))
|
||||
|
||||
Credential
|
||||
critical = 8
|
||||
|
||||
regex_web = "((?:https?:\/\/)[-_0-9a-zA-Z]+\.[0-9a-zA-Z]+)"
|
||||
|
|
@ -134,7 +135,7 @@ if __name__ == "__main__":
|
|||
cred = cred.split('@')[0]
|
||||
|
||||
#unique number attached to unique path
|
||||
uniq_num_path = server_cred.incr(REDIS_KEY_ALL_PATH_SET)
|
||||
uniq_num_path = server_cred.incr(REDIS_KEY_NUM_PATH)
|
||||
print(REDIS_KEY_ALL_PATH_SET, {filepath: uniq_num_path})
|
||||
server_cred.hmset(REDIS_KEY_ALL_PATH_SET, {filepath: uniq_num_path})
|
||||
server_cred.hmset(REDIS_KEY_ALL_PATH_SET_REV, {uniq_num_path: filepath})
|
||||
|
|
@ -151,6 +152,7 @@ if __name__ == "__main__":
|
|||
splitedCred = re.findall(REGEX_CRED, cred)
|
||||
print(splitedCred)
|
||||
for partCred in splitedCred:
|
||||
if len(partCred) > MINIMUMSIZETHRESHOLD:
|
||||
server_cred.sadd(partCred, uniq_num_cred)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ nltk
|
|||
crcmod
|
||||
mmh3
|
||||
ssdeep
|
||||
python-Levenshtein
|
||||
|
||||
#Others
|
||||
python-magic
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from flask import Flask, render_template, jsonify, request, Blueprint
|
|||
import re
|
||||
import Paste
|
||||
from pprint import pprint
|
||||
import Levenshtein
|
||||
|
||||
# ============ VARIABLES ============
|
||||
import Flask_config
|
||||
|
|
@ -54,6 +55,41 @@ def Term_getValueOverRange(word, startDate, num_day, per_paste=""):
|
|||
passed_days += 1
|
||||
return to_return
|
||||
|
||||
def mixUserName(supplied):
|
||||
#e.g.: John Smith
|
||||
terms = supplied.split()[:2]
|
||||
usernames = []
|
||||
if len(terms) == 1:
|
||||
terms.append(' ')
|
||||
|
||||
#john, smith, John, Smith, JOHN, SMITH
|
||||
usernames += [terms[0].lower()]
|
||||
usernames += [terms[1].lower()]
|
||||
usernames += [terms[0][0].upper() + terms[0][1:].lower()]
|
||||
usernames += [terms[1][0].upper() + terms[1][1:].lower()]
|
||||
usernames += [terms[0].upper()]
|
||||
usernames += [terms[1].upper()]
|
||||
|
||||
#johnsmith, smithjohn, JOHNsmith, johnSMITH, SMITHjohn, smithJOHN
|
||||
usernames += [(terms[0].lower() + terms[1].lower()).strip()]
|
||||
usernames += [(terms[1].lower() + terms[0].lower()).strip()]
|
||||
usernames += [(terms[0].upper() + terms[1].lower()).strip()]
|
||||
usernames += [(terms[0].lower() + terms[1].upper()).strip()]
|
||||
usernames += [(terms[1].upper() + terms[0].lower()).strip()]
|
||||
usernames += [(terms[1].lower() + terms[0].upper()).strip()]
|
||||
#Jsmith, JSmith, jsmith, jSmith, johnS, Js, JohnSmith, Johnsmith, johnSmith
|
||||
usernames += [(terms[0][0].upper() + terms[1][0].lower() + terms[1][1:].lower()).strip()]
|
||||
usernames += [(terms[0][0].upper() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
|
||||
usernames += [(terms[0][0].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()]
|
||||
usernames += [(terms[0][0].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
|
||||
usernames += [(terms[0].lower() + terms[1][0].upper()).strip()]
|
||||
usernames += [(terms[0].upper() + terms[1][0].lower()).strip()]
|
||||
usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
|
||||
usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()]
|
||||
usernames += [(terms[0][0].lower() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
|
||||
|
||||
return usernames
|
||||
|
||||
|
||||
# ============ ROUTES ============
|
||||
|
||||
|
|
@ -359,40 +395,6 @@ def credentials_management_query_paste():
|
|||
cred = request.args.get('cred')
|
||||
return 1
|
||||
|
||||
def mixUserName(supplied):
|
||||
#e.g.: John Smith
|
||||
terms = supplied.split()[:2]
|
||||
usernames = []
|
||||
if len(terms) == 1:
|
||||
terms.append(' ')
|
||||
|
||||
#john, smith, John, Smith, JOHN, SMITH
|
||||
usernames += [terms[0].lower()]
|
||||
usernames += [terms[1].lower()]
|
||||
usernames += [terms[0][0].upper() + terms[0][1:].lower()]
|
||||
usernames += [terms[1][0].upper() + terms[1][1:].lower()]
|
||||
usernames += [terms[0].upper()]
|
||||
usernames += [terms[1].upper()]
|
||||
|
||||
#johnsmith, smithjohn, JOHNsmith, johnSMITH, SMITHjohn, smithJOHN
|
||||
usernames += [(terms[0].lower() + terms[1].lower()).strip()]
|
||||
usernames += [(terms[1].lower() + terms[0].lower()).strip()]
|
||||
usernames += [(terms[0].upper() + terms[1].lower()).strip()]
|
||||
usernames += [(terms[0].lower() + terms[1].upper()).strip()]
|
||||
usernames += [(terms[1].upper() + terms[0].lower()).strip()]
|
||||
usernames += [(terms[1].lower() + terms[0].upper()).strip()]
|
||||
#Jsmith, JSmith, jsmith, jSmith, johnS, Js, JohnSmith, Johnsmith, johnSmith
|
||||
usernames += [(terms[0][0].upper() + terms[1][0].lower() + terms[1][1:].lower()).strip()]
|
||||
usernames += [(terms[0][0].upper() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
|
||||
usernames += [(terms[0][0].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()]
|
||||
usernames += [(terms[0][0].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
|
||||
usernames += [(terms[0].lower() + terms[1][0].upper()).strip()]
|
||||
usernames += [(terms[0].upper() + terms[1][0].lower()).strip()]
|
||||
usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
|
||||
usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()]
|
||||
usernames += [(terms[0][0].lower() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
|
||||
|
||||
return usernames
|
||||
|
||||
|
||||
|
||||
|
|
@ -407,7 +409,7 @@ def cred_management_action():
|
|||
REDIS_KEY_ALL_PATH_SET_REV = 'AllPath'
|
||||
REDIS_KEY_MAP_CRED_TO_PATH = 'CredToPathMapping'
|
||||
|
||||
supplied = request.args.get('term')
|
||||
supplied = request.args.get('term').encode('utf-8')
|
||||
action = request.args.get('action')
|
||||
section = request.args.get('section')
|
||||
|
||||
|
|
@ -419,12 +421,22 @@ def cred_management_action():
|
|||
for num in r_serv_cred.smembers(poss):
|
||||
uniq_num_set.add(num)
|
||||
|
||||
data = {'usr': [], 'path': []}
|
||||
data = {'usr': [], 'path': [], 'numPaste': [], 'simil': []}
|
||||
for Unum in uniq_num_set:
|
||||
data['usr'].append(r_serv_cred.hget(REDIS_KEY_ALL_CRED_SET_REV, Unum))
|
||||
data['path'].append(r_serv_cred.hget(REDIS_KEY_MAP_CRED_TO_PATH, Unum))
|
||||
username = r_serv_cred.hget(REDIS_KEY_ALL_CRED_SET_REV, Unum)
|
||||
|
||||
# Calculate Levenshtein distance, ignore negative ratio
|
||||
levenDist = float(Levenshtein.distance(supplied, username))
|
||||
levenRatio = levenDist / float(len(supplied))
|
||||
levenRatioStr = "{:.1%}".format(1.0 - levenRatio)
|
||||
if levenRatio >= 1.0:
|
||||
continue
|
||||
|
||||
data['usr'].append(username)
|
||||
data['path'].append(r_serv_cred.hget(REDIS_KEY_MAP_CRED_TO_PATH, Unum))
|
||||
data['numPaste'].append(len(uniq_num_set))
|
||||
data['simil'].append(levenRatioStr)
|
||||
|
||||
pprint(data)
|
||||
to_return = {}
|
||||
to_return["section"] = section
|
||||
to_return["action"] = action
|
||||
|
|
|
|||
|
|
@ -25,9 +25,31 @@
|
|||
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
|
||||
|
||||
<style>
|
||||
.btn-link {
|
||||
color: #000000
|
||||
}
|
||||
</style>
|
||||
|
||||
</head>
|
||||
<body>
|
||||
<!-- Modal -->
|
||||
<div id="mymodal" class="modal fade" role="dialog">
|
||||
<div class="modal-dialog modal-lg">
|
||||
|
||||
<!-- Modal content-->
|
||||
<div id="mymodalcontent" class="modal-content">
|
||||
<div id="mymodalbody" class="modal-body" max-width="8500px">
|
||||
<p>Loading paste information...</p>
|
||||
<img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" height="26" width="26" style="margin: 4px;">
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<a id="button_show_plot" target="_blank" href=""><button type="button" class="btn btn-info">Plot term</button></a>
|
||||
<button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% include 'navbar.html' %}
|
||||
|
||||
<div id="page-wrapper">
|
||||
|
|
@ -48,36 +70,22 @@
|
|||
<div class="panel-body">
|
||||
|
||||
<div class="form-group input-group" style="margin-bottom: 30px;">
|
||||
<span class="input-group-addon"><span class="fa fa-eye"></span></span>
|
||||
<span class="input-group-addon"><span class="glyphicon glyphicon-screenshot"></span></span>
|
||||
<input id="seekInput" class="form-control" placeholder="Credential to seek." type="text" style="max-width: 400px;">
|
||||
<button id="followTermBtn" class="btn btn-success btn-interaction" style="margin-left: 10px;" data-section="seek" data-action="seek"> Seek</button>
|
||||
<span id="nodata" class="alert alert-danger" style="margin-left: 10px; display: none; padding: 6px;">No data</span>
|
||||
<span id="nodata" class="alert alert-info" style="margin-left: 10px; display: none; padding: 6px;">No data</span>
|
||||
</div>
|
||||
|
||||
<table class="table table-striped table-bordered table-hover" id="myTable">
|
||||
<thead>
|
||||
<tr>
|
||||
<th style="max-width: 800px;">Credential</th>
|
||||
<th>Date</th>
|
||||
<th># line in the paste</th>
|
||||
<th># tracked paste</th>
|
||||
<th style="max-width: 800px;">Username</th>
|
||||
<th>Similarity</th>
|
||||
<th># concerned paste(s)</th>
|
||||
<th>Action</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<!-- SET -->
|
||||
{% for set in trackSet_list %}
|
||||
<tr style="background-color: #cdffca;">
|
||||
<td>{{ set }}</td>
|
||||
<td>{{ trackSet_list_values[loop.index0][0] }}</td>
|
||||
<td>{{ trackSet_list_values[loop.index0][1] }}</td>
|
||||
<td>{{ trackSet_list_num_of_paste[loop.index0] }}</td>
|
||||
<td><p style="margin: 0px;">
|
||||
<span data-toggle="modal" data-target="#mymodal" data-term="{{ set }}" ><button class="btn-link" data-toggle="tooltip" data-placement="right" title="Show concerned paste(s)"><span class="glyphicon glyphicon-info-sign"></span></button></span>
|
||||
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Track this credentials" data-content="{{ set }}" data-section="followTerm" data-action="add"><span class="fa fa-eye"></span></button>
|
||||
</p></td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
<!-- /.panel-body -->
|
||||
|
|
@ -88,9 +96,10 @@
|
|||
<!-- /.panel -->
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<!-- /.row -->
|
||||
</div>
|
||||
|
||||
</div>
|
||||
<!-- /.row -->
|
||||
</div>
|
||||
<!-- /#page-wrapper -->
|
||||
|
||||
<script>
|
||||
|
|
@ -99,17 +108,16 @@
|
|||
$("#"+activePage).addClass("active");
|
||||
|
||||
$('[data-toggle="tooltip"]').tooltip();
|
||||
table_track = $('#myTable').DataTable();
|
||||
table_black = $('#myTable2').DataTable();
|
||||
table_track = $('#myTable').DataTable({ "order": [[ 1, "dec" ]] });
|
||||
|
||||
table_track.on( 'draw.dt', function () {
|
||||
perform_binding();
|
||||
});
|
||||
|
||||
$("#followCredInput").keyup(function(event){
|
||||
$("#seekInput").keyup(function(event){
|
||||
if(event.keyCode == 13){
|
||||
$("#followTermBtn").click();
|
||||
$("#followCredInput").val("");
|
||||
$("#seekInput").val("");
|
||||
}
|
||||
});
|
||||
|
||||
|
|
@ -193,25 +201,37 @@
|
|||
$.get("{{ url_for('terms.cred_management_action') }}", data_to_send, function(data, status){
|
||||
if(status == "success") {
|
||||
var json = data;
|
||||
console.log(json)
|
||||
|
||||
if(json.action == "add") {
|
||||
|
||||
} else if (json.action == "seek") {
|
||||
var rep = json.data
|
||||
var action_button = "<button class=\"btn-link btn-interaction\" data-toggle=\"tooltip\" data-placement=\"left\" title=\"Remove this term\" data-content=\"" + json.term + "\" data-section=\"followTerm\" data-action=\"delete\"><span class=\"glyphicon glyphicon-trash\"></span></button>"
|
||||
table_track.clear().draw();
|
||||
var rep = json.data;
|
||||
var action_button = "<button class=\"btn-link btn-interaction\" data-toggle=\"tooltip\" data-placement=\"left\" title=\"Track this term\" data-content=\"" + json.term + "\" data-section=\"followTerm\" data-action=\"add\">";
|
||||
if (rep.usr.length == 0) {
|
||||
console.log('noData');
|
||||
$( "#nodata" ).removeClass('alert-info').addClass('alert-danger');
|
||||
$( "#nodata" ).text("No data");
|
||||
$( "#nodata" ).fadeIn( "fast", function() {
|
||||
setTimeout(function() {
|
||||
$( "#nodata" ).fadeOut("fast");
|
||||
}, 2000);
|
||||
});
|
||||
} else {
|
||||
for(i=0; i < rep.length; i++) {
|
||||
table_track.row.add( [ json.term, "xx/xx/xxxx", "xxxx", "xxxx", action_button ] ).draw( false );
|
||||
console.log(json);
|
||||
console.log(table_track);
|
||||
for(i=0; i < rep.usr.length; i++) {
|
||||
$( "#nodata" ).removeClass('alert-danger').addClass('alert-info');
|
||||
$( "#nodata" ).text(curr_term);
|
||||
$( "#nodata" ).fadeIn( "fast");
|
||||
toAdd = "<span class=\"fa fa-eye\"></span></button><span data-toggle=\"modal\" data-target=\"#mymodal\" data-term=\""+rep.usr[i]+"\" ><button class=\"btn-link\" data-toggle=\"tooltip\" data-placement=\"right\" title=\"Show concerned paste(s)\"><span class=\"glyphicon glyphicon-info-sign\"></span></button></span>";
|
||||
table_track.row.add( [
|
||||
rep.usr[i],
|
||||
rep.simil[i],
|
||||
rep.numPaste[i],
|
||||
toAdd+action_button ] ).draw( false );
|
||||
}
|
||||
//perform_binding();
|
||||
perform_binding();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -220,7 +240,6 @@
|
|||
}
|
||||
</script>
|
||||
|
||||
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
<span class="caret"></span></a>
|
||||
<ul class="dropdown-menu">
|
||||
<li><a href="{{ url_for('terms.terms_management') }}"><i class="fa fa-gear "> </i> Terms managements</a></li>
|
||||
<li><a href="{{ url_for('terms.credentials_tracker') }}"><i class="glyphicon glyphicon-screenshot"> </i> Credentials tracker</a></li>
|
||||
<li><a href="{{ url_for('terms.credentials_tracker') }}"><i class="glyphicon glyphicon-screenshot"> </i> Credentials seeker</a></li>
|
||||
<li><a href="{{ url_for('terms.terms_plot_top') }}"><i class="glyphicon glyphicon-fire"> </i> Terms plot top</a></li>
|
||||
<li><a href="{{ url_for('terms.terms_plot_tool') }}"><i class="fa fa-wrench"> </i> Terms plot tool</a></li>
|
||||
</ul>
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@
|
|||
<span class="caret"></span></a>
|
||||
<ul class="dropdown-menu">
|
||||
<li><a href="{{ url_for('terms.terms_management') }}"><i class="fa fa-gear "> </i> Terms managements</a></li>
|
||||
<li><a href="{{ url_for('terms.credentials_tracker') }}"><i class="glyphicon glyphicon-screenshot"> </i> Credentials tracker</a></li>
|
||||
<li><a href="{{ url_for('terms.credentials_tracker') }}"><i class="glyphicon glyphicon-screenshot"> </i> Credentials seeker</a></li>
|
||||
<li><a href="{{ url_for('terms.terms_plot_top') }}"><i class="glyphicon glyphicon-fire"> </i> Terms plot top</a></li>
|
||||
<li><a href="{{ url_for('terms.terms_plot_tool') }}"><i class="fa fa-wrench"> </i> Terms plot tool</a></li>
|
||||
</ul>
|
||||
|
|
|
|||
Loading…
Reference in a new issue