Draft module + web link

bin/Credential.py

@@ -9,6 +9,17 @@ This module is consuming the Redis-list created by the Categ module.
 
 It apply credential regexes on paste content and warn if above a threshold.
 
+It also split the username and store it into redis for searching purposes.
+
+Redis organization:
+    uniqNumForUsername: unique number attached to unique username
+    uniqNumForPath: unique number attached to unique path
+    AllCredentials: hashed set where keys are username and value are their uniq number
+    AllCredentialsRev: the opposite of AllCredentials, uniqNum -> username
+    AllPath: hashed set where keys are path and value are their uniq number
+    AllPathRev: the opposite of AllPath, uniqNum -> path
+    splitedCred -> uniq_num (set)
+
 """
 
 import time
@@ -17,8 +28,19 @@ from packages import Paste
 from pubsublogger import publisher
 from Helper import Process
 import re
+import redis
 from pyfaup.faup import Faup
 
+#split username with spec. char or with upper case, distinguish start with upper
+REGEX_CRED = "[a-z]+|[A-Z]{3,}|[A-Z]{1,2}[a-z]+|[0-9]+"
+REDIS_KEY_NUM_USERNAME = 'uniqNumForUsername'
+REDIS_KEY_NUM_PATH = 'uniqNumForUsername'
+REDIS_KEY_ALL_CRED_SET = 'AllCredentials'
+REDIS_KEY_ALL_CRED_SET_REV = 'AllCredentialsRev'
+REDIS_KEY_ALL_PATH_SET = 'AllPath'
+REDIS_KEY_ALL_PATH_SET_REV = 'AllPathRev'
+REDIS_KEY_MAP_CRED_TO_PATH = 'CredToPathMapping'
+
 if __name__ == "__main__":
     publisher.port = 6380
     publisher.channel = "Script"
@@ -27,6 +49,10 @@ if __name__ == "__main__":
     publisher.info("Find credentials")
 
     faup = Faup()
+    server_cred = redis.StrictRedis(
+        host=p.config.get("Redis_Level_DB_TermCred", "host"),
+        port=p.config.get("Redis_Level_DB_TermCred", "port"),
+        db=p.config.get("Redis_Level_DB_TermCred", "db"))
 
     critical = 8
 
@@ -37,6 +63,7 @@ if __name__ == "__main__":
         message = p.get_from_set()
         if message is None:
             publisher.debug("Script Credential is Idling 10s")
+            print('sleeping 10s')
             time.sleep(10)
             continue
 
@@ -44,6 +71,7 @@ if __name__ == "__main__":
 
         if count < 5:
             # Less than 5 matches from the top password list, false positive.
+            print("false positive:", count)
             continue
 
         paste = Paste.Paste(filepath)
@@ -63,6 +91,7 @@ if __name__ == "__main__":
 
         print('\n '.join(creds))
 
+        #num of creds above tresh, publish an alert
         if len(creds) > critical:
             print("========> Found more than 10 credentials in this file : {}".format(filepath))
             publisher.warning(to_print)
@@ -97,3 +126,31 @@ if __name__ == "__main__":
                 print("=======> Probably on : {}".format(', '.join(sites_set)))
         else:
             publisher.info(to_print)
+            print('found {} credentials'.format(len(creds)))
+
+
+        #for searching credential in cred seeker
+        for cred in creds:
+            cred = cred.split('@')[0]
+
+            #unique number attached to unique path
+            uniq_num_path = server_cred.incr(REDIS_KEY_ALL_PATH_SET)
+            print(REDIS_KEY_ALL_PATH_SET, {filepath: uniq_num_path})
+            server_cred.hmset(REDIS_KEY_ALL_PATH_SET, {filepath: uniq_num_path})
+            server_cred.hmset(REDIS_KEY_ALL_PATH_SET_REV, {uniq_num_path: filepath})
+
+            #unique number attached to unique username
+            uniq_num_cred = server_cred.hget(REDIS_KEY_ALL_CRED_SET, cred)
+            if uniq_num_cred is None: #cred do not exist, create new entries
+                uniq_num_cred = server_cred.incr(REDIS_KEY_NUM_USERNAME)
+                server_cred.hmset(REDIS_KEY_ALL_CRED_SET, {cred: uniq_num_cred})
+                server_cred.hmset(REDIS_KEY_ALL_CRED_SET_REV, {uniq_num_cred: cred})
+        
+            server_cred.hmset(REDIS_KEY_MAP_CRED_TO_PATH, {uniq_num_cred: uniq_num_path})
+
+            splitedCred = re.findall(REGEX_CRED, cred)
+            print(splitedCred)
+            for partCred in splitedCred:
+                server_cred.sadd(partCred, uniq_num_cred)
+
+

bin/packages/config.cfg.sample

@@ -97,6 +97,11 @@ host = localhost
 port = 6382
 db = 2
 
+[Redis_Level_DB_TermCred]
+host = localhost
+port = 6382
+db = 5
+
 [Redis_Level_DB]
 host = localhost
 port = 2016

var/www/modules/Flask_config.py

@@ -53,6 +53,11 @@ r_serv_term = redis.StrictRedis(
         port=cfg.getint("Redis_Level_DB_TermFreq", "port"),
         db=cfg.getint("Redis_Level_DB_TermFreq", "db"))
 
+r_serv_cred = redis.StrictRedis(
+        host=cfg.get("Redis_Level_DB_TermCred", "host"),
+        port=cfg.getint("Redis_Level_DB_TermCred", "port"),
+        db=cfg.getint("Redis_Level_DB_TermCred", "db"))
+
 r_serv_pasteName = redis.StrictRedis(
     host=cfg.get("Redis_Paste_Name", "host"),
     port=cfg.getint("Redis_Paste_Name", "port"),

var/www/modules/terms/Flask_terms.py

@@ -11,6 +11,7 @@ import flask
 from flask import Flask, render_template, jsonify, request, Blueprint
 import re
 import Paste
+from pprint import pprint
 
 # ============ VARIABLES ============
 import Flask_config
@@ -18,6 +19,7 @@ import Flask_config
 app = Flask_config.app
 cfg = Flask_config.cfg
 r_serv_term = Flask_config.r_serv_term
+r_serv_cred = Flask_config.r_serv_cred
 
 terms = Blueprint('terms', __name__, template_folder='templates')
 
@@ -357,11 +359,80 @@ def credentials_management_query_paste():
     cred =  request.args.get('cred')
     return 1
 
+def mixUserName(supplied):
+    #e.g.: John Smith
+    terms = supplied.split()[:2]
+    usernames = []
+    if len(terms) == 1:
+        terms.append(' ')
+
+    #john, smith, John, Smith, JOHN, SMITH
+    usernames += [terms[0].lower()]
+    usernames += [terms[1].lower()]
+    usernames += [terms[0][0].upper() + terms[0][1:].lower()]
+    usernames += [terms[1][0].upper() + terms[1][1:].lower()]
+    usernames += [terms[0].upper()]
+    usernames += [terms[1].upper()]
+
+    #johnsmith, smithjohn, JOHNsmith, johnSMITH, SMITHjohn, smithJOHN
+    usernames += [(terms[0].lower() + terms[1].lower()).strip()]
+    usernames += [(terms[1].lower() + terms[0].lower()).strip()]
+    usernames += [(terms[0].upper() + terms[1].lower()).strip()]
+    usernames += [(terms[0].lower() + terms[1].upper()).strip()]
+    usernames += [(terms[1].upper() + terms[0].lower()).strip()]
+    usernames += [(terms[1].lower() + terms[0].upper()).strip()]
+    #Jsmith, JSmith, jsmith, jSmith, johnS, Js, JohnSmith, Johnsmith, johnSmith
+    usernames += [(terms[0][0].upper() + terms[1][0].lower() + terms[1][1:].lower()).strip()]
+    usernames += [(terms[0][0].upper() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
+    usernames += [(terms[0][0].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()]
+    usernames += [(terms[0][0].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
+    usernames += [(terms[0].lower() + terms[1][0].upper()).strip()]
+    usernames += [(terms[0].upper() + terms[1][0].lower()).strip()]
+    usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
+    usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()]
+    usernames += [(terms[0][0].lower() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
+
+    return usernames
+    
+
+
 @terms.route("/credentials_management_action/", methods=['GET'])
 def cred_management_action():
-    cred =  request.args.get('cred')
+    REGEX_CRED = '[a-z]+|[A-Z]{3,}|[A-Z]{1,2}[a-z]+|[0-9]+'
+    REDIS_KEY_NUM_USERNAME = 'uniqNumForUsername'
+    REDIS_KEY_NUM_PATH = 'uniqNumForUsername'
+    REDIS_KEY_ALL_CRED_SET = 'AllCredentials'
+    REDIS_KEY_ALL_CRED_SET_REV = 'AllCredentialsRev'
+    REDIS_KEY_ALL_PATH_SET = 'AllPath'
+    REDIS_KEY_ALL_PATH_SET_REV = 'AllPath'
+    REDIS_KEY_MAP_CRED_TO_PATH = 'CredToPathMapping'
+
+    supplied =  request.args.get('term')
     action = request.args.get('action')
-    return 1
+    section = request.args.get('section')
+
+    #splitedCred = re.findall(REGEX_CRED, cred)
+    uniq_num_set = set()
+    if action == "seek":
+        possibilities = mixUserName(supplied)
+        for poss in possibilities:
+            for num in r_serv_cred.smembers(poss):
+                uniq_num_set.add(num)
+
+    data = {'usr': [], 'path': []}
+    for Unum in uniq_num_set:
+        data['usr'].append(r_serv_cred.hget(REDIS_KEY_ALL_CRED_SET_REV, Unum))
+        data['path'].append(r_serv_cred.hget(REDIS_KEY_MAP_CRED_TO_PATH, Unum))
+
+    pprint(data)
+    to_return = {}
+    to_return["section"] = section
+    to_return["action"] = action
+    to_return["term"] = supplied
+    to_return["data"] = data
+
+    return jsonify(to_return)
+
 
 @terms.route("/credentials_management_query/")
 def cred_management_query():

var/www/modules/terms/templates/credentials_tracker.html

@@ -33,7 +33,7 @@
             <div id="page-wrapper">
 				<div class="row">
 				  <div class="col-lg-12">
-				  <h1 class="page-header" data-page="page-termsfrequency" >Credentials tracking</h1>
+				  <h1 class="page-header" data-page="page-termsfrequency" >Credential seeker</h1>
 				  </div>
 				  <!-- /.col-lg-12 -->
     <!-- Panel OPTIONS -->
@@ -43,24 +43,23 @@
                 <div class="col-lg-12">
                     <div id="panel-today" class="panel panel-success">
                         <div class="panel-heading">
-                            <strong>Manage tracked credentials</strong>
+                            <strong>Credential seeker</strong>
                         </div>
                         <div class="panel-body">
 
                             <div class="form-group input-group" style="margin-bottom: 30px;">
                                 <span class="input-group-addon"><span class="fa fa-eye"></span></span>
-                                <input id="followCredInput" class="form-control" placeholder="Credential to track." type="text" style="max-width: 400px;">
+                                <input id="seekInput" class="form-control" placeholder="Credential to seek." type="text" style="max-width: 400px;">
-                                <button id="followTermBtn" class="btn btn-success btn-interaction" style="margin-left: 10px;" data-section="followTerm" data-action="add"> Track</button>
+                                <button id="followTermBtn" class="btn btn-success btn-interaction" style="margin-left: 10px;" data-section="seek" data-action="seek"> Seek</button>
+                                <span id="nodata" class="alert alert-danger" style="margin-left: 10px; display: none; padding: 6px;">No data</span>
                             </div>
 
                             <table class="table table-striped table-bordered table-hover" id="myTable">
                                 <thead>
                                      <tr>
                                         <th style="max-width: 800px;">Credential</th>
-                                        <th>Added date</th>
+                                        <th>Date</th>
-                                        <th>Day occurence</th>
+                                        <th># line in the paste</th>
-                                        <th>Week occurence</th>
-                                        <th>Month occurence</th>
                                         <th># tracked paste</th>
                                         <th>Action</th>
                                      </tr>
@@ -70,14 +69,12 @@
                                 {% for set in trackSet_list %}
                                   <tr style="background-color: #cdffca;">
                                       <td>{{ set }}</td>
-                                      <td>{{ trackSet_list_values[loop.index0][3] }}</td>
                                       <td>{{ trackSet_list_values[loop.index0][0] }}</td>
                                       <td>{{ trackSet_list_values[loop.index0][1] }}</td>
-                                      <td>{{ trackSet_list_values[loop.index0][2] }}</td>
                                       <td>{{ trackSet_list_num_of_paste[loop.index0] }}</td>
                                       <td><p style="margin: 0px;">
                                           <span data-toggle="modal" data-target="#mymodal" data-term="{{ set }}" ><button class="btn-link" data-toggle="tooltip" data-placement="right" title="Show concerned paste(s)"><span class="glyphicon glyphicon-info-sign"></span></button></span>
-                                          <button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ set }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
+                                          <button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Track this credentials" data-content="{{ set }}" data-section="followTerm" data-action="add"><span class="fa fa-eye"></span></button>
                                       </p></td>
                                   </tr>
                                 {% endfor %}
@@ -184,32 +181,37 @@
                     var curr_action = $(this).attr('data-action'); 
                     if (curr_action == "add") {
                         var curr_term = $('#'+curr_section+'Input').val();
+                    } else if (curr_action == "seek") {
+                        var curr_term = $('#'+curr_section+'Input').val();
                     } else {
                         var curr_term = $(this).attr('data-content');
                     }
                     var data_to_send = { section: curr_section, action:curr_action, term: curr_term};
                 
                     if (curr_term != "") {
-                        console.log(data_to_send);
+                        //console.log(data_to_send);
                         $.get("{{ url_for('terms.cred_management_action') }}", data_to_send, function(data, status){
                             if(status == "success") {
                                 var json = data;
+                                console.log(json)
                 
-                                if(json.section == "followTerm") {
+                                if(json.action == "add") {
-                                    if(json.action == "add") {
+
-                                        // query data
+                                } else if (json.action == "seek") {
-                                        $.get("{{ url_for('terms.cred_management_query') }}", { term: json.term, section: json.section }, function(data2, status){
+                                    var rep = json.data
-                                            var action_button = "<button class=\"btn-link btn-interaction\" data-toggle=\"tooltip\" data-placement=\"left\" title=\"Remove this term\" data-content=\"" + json.term + "\" data-section=\"followTerm\" data-action=\"delete\"><span class=\"glyphicon glyphicon-trash\"></span></button>"
+                                    var action_button = "<button class=\"btn-link btn-interaction\" data-toggle=\"tooltip\" data-placement=\"left\" title=\"Remove this term\" data-content=\"" + json.term + "\" data-section=\"followTerm\" data-action=\"delete\"><span class=\"glyphicon glyphicon-trash\"></span></button>"
-                                            table_track.row.add( [ json.term, data2[3], data2[0], data2[1], data2[2], 0, action_button ] ).draw( false );
+                                    if (rep.usr.length == 0) {
-                                            perform_binding();
+                                        console.log('noData');
+                                        $( "#nodata" ).fadeIn( "fast", function() {
+                                            setTimeout(function() {
+                                                $( "#nodata" ).fadeOut("fast");
+                                            }, 2000);
                                         });
-                                   } else if (json.action == "delete") {
+                                    } else {
-                                       // Find indexes of row which have the term in the first column
+                                        for(i=0; i < rep.length; i++) {
-                                       var index = table_track.rows().eq( 0 ).filter( function (rowIdx) {
+                                            table_track.row.add( [ json.term, "xx/xx/xxxx", "xxxx", "xxxx", action_button ] ).draw( false );
-                                           console.log(table_track.cell( rowIdx, 0 ).data())
+                                        }
-                                           return table_track.cell( rowIdx, 0 ).data() === json.term;
+                                        //perform_binding();
-                                       } );
-                                       table_track.rows(index).remove().draw( false );
                                     }
                                 }                
                             }