Added similarity checks + started link webInterface

bin/Credential.py

@@ -40,6 +40,7 @@ REDIS_KEY_ALL_CRED_SET_REV = 'AllCredentialsRev'
 REDIS_KEY_ALL_PATH_SET = 'AllPath'
 REDIS_KEY_ALL_PATH_SET_REV = 'AllPathRev'
 REDIS_KEY_MAP_CRED_TO_PATH = 'CredToPathMapping'
+MINIMUMSIZETHRESHOLD = 3
 
 if __name__ == "__main__":
     publisher.port = 6380
@@ -53,7 +54,7 @@ if __name__ == "__main__":
         host=p.config.get("Redis_Level_DB_TermCred", "host"),
         port=p.config.get("Redis_Level_DB_TermCred", "port"),
         db=p.config.get("Redis_Level_DB_TermCred", "db"))
-
+Credential
     critical = 8
 
     regex_web = "((?:https?:\/\/)[-_0-9a-zA-Z]+\.[0-9a-zA-Z]+)"
@@ -134,7 +135,7 @@ if __name__ == "__main__":
             cred = cred.split('@')[0]
 
             #unique number attached to unique path
-            uniq_num_path = server_cred.incr(REDIS_KEY_ALL_PATH_SET)
+            uniq_num_path = server_cred.incr(REDIS_KEY_NUM_PATH)
             print(REDIS_KEY_ALL_PATH_SET, {filepath: uniq_num_path})
             server_cred.hmset(REDIS_KEY_ALL_PATH_SET, {filepath: uniq_num_path})
             server_cred.hmset(REDIS_KEY_ALL_PATH_SET_REV, {uniq_num_path: filepath})
@@ -151,6 +152,7 @@ if __name__ == "__main__":
             splitedCred = re.findall(REGEX_CRED, cred)
             print(splitedCred)
             for partCred in splitedCred:
+                if len(partCred) > MINIMUMSIZETHRESHOLD:
                     server_cred.sadd(partCred, uniq_num_cred)
 
 

pip_packages_requirement.txt

@@ -21,6 +21,7 @@ nltk
 crcmod
 mmh3
 ssdeep
+python-Levenshtein
 
 #Others
 python-magic

var/www/modules/terms/Flask_terms.py

@@ -12,6 +12,7 @@ from flask import Flask, render_template, jsonify, request, Blueprint
 import re
 import Paste
 from pprint import pprint
+import Levenshtein
 
 # ============ VARIABLES ============
 import Flask_config
@@ -54,6 +55,41 @@ def Term_getValueOverRange(word, startDate, num_day, per_paste=""):
         passed_days += 1
     return to_return
 
+def mixUserName(supplied):
+    #e.g.: John Smith
+    terms = supplied.split()[:2]
+    usernames = []
+    if len(terms) == 1:
+        terms.append(' ')
+
+    #john, smith, John, Smith, JOHN, SMITH
+    usernames += [terms[0].lower()]
+    usernames += [terms[1].lower()]
+    usernames += [terms[0][0].upper() + terms[0][1:].lower()]
+    usernames += [terms[1][0].upper() + terms[1][1:].lower()]
+    usernames += [terms[0].upper()]
+    usernames += [terms[1].upper()]
+
+    #johnsmith, smithjohn, JOHNsmith, johnSMITH, SMITHjohn, smithJOHN
+    usernames += [(terms[0].lower() + terms[1].lower()).strip()]
+    usernames += [(terms[1].lower() + terms[0].lower()).strip()]
+    usernames += [(terms[0].upper() + terms[1].lower()).strip()]
+    usernames += [(terms[0].lower() + terms[1].upper()).strip()]
+    usernames += [(terms[1].upper() + terms[0].lower()).strip()]
+    usernames += [(terms[1].lower() + terms[0].upper()).strip()]
+    #Jsmith, JSmith, jsmith, jSmith, johnS, Js, JohnSmith, Johnsmith, johnSmith
+    usernames += [(terms[0][0].upper() + terms[1][0].lower() + terms[1][1:].lower()).strip()]
+    usernames += [(terms[0][0].upper() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
+    usernames += [(terms[0][0].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()]
+    usernames += [(terms[0][0].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
+    usernames += [(terms[0].lower() + terms[1][0].upper()).strip()]
+    usernames += [(terms[0].upper() + terms[1][0].lower()).strip()]
+    usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
+    usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()]
+    usernames += [(terms[0][0].lower() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
+
+    return usernames
+ 
 
 # ============ ROUTES ============
 
@@ -359,40 +395,6 @@ def credentials_management_query_paste():
     cred =  request.args.get('cred')
     return 1
 
-def mixUserName(supplied):
-    #e.g.: John Smith
-    terms = supplied.split()[:2]
-    usernames = []
-    if len(terms) == 1:
-        terms.append(' ')
-
-    #john, smith, John, Smith, JOHN, SMITH
-    usernames += [terms[0].lower()]
-    usernames += [terms[1].lower()]
-    usernames += [terms[0][0].upper() + terms[0][1:].lower()]
-    usernames += [terms[1][0].upper() + terms[1][1:].lower()]
-    usernames += [terms[0].upper()]
-    usernames += [terms[1].upper()]
-
-    #johnsmith, smithjohn, JOHNsmith, johnSMITH, SMITHjohn, smithJOHN
-    usernames += [(terms[0].lower() + terms[1].lower()).strip()]
-    usernames += [(terms[1].lower() + terms[0].lower()).strip()]
-    usernames += [(terms[0].upper() + terms[1].lower()).strip()]
-    usernames += [(terms[0].lower() + terms[1].upper()).strip()]
-    usernames += [(terms[1].upper() + terms[0].lower()).strip()]
-    usernames += [(terms[1].lower() + terms[0].upper()).strip()]
-    #Jsmith, JSmith, jsmith, jSmith, johnS, Js, JohnSmith, Johnsmith, johnSmith
-    usernames += [(terms[0][0].upper() + terms[1][0].lower() + terms[1][1:].lower()).strip()]
-    usernames += [(terms[0][0].upper() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
-    usernames += [(terms[0][0].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()]
-    usernames += [(terms[0][0].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
-    usernames += [(terms[0].lower() + terms[1][0].upper()).strip()]
-    usernames += [(terms[0].upper() + terms[1][0].lower()).strip()]
-    usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
-    usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()]
-    usernames += [(terms[0][0].lower() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
-
-    return usernames
    
 
 
@@ -407,7 +409,7 @@ def cred_management_action():
     REDIS_KEY_ALL_PATH_SET_REV = 'AllPath'
     REDIS_KEY_MAP_CRED_TO_PATH = 'CredToPathMapping'
 
-    supplied =  request.args.get('term')
+    supplied =  request.args.get('term').encode('utf-8')
     action = request.args.get('action')
     section = request.args.get('section')
 
@@ -419,12 +421,22 @@ def cred_management_action():
             for num in r_serv_cred.smembers(poss):
                 uniq_num_set.add(num)
 
-    data = {'usr': [], 'path': []}
+    data = {'usr': [], 'path': [], 'numPaste': [], 'simil': []}
     for Unum in uniq_num_set:
-        data['usr'].append(r_serv_cred.hget(REDIS_KEY_ALL_CRED_SET_REV, Unum))
+        username = r_serv_cred.hget(REDIS_KEY_ALL_CRED_SET_REV, Unum)
-        data['path'].append(r_serv_cred.hget(REDIS_KEY_MAP_CRED_TO_PATH, Unum))
+        
+        # Calculate Levenshtein distance, ignore negative ratio
+        levenDist = float(Levenshtein.distance(supplied, username))
+        levenRatio = levenDist / float(len(supplied))
+        levenRatioStr = "{:.1%}".format(1.0 - levenRatio)
+        if levenRatio >= 1.0:
+            continue
+
+        data['usr'].append(username)
+        data['path'].append(r_serv_cred.hget(REDIS_KEY_MAP_CRED_TO_PATH, Unum))
+        data['numPaste'].append(len(uniq_num_set))
+        data['simil'].append(levenRatioStr)
 
-    pprint(data)
     to_return = {}
     to_return["section"] = section
     to_return["action"] = action

var/www/modules/terms/templates/credentials_tracker.html

@@ -25,9 +25,31 @@
         <script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
         <script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
     
+        <style>
+          .btn-link {
+              color: #000000
+          }
+        </style>
+    
     </head>
     <body>
+        <!-- Modal -->
+        <div id="mymodal" class="modal fade" role="dialog">
+          <div class="modal-dialog modal-lg">
         
+            <!-- Modal content-->
+            <div id="mymodalcontent" class="modal-content">
+              <div id="mymodalbody" class="modal-body" max-width="8500px">
+                <p>Loading paste information...</p>
+                <img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" height="26" width="26" style="margin: 4px;">
+              </div>
+              <div class="modal-footer">
+                <a id="button_show_plot" target="_blank" href=""><button type="button" class="btn btn-info">Plot term</button></a>
+                <button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
+              </div>
+            </div>
+          </div>
+        </div>
         {% include 'navbar.html' %}
     
         <div id="page-wrapper">
@@ -48,36 +70,22 @@
                         <div class="panel-body">
 
                             <div class="form-group input-group" style="margin-bottom: 30px;">
-                                <span class="input-group-addon"><span class="fa fa-eye"></span></span>
+                                <span class="input-group-addon"><span class="glyphicon glyphicon-screenshot"></span></span>
                                 <input id="seekInput" class="form-control" placeholder="Credential to seek." type="text" style="max-width: 400px;">
                                 <button id="followTermBtn" class="btn btn-success btn-interaction" style="margin-left: 10px;" data-section="seek" data-action="seek"> Seek</button>
-                                <span id="nodata" class="alert alert-danger" style="margin-left: 10px; display: none; padding: 6px;">No data</span>
+                                <span id="nodata" class="alert alert-info" style="margin-left: 10px; display: none; padding: 6px;">No data</span>
                             </div>
 
                             <table class="table table-striped table-bordered table-hover" id="myTable">
                                 <thead>
                                      <tr>
-                                        <th style="max-width: 800px;">Credential</th>
+                                        <th style="max-width: 800px;">Username</th>
-                                        <th>Date</th>
+                                        <th>Similarity</th>
-                                        <th># line in the paste</th>
+                                        <th># concerned paste(s)</th>
-                                        <th># tracked paste</th>
                                         <th>Action</th>
                                      </tr>
                                 </thead>
                                 <tbody>
-                                <!-- SET -->
-                                {% for set in trackSet_list %}
-                                  <tr style="background-color: #cdffca;">
-                                      <td>{{ set }}</td>
-                                      <td>{{ trackSet_list_values[loop.index0][0] }}</td>
-                                      <td>{{ trackSet_list_values[loop.index0][1] }}</td>
-                                      <td>{{ trackSet_list_num_of_paste[loop.index0] }}</td>
-                                      <td><p style="margin: 0px;">
-                                          <span data-toggle="modal" data-target="#mymodal" data-term="{{ set }}" ><button class="btn-link" data-toggle="tooltip" data-placement="right" title="Show concerned paste(s)"><span class="glyphicon glyphicon-info-sign"></span></button></span>
-                                          <button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Track this credentials" data-content="{{ set }}" data-section="followTerm" data-action="add"><span class="fa fa-eye"></span></button>
-                                      </p></td>
-                                  </tr>
-                                {% endfor %}
                                 </tbody>
                             </table>
                         <!-- /.panel-body -->
@@ -88,9 +96,10 @@
             <!-- /.panel -->
         </div>
     </div>
-				</div>
+
-                <!-- /.row -->
+</div>
-            </div>
+<!-- /.row -->
+</div>
             <!-- /#page-wrapper -->
 
             <script>
@@ -99,17 +108,16 @@
                     $("#"+activePage).addClass("active");
 
                     $('[data-toggle="tooltip"]').tooltip();
-               	    table_track = $('#myTable').DataTable();
+                    table_track = $('#myTable').DataTable({ "order": [[ 1, "dec" ]] });
-                    table_black = $('#myTable2').DataTable();
             
                     table_track.on( 'draw.dt', function () {
                         perform_binding();
                     });
 
-                    $("#followCredInput").keyup(function(event){
+                    $("#seekInput").keyup(function(event){
                         if(event.keyCode == 13){
                             $("#followTermBtn").click();
-                            $("#followCredInput").val("");
+                            $("#seekInput").val("");
                         }
                     });
 
@@ -193,25 +201,37 @@
                         $.get("{{ url_for('terms.cred_management_action') }}", data_to_send, function(data, status){
                             if(status == "success") {
                                 var json = data;
-                                console.log(json)
                 
                                 if(json.action == "add") {
 
                                 } else if (json.action == "seek") {
-                                    var rep = json.data
+                                    table_track.clear().draw();
-                                    var action_button = "<button class=\"btn-link btn-interaction\" data-toggle=\"tooltip\" data-placement=\"left\" title=\"Remove this term\" data-content=\"" + json.term + "\" data-section=\"followTerm\" data-action=\"delete\"><span class=\"glyphicon glyphicon-trash\"></span></button>"
+                                    var rep = json.data;
+                                    var action_button = "<button class=\"btn-link btn-interaction\" data-toggle=\"tooltip\" data-placement=\"left\" title=\"Track this term\" data-content=\"" + json.term + "\" data-section=\"followTerm\" data-action=\"add\">";
                                     if (rep.usr.length == 0) {
                                         console.log('noData');
+                                        $( "#nodata" ).removeClass('alert-info').addClass('alert-danger');
+                                        $( "#nodata" ).text("No data");
                                         $( "#nodata" ).fadeIn( "fast", function() {
                                             setTimeout(function() {
                                                 $( "#nodata" ).fadeOut("fast");
                                             }, 2000);
                                         });
                                     } else {
-                                        for(i=0; i < rep.length; i++) {
+                                        console.log(json);
-                                            table_track.row.add( [ json.term, "xx/xx/xxxx", "xxxx", "xxxx", action_button ] ).draw( false );
+                                        console.log(table_track);
+                                        for(i=0; i < rep.usr.length; i++) {
+                                            $( "#nodata" ).removeClass('alert-danger').addClass('alert-info');
+                                            $( "#nodata" ).text(curr_term);
+                                            $( "#nodata" ).fadeIn( "fast");
+                                            toAdd = "<span class=\"fa fa-eye\"></span></button><span data-toggle=\"modal\" data-target=\"#mymodal\" data-term=\""+rep.usr[i]+"\" ><button class=\"btn-link\" data-toggle=\"tooltip\" data-placement=\"right\" title=\"Show concerned paste(s)\"><span class=\"glyphicon glyphicon-info-sign\"></span></button></span>";
+                                            table_track.row.add( [ 
+                                                    rep.usr[i], 
+                                                    rep.simil[i], 
+                                                    rep.numPaste[i], 
+                                                    toAdd+action_button ] ).draw( false );
                                         }
-                                        //perform_binding();
+                                        perform_binding();
                                     }
                                 }                
                             }
@@ -220,7 +240,6 @@
                 }
             </script>
 
-            <script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
         </body>
 
 </html>

var/www/modules/terms/templates/header_terms.html

@@ -2,7 +2,7 @@
     <span class="caret"></span></a>
     <ul class="dropdown-menu">
       <li><a href="{{ url_for('terms.terms_management') }}"><i class="fa fa-gear "> </i> Terms managements</a></li>
-      <li><a href="{{ url_for('terms.credentials_tracker') }}"><i class="glyphicon glyphicon-screenshot"> </i> Credentials tracker</a></li>
+      <li><a href="{{ url_for('terms.credentials_tracker') }}"><i class="glyphicon glyphicon-screenshot"> </i> Credentials seeker</a></li>
       <li><a href="{{ url_for('terms.terms_plot_top') }}"><i class="glyphicon glyphicon-fire"> </i> Terms plot top</a></li>
       <li><a href="{{ url_for('terms.terms_plot_tool') }}"><i class="fa fa-wrench"> </i> Terms plot tool</a></li>
     </ul>

var/www/templates/header.html

@@ -20,7 +20,7 @@
     <span class="caret"></span></a>
     <ul class="dropdown-menu">
       <li><a href="{{ url_for('terms.terms_management') }}"><i class="fa fa-gear "> </i> Terms managements</a></li>
-      <li><a href="{{ url_for('terms.credentials_tracker') }}"><i class="glyphicon glyphicon-screenshot"> </i> Credentials tracker</a></li>
+      <li><a href="{{ url_for('terms.credentials_tracker') }}"><i class="glyphicon glyphicon-screenshot"> </i> Credentials seeker</a></li>
       <li><a href="{{ url_for('terms.terms_plot_top') }}"><i class="glyphicon glyphicon-fire"> </i> Terms plot top</a></li>
       <li><a href="{{ url_for('terms.terms_plot_tool') }}"><i class="fa fa-wrench"> </i> Terms plot tool</a></li>
     </ul>