add: [tracker] typo-squatting

bin/lib/Tracker.py

@@ -11,6 +11,9 @@ import yara
 import datetime
 import base64
 
+from ail_typo_squatting import runAll
+import math
+
 
 from flask import escape
 
@@ -400,6 +403,16 @@ def api_validate_tracker_to_add(tracker , tracker_type, nb_words=1):
 
             tracker = ",".join(words_set)
             tracker = "{};{}".format(tracker, nb_words)
+    elif tracker_type == 'typosquat':
+        tracker = tracker.lower()
+        # Take only the first term
+        domain = tracker.split(" ")[0]
+        
+        typo_generation = runAll(domain=domain, limit=math.inf, formatoutput="text", pathOutput="-", verbose=False)
+        #typo_generation = domain
+            
+        tracker = ",".join(typo_generation)
+        tracker = "{};{}".format(tracker, len(typo_generation))
 
     elif tracker_type=='yara_custom':
         if not is_valid_yara_rule(tracker):

bin/packages/Term.py

@@ -14,6 +14,9 @@ from collections import defaultdict
 from nltk.tokenize import RegexpTokenizer
 from textblob import TextBlob
 
+from ail_typo_squatting import runAll
+import math
+
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
 import ConfigLoader
 import Tracker
@@ -114,6 +117,16 @@ def get_set_tracked_words_list():
         all_set_list.append((ter_set, num_words, elem))
     return all_set_list
 
+def get_typosquat_tracked_words_list():
+    set_list = r_serv_term.smembers('all:tracker:typosquat')
+    all_set_list = []
+    for elem in set_list:
+        res = elem.split(';')
+        num_words = int(res[1])
+        ter_set = res[0].split(',')
+        all_set_list.append((ter_set, num_words, elem))
+    return all_set_list
+
 def get_regex_tracked_words_dict():
     regex_list = r_serv_term.smembers('all:tracker:regex')
     dict_tracked_regex = {}
@@ -227,6 +240,16 @@ def parse_tracked_term_to_add(term , term_type, nb_words=1):
 
             term = ",".join(words_set)
             term = "{};{}".format(term, nb_words)
+    elif term_type == 'typosquat':
+        term = term.lower()
+        # Take only the first term
+        domain = term.split(" ")[0]
+        
+        typo_generation = runAll(domain=domain, limit=math.inf, formatoutput="text", pathOutput="-", verbose=False)
+        #typo_generation = domain
+            
+        term = ",".join(typo_generation)
+        term = "{};{}".format(term, len(typo_generation))
 
     elif term_type=='yara_custom':
         if not Tracker.is_valid_yara_rule(term):

bin/trackers/Tracker_Term.py

@@ -58,6 +58,8 @@ class Tracker_Term(AbstractModule):
         self.last_refresh_word = time.time()
         self.set_tracked_words_list = Term.get_set_tracked_words_list()
         self.last_refresh_set = time.time()
+        self.typosquat_tracked_words_list = Term.get_set_tracked_words_list()
+        self.last_refresh_typosquat = time.time()
 
         self.redis_logger.info(f"Module: {self.module_name} Launched")
 
@@ -75,6 +77,12 @@ class Tracker_Term(AbstractModule):
             self.redis_logger.debug('Tracked set refreshed')
             print('Tracked set refreshed')
 
+        if self.last_refresh_typosquat < Term.get_tracked_term_last_updated_by_type('set'):
+            self.typosquat_tracked_words_list = Term.get_typosquat_tracked_words_list()
+            self.last_refresh_typosquat = time.time()
+            self.redis_logger.debug('Tracked set refreshed')
+            print('Tracked set refreshed')
+
         # Cast message as Item
         item = Item(item_id)
         item_date = item.get_date()
@@ -113,6 +121,18 @@ class Tracker_Term(AbstractModule):
                         nb_uniq_word += 1
                 if nb_uniq_word >= nb_words_threshold:
                     self.new_term_found(word_set, 'set', item)
+            
+            for elem in self.typosquat_tracked_words_list:
+                list_words = elem[0]
+                nb_words_threshold = elem[1]
+                word_set = elem[2]
+                nb_uniq_word = 0
+
+                for word in list_words:
+                    if word in dict_words_freq:
+                        nb_uniq_word += 1
+                if nb_uniq_word >= nb_words_threshold:
+                    self.new_term_found(word_set, 'typosquat', item)
 
     def new_term_found(self, term, term_type, item):
         uuid_list = Term.get_term_uuid_list(term, term_type)

requirements.txt

@@ -70,6 +70,8 @@ flask>=1.1.4
 flask-login
 bcrypt>3.1.6
 
+# Ail typo squatting
+ail_typo_squatting
 
 # Tests
 nose>=1.3.7

var/www/modules/hunter/Flask_hunter.py

@@ -85,6 +85,16 @@ def tracked_menu_yara():
     global_term = Term.get_all_global_tracked_terms(filter_type=filter_type)
     return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type)
 
+@hunter.route("/trackers/typosquat")
+@login_required
+@login_read_only
+def tracked_menu_typosquat():
+    filter_type = 'typosquat'
+    user_id = current_user.get_id()
+    user_term = Term.get_all_user_tracked_terms(user_id, filter_type=filter_type)
+    global_term = Term.get_all_global_tracked_terms(filter_type=filter_type)
+    return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type)
+
 
 @hunter.route("/tracker/add", methods=['GET', 'POST'])
 @login_required

var/www/modules/hunter/templates/edit_tracker.html

@@ -94,6 +94,7 @@
 										<option value="set">Set</option>
 										<option value="regex">Regex</option>
 										<option value="yara">YARA rule</option>
+										<option value="typosquat">Typo-squatting</option>
 									</select>
 
 									<p id="tracker_desc">Terms to track (space separated)</p>
@@ -199,6 +200,12 @@ $(document).ready(function(){
 			$("#tracker").hide();
 			$("#nb_word").hide();
 			$("#yara_rule").show();
+		} else if (tracker_type=="typosquat") {
+			$("#tracker_desc").text("Generation of variation for domain name. Only one domain name at a time.");
+			$("#tracker_desc").show();
+			$("#tracker").show();
+			$("#nb_word").hide();
+			$("#yara_rule").hide();
 		}
 	});
 

var/www/templates/hunter/menu_sidebar.html

@@ -42,6 +42,12 @@
                 <span class="bg-danger text-white font-weight-bold" style="font-size: 120%">&nbsp;{ </span>
                 <span>&nbsp;YARA</span>
       	      </a>
+      	    </li>
+			<li class="nav-item">
+      	      <a class="nav-link" href="{{url_for('hunter.tracked_menu_typosquat')}}" id="nav_tracker_typosquat">
+                <i class="fa fa-clone"></i>
+                <span>Typo-squatting</span>
+      	      </a>
       	    </li>
           </ul>
           <h5 class="d-flex text-muted w-100" id="nav_title_retro_hunt">