fix: [Typo] tracker typo

bin/LAUNCH.sh

@@ -234,6 +234,8 @@ function launching_scripts {
     ##################################
     #       TRACKERS MODULES         #
     ##################################
+    screen -S "Script_AIL" -X screen -t "Tracker_Typo" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Typo.py; read x"
+    sleep 0.1
     screen -S "Script_AIL" -X screen -t "Tracker_Term" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Term.py; read x"
     sleep 0.1
     screen -S "Script_AIL" -X screen -t "Tracker_Regex" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Regex.py; read x"

bin/lib/Tracker.py

@@ -78,8 +78,8 @@ def get_all_tracker_type():
 def get_all_tracker_uuid():
     return r_serv_tracker.smembers(f'trackers:all')
 
-def get_all_tracker_by_type(tracker_type):
+def get_all_tracker_uuid_by_type(tracker_type):
-    r_serv_tracker.smembers(f'trackers:all:{tracker_type}')
+    return r_serv_tracker.smembers(f'trackers:all:{tracker_type}')
 
 # def get_all_tracker():
 #     l_keys_name = []
@@ -214,6 +214,20 @@ def get_tracker_items_by_daterange(tracker_uuid, date_from, date_to):
                     all_item_id |= r_serv_tracker.smembers(f'tracker:item:{tracker_uuid}:{date_day}')
     return all_item_id
 
+def get_tracker_typosquatting_domains(tracker_uuid):
+    return r_serv_tracker.smembers(f'tracker:typosquatting:{tracker_uuid}')
+
+def get_typosquatting_tracked_words_list():
+    all_typo = dict()
+    typos_uuid = get_all_tracker_uuid_by_type("typosquatting")
+
+    for typo_uuid in typos_uuid:
+        tracker = get_tracker_by_uuid(typo_uuid)
+        all_typo[tracker] = get_tracker_typosquatting_domains(typo_uuid)
+
+    return all_typo
+
+
 def add_tracked_item(tracker_uuid, item_id):
     item_date = item_basic.get_item_date(item_id)
     # track item
@@ -403,16 +417,15 @@ def api_validate_tracker_to_add(tracker , tracker_type, nb_words=1):
 
             tracker = ",".join(words_set)
             tracker = "{};{}".format(tracker, nb_words)
-    elif tracker_type == 'typosquat':
+    elif tracker_type == 'typosquatting':
         tracker = tracker.lower()
         # Take only the first term
-        domain = tracker.split(" ")[0]
+        domain = tracker.split(" ")
+        if len(domain) > 1:
+            return {"status": "error", "reason": "Only one domain is accepted at a time"}, 400
+        if not "." in tracker:
+            return {"status": "error", "reason": "Invalid domain name"}, 400
             
-        typo_generation = runAll(domain=domain, limit=math.inf, formatoutput="text", pathOutput="-", verbose=False)
-        #typo_generation = domain
-            
-        tracker = ",".join(typo_generation)
-        tracker = "{};{}".format(tracker, len(typo_generation))
 
     elif tracker_type=='yara_custom':
         if not is_valid_yara_rule(tracker):
@@ -453,6 +466,12 @@ def create_tracker(tracker, tracker_type, user_id, level, tags, mails, descripti
         tracker = save_yara_rule(tracker_type, tracker, tracker_uuid=tracker_uuid)
         tracker_type = 'yara'
 
+    elif tracker_type == 'typosquatting':
+        domain = tracker.split(" ")[0]
+        typo_generation = runAll(domain=domain, limit=math.inf, formatoutput="text", pathOutput="-", verbose=False)
+        for typo in typo_generation:
+            r_serv_tracker.sadd(f'tracker:typosquatting:{tracker_uuid}', typo)
+
     # create metadata
     r_serv_tracker.hset('tracker:{}'.format(tracker_uuid), 'tracked', tracker)
     r_serv_tracker.hset('tracker:{}'.format(tracker_uuid), 'type', tracker_type)

bin/packages/Term.py

@@ -107,16 +107,6 @@ def get_text_word_frequency(item_content, filtering=True):
 def get_tracked_words_list():
     return list(r_serv_term.smembers('all:tracker:word'))
 
-def get_set_tracked_words_list():
-    set_list = r_serv_term.smembers('all:tracker:set')
-    all_set_list = []
-    for elem in set_list:
-        res = elem.split(';')
-        num_words = int(res[1])
-        ter_set = res[0].split(',')
-        all_set_list.append((ter_set, num_words, elem))
-    return all_set_list
-
 def get_typosquat_tracked_words_list():
     set_list = r_serv_term.smembers('all:tracker:typosquat')
     all_set_list = []
@@ -240,16 +230,6 @@ def parse_tracked_term_to_add(term , term_type, nb_words=1):
 
             term = ",".join(words_set)
             term = "{};{}".format(term, nb_words)
-    elif term_type == 'typosquat':
-        term = term.lower()
-        # Take only the first term
-        domain = term.split(" ")[0]
-        
-        typo_generation = runAll(domain=domain, limit=math.inf, formatoutput="text", pathOutput="-", verbose=False)
-        #typo_generation = domain
-            
-        term = ",".join(typo_generation)
-        term = "{};{}".format(term, len(typo_generation))
 
     elif term_type=='yara_custom':
         if not Tracker.is_valid_yara_rule(term):

bin/packages/modules.cfg

@@ -37,6 +37,10 @@ subscribe = Redis_D4_client
 subscribe = Redis
 publish = Redis_Tags
 
+[Tracker_Typo]
+subscribe = Redis_Host
+publish = Redis_Tags
+
 [Tracker_Term]
 subscribe = Redis_Global
 publish = Redis_Tags

bin/trackers/Tracker_Term.py

@@ -58,8 +58,6 @@ class Tracker_Term(AbstractModule):
         self.last_refresh_word = time.time()
         self.set_tracked_words_list = Term.get_set_tracked_words_list()
         self.last_refresh_set = time.time()
-        self.typosquat_tracked_words_list = Term.get_set_tracked_words_list()
-        self.last_refresh_typosquat = time.time()
 
         self.redis_logger.info(f"Module: {self.module_name} Launched")
 
@@ -77,12 +75,6 @@ class Tracker_Term(AbstractModule):
             self.redis_logger.debug('Tracked set refreshed')
             print('Tracked set refreshed')
 
-        if self.last_refresh_typosquat < Term.get_tracked_term_last_updated_by_type('set'):
-            self.typosquat_tracked_words_list = Term.get_typosquat_tracked_words_list()
-            self.last_refresh_typosquat = time.time()
-            self.redis_logger.debug('Tracked set refreshed')
-            print('Tracked set refreshed')
-
         # Cast message as Item
         item = Item(item_id)
         item_date = item.get_date()
@@ -122,18 +114,6 @@ class Tracker_Term(AbstractModule):
                 if nb_uniq_word >= nb_words_threshold:
                     self.new_term_found(word_set, 'set', item)
             
-            for elem in self.typosquat_tracked_words_list:
-                list_words = elem[0]
-                nb_words_threshold = elem[1]
-                word_set = elem[2]
-                nb_uniq_word = 0
-
-                for word in list_words:
-                    if word in dict_words_freq:
-                        nb_uniq_word += 1
-                if nb_uniq_word >= nb_words_threshold:
-                    self.new_term_found(word_set, 'typosquat', item)
-
     def new_term_found(self, term, term_type, item):
         uuid_list = Term.get_term_uuid_list(term, term_type)
 

bin/trackers/Tracker_Typo.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+"""
+The Tracker_Typo Module
+===================
+
+"""
+
+##################################
+# Import External packages
+##################################
+import os
+import sys
+import time
+import signal
+import requests
+
+
+sys.path.append(os.environ['AIL_BIN'])
+##################################
+# Import Project packages
+##################################
+from modules.abstract_module import AbstractModule
+import NotificationHelper
+from packages.Item import Item
+from packages import Term
+from lib import Tracker
+
+class TimeoutException(Exception):
+    pass
+
+
+def timeout_handler(signum, frame):
+    raise TimeoutException
+
+
+signal.signal(signal.SIGALRM, timeout_handler)
+
+
+class Tracker_Typo(AbstractModule):
+    mail_body_template = "AIL Framework,\nNew occurrence for tracked Typo: {}\nitem id: {}\nurl: {}{}"
+
+    """
+    Tracker_Typo module for AIL framework
+    """
+
+    def __init__(self):
+        super(Tracker_Typo, self).__init__()
+
+        self.pending_seconds = 5
+
+        #self.max_execution_time = self.process.config.getint('Tracker_Typo', "max_execution_time")
+
+        self.full_item_url = self.process.config.get("Notifications", "ail_domain") + "/object/item?id="
+
+        # loads tracked words
+        self.typosquat_tracked_words_list = Tracker.get_typosquatting_tracked_words_list()
+        print(len(self.typosquat_tracked_words_list["google.com"]))
+        self.last_refresh_typosquat = time.time()
+
+        self.redis_logger.info(f"Module: {self.module_name} Launched")
+
+    def compute(self, message):
+        # refresh Tracked typo
+        if self.last_refresh_typosquat < Term.get_tracked_term_last_updated_by_type('typosquatting'):
+            self.typosquat_tracked_words_list = Tracker.get_typosquatting_tracked_words_list()
+            self.last_refresh_typosquat = time.time()
+            self.redis_logger.debug('Tracked typosquatting refreshed')
+            print('Tracked typosquatting refreshed')
+
+        host, id = message.split()
+        item = Item(id)
+        
+        # Cast message as Item
+        for key in self.typosquat_tracked_words_list.keys():
+            #print(key)
+            if host in self.typosquat_tracked_words_list[key]:
+                self.new_term_found(key, 'typosquatting', item)
+
+    def new_term_found(self, term, term_type, item):
+        uuid_list = Term.get_term_uuid_list(term, term_type)
+
+        item_id = item.get_id()
+        item_date = item.get_date()
+        item_source = item.get_source()
+        self.redis_logger.info(f'new tracked typo found: {term} in {item_id}')
+        print(f'new tracked typo found: {term} in {item_id}')
+        for term_uuid in uuid_list:
+            tracker_sources = Tracker.get_tracker_uuid_sources(term_uuid)
+            if not tracker_sources or item_source in tracker_sources:
+                Tracker.add_tracked_item(term_uuid, item_id)
+
+                tags_to_add = Term.get_term_tags(term_uuid)
+                for tag in tags_to_add:
+                    msg = '{};{}'.format(tag, item_id)
+                    self.send_message_to_queue(msg, 'Tags')
+
+                mail_to_notify = Term.get_term_mails(term_uuid)
+                if mail_to_notify:
+                    mail_subject = Tracker.get_email_subject(term_uuid)
+                    mail_body = Tracker_Typo.mail_body_template.format(term, item_id, self.full_item_url, item_id)
+                for mail in mail_to_notify:
+                    self.redis_logger.debug(f'Send Mail {mail_subject}')
+                    print(f'S        print(item_content)end Mail {mail_subject}')
+                    NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body)
+
+                # Webhook
+                webhook_to_post = Term.get_term_webhook(term_uuid)
+                if webhook_to_post:
+                    json_request = {"trackerId": term_uuid,
+                                    "itemId": item_id,
+                                    "itemURL": self.full_item_url + item_id,
+                                    "term": term,
+                                    "itemSource": item_source,
+                                    "itemDate": item_date,
+                                    "tags": tags_to_add,
+                                    "emailNotification": f'{mail_to_notify}',
+                                    "trackerType": term_type
+                                    }
+                    try:
+                        response = requests.post(webhook_to_post, json=json_request)
+                        if response.status_code >= 400:
+                            self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: {response.reason}")
+                    except:
+                        self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: Something went wrong")
+
+
+
+if __name__ == '__main__':
+    module = Tracker_Typo()
+    module.run()

var/www/modules/hunter/Flask_hunter.py

@@ -85,11 +85,11 @@ def tracked_menu_yara():
     global_term = Term.get_all_global_tracked_terms(filter_type=filter_type)
     return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type)
 
-@hunter.route("/trackers/typosquat")
+@hunter.route("/trackers/typosquatting")
 @login_required
 @login_read_only
-def tracked_menu_typosquat():
+def tracked_menu_typosquatting():
-    filter_type = 'typosquat'
+    filter_type = 'typosquatting'
     user_id = current_user.get_id()
     user_term = Term.get_all_user_tracked_terms(user_id, filter_type=filter_type)
     global_term = Term.get_all_global_tracked_terms(filter_type=filter_type)
@@ -218,6 +218,13 @@ def show_tracker():
     else:
         yara_rule_content = None
    
+    if tracker_metadata['type'] == 'typosquatting':
+        typo_squatting = list(Tracker.get_tracker_typosquatting_domains(tracker_uuid))
+        typo_squatting.sort()
+    else:
+        typo_squatting = None
+
+
     if date_from:
         res = Term.parse_get_tracker_term_item({'uuid': tracker_uuid, 'date_from': date_from, 'date_to': date_to}, user_id)
         if res[1] !=200:
@@ -234,6 +241,7 @@ def show_tracker():
 
     return render_template("showTracker.html", tracker_metadata=tracker_metadata,
                                     yara_rule_content=yara_rule_content,
+                                    typo_squatting=typo_squatting,
                                     bootstrap_label=bootstrap_label)
 
 @hunter.route("/tracker/update_tracker_description", methods=['POST'])

var/www/modules/hunter/templates/edit_tracker.html

@@ -94,7 +94,7 @@
 										<option value="set">Set</option>
 										<option value="regex">Regex</option>
 										<option value="yara">YARA rule</option>
-										<option value="typosquat">Typo-squatting</option>
+										<option value="typosquatting">Typo-squatting</option>
 									</select>
 
 									<p id="tracker_desc">Terms to track (space separated)</p>
@@ -200,7 +200,7 @@ $(document).ready(function(){
 			$("#tracker").hide();
 			$("#nb_word").hide();
 			$("#yara_rule").show();
-		} else if (tracker_type=="typosquat") {
+		} else if (tracker_type=="typosquatting") {
 			$("#tracker_desc").text("Generation of variation for domain name. Only one domain name at a time.");
 			$("#tracker_desc").show();
 			$("#tracker").show();

var/www/modules/hunter/templates/showTracker.html

@@ -91,14 +91,19 @@
                                         <tbody>
                                         <tr>
                                             <td>{{ tracker_metadata['type'] }}</td>
-                                            {% if tracker_metadata['type'] == 'typosquat' %}
+                                            {% if tracker_metadata['type'] == 'typosquatting' %}
-                                                <td style="word-break: break-word;">
+                                                <td>
-                                                    <a class="btn btn-primary" data-toggle="collapse" href="#collapseExample" role="button" aria-expanded="false" aria-controls="collapseExample">
+                                                    <a class="btn btn-primary" data-toggle="collapse" href="#collapseTypo" role="button" aria-expanded="false" aria-controls="collapseTypo">
                                                         {{ tracker_metadata['tracker'].split(",")[0] }}
                                                     </a>
-                                                    <div class="collapse" id="collapseExample">
+                                                    <div class="collapse" id="collapseTypo">
                                                         <div class="card card-body">
-                                                            {{ tracker_metadata['tracker'] }}
+                                                            {% if typo_squatting %}
+                                                                {% for typo in typo_squatting %}
+                                                                    {{typo}}
+                                                                    <br/>
+                                                                {% endfor %}
+                                                            {%endif%}
                                                         </div>
                                                     </div>
                                                 </td>

var/www/templates/hunter/menu_sidebar.html

@@ -44,7 +44,7 @@
       	      </a>
       	    </li>
 			<li class="nav-item">
-      	      <a class="nav-link" href="{{url_for('hunter.tracked_menu_typosquat')}}" id="nav_tracker_typosquat">
+      	      <a class="nav-link" href="{{url_for('hunter.tracked_menu_typosquatting')}}" id="nav_tracker_typosquatting">
                 <i class="fa fa-clone"></i>
                 <span>Typo-squatting</span>
       	      </a>