diff --git a/bin/lib/Tracker.py b/bin/lib/Tracker.py index 2f8e8d42..57fee425 100755 --- a/bin/lib/Tracker.py +++ b/bin/lib/Tracker.py @@ -11,6 +11,9 @@ import yara import datetime import base64 +from ail_typo_squatting import runAll +import math + from flask import escape @@ -400,6 +403,16 @@ def api_validate_tracker_to_add(tracker , tracker_type, nb_words=1): tracker = ",".join(words_set) tracker = "{};{}".format(tracker, nb_words) + elif tracker_type == 'typosquat': + tracker = tracker.lower() + # Take only the first term + domain = tracker.split(" ")[0] + + typo_generation = runAll(domain=domain, limit=math.inf, formatoutput="text", pathOutput="-", verbose=False) + #typo_generation = domain + + tracker = ",".join(typo_generation) + tracker = "{};{}".format(tracker, len(typo_generation)) elif tracker_type=='yara_custom': if not is_valid_yara_rule(tracker): diff --git a/bin/packages/Term.py b/bin/packages/Term.py index f46ec4ea..cc3d3b3b 100755 --- a/bin/packages/Term.py +++ b/bin/packages/Term.py @@ -14,6 +14,9 @@ from collections import defaultdict from nltk.tokenize import RegexpTokenizer from textblob import TextBlob +from ail_typo_squatting import runAll +import math + sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) import ConfigLoader import Tracker @@ -114,6 +117,16 @@ def get_set_tracked_words_list(): all_set_list.append((ter_set, num_words, elem)) return all_set_list +def get_typosquat_tracked_words_list(): + set_list = r_serv_term.smembers('all:tracker:typosquat') + all_set_list = [] + for elem in set_list: + res = elem.split(';') + num_words = int(res[1]) + ter_set = res[0].split(',') + all_set_list.append((ter_set, num_words, elem)) + return all_set_list + def get_regex_tracked_words_dict(): regex_list = r_serv_term.smembers('all:tracker:regex') dict_tracked_regex = {} @@ -227,6 +240,16 @@ def parse_tracked_term_to_add(term , term_type, nb_words=1): term = ",".join(words_set) term = "{};{}".format(term, nb_words) + elif term_type == 'typosquat': + term = term.lower() + # Take only the first term + domain = term.split(" ")[0] + + typo_generation = runAll(domain=domain, limit=math.inf, formatoutput="text", pathOutput="-", verbose=False) + #typo_generation = domain + + term = ",".join(typo_generation) + term = "{};{}".format(term, len(typo_generation)) elif term_type=='yara_custom': if not Tracker.is_valid_yara_rule(term): diff --git a/bin/trackers/Tracker_Term.py b/bin/trackers/Tracker_Term.py index c23b1077..4f3e1e66 100755 --- a/bin/trackers/Tracker_Term.py +++ b/bin/trackers/Tracker_Term.py @@ -58,6 +58,8 @@ class Tracker_Term(AbstractModule): self.last_refresh_word = time.time() self.set_tracked_words_list = Term.get_set_tracked_words_list() self.last_refresh_set = time.time() + self.typosquat_tracked_words_list = Term.get_set_tracked_words_list() + self.last_refresh_typosquat = time.time() self.redis_logger.info(f"Module: {self.module_name} Launched") @@ -75,6 +77,12 @@ class Tracker_Term(AbstractModule): self.redis_logger.debug('Tracked set refreshed') print('Tracked set refreshed') + if self.last_refresh_typosquat < Term.get_tracked_term_last_updated_by_type('set'): + self.typosquat_tracked_words_list = Term.get_typosquat_tracked_words_list() + self.last_refresh_typosquat = time.time() + self.redis_logger.debug('Tracked set refreshed') + print('Tracked set refreshed') + # Cast message as Item item = Item(item_id) item_date = item.get_date() @@ -113,6 +121,18 @@ class Tracker_Term(AbstractModule): nb_uniq_word += 1 if nb_uniq_word >= nb_words_threshold: self.new_term_found(word_set, 'set', item) + + for elem in self.typosquat_tracked_words_list: + list_words = elem[0] + nb_words_threshold = elem[1] + word_set = elem[2] + nb_uniq_word = 0 + + for word in list_words: + if word in dict_words_freq: + nb_uniq_word += 1 + if nb_uniq_word >= nb_words_threshold: + self.new_term_found(word_set, 'typosquat', item) def new_term_found(self, term, term_type, item): uuid_list = Term.get_term_uuid_list(term, term_type) diff --git a/requirements.txt b/requirements.txt index f2ebbc1c..2ea2f499 100644 --- a/requirements.txt +++ b/requirements.txt @@ -70,6 +70,8 @@ flask>=1.1.4 flask-login bcrypt>3.1.6 +# Ail typo squatting +ail_typo_squatting # Tests nose>=1.3.7 diff --git a/var/www/modules/hunter/Flask_hunter.py b/var/www/modules/hunter/Flask_hunter.py index 1a52e31b..66e7d21e 100644 --- a/var/www/modules/hunter/Flask_hunter.py +++ b/var/www/modules/hunter/Flask_hunter.py @@ -85,6 +85,16 @@ def tracked_menu_yara(): global_term = Term.get_all_global_tracked_terms(filter_type=filter_type) return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type) +@hunter.route("/trackers/typosquat") +@login_required +@login_read_only +def tracked_menu_typosquat(): + filter_type = 'typosquat' + user_id = current_user.get_id() + user_term = Term.get_all_user_tracked_terms(user_id, filter_type=filter_type) + global_term = Term.get_all_global_tracked_terms(filter_type=filter_type) + return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type) + @hunter.route("/tracker/add", methods=['GET', 'POST']) @login_required diff --git a/var/www/modules/hunter/templates/edit_tracker.html b/var/www/modules/hunter/templates/edit_tracker.html index cc3c8f80..1d6012c6 100644 --- a/var/www/modules/hunter/templates/edit_tracker.html +++ b/var/www/modules/hunter/templates/edit_tracker.html @@ -94,6 +94,7 @@ +
Terms to track (space separated)
@@ -199,6 +200,12 @@ $(document).ready(function(){ $("#tracker").hide(); $("#nb_word").hide(); $("#yara_rule").show(); + } else if (tracker_type=="typosquat") { + $("#tracker_desc").text("Generation of variation for domain name. Only one domain name at a time."); + $("#tracker_desc").show(); + $("#tracker").show(); + $("#nb_word").hide(); + $("#yara_rule").hide(); } }); diff --git a/var/www/templates/hunter/menu_sidebar.html b/var/www/templates/hunter/menu_sidebar.html index 7410b5f0..3e1eb59c 100644 --- a/var/www/templates/hunter/menu_sidebar.html +++ b/var/www/templates/hunter/menu_sidebar.html @@ -42,6 +42,12 @@ { YARA + +