From 6e1c92e9f0ccd4611efbdc2280a78dfacee4e3ed Mon Sep 17 00:00:00 2001 From: David Cruciani Date: Tue, 7 Jun 2022 16:18:52 +0200 Subject: [PATCH] fix: [Typo] tracker typo --- bin/LAUNCH.sh | 2 + bin/lib/Tracker.py | 37 +++-- bin/packages/Term.py | 20 --- bin/packages/modules.cfg | 4 + bin/trackers/Tracker_Term.py | 20 --- bin/trackers/Tracker_Typo.py | 131 ++++++++++++++++++ var/www/modules/hunter/Flask_hunter.py | 14 +- .../hunter/templates/edit_tracker.html | 4 +- .../modules/hunter/templates/showTracker.html | 15 +- var/www/templates/hunter/menu_sidebar.html | 2 +- 10 files changed, 189 insertions(+), 60 deletions(-) create mode 100644 bin/trackers/Tracker_Typo.py diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index b0942517..1531eb67 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -234,6 +234,8 @@ function launching_scripts { ################################## # TRACKERS MODULES # ################################## + screen -S "Script_AIL" -X screen -t "Tracker_Typo" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Typo.py; read x" + sleep 0.1 screen -S "Script_AIL" -X screen -t "Tracker_Term" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Term.py; read x" sleep 0.1 screen -S "Script_AIL" -X screen -t "Tracker_Regex" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Regex.py; read x" diff --git a/bin/lib/Tracker.py b/bin/lib/Tracker.py index 57fee425..3a683b2f 100755 --- a/bin/lib/Tracker.py +++ b/bin/lib/Tracker.py @@ -78,8 +78,8 @@ def get_all_tracker_type(): def get_all_tracker_uuid(): return r_serv_tracker.smembers(f'trackers:all') -def get_all_tracker_by_type(tracker_type): - r_serv_tracker.smembers(f'trackers:all:{tracker_type}') +def get_all_tracker_uuid_by_type(tracker_type): + return r_serv_tracker.smembers(f'trackers:all:{tracker_type}') # def get_all_tracker(): # l_keys_name = [] @@ -214,6 +214,20 @@ def get_tracker_items_by_daterange(tracker_uuid, date_from, date_to): all_item_id |= r_serv_tracker.smembers(f'tracker:item:{tracker_uuid}:{date_day}') return all_item_id +def get_tracker_typosquatting_domains(tracker_uuid): + return r_serv_tracker.smembers(f'tracker:typosquatting:{tracker_uuid}') + +def get_typosquatting_tracked_words_list(): + all_typo = dict() + typos_uuid = get_all_tracker_uuid_by_type("typosquatting") + + for typo_uuid in typos_uuid: + tracker = get_tracker_by_uuid(typo_uuid) + all_typo[tracker] = get_tracker_typosquatting_domains(typo_uuid) + + return all_typo + + def add_tracked_item(tracker_uuid, item_id): item_date = item_basic.get_item_date(item_id) # track item @@ -403,16 +417,15 @@ def api_validate_tracker_to_add(tracker , tracker_type, nb_words=1): tracker = ",".join(words_set) tracker = "{};{}".format(tracker, nb_words) - elif tracker_type == 'typosquat': + elif tracker_type == 'typosquatting': tracker = tracker.lower() # Take only the first term - domain = tracker.split(" ")[0] - - typo_generation = runAll(domain=domain, limit=math.inf, formatoutput="text", pathOutput="-", verbose=False) - #typo_generation = domain + domain = tracker.split(" ") + if len(domain) > 1: + return {"status": "error", "reason": "Only one domain is accepted at a time"}, 400 + if not "." in tracker: + return {"status": "error", "reason": "Invalid domain name"}, 400 - tracker = ",".join(typo_generation) - tracker = "{};{}".format(tracker, len(typo_generation)) elif tracker_type=='yara_custom': if not is_valid_yara_rule(tracker): @@ -453,6 +466,12 @@ def create_tracker(tracker, tracker_type, user_id, level, tags, mails, descripti tracker = save_yara_rule(tracker_type, tracker, tracker_uuid=tracker_uuid) tracker_type = 'yara' + elif tracker_type == 'typosquatting': + domain = tracker.split(" ")[0] + typo_generation = runAll(domain=domain, limit=math.inf, formatoutput="text", pathOutput="-", verbose=False) + for typo in typo_generation: + r_serv_tracker.sadd(f'tracker:typosquatting:{tracker_uuid}', typo) + # create metadata r_serv_tracker.hset('tracker:{}'.format(tracker_uuid), 'tracked', tracker) r_serv_tracker.hset('tracker:{}'.format(tracker_uuid), 'type', tracker_type) diff --git a/bin/packages/Term.py b/bin/packages/Term.py index cc3d3b3b..0f498b68 100755 --- a/bin/packages/Term.py +++ b/bin/packages/Term.py @@ -107,16 +107,6 @@ def get_text_word_frequency(item_content, filtering=True): def get_tracked_words_list(): return list(r_serv_term.smembers('all:tracker:word')) -def get_set_tracked_words_list(): - set_list = r_serv_term.smembers('all:tracker:set') - all_set_list = [] - for elem in set_list: - res = elem.split(';') - num_words = int(res[1]) - ter_set = res[0].split(',') - all_set_list.append((ter_set, num_words, elem)) - return all_set_list - def get_typosquat_tracked_words_list(): set_list = r_serv_term.smembers('all:tracker:typosquat') all_set_list = [] @@ -240,16 +230,6 @@ def parse_tracked_term_to_add(term , term_type, nb_words=1): term = ",".join(words_set) term = "{};{}".format(term, nb_words) - elif term_type == 'typosquat': - term = term.lower() - # Take only the first term - domain = term.split(" ")[0] - - typo_generation = runAll(domain=domain, limit=math.inf, formatoutput="text", pathOutput="-", verbose=False) - #typo_generation = domain - - term = ",".join(typo_generation) - term = "{};{}".format(term, len(typo_generation)) elif term_type=='yara_custom': if not Tracker.is_valid_yara_rule(term): diff --git a/bin/packages/modules.cfg b/bin/packages/modules.cfg index 4da8fd2e..7784ebe8 100644 --- a/bin/packages/modules.cfg +++ b/bin/packages/modules.cfg @@ -37,6 +37,10 @@ subscribe = Redis_D4_client subscribe = Redis publish = Redis_Tags +[Tracker_Typo] +subscribe = Redis_Host +publish = Redis_Tags + [Tracker_Term] subscribe = Redis_Global publish = Redis_Tags diff --git a/bin/trackers/Tracker_Term.py b/bin/trackers/Tracker_Term.py index 4f3e1e66..7cfdc9b7 100755 --- a/bin/trackers/Tracker_Term.py +++ b/bin/trackers/Tracker_Term.py @@ -58,8 +58,6 @@ class Tracker_Term(AbstractModule): self.last_refresh_word = time.time() self.set_tracked_words_list = Term.get_set_tracked_words_list() self.last_refresh_set = time.time() - self.typosquat_tracked_words_list = Term.get_set_tracked_words_list() - self.last_refresh_typosquat = time.time() self.redis_logger.info(f"Module: {self.module_name} Launched") @@ -77,12 +75,6 @@ class Tracker_Term(AbstractModule): self.redis_logger.debug('Tracked set refreshed') print('Tracked set refreshed') - if self.last_refresh_typosquat < Term.get_tracked_term_last_updated_by_type('set'): - self.typosquat_tracked_words_list = Term.get_typosquat_tracked_words_list() - self.last_refresh_typosquat = time.time() - self.redis_logger.debug('Tracked set refreshed') - print('Tracked set refreshed') - # Cast message as Item item = Item(item_id) item_date = item.get_date() @@ -122,18 +114,6 @@ class Tracker_Term(AbstractModule): if nb_uniq_word >= nb_words_threshold: self.new_term_found(word_set, 'set', item) - for elem in self.typosquat_tracked_words_list: - list_words = elem[0] - nb_words_threshold = elem[1] - word_set = elem[2] - nb_uniq_word = 0 - - for word in list_words: - if word in dict_words_freq: - nb_uniq_word += 1 - if nb_uniq_word >= nb_words_threshold: - self.new_term_found(word_set, 'typosquat', item) - def new_term_found(self, term, term_type, item): uuid_list = Term.get_term_uuid_list(term, term_type) diff --git a/bin/trackers/Tracker_Typo.py b/bin/trackers/Tracker_Typo.py new file mode 100644 index 00000000..d8273520 --- /dev/null +++ b/bin/trackers/Tracker_Typo.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* +""" +The Tracker_Typo Module +=================== + +""" + +################################## +# Import External packages +################################## +import os +import sys +import time +import signal +import requests + + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from modules.abstract_module import AbstractModule +import NotificationHelper +from packages.Item import Item +from packages import Term +from lib import Tracker + +class TimeoutException(Exception): + pass + + +def timeout_handler(signum, frame): + raise TimeoutException + + +signal.signal(signal.SIGALRM, timeout_handler) + + +class Tracker_Typo(AbstractModule): + mail_body_template = "AIL Framework,\nNew occurrence for tracked Typo: {}\nitem id: {}\nurl: {}{}" + + """ + Tracker_Typo module for AIL framework + """ + + def __init__(self): + super(Tracker_Typo, self).__init__() + + self.pending_seconds = 5 + + #self.max_execution_time = self.process.config.getint('Tracker_Typo', "max_execution_time") + + self.full_item_url = self.process.config.get("Notifications", "ail_domain") + "/object/item?id=" + + # loads tracked words + self.typosquat_tracked_words_list = Tracker.get_typosquatting_tracked_words_list() + print(len(self.typosquat_tracked_words_list["google.com"])) + self.last_refresh_typosquat = time.time() + + self.redis_logger.info(f"Module: {self.module_name} Launched") + + def compute(self, message): + # refresh Tracked typo + if self.last_refresh_typosquat < Term.get_tracked_term_last_updated_by_type('typosquatting'): + self.typosquat_tracked_words_list = Tracker.get_typosquatting_tracked_words_list() + self.last_refresh_typosquat = time.time() + self.redis_logger.debug('Tracked typosquatting refreshed') + print('Tracked typosquatting refreshed') + + host, id = message.split() + item = Item(id) + + # Cast message as Item + for key in self.typosquat_tracked_words_list.keys(): + #print(key) + if host in self.typosquat_tracked_words_list[key]: + self.new_term_found(key, 'typosquatting', item) + + def new_term_found(self, term, term_type, item): + uuid_list = Term.get_term_uuid_list(term, term_type) + + item_id = item.get_id() + item_date = item.get_date() + item_source = item.get_source() + self.redis_logger.info(f'new tracked typo found: {term} in {item_id}') + print(f'new tracked typo found: {term} in {item_id}') + for term_uuid in uuid_list: + tracker_sources = Tracker.get_tracker_uuid_sources(term_uuid) + if not tracker_sources or item_source in tracker_sources: + Tracker.add_tracked_item(term_uuid, item_id) + + tags_to_add = Term.get_term_tags(term_uuid) + for tag in tags_to_add: + msg = '{};{}'.format(tag, item_id) + self.send_message_to_queue(msg, 'Tags') + + mail_to_notify = Term.get_term_mails(term_uuid) + if mail_to_notify: + mail_subject = Tracker.get_email_subject(term_uuid) + mail_body = Tracker_Typo.mail_body_template.format(term, item_id, self.full_item_url, item_id) + for mail in mail_to_notify: + self.redis_logger.debug(f'Send Mail {mail_subject}') + print(f'S print(item_content)end Mail {mail_subject}') + NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body) + + # Webhook + webhook_to_post = Term.get_term_webhook(term_uuid) + if webhook_to_post: + json_request = {"trackerId": term_uuid, + "itemId": item_id, + "itemURL": self.full_item_url + item_id, + "term": term, + "itemSource": item_source, + "itemDate": item_date, + "tags": tags_to_add, + "emailNotification": f'{mail_to_notify}', + "trackerType": term_type + } + try: + response = requests.post(webhook_to_post, json=json_request) + if response.status_code >= 400: + self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: {response.reason}") + except: + self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: Something went wrong") + + + +if __name__ == '__main__': + module = Tracker_Typo() + module.run() diff --git a/var/www/modules/hunter/Flask_hunter.py b/var/www/modules/hunter/Flask_hunter.py index 66e7d21e..41019d49 100644 --- a/var/www/modules/hunter/Flask_hunter.py +++ b/var/www/modules/hunter/Flask_hunter.py @@ -85,11 +85,11 @@ def tracked_menu_yara(): global_term = Term.get_all_global_tracked_terms(filter_type=filter_type) return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type) -@hunter.route("/trackers/typosquat") +@hunter.route("/trackers/typosquatting") @login_required @login_read_only -def tracked_menu_typosquat(): - filter_type = 'typosquat' +def tracked_menu_typosquatting(): + filter_type = 'typosquatting' user_id = current_user.get_id() user_term = Term.get_all_user_tracked_terms(user_id, filter_type=filter_type) global_term = Term.get_all_global_tracked_terms(filter_type=filter_type) @@ -217,6 +217,13 @@ def show_tracker(): yara_rule_content = Tracker.get_yara_rule_content(tracker_metadata['tracker']) else: yara_rule_content = None + + if tracker_metadata['type'] == 'typosquatting': + typo_squatting = list(Tracker.get_tracker_typosquatting_domains(tracker_uuid)) + typo_squatting.sort() + else: + typo_squatting = None + if date_from: res = Term.parse_get_tracker_term_item({'uuid': tracker_uuid, 'date_from': date_from, 'date_to': date_to}, user_id) @@ -234,6 +241,7 @@ def show_tracker(): return render_template("showTracker.html", tracker_metadata=tracker_metadata, yara_rule_content=yara_rule_content, + typo_squatting=typo_squatting, bootstrap_label=bootstrap_label) @hunter.route("/tracker/update_tracker_description", methods=['POST']) diff --git a/var/www/modules/hunter/templates/edit_tracker.html b/var/www/modules/hunter/templates/edit_tracker.html index 1d6012c6..2843ebfe 100644 --- a/var/www/modules/hunter/templates/edit_tracker.html +++ b/var/www/modules/hunter/templates/edit_tracker.html @@ -94,7 +94,7 @@ - +

Terms to track (space separated)

@@ -200,7 +200,7 @@ $(document).ready(function(){ $("#tracker").hide(); $("#nb_word").hide(); $("#yara_rule").show(); - } else if (tracker_type=="typosquat") { + } else if (tracker_type=="typosquatting") { $("#tracker_desc").text("Generation of variation for domain name. Only one domain name at a time."); $("#tracker_desc").show(); $("#tracker").show(); diff --git a/var/www/modules/hunter/templates/showTracker.html b/var/www/modules/hunter/templates/showTracker.html index ca6eb2bc..ca6fdf6b 100644 --- a/var/www/modules/hunter/templates/showTracker.html +++ b/var/www/modules/hunter/templates/showTracker.html @@ -91,14 +91,19 @@ {{ tracker_metadata['type'] }} - {% if tracker_metadata['type'] == 'typosquat' %} - - -
+
- {{ tracker_metadata['tracker'] }} + {% if typo_squatting %} + {% for typo in typo_squatting %} + {{typo}} +
+ {% endfor %} + {%endif%}
diff --git a/var/www/templates/hunter/menu_sidebar.html b/var/www/templates/hunter/menu_sidebar.html index 3e1eb59c..d727810e 100644 --- a/var/www/templates/hunter/menu_sidebar.html +++ b/var/www/templates/hunter/menu_sidebar.html @@ -44,7 +44,7 @@
  • - + Typo-squatting