diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 9fbec934..d6a5fbe6 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -226,6 +226,8 @@ function launching_scripts { ################################## # TRACKERS MODULES # ################################## + screen -S "Script_AIL" -X screen -t "Tracker_Typo_Squatting" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Typo_Squatting.py; read x" + sleep 0.1 screen -S "Script_AIL" -X screen -t "Tracker_Term" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Term.py; read x" sleep 0.1 screen -S "Script_AIL" -X screen -t "Tracker_Regex" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Regex.py; read x" diff --git a/bin/lib/Tracker.py b/bin/lib/Tracker.py index 06d25be9..f06f9df5 100755 --- a/bin/lib/Tracker.py +++ b/bin/lib/Tracker.py @@ -11,6 +11,9 @@ import yara import datetime import base64 +from ail_typo_squatting import runAll +import math + from flask import escape @@ -80,8 +83,8 @@ def get_all_tracker_type(): def get_all_tracker_uuid(): return r_serv_tracker.smembers(f'trackers:all') -def get_all_tracker_by_type(tracker_type): - r_serv_tracker.smembers(f'trackers:all:{tracker_type}') +def get_all_tracker_uuid_by_type(tracker_type): + return r_serv_tracker.smembers(f'trackers:all:{tracker_type}') # def get_all_tracker(): # l_keys_name = [] @@ -216,6 +219,20 @@ def get_tracker_items_by_daterange(tracker_uuid, date_from, date_to): all_item_id |= r_serv_tracker.smembers(f'tracker:item:{tracker_uuid}:{date_day}') return all_item_id +def get_tracker_typosquatting_domains(tracker_uuid): + return r_serv_tracker.smembers(f'tracker:typosquatting:{tracker_uuid}') + +def get_typosquatting_tracked_words_list(): + all_typo = dict() + typos_uuid = get_all_tracker_uuid_by_type("typosquatting") + + for typo_uuid in typos_uuid: + tracker = get_tracker_by_uuid(typo_uuid) + all_typo[tracker] = get_tracker_typosquatting_domains(typo_uuid) + + return all_typo + + def add_tracked_item(tracker_uuid, item_id): item_date = item_basic.get_item_date(item_id) # track item @@ -409,6 +426,15 @@ def api_validate_tracker_to_add(tracker , tracker_type, nb_words=1): tracker = ",".join(words_set) tracker = "{};{}".format(tracker, nb_words) + elif tracker_type == 'typosquatting': + tracker = tracker.lower() + # Take only the first term + domain = tracker.split(" ") + if len(domain) > 1: + return {"status": "error", "reason": "Only one domain is accepted at a time"}, 400 + if not "." in tracker: + return {"status": "error", "reason": "Invalid domain name"}, 400 + elif tracker_type=='yara_custom': if not is_valid_yara_rule(tracker): @@ -449,6 +475,12 @@ def create_tracker(tracker, tracker_type, user_id, level, tags, mails, descripti tracker = save_yara_rule(tracker_type, tracker, tracker_uuid=tracker_uuid) tracker_type = 'yara' + elif tracker_type == 'typosquatting': + domain = tracker.split(" ")[0] + typo_generation = runAll(domain=domain, limit=math.inf, formatoutput="text", pathOutput="-", verbose=False) + for typo in typo_generation: + r_serv_tracker.sadd(f'tracker:typosquatting:{tracker_uuid}', typo) + # create metadata r_serv_tracker.hset('tracker:{}'.format(tracker_uuid), 'tracked', tracker) r_serv_tracker.hset('tracker:{}'.format(tracker_uuid), 'type', tracker_type) diff --git a/bin/packages/Term.py b/bin/packages/Term.py index f46ec4ea..0cce6e9c 100755 --- a/bin/packages/Term.py +++ b/bin/packages/Term.py @@ -103,7 +103,7 @@ def get_text_word_frequency(item_content, filtering=True): # # TODO: create all tracked words def get_tracked_words_list(): return list(r_serv_term.smembers('all:tracker:word')) - + def get_set_tracked_words_list(): set_list = r_serv_term.smembers('all:tracker:set') all_set_list = [] diff --git a/bin/packages/modules.cfg b/bin/packages/modules.cfg index 0a2ae61a..654df7bd 100644 --- a/bin/packages/modules.cfg +++ b/bin/packages/modules.cfg @@ -37,6 +37,10 @@ subscribe = Redis_D4_client subscribe = Redis publish = Redis_Tags +[Tracker_Typo_Squatting] +subscribe = Redis_Host +publish = Redis_Tags + [Tracker_Term] subscribe = Redis_Global publish = Redis_Tags diff --git a/bin/trackers/Tracker_Term.py b/bin/trackers/Tracker_Term.py index c23b1077..7cfdc9b7 100755 --- a/bin/trackers/Tracker_Term.py +++ b/bin/trackers/Tracker_Term.py @@ -113,7 +113,7 @@ class Tracker_Term(AbstractModule): nb_uniq_word += 1 if nb_uniq_word >= nb_words_threshold: self.new_term_found(word_set, 'set', item) - + def new_term_found(self, term, term_type, item): uuid_list = Term.get_term_uuid_list(term, term_type) diff --git a/bin/trackers/Tracker_Typo_Squatting.py b/bin/trackers/Tracker_Typo_Squatting.py new file mode 100644 index 00000000..bdbfaaf5 --- /dev/null +++ b/bin/trackers/Tracker_Typo_Squatting.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* +""" +The Tracker_Typo_Squatting Module +=================== + +""" + +################################## +# Import External packages +################################## +import os +import sys +import time +import requests + + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from modules.abstract_module import AbstractModule +import NotificationHelper +from packages.Item import Item +from packages import Term +from lib import Tracker + +class Tracker_Typo_Squatting(AbstractModule): + mail_body_template = "AIL Framework,\nNew occurrence for tracked Typo: {}\nitem id: {}\nurl: {}{}" + + """ + Tracker_Typo_Squatting module for AIL framework + """ + + def __init__(self): + super(Tracker_Typo_Squatting, self).__init__() + + self.pending_seconds = 5 + + self.full_item_url = self.process.config.get("Notifications", "ail_domain") + "/object/item?id=" + + # loads typosquatting + self.typosquat_tracked_words_list = Tracker.get_typosquatting_tracked_words_list() + self.last_refresh_typosquat = time.time() + + self.redis_logger.info(f"Module: {self.module_name} Launched") + + def compute(self, message): + # refresh Tracked typo + if self.last_refresh_typosquat < Term.get_tracked_term_last_updated_by_type('typosquatting'): + self.typosquat_tracked_words_list = Tracker.get_typosquatting_tracked_words_list() + self.last_refresh_typosquat = time.time() + self.redis_logger.debug('Tracked typosquatting refreshed') + print('Tracked typosquatting refreshed') + + host, id = message.split() + item = Item(id) + + # Cast message as Item + for key in self.typosquat_tracked_words_list.keys(): + #print(key) + if host in self.typosquat_tracked_words_list[key]: + self.new_term_found(key, 'typosquatting', item) + + def new_term_found(self, term, term_type, item): + uuid_list = Term.get_term_uuid_list(term, term_type) + + item_id = item.get_id() + item_date = item.get_date() + item_source = item.get_source() + self.redis_logger.info(f'new tracked typo found: {term} in {item_id}') + print(f'new tracked typo found: {term} in {item_id}') + for term_uuid in uuid_list: + tracker_sources = Tracker.get_tracker_uuid_sources(term_uuid) + if not tracker_sources or item_source in tracker_sources: + Tracker.add_tracked_item(term_uuid, item_id) + + tags_to_add = Term.get_term_tags(term_uuid) + for tag in tags_to_add: + msg = '{};{}'.format(tag, item_id) + self.send_message_to_queue(msg, 'Tags') + + mail_to_notify = Term.get_term_mails(term_uuid) + if mail_to_notify: + mail_subject = Tracker.get_email_subject(term_uuid) + mail_body = Tracker_Typo_Squatting.mail_body_template.format(term, item_id, self.full_item_url, item_id) + for mail in mail_to_notify: + self.redis_logger.debug(f'Send Mail {mail_subject}') + print(f'S print(item_content)end Mail {mail_subject}') + NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body) + + # Webhook + webhook_to_post = Term.get_term_webhook(term_uuid) + if webhook_to_post: + json_request = {"trackerId": term_uuid, + "itemId": item_id, + "itemURL": self.full_item_url + item_id, + "term": term, + "itemSource": item_source, + "itemDate": item_date, + "tags": tags_to_add, + "emailNotification": f'{mail_to_notify}', + "trackerType": term_type + } + try: + response = requests.post(webhook_to_post, json=json_request) + if response.status_code >= 400: + self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: {response.reason}") + except: + self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: Something went wrong") + + + +if __name__ == '__main__': + module = Tracker_Typo_Squatting() + module.run() diff --git a/requirements.txt b/requirements.txt index f2ebbc1c..2ea2f499 100644 --- a/requirements.txt +++ b/requirements.txt @@ -70,6 +70,8 @@ flask>=1.1.4 flask-login bcrypt>3.1.6 +# Ail typo squatting +ail_typo_squatting # Tests nose>=1.3.7 diff --git a/var/www/modules/hunter/Flask_hunter.py b/var/www/modules/hunter/Flask_hunter.py index 1a52e31b..41019d49 100644 --- a/var/www/modules/hunter/Flask_hunter.py +++ b/var/www/modules/hunter/Flask_hunter.py @@ -85,6 +85,16 @@ def tracked_menu_yara(): global_term = Term.get_all_global_tracked_terms(filter_type=filter_type) return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type) +@hunter.route("/trackers/typosquatting") +@login_required +@login_read_only +def tracked_menu_typosquatting(): + filter_type = 'typosquatting' + user_id = current_user.get_id() + user_term = Term.get_all_user_tracked_terms(user_id, filter_type=filter_type) + global_term = Term.get_all_global_tracked_terms(filter_type=filter_type) + return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type) + @hunter.route("/tracker/add", methods=['GET', 'POST']) @login_required @@ -207,6 +217,13 @@ def show_tracker(): yara_rule_content = Tracker.get_yara_rule_content(tracker_metadata['tracker']) else: yara_rule_content = None + + if tracker_metadata['type'] == 'typosquatting': + typo_squatting = list(Tracker.get_tracker_typosquatting_domains(tracker_uuid)) + typo_squatting.sort() + else: + typo_squatting = None + if date_from: res = Term.parse_get_tracker_term_item({'uuid': tracker_uuid, 'date_from': date_from, 'date_to': date_to}, user_id) @@ -224,6 +241,7 @@ def show_tracker(): return render_template("showTracker.html", tracker_metadata=tracker_metadata, yara_rule_content=yara_rule_content, + typo_squatting=typo_squatting, bootstrap_label=bootstrap_label) @hunter.route("/tracker/update_tracker_description", methods=['POST']) diff --git a/var/www/modules/hunter/templates/edit_tracker.html b/var/www/modules/hunter/templates/edit_tracker.html index cc3c8f80..2843ebfe 100644 --- a/var/www/modules/hunter/templates/edit_tracker.html +++ b/var/www/modules/hunter/templates/edit_tracker.html @@ -94,6 +94,7 @@ +

Terms to track (space separated)

@@ -199,6 +200,12 @@ $(document).ready(function(){ $("#tracker").hide(); $("#nb_word").hide(); $("#yara_rule").show(); + } else if (tracker_type=="typosquatting") { + $("#tracker_desc").text("Generation of variation for domain name. Only one domain name at a time."); + $("#tracker_desc").show(); + $("#tracker").show(); + $("#nb_word").hide(); + $("#yara_rule").hide(); } }); diff --git a/var/www/modules/hunter/templates/showTracker.html b/var/www/modules/hunter/templates/showTracker.html index 5f551d24..ca6fdf6b 100644 --- a/var/www/modules/hunter/templates/showTracker.html +++ b/var/www/modules/hunter/templates/showTracker.html @@ -69,29 +69,47 @@
- - - - - - - - - {% if tracker_metadata['webhook'] %} - - {% endif %} - - - + + + + + + + + + {% if tracker_metadata['webhook'] %} + + {% endif %} + + + - + {% if tracker_metadata['type'] == 'typosquatting' %} + + {% else %} + + {% endif %}
TypeTrackerDate addedAccess LevelCreated byFirst seenLast seenWebhookTags Email
TypeTrackerDate addedAccess LevelCreated byFirst seenLast seenWebhookTags Email
{{ tracker_metadata['type'] }}{{ tracker_metadata['tracker'] }} + +
+
+ {% if typo_squatting %} + {% for typo in typo_squatting %} + {{typo}} +
+ {% endfor %} + {%endif%} +
+
+ 		{{ tracker_metadata['tracker'] }}{{ tracker_metadata['date'][0:4] }}/{{ tracker_metadata['date'][4:6] }}/{{ tracker_metadata['date'][6:8] }} {% if tracker_metadata['level'] == 0 %} diff --git a/var/www/modules/hunter/templates/trackersManagement.html b/var/www/modules/hunter/templates/trackersManagement.html index aedc790d..09e19994 100644 --- a/var/www/modules/hunter/templates/trackersManagement.html +++ b/var/www/modules/hunter/templates/trackersManagement.html @@ -70,10 +70,14 @@ {% if dict_uuid['term'] %} - {% if dict_uuid['term']|length > 256 %} - {{ dict_uuid['term'][0:256] }}... + {% if dict_uuid['term']|length > 2000 %} + {{ dict_uuid['term'][0:50] }}... {% else %} - {{ dict_uuid['term'] }} + {% if dict_uuid['term']|length > 100 %} + {{ dict_uuid['term'][0:100] }}... + {% else %} + {{ dict_uuid['term'] }} + {% endif %} {% endif %} {% endif %} @@ -135,10 +139,14 @@ {% if dict_uuid['term'] %} - {% if dict_uuid['term']|length > 256 %} - {{ dict_uuid['term'][0:256] }}... + {% if dict_uuid['term']|length > 2000 %} + {{ dict_uuid['term'][0:50] }}... {% else %} - {{ dict_uuid['term'] }} + {% if dict_uuid['term']|length > 100 %} + {{ dict_uuid['term'][0:100] }}... + {% else %} + {{ dict_uuid['term'] }} + {% endif %} {% endif %} {% endif %} diff --git a/var/www/templates/hunter/menu_sidebar.html b/var/www/templates/hunter/menu_sidebar.html index 7410b5f0..d727810e 100644 --- a/var/www/templates/hunter/menu_sidebar.html +++ b/var/www/templates/hunter/menu_sidebar.html @@ -42,6 +42,12 @@  {  YARA + +
  • + + + Typo-squatting +