mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 08:38:28 +00:00
Merge pull request #147 from ail-project/typo
Integration of the typo-squatting tracker
This commit is contained in:
Thirion Aurélien
GitHub
commit
a597eece83
12 changed files with 242 additions and 29 deletions
|
|
@ -226,6 +226,8 @@ function launching_scripts {
|
|||
##################################
|
||||
# TRACKERS MODULES #
|
||||
##################################
|
||||
screen -S "Script_AIL" -X screen -t "Tracker_Typo_Squatting" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Typo_Squatting.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Tracker_Term" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Term.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Tracker_Regex" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Regex.py; read x"
|
||||
|
|
|
|||
|
|
@ -11,6 +11,9 @@ import yara
|
|||
import datetime
|
||||
import base64
|
||||
|
||||
from ail_typo_squatting import runAll
|
||||
import math
|
||||
|
||||
|
||||
from flask import escape
|
||||
|
||||
|
|
@ -80,8 +83,8 @@ def get_all_tracker_type():
|
|||
def get_all_tracker_uuid():
|
||||
return r_serv_tracker.smembers(f'trackers:all')
|
||||
|
||||
def get_all_tracker_by_type(tracker_type):
|
||||
r_serv_tracker.smembers(f'trackers:all:{tracker_type}')
|
||||
def get_all_tracker_uuid_by_type(tracker_type):
|
||||
return r_serv_tracker.smembers(f'trackers:all:{tracker_type}')
|
||||
|
||||
# def get_all_tracker():
|
||||
# l_keys_name = []
|
||||
|
|
@ -216,6 +219,20 @@ def get_tracker_items_by_daterange(tracker_uuid, date_from, date_to):
|
|||
all_item_id |= r_serv_tracker.smembers(f'tracker:item:{tracker_uuid}:{date_day}')
|
||||
return all_item_id
|
||||
|
||||
def get_tracker_typosquatting_domains(tracker_uuid):
|
||||
return r_serv_tracker.smembers(f'tracker:typosquatting:{tracker_uuid}')
|
||||
|
||||
def get_typosquatting_tracked_words_list():
|
||||
all_typo = dict()
|
||||
typos_uuid = get_all_tracker_uuid_by_type("typosquatting")
|
||||
|
||||
for typo_uuid in typos_uuid:
|
||||
tracker = get_tracker_by_uuid(typo_uuid)
|
||||
all_typo[tracker] = get_tracker_typosquatting_domains(typo_uuid)
|
||||
|
||||
return all_typo
|
||||
|
||||
|
||||
def add_tracked_item(tracker_uuid, item_id):
|
||||
item_date = item_basic.get_item_date(item_id)
|
||||
# track item
|
||||
|
|
@ -409,6 +426,15 @@ def api_validate_tracker_to_add(tracker , tracker_type, nb_words=1):
|
|||
|
||||
tracker = ",".join(words_set)
|
||||
tracker = "{};{}".format(tracker, nb_words)
|
||||
elif tracker_type == 'typosquatting':
|
||||
tracker = tracker.lower()
|
||||
# Take only the first term
|
||||
domain = tracker.split(" ")
|
||||
if len(domain) > 1:
|
||||
return {"status": "error", "reason": "Only one domain is accepted at a time"}, 400
|
||||
if not "." in tracker:
|
||||
return {"status": "error", "reason": "Invalid domain name"}, 400
|
||||
|
||||
|
||||
elif tracker_type=='yara_custom':
|
||||
if not is_valid_yara_rule(tracker):
|
||||
|
|
@ -449,6 +475,12 @@ def create_tracker(tracker, tracker_type, user_id, level, tags, mails, descripti
|
|||
tracker = save_yara_rule(tracker_type, tracker, tracker_uuid=tracker_uuid)
|
||||
tracker_type = 'yara'
|
||||
|
||||
elif tracker_type == 'typosquatting':
|
||||
domain = tracker.split(" ")[0]
|
||||
typo_generation = runAll(domain=domain, limit=math.inf, formatoutput="text", pathOutput="-", verbose=False)
|
||||
for typo in typo_generation:
|
||||
r_serv_tracker.sadd(f'tracker:typosquatting:{tracker_uuid}', typo)
|
||||
|
||||
# create metadata
|
||||
r_serv_tracker.hset('tracker:{}'.format(tracker_uuid), 'tracked', tracker)
|
||||
r_serv_tracker.hset('tracker:{}'.format(tracker_uuid), 'type', tracker_type)
|
||||
|
|
|
|||
|
|
@ -103,7 +103,7 @@ def get_text_word_frequency(item_content, filtering=True):
|
|||
# # TODO: create all tracked words
|
||||
def get_tracked_words_list():
|
||||
return list(r_serv_term.smembers('all:tracker:word'))
|
||||
|
||||
|
||||
def get_set_tracked_words_list():
|
||||
set_list = r_serv_term.smembers('all:tracker:set')
|
||||
all_set_list = []
|
||||
|
|
|
|||
|
|
@ -37,6 +37,10 @@ subscribe = Redis_D4_client
|
|||
subscribe = Redis
|
||||
publish = Redis_Tags
|
||||
|
||||
[Tracker_Typo_Squatting]
|
||||
subscribe = Redis_Host
|
||||
publish = Redis_Tags
|
||||
|
||||
[Tracker_Term]
|
||||
subscribe = Redis_Global
|
||||
publish = Redis_Tags
|
||||
|
|
|
|||
|
|
@ -113,7 +113,7 @@ class Tracker_Term(AbstractModule):
|
|||
nb_uniq_word += 1
|
||||
if nb_uniq_word >= nb_words_threshold:
|
||||
self.new_term_found(word_set, 'set', item)
|
||||
|
||||
|
||||
def new_term_found(self, term, term_type, item):
|
||||
uuid_list = Term.get_term_uuid_list(term, term_type)
|
||||
|
||||
|
|
|
|||
116
bin/trackers/Tracker_Typo_Squatting.py
Normal file
116
bin/trackers/Tracker_Typo_Squatting.py
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
The Tracker_Typo_Squatting Module
|
||||
===================
|
||||
|
||||
"""
|
||||
|
||||
##################################
|
||||
# Import External packages
|
||||
##################################
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import requests
|
||||
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from modules.abstract_module import AbstractModule
|
||||
import NotificationHelper
|
||||
from packages.Item import Item
|
||||
from packages import Term
|
||||
from lib import Tracker
|
||||
|
||||
class Tracker_Typo_Squatting(AbstractModule):
|
||||
mail_body_template = "AIL Framework,\nNew occurrence for tracked Typo: {}\nitem id: {}\nurl: {}{}"
|
||||
|
||||
"""
|
||||
Tracker_Typo_Squatting module for AIL framework
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super(Tracker_Typo_Squatting, self).__init__()
|
||||
|
||||
self.pending_seconds = 5
|
||||
|
||||
self.full_item_url = self.process.config.get("Notifications", "ail_domain") + "/object/item?id="
|
||||
|
||||
# loads typosquatting
|
||||
self.typosquat_tracked_words_list = Tracker.get_typosquatting_tracked_words_list()
|
||||
self.last_refresh_typosquat = time.time()
|
||||
|
||||
self.redis_logger.info(f"Module: {self.module_name} Launched")
|
||||
|
||||
def compute(self, message):
|
||||
# refresh Tracked typo
|
||||
if self.last_refresh_typosquat < Term.get_tracked_term_last_updated_by_type('typosquatting'):
|
||||
self.typosquat_tracked_words_list = Tracker.get_typosquatting_tracked_words_list()
|
||||
self.last_refresh_typosquat = time.time()
|
||||
self.redis_logger.debug('Tracked typosquatting refreshed')
|
||||
print('Tracked typosquatting refreshed')
|
||||
|
||||
host, id = message.split()
|
||||
item = Item(id)
|
||||
|
||||
# Cast message as Item
|
||||
for key in self.typosquat_tracked_words_list.keys():
|
||||
#print(key)
|
||||
if host in self.typosquat_tracked_words_list[key]:
|
||||
self.new_term_found(key, 'typosquatting', item)
|
||||
|
||||
def new_term_found(self, term, term_type, item):
|
||||
uuid_list = Term.get_term_uuid_list(term, term_type)
|
||||
|
||||
item_id = item.get_id()
|
||||
item_date = item.get_date()
|
||||
item_source = item.get_source()
|
||||
self.redis_logger.info(f'new tracked typo found: {term} in {item_id}')
|
||||
print(f'new tracked typo found: {term} in {item_id}')
|
||||
for term_uuid in uuid_list:
|
||||
tracker_sources = Tracker.get_tracker_uuid_sources(term_uuid)
|
||||
if not tracker_sources or item_source in tracker_sources:
|
||||
Tracker.add_tracked_item(term_uuid, item_id)
|
||||
|
||||
tags_to_add = Term.get_term_tags(term_uuid)
|
||||
for tag in tags_to_add:
|
||||
msg = '{};{}'.format(tag, item_id)
|
||||
self.send_message_to_queue(msg, 'Tags')
|
||||
|
||||
mail_to_notify = Term.get_term_mails(term_uuid)
|
||||
if mail_to_notify:
|
||||
mail_subject = Tracker.get_email_subject(term_uuid)
|
||||
mail_body = Tracker_Typo_Squatting.mail_body_template.format(term, item_id, self.full_item_url, item_id)
|
||||
for mail in mail_to_notify:
|
||||
self.redis_logger.debug(f'Send Mail {mail_subject}')
|
||||
print(f'S print(item_content)end Mail {mail_subject}')
|
||||
NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body)
|
||||
|
||||
# Webhook
|
||||
webhook_to_post = Term.get_term_webhook(term_uuid)
|
||||
if webhook_to_post:
|
||||
json_request = {"trackerId": term_uuid,
|
||||
"itemId": item_id,
|
||||
"itemURL": self.full_item_url + item_id,
|
||||
"term": term,
|
||||
"itemSource": item_source,
|
||||
"itemDate": item_date,
|
||||
"tags": tags_to_add,
|
||||
"emailNotification": f'{mail_to_notify}',
|
||||
"trackerType": term_type
|
||||
}
|
||||
try:
|
||||
response = requests.post(webhook_to_post, json=json_request)
|
||||
if response.status_code >= 400:
|
||||
self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: {response.reason}")
|
||||
except:
|
||||
self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: Something went wrong")
|
||||
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
module = Tracker_Typo_Squatting()
|
||||
module.run()
|
||||
|
|
@ -70,6 +70,8 @@ flask>=1.1.4
|
|||
flask-login
|
||||
bcrypt>3.1.6
|
||||
|
||||
# Ail typo squatting
|
||||
ail_typo_squatting
|
||||
|
||||
# Tests
|
||||
nose>=1.3.7
|
||||
|
|
|
|||
|
|
@ -85,6 +85,16 @@ def tracked_menu_yara():
|
|||
global_term = Term.get_all_global_tracked_terms(filter_type=filter_type)
|
||||
return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type)
|
||||
|
||||
@hunter.route("/trackers/typosquatting")
|
||||
@login_required
|
||||
@login_read_only
|
||||
def tracked_menu_typosquatting():
|
||||
filter_type = 'typosquatting'
|
||||
user_id = current_user.get_id()
|
||||
user_term = Term.get_all_user_tracked_terms(user_id, filter_type=filter_type)
|
||||
global_term = Term.get_all_global_tracked_terms(filter_type=filter_type)
|
||||
return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type)
|
||||
|
||||
|
||||
@hunter.route("/tracker/add", methods=['GET', 'POST'])
|
||||
@login_required
|
||||
|
|
@ -207,6 +217,13 @@ def show_tracker():
|
|||
yara_rule_content = Tracker.get_yara_rule_content(tracker_metadata['tracker'])
|
||||
else:
|
||||
yara_rule_content = None
|
||||
|
||||
if tracker_metadata['type'] == 'typosquatting':
|
||||
typo_squatting = list(Tracker.get_tracker_typosquatting_domains(tracker_uuid))
|
||||
typo_squatting.sort()
|
||||
else:
|
||||
typo_squatting = None
|
||||
|
||||
|
||||
if date_from:
|
||||
res = Term.parse_get_tracker_term_item({'uuid': tracker_uuid, 'date_from': date_from, 'date_to': date_to}, user_id)
|
||||
|
|
@ -224,6 +241,7 @@ def show_tracker():
|
|||
|
||||
return render_template("showTracker.html", tracker_metadata=tracker_metadata,
|
||||
yara_rule_content=yara_rule_content,
|
||||
typo_squatting=typo_squatting,
|
||||
bootstrap_label=bootstrap_label)
|
||||
|
||||
@hunter.route("/tracker/update_tracker_description", methods=['POST'])
|
||||
|
|
|
|||
|
|
@ -94,6 +94,7 @@
|
|||
<option value="set">Set</option>
|
||||
<option value="regex">Regex</option>
|
||||
<option value="yara">YARA rule</option>
|
||||
<option value="typosquatting">Typo-squatting</option>
|
||||
</select>
|
||||
|
||||
<p id="tracker_desc">Terms to track (space separated)</p>
|
||||
|
|
@ -199,6 +200,12 @@ $(document).ready(function(){
|
|||
$("#tracker").hide();
|
||||
$("#nb_word").hide();
|
||||
$("#yara_rule").show();
|
||||
} else if (tracker_type=="typosquatting") {
|
||||
$("#tracker_desc").text("Generation of variation for domain name. Only one domain name at a time.");
|
||||
$("#tracker_desc").show();
|
||||
$("#tracker").show();
|
||||
$("#nb_word").hide();
|
||||
$("#yara_rule").hide();
|
||||
}
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -69,29 +69,47 @@
|
|||
<div class="col-md-10">
|
||||
<table class="table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Type</th>
|
||||
<th>Tracker</th>
|
||||
<th>Date added</th>
|
||||
<th>Access Level</th>
|
||||
<th>Created by</th>
|
||||
<th>First seen</th>
|
||||
<th>Last seen</th>
|
||||
{% if tracker_metadata['webhook'] %}
|
||||
<th>Webhook</th>
|
||||
{% endif %}
|
||||
<th>Tags <span class="btn-link btn-interaction mouse_pointer"
|
||||
title="Edit Tags List" onclick="edit_tags();"><i
|
||||
class="fas fa-pencil-alt" style="color:Red;"></i></span></th>
|
||||
<th>Email <span class="btn-link btn-interaction mouse_pointer"
|
||||
title="Edit Email List" onclick="edit_mails();"><i
|
||||
class="fas fa-pencil-alt" style="color:Red;"></i></span></th>
|
||||
</tr>
|
||||
<tr>
|
||||
<th>Type</th>
|
||||
<th>Tracker</th>
|
||||
<th>Date added</th>
|
||||
<th>Access Level</th>
|
||||
<th>Created by</th>
|
||||
<th>First seen</th>
|
||||
<th>Last seen</th>
|
||||
{% if tracker_metadata['webhook'] %}
|
||||
<th>Webhook</th>
|
||||
{% endif %}
|
||||
<th>Tags <span class="btn-link btn-interaction mouse_pointer"
|
||||
title="Edit Tags List" onclick="edit_tags();"><i
|
||||
class="fas fa-pencil-alt" style="color:Red;"></i></span></th>
|
||||
<th>Email <span class="btn-link btn-interaction mouse_pointer"
|
||||
title="Edit Email List" onclick="edit_mails();"><i
|
||||
class="fas fa-pencil-alt" style="color:Red;"></i></span></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>{{ tracker_metadata['type'] }}</td>
|
||||
<td>{{ tracker_metadata['tracker'] }}</td>
|
||||
{% if tracker_metadata['type'] == 'typosquatting' %}
|
||||
<td>
|
||||
<a class="btn btn-primary" data-toggle="collapse" href="#collapseTypo" role="button" aria-expanded="false" aria-controls="collapseTypo">
|
||||
{{ tracker_metadata['tracker'].split(",")[0] }}
|
||||
</a>
|
||||
<div class="collapse" id="collapseTypo">
|
||||
<div class="card card-body">
|
||||
{% if typo_squatting %}
|
||||
{% for typo in typo_squatting %}
|
||||
{{typo}}
|
||||
<br/>
|
||||
{% endfor %}
|
||||
{%endif%}
|
||||
</div>
|
||||
</div>
|
||||
</td>
|
||||
{% else %}
|
||||
<td>{{ tracker_metadata['tracker'] }}</td>
|
||||
{% endif %}
|
||||
<td>{{ tracker_metadata['date'][0:4] }}/{{ tracker_metadata['date'][4:6] }}/{{ tracker_metadata['date'][6:8] }}</td>
|
||||
<td>
|
||||
{% if tracker_metadata['level'] == 0 %}
|
||||
|
|
|
|||
|
|
@ -70,10 +70,14 @@
|
|||
<span>
|
||||
<a target="_blank" href="{{ url_for('hunter.show_tracker') }}?uuid={{ dict_uuid['uuid'] }}">
|
||||
{% if dict_uuid['term'] %}
|
||||
{% if dict_uuid['term']|length > 256 %}
|
||||
{{ dict_uuid['term'][0:256] }}...
|
||||
{% if dict_uuid['term']|length > 2000 %}
|
||||
{{ dict_uuid['term'][0:50] }}...
|
||||
{% else %}
|
||||
{{ dict_uuid['term'] }}
|
||||
{% if dict_uuid['term']|length > 100 %}
|
||||
{{ dict_uuid['term'][0:100] }}...
|
||||
{% else %}
|
||||
{{ dict_uuid['term'] }}
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
</a>
|
||||
|
|
@ -135,10 +139,14 @@
|
|||
<span>
|
||||
<a target="_blank" href="{{ url_for('hunter.show_tracker') }}?uuid={{ dict_uuid['uuid'] }}">
|
||||
{% if dict_uuid['term'] %}
|
||||
{% if dict_uuid['term']|length > 256 %}
|
||||
{{ dict_uuid['term'][0:256] }}...
|
||||
{% if dict_uuid['term']|length > 2000 %}
|
||||
{{ dict_uuid['term'][0:50] }}...
|
||||
{% else %}
|
||||
{{ dict_uuid['term'] }}
|
||||
{% if dict_uuid['term']|length > 100 %}
|
||||
{{ dict_uuid['term'][0:100] }}...
|
||||
{% else %}
|
||||
{{ dict_uuid['term'] }}
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
</a>
|
||||
|
|
|
|||
|
|
@ -42,6 +42,12 @@
|
|||
<span class="bg-danger text-white font-weight-bold" style="font-size: 120%"> { </span>
|
||||
<span> YARA</span>
|
||||
</a>
|
||||
</li>
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href="{{url_for('hunter.tracked_menu_typosquatting')}}" id="nav_tracker_typosquatting">
|
||||
<i class="fa fa-clone"></i>
|
||||
<span>Typo-squatting</span>
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
<h5 class="d-flex text-muted w-100" id="nav_title_retro_hunt">
|
||||
|
|
|
|||
Loading…
Reference in a new issue