AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-22 22:27:17 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix: [Typo] tracker typo

Browse source
This commit is contained in:
David Cruciani 2022-06-07 16:18:52 +02:00
parent 9d78721f7a
commit 6e1c92e9f0
10 changed files with 189 additions and 60 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
bin/LAUNCH.sh
View file

@ -234,6 +234,8 @@ function launching_scripts {
##################################
# TRACKERS MODULES #
##################################
screen -S "Script_AIL" -X screen -t "Tracker_Typo" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Typo.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Tracker_Term" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Term.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Tracker_Regex" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Regex.py; read x"

37
bin/lib/Tracker.py
View file

@ -78,8 +78,8 @@ def get_all_tracker_type():
def get_all_tracker_uuid():
return r_serv_tracker.smembers(f'trackers:all')
def get_all_tracker_by_type(tracker_type):
r_serv_tracker.smembers(f'trackers:all:{tracker_type}')
def get_all_tracker_uuid_by_type(tracker_type):
return r_serv_tracker.smembers(f'trackers:all:{tracker_type}')
# def get_all_tracker():
# l_keys_name = []
@ -214,6 +214,20 @@ def get_tracker_items_by_daterange(tracker_uuid, date_from, date_to):
all_item_id |= r_serv_tracker.smembers(f'tracker:item:{tracker_uuid}:{date_day}')
return all_item_id
def get_tracker_typosquatting_domains(tracker_uuid):
return r_serv_tracker.smembers(f'tracker:typosquatting:{tracker_uuid}')
def get_typosquatting_tracked_words_list():
all_typo = dict()
typos_uuid = get_all_tracker_uuid_by_type("typosquatting")
for typo_uuid in typos_uuid:
tracker = get_tracker_by_uuid(typo_uuid)
all_typo[tracker] = get_tracker_typosquatting_domains(typo_uuid)
return all_typo
def add_tracked_item(tracker_uuid, item_id):
item_date = item_basic.get_item_date(item_id)
# track item
@ -403,16 +417,15 @@ def api_validate_tracker_to_add(tracker , tracker_type, nb_words=1):
tracker = ",".join(words_set)
tracker = "{};{}".format(tracker, nb_words)
elif tracker_type == 'typosquat':
elif tracker_type == 'typosquatting':
tracker = tracker.lower()
# Take only the first term
domain = tracker.split(" ")[0]
typo_generation = runAll(domain=domain, limit=math.inf, formatoutput="text", pathOutput="-", verbose=False)
#typo_generation = domain
domain = tracker.split(" ")
if len(domain) > 1:
return {"status": "error", "reason": "Only one domain is accepted at a time"}, 400
if not "." in tracker:
return {"status": "error", "reason": "Invalid domain name"}, 400
tracker = ",".join(typo_generation)
tracker = "{};{}".format(tracker, len(typo_generation))
elif tracker_type=='yara_custom':
if not is_valid_yara_rule(tracker):
@ -453,6 +466,12 @@ def create_tracker(tracker, tracker_type, user_id, level, tags, mails, descripti
tracker = save_yara_rule(tracker_type, tracker, tracker_uuid=tracker_uuid)
tracker_type = 'yara'
elif tracker_type == 'typosquatting':
domain = tracker.split(" ")[0]
typo_generation = runAll(domain=domain, limit=math.inf, formatoutput="text", pathOutput="-", verbose=False)
for typo in typo_generation:
r_serv_tracker.sadd(f'tracker:typosquatting:{tracker_uuid}', typo)
# create metadata
r_serv_tracker.hset('tracker:{}'.format(tracker_uuid), 'tracked', tracker)
r_serv_tracker.hset('tracker:{}'.format(tracker_uuid), 'type', tracker_type)

20
bin/packages/Term.py
View file

@ -107,16 +107,6 @@ def get_text_word_frequency(item_content, filtering=True):
def get_tracked_words_list():
return list(r_serv_term.smembers('all:tracker:word'))
def get_set_tracked_words_list():
set_list = r_serv_term.smembers('all:tracker:set')
all_set_list = []
for elem in set_list:
res = elem.split(';')
num_words = int(res[1])
ter_set = res[0].split(',')
all_set_list.append((ter_set, num_words, elem))
return all_set_list
def get_typosquat_tracked_words_list():
set_list = r_serv_term.smembers('all:tracker:typosquat')
all_set_list = []
@ -240,16 +230,6 @@ def parse_tracked_term_to_add(term , term_type, nb_words=1):
term = ",".join(words_set)
term = "{};{}".format(term, nb_words)
elif term_type == 'typosquat':
term = term.lower()
# Take only the first term
domain = term.split(" ")[0]
typo_generation = runAll(domain=domain, limit=math.inf, formatoutput="text", pathOutput="-", verbose=False)
#typo_generation = domain
term = ",".join(typo_generation)
term = "{};{}".format(term, len(typo_generation))
elif term_type=='yara_custom':
if not Tracker.is_valid_yara_rule(term):

4
bin/packages/modules.cfg
View file

@ -37,6 +37,10 @@ subscribe = Redis_D4_client
subscribe = Redis
publish = Redis_Tags
[Tracker_Typo]
subscribe = Redis_Host
publish = Redis_Tags
[Tracker_Term]
subscribe = Redis_Global
publish = Redis_Tags

20
bin/trackers/Tracker_Term.py
View file

@ -58,8 +58,6 @@ class Tracker_Term(AbstractModule):
self.last_refresh_word = time.time()
self.set_tracked_words_list = Term.get_set_tracked_words_list()
self.last_refresh_set = time.time()
self.typosquat_tracked_words_list = Term.get_set_tracked_words_list()
self.last_refresh_typosquat = time.time()
self.redis_logger.info(f"Module: {self.module_name} Launched")
@ -77,12 +75,6 @@ class Tracker_Term(AbstractModule):
self.redis_logger.debug('Tracked set refreshed')
print('Tracked set refreshed')
if self.last_refresh_typosquat < Term.get_tracked_term_last_updated_by_type('set'):
self.typosquat_tracked_words_list = Term.get_typosquat_tracked_words_list()
self.last_refresh_typosquat = time.time()
self.redis_logger.debug('Tracked set refreshed')
print('Tracked set refreshed')
# Cast message as Item
item = Item(item_id)
item_date = item.get_date()
@ -122,18 +114,6 @@ class Tracker_Term(AbstractModule):
if nb_uniq_word >= nb_words_threshold:
self.new_term_found(word_set, 'set', item)
for elem in self.typosquat_tracked_words_list:
list_words = elem[0]
nb_words_threshold = elem[1]
word_set = elem[2]
nb_uniq_word = 0
for word in list_words:
if word in dict_words_freq:
nb_uniq_word += 1
if nb_uniq_word >= nb_words_threshold:
self.new_term_found(word_set, 'typosquat', item)
def new_term_found(self, term, term_type, item):
uuid_list = Term.get_term_uuid_list(term, term_type)

131
bin/trackers/Tracker_Typo.py Normal file
View file

@ -0,0 +1,131 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
"""
The Tracker_Typo Module
===================
"""
##################################
# Import External packages
##################################
import os
import sys
import time
import signal
import requests
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from modules.abstract_module import AbstractModule
import NotificationHelper
from packages.Item import Item
from packages import Term
from lib import Tracker
class TimeoutException(Exception):
pass
def timeout_handler(signum, frame):
raise TimeoutException
signal.signal(signal.SIGALRM, timeout_handler)
class Tracker_Typo(AbstractModule):
mail_body_template = "AIL Framework,\nNew occurrence for tracked Typo: {}\nitem id: {}\nurl: {}{}"
"""
Tracker_Typo module for AIL framework
"""
def __init__(self):
super(Tracker_Typo, self).__init__()
self.pending_seconds = 5
#self.max_execution_time = self.process.config.getint('Tracker_Typo', "max_execution_time")
self.full_item_url = self.process.config.get("Notifications", "ail_domain") + "/object/item?id="
# loads tracked words
self.typosquat_tracked_words_list = Tracker.get_typosquatting_tracked_words_list()
print(len(self.typosquat_tracked_words_list["google.com"]))
self.last_refresh_typosquat = time.time()
self.redis_logger.info(f"Module: {self.module_name} Launched")
def compute(self, message):
# refresh Tracked typo
if self.last_refresh_typosquat < Term.get_tracked_term_last_updated_by_type('typosquatting'):
self.typosquat_tracked_words_list = Tracker.get_typosquatting_tracked_words_list()
self.last_refresh_typosquat = time.time()
self.redis_logger.debug('Tracked typosquatting refreshed')
print('Tracked typosquatting refreshed')
host, id = message.split()
item = Item(id)
# Cast message as Item
for key in self.typosquat_tracked_words_list.keys():
#print(key)
if host in self.typosquat_tracked_words_list[key]:
self.new_term_found(key, 'typosquatting', item)
def new_term_found(self, term, term_type, item):
uuid_list = Term.get_term_uuid_list(term, term_type)
item_id = item.get_id()
item_date = item.get_date()
item_source = item.get_source()
self.redis_logger.info(f'new tracked typo found: {term} in {item_id}')
print(f'new tracked typo found: {term} in {item_id}')
for term_uuid in uuid_list:
tracker_sources = Tracker.get_tracker_uuid_sources(term_uuid)
if not tracker_sources or item_source in tracker_sources:
Tracker.add_tracked_item(term_uuid, item_id)
tags_to_add = Term.get_term_tags(term_uuid)
for tag in tags_to_add:
msg = '{};{}'.format(tag, item_id)
self.send_message_to_queue(msg, 'Tags')
mail_to_notify = Term.get_term_mails(term_uuid)
if mail_to_notify:
mail_subject = Tracker.get_email_subject(term_uuid)
mail_body = Tracker_Typo.mail_body_template.format(term, item_id, self.full_item_url, item_id)
for mail in mail_to_notify:
self.redis_logger.debug(f'Send Mail {mail_subject}')
print(f'S print(item_content)end Mail {mail_subject}')
NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body)
# Webhook
webhook_to_post = Term.get_term_webhook(term_uuid)
if webhook_to_post:
json_request = {"trackerId": term_uuid,
"itemId": item_id,
"itemURL": self.full_item_url + item_id,
"term": term,
"itemSource": item_source,
"itemDate": item_date,
"tags": tags_to_add,
"emailNotification": f'{mail_to_notify}',
"trackerType": term_type
}
try:
response = requests.post(webhook_to_post, json=json_request)
if response.status_code >= 400:
self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: {response.reason}")
except:
self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: Something went wrong")
if __name__ == '__main__':
module = Tracker_Typo()
module.run()

14
var/www/modules/hunter/Flask_hunter.py
View file

@ -85,11 +85,11 @@ def tracked_menu_yara():
global_term = Term.get_all_global_tracked_terms(filter_type=filter_type)
return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type)
@hunter.route("/trackers/typosquat")
@hunter.route("/trackers/typosquatting")
@login_required
@login_read_only
def tracked_menu_typosquat():
filter_type = 'typosquat'
def tracked_menu_typosquatting():
filter_type = 'typosquatting'
user_id = current_user.get_id()
user_term = Term.get_all_user_tracked_terms(user_id, filter_type=filter_type)
global_term = Term.get_all_global_tracked_terms(filter_type=filter_type)
@ -217,6 +217,13 @@ def show_tracker():
yara_rule_content = Tracker.get_yara_rule_content(tracker_metadata['tracker'])
else:
yara_rule_content = None
if tracker_metadata['type'] == 'typosquatting':
typo_squatting = list(Tracker.get_tracker_typosquatting_domains(tracker_uuid))
typo_squatting.sort()
else:
typo_squatting = None
if date_from:
res = Term.parse_get_tracker_term_item({'uuid': tracker_uuid, 'date_from': date_from, 'date_to': date_to}, user_id)
@ -234,6 +241,7 @@ def show_tracker():
return render_template("showTracker.html", tracker_metadata=tracker_metadata,
yara_rule_content=yara_rule_content,
typo_squatting=typo_squatting,
bootstrap_label=bootstrap_label)
@hunter.route("/tracker/update_tracker_description", methods=['POST'])

4
var/www/modules/hunter/templates/edit_tracker.html
View file

@ -94,7 +94,7 @@
<option value="set">Set</option>
<option value="regex">Regex</option>
<option value="yara">YARA rule</option>
<option value="typosquat">Typo-squatting</option>
<option value="typosquatting">Typo-squatting</option>
</select>
<p id="tracker_desc">Terms to track (space separated)</p>
@ -200,7 +200,7 @@ $(document).ready(function(){
$("#tracker").hide();
$("#nb_word").hide();
$("#yara_rule").show();
} else if (tracker_type=="typosquat") {
} else if (tracker_type=="typosquatting") {
$("#tracker_desc").text("Generation of variation for domain name. Only one domain name at a time.");
$("#tracker_desc").show();
$("#tracker").show();

15
var/www/modules/hunter/templates/showTracker.html
View file

@ -91,14 +91,19 @@
<tbody>
<tr>
<td>{{ tracker_metadata['type'] }}</td>
{% if tracker_metadata['type'] == 'typosquat' %}
<td style="word-break: break-word;">
<a class="btn btn-primary" data-toggle="collapse" href="#collapseExample" role="button" aria-expanded="false" aria-controls="collapseExample">
{% if tracker_metadata['type'] == 'typosquatting' %}
<td>
<a class="btn btn-primary" data-toggle="collapse" href="#collapseTypo" role="button" aria-expanded="false" aria-controls="collapseTypo">
{{ tracker_metadata['tracker'].split(",")[0] }}
</a>
<div class="collapse" id="collapseExample">
<div class="collapse" id="collapseTypo">
<div class="card card-body">
{{ tracker_metadata['tracker'] }}
{% if typo_squatting %}
{% for typo in typo_squatting %}
{{typo}}
<br/>
{% endfor %}
{%endif%}
</div>
</div>
</td>

2
var/www/templates/hunter/menu_sidebar.html
View file

@ -44,7 +44,7 @@
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="{{url_for('hunter.tracked_menu_typosquat')}}" id="nav_tracker_typosquat">
<a class="nav-link" href="{{url_for('hunter.tracked_menu_typosquatting')}}" id="nav_tracker_typosquatting">
<i class="fa fa-clone"></i>
<span>Typo-squatting</span>
</a>