mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-12 17:48:22 +00:00
chg: [core] merge conflict
This commit is contained in:
Terrtia
commit
524a404dc8
21 changed files with 468 additions and 52 deletions
|
|
@ -117,16 +117,39 @@ function launching_kvrocks {
|
|||
}
|
||||
|
||||
function launching_logs {
|
||||
conf_dir="${AIL_HOME}/configs/"
|
||||
syslog_cmd=""
|
||||
syslog_enabled=`cat $conf_dir/core.cfg | grep 'ail_logs_syslog' | cut -d " " -f 3 `
|
||||
if [ "$syslog_enabled" = "True" ]; then
|
||||
syslog_cmd="--syslog"
|
||||
fi
|
||||
syslog_server=`cat $conf_dir/core.cfg | grep 'ail_logs_syslog_server' | cut -d " " -f 3 `
|
||||
syslog_port=`cat $conf_dir/core.cfg | grep 'ail_logs_syslog_port' | cut -d " " -f 3 `
|
||||
if [ ! -z "$syslog_server" -a "$str" != " " ]; then
|
||||
syslog_cmd="${syslog_cmd} -ss ${syslog_server}"
|
||||
if [ ! -z "$syslog_port" -a "$str" != " " ]; then
|
||||
syslog_cmd="${syslog_cmd} -sp ${syslog_port}"
|
||||
fi
|
||||
fi
|
||||
syslog_facility=`cat $conf_dir/core.cfg | grep 'ail_logs_syslog_facility' | cut -d " " -f 3 `
|
||||
if [ ! -z "$syslog_facility" -a "$str" != " " ]; then
|
||||
syslog_cmd="${syslog_cmd} -sf ${syslog_facility}"
|
||||
fi
|
||||
syslog_level=`cat $conf_dir/core.cfg | grep 'ail_logs_syslog_level' | cut -d " " -f 3 `
|
||||
if [ ! -z "$syslog_level" -a "$str" != " " ]; then
|
||||
syslog_cmd="${syslog_cmd} -sl ${syslog_level}"
|
||||
fi
|
||||
|
||||
screen -dmS "Logging_AIL"
|
||||
sleep 0.1
|
||||
echo -e $GREEN"\t* Launching logging process"$DEFAULT
|
||||
screen -S "Logging_AIL" -X screen -t "LogQueue" bash -c "cd ${AIL_BIN}; ${AIL_VENV}/bin/log_subscriber -p 6380 -c Queuing -l ../logs/; read x"
|
||||
screen -S "Logging_AIL" -X screen -t "LogQueue" bash -c "cd ${AIL_BIN}; ${AIL_VENV}/bin/log_subscriber -p 6380 -c Queuing -l ../logs/ ${syslog_cmd}; read x"
|
||||
sleep 0.1
|
||||
screen -S "Logging_AIL" -X screen -t "LogScript" bash -c "cd ${AIL_BIN}; ${AIL_VENV}/bin/log_subscriber -p 6380 -c Script -l ../logs/; read x"
|
||||
screen -S "Logging_AIL" -X screen -t "LogScript" bash -c "cd ${AIL_BIN}; ${AIL_VENV}/bin/log_subscriber -p 6380 -c Script -l ../logs/ ${syslog_cmd}; read x"
|
||||
sleep 0.1
|
||||
screen -S "Logging_AIL" -X screen -t "LogScript" bash -c "cd ${AIL_BIN}; ${AIL_VENV}/bin/log_subscriber -p 6380 -c Sync -l ../logs/; read x"
|
||||
screen -S "Logging_AIL" -X screen -t "LogSync" bash -c "cd ${AIL_BIN}; ${AIL_VENV}/bin/log_subscriber -p 6380 -c Sync -l ../logs/ ${syslog_cmd}; read x"
|
||||
sleep 0.1
|
||||
screen -S "Logging_AIL" -X screen -t "LogScript" bash -c "cd ${AIL_BIN}; ${AIL_VENV}/bin/log_subscriber -p 6380 -c Crawler -l ../logs/; read x"
|
||||
screen -S "Logging_AIL" -X screen -t "LogCrawler" bash -c "cd ${AIL_BIN}; ${AIL_VENV}/bin/log_subscriber -p 6380 -c Crawler -l ../logs/ ${syslog_cmd}; read x"
|
||||
}
|
||||
|
||||
function launching_queues {
|
||||
|
|
@ -259,6 +282,8 @@ function launching_scripts {
|
|||
##################################
|
||||
# TRACKERS MODULES #
|
||||
##################################
|
||||
screen -S "Script_AIL" -X screen -t "Tracker_Typo_Squatting" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Typo_Squatting.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Tracker_Term" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Term.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Tracker_Regex" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Regex.py; read x"
|
||||
|
|
|
|||
57
bin/import/ail_json_importer/Ail_bgp_monitor.py
Executable file
57
bin/import/ail_json_importer/Ail_bgp_monitor.py
Executable file
|
|
@ -0,0 +1,57 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
The JSON Receiver Module
|
||||
================
|
||||
|
||||
Recieve Json Items (example: Twitter feeder)
|
||||
|
||||
"""
|
||||
import os
|
||||
import json
|
||||
import sys
|
||||
import datetime
|
||||
import uuid
|
||||
|
||||
from packages import Tag
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
|
||||
import item_basic
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'import', 'ail_json_importer'))
|
||||
from Default_json import Default_json
|
||||
|
||||
class Ail_bgp_monitor(Default_json):
|
||||
"""urlextract Feeder functions"""
|
||||
|
||||
def __init__(self, name, json_item):
|
||||
super().__init__(name, json_item)
|
||||
|
||||
def get_feeder_name(self):
|
||||
return 'bgp_monitor'
|
||||
|
||||
# # define item id
|
||||
# def get_item_id(self):
|
||||
# # use twitter timestamp ?
|
||||
# item_date = datetime.date.today().strftime("%Y/%m/%d")
|
||||
# item_id = str(self.json_item['meta']['twitter:url-extracted'])
|
||||
# item_id = item_id.split('//')
|
||||
# if len(item_id) > 1:
|
||||
# item_id = ''.join(item_id[1:])
|
||||
# else:
|
||||
# item_id = item_id[0]
|
||||
# item_id = item_id.replace('/', '_')
|
||||
# if len(item_id) > 215:
|
||||
# item_id = '{}{}.gz'.format(item_id[:215], str(uuid.uuid4()))
|
||||
# else:
|
||||
# item_id = '{}{}.gz'.format(item_id, str(uuid.uuid4()))
|
||||
# return os.path.join('urlextract', item_date, item_id)
|
||||
|
||||
def process_json_meta(self, process, item_id):
|
||||
'''
|
||||
Process JSON meta filed.
|
||||
'''
|
||||
json_meta = self.get_json_meta()
|
||||
|
||||
tag = 'infoleak:automatic-detection=bgp_monitor'
|
||||
Tag.add_tag('item', tag, item_id)
|
||||
|
|
@ -10,6 +10,9 @@ import yara
|
|||
import datetime
|
||||
import base64
|
||||
|
||||
from ail_typo_squatting import runAll
|
||||
import math
|
||||
|
||||
|
||||
from flask import escape
|
||||
|
||||
|
|
@ -182,8 +185,8 @@ def get_all_tracker_type():
|
|||
def get_all_tracker_uuid():
|
||||
return r_serv_tracker.smembers(f'trackers:all')
|
||||
|
||||
def get_all_tracker_by_type(tracker_type):
|
||||
r_serv_tracker.smembers(f'trackers:all:{tracker_type}')
|
||||
def get_all_tracker_uuid_by_type(tracker_type):
|
||||
return r_serv_tracker.smembers(f'trackers:all:{tracker_type}')
|
||||
|
||||
# def get_all_tracker():
|
||||
# l_keys_name = []
|
||||
|
|
@ -231,7 +234,7 @@ def get_tracker_mails(tracker_uuid):
|
|||
return list(r_serv_tracker.smembers('tracker:mail:{}'.format(tracker_uuid)))
|
||||
|
||||
def get_tracker_webhook(tracker_uuid):
|
||||
return r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'webhook')
|
||||
return r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'webhook')
|
||||
|
||||
def get_tracker_uuid_sources(tracker_uuid):
|
||||
return list(r_serv_tracker.smembers(f'tracker:sources:{tracker_uuid}'))
|
||||
|
|
@ -345,6 +348,20 @@ def get_tracker_items_by_daterange(tracker_uuid, date_from, date_to):
|
|||
all_item_id |= r_serv_tracker.smembers(f'tracker:item:{tracker_uuid}:{date_day}')
|
||||
return all_item_id
|
||||
|
||||
def get_tracker_typosquatting_domains(tracker_uuid):
|
||||
return r_serv_tracker.smembers(f'tracker:typosquatting:{tracker_uuid}')
|
||||
|
||||
def get_typosquatting_tracked_words_list():
|
||||
typosquattings = {}
|
||||
typos_uuid = get_all_tracker_uuid_by_type("typosquatting")
|
||||
|
||||
for typo_uuid in typos_uuid:
|
||||
tracker = get_tracker_by_uuid(typo_uuid)
|
||||
typosquattings[tracker] = get_tracker_typosquatting_domains(typo_uuid)
|
||||
|
||||
return typosquattings
|
||||
|
||||
|
||||
def add_tracked_item(tracker_uuid, item_id):
|
||||
item_date = item_basic.get_item_date(item_id)
|
||||
# track item
|
||||
|
|
@ -416,14 +433,14 @@ def get_email_subject(tracker_uuid):
|
|||
return 'AIL framework: {}'.format(tracker_description)
|
||||
|
||||
def get_tracker_last_updated_by_type(tracker_type):
|
||||
epoch_update = r_serv_tracker.get('tracker:refresh:{}'.format(tracker_type))
|
||||
epoch_update = r_cache.get(f'tracker:refresh:{tracker_type}')
|
||||
if not epoch_update:
|
||||
epoch_update = 0
|
||||
return float(epoch_update)
|
||||
|
||||
# # TODO: check type API
|
||||
def trigger_trackers_refresh(tracker_type):
|
||||
r_serv_tracker.set(f'tracker:refresh:{tracker_type}', time.time())
|
||||
r_cache.set(f'tracker:refresh:{tracker_type}', time.time())
|
||||
|
||||
######################
|
||||
#### TRACKERS ACL ####
|
||||
|
|
@ -542,6 +559,15 @@ def api_validate_tracker_to_add(tracker , tracker_type, nb_words=1):
|
|||
|
||||
tracker = ",".join(words_set)
|
||||
tracker = "{};{}".format(tracker, nb_words)
|
||||
elif tracker_type == 'typosquatting':
|
||||
tracker = tracker.lower()
|
||||
# Take only the first term
|
||||
domain = tracker.split(" ")
|
||||
if len(domain) > 1:
|
||||
return {"status": "error", "reason": "Only one domain is accepted at a time"}, 400
|
||||
if not "." in tracker:
|
||||
return {"status": "error", "reason": "Invalid domain name"}, 400
|
||||
|
||||
|
||||
elif tracker_type=='yara_custom':
|
||||
if not is_valid_yara_rule(tracker):
|
||||
|
|
@ -594,6 +620,12 @@ def create_tracker(tracker, tracker_type, user_id, level, tags, mails, descripti
|
|||
tracker = save_yara_rule(tracker_type, tracker, tracker_uuid=tracker_uuid)
|
||||
tracker_type = 'yara'
|
||||
|
||||
elif tracker_type == 'typosquatting':
|
||||
domain = tracker.split(" ")[0]
|
||||
typo_generation = runAll(domain=domain, limit=math.inf, formatoutput="text", pathOutput="-", verbose=False)
|
||||
for typo in typo_generation:
|
||||
r_serv_tracker.sadd(f'tracker:typosquatting:{tracker_uuid}', typo)
|
||||
|
||||
# create metadata
|
||||
r_serv_tracker.hset(f'tracker:{tracker_uuid}', 'tracked', tracker)
|
||||
r_serv_tracker.hset(f'tracker:{tracker_uuid}', 'type', tracker_type)
|
||||
|
|
@ -666,9 +698,9 @@ def create_tracker(tracker, tracker_type, user_id, level, tags, mails, descripti
|
|||
# escape source ?
|
||||
r_serv_tracker.sadd(f'tracker:sources:{tracker_uuid}', escape(source))
|
||||
# toggle refresh module tracker list/set
|
||||
r_serv_tracker.set('tracker:refresh:{}'.format(tracker_type), time.time())
|
||||
trigger_trackers_refresh(tracker_type)
|
||||
if tracker_type != old_type: # toggle old type refresh
|
||||
r_serv_tracker.set('tracker:refresh:{}'.format(old_type), time.time())
|
||||
trigger_trackers_refresh(old_type)
|
||||
return tracker_uuid
|
||||
|
||||
def api_add_tracker(dict_input, user_id):
|
||||
|
|
|
|||
|
|
@ -1091,7 +1091,6 @@ def get_captures_status():
|
|||
|
||||
##-- CRAWLER STATE --##
|
||||
|
||||
#### CRAWLER TASKS ####
|
||||
|
||||
#### CRAWLER TASK ####
|
||||
|
||||
|
|
@ -1147,6 +1146,7 @@ class CrawlerTask:
|
|||
def get_proxy(self):
|
||||
return r_crawler.hget(f'crawler:task:{self.uuid}', 'proxy')
|
||||
|
||||
<<<<<<< HEAD
|
||||
def get_parent(self):
|
||||
return r_crawler.hget(f'crawler:task:{self.uuid}', 'parent')
|
||||
|
||||
|
|
@ -1316,6 +1316,22 @@ def create_task(url, depth=1, har=True, screenshot=True, header=None, cookiejar=
|
|||
|
||||
|
||||
## -- CRAWLER TASK -- ##
|
||||
=======
|
||||
def send_url_to_crawl_in_queue(crawler_mode, crawler_type, url):
|
||||
print(f'{crawler_type}_crawler_priority_queue', f'{url};{crawler_mode}')
|
||||
r_serv_onion.sadd(f'{crawler_type}_crawler_priority_queue', f'{url};{crawler_mode}')
|
||||
# add auto crawled url for user UI
|
||||
if crawler_mode == 'auto':
|
||||
r_serv_onion.sadd(f'auto_crawler_url:{crawler_type}', url)
|
||||
|
||||
def add_url_to_crawl_in_queue(url, crawler_mode='manual'): # crawler_type
|
||||
#print(f'{crawler_type}_crawler_priority_queue', f'{url};{crawler_mode}')
|
||||
r_serv_onion.sadd(f'{crawler_type}_crawler_priority_queue', f'{url};{crawler_mode}')
|
||||
# CURRENTLY DISABLED
|
||||
# # add auto crawled url for user UI
|
||||
# if crawler_mode == 'auto':
|
||||
# r_serv_onion.sadd(f'auto_crawler_url:{crawler_type}', url)
|
||||
>>>>>>> master
|
||||
|
||||
#### CRAWLER TASK API ####
|
||||
|
||||
|
|
@ -1666,6 +1682,20 @@ def test_ail_crawlers():
|
|||
|
||||
#### ---- ####
|
||||
|
||||
# TODO CHECK MIGRATION - Rest API
|
||||
|
||||
# def add_auto_crawler_in_queue(domain, domain_type, port, epoch, delta, message):
|
||||
# r_serv_onion.zadd('crawler_auto_queue', int(time.time() + delta) , f'{message};{domain_type}')
|
||||
# # update list, last auto crawled domains
|
||||
# r_serv_onion.lpush('last_auto_crawled', f'{domain}:{port};{epoch}')
|
||||
# r_serv_onion.ltrim('last_auto_crawled', 0, 9)
|
||||
|
||||
# TODO MIGRATE ME
|
||||
# def api_create_crawler_task(user_id, url, screenshot=True, har=True, depth_limit=1, max_pages=100, auto_crawler=False, crawler_delta=3600, crawler_type=None, cookiejar_uuid=None, user_agent=None):
|
||||
# # validate url
|
||||
# if url is None or url=='' or url=='\n':
|
||||
# return ({'error':'invalid depth limit'}, 400)
|
||||
|
||||
|
||||
# TODO MOVE ME IN CRAWLER OR FLASK
|
||||
load_blacklist()
|
||||
|
|
|
|||
|
|
@ -24,13 +24,11 @@ Requirements
|
|||
# Import External packages
|
||||
##################################
|
||||
import base64
|
||||
import hashlib
|
||||
import io
|
||||
import gzip
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import datetime
|
||||
|
||||
from hashlib import md5
|
||||
from uuid import uuid4
|
||||
|
|
@ -40,7 +38,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
# Import Project packages
|
||||
##################################
|
||||
from modules.abstract_module import AbstractModule
|
||||
from lib.ConfigLoader import ConfigLoader
|
||||
from lib.ail_core import get_ail_uuid
|
||||
from lib.data_retention_engine import update_obj_date
|
||||
from lib import item_basic
|
||||
# from lib import Statistics
|
||||
|
|
@ -67,6 +65,8 @@ class Global(AbstractModule):
|
|||
|
||||
# Send module state to logs
|
||||
self.redis_logger.info(f"Module {self.module_name} initialized")
|
||||
# Send module state to logs
|
||||
self.redis_logger.critical(f"AIL {get_ail_uuid()} started")
|
||||
|
||||
def computeNone(self):
|
||||
difftime = time.time() - self.time_last_stats
|
||||
|
|
|
|||
|
|
@ -25,6 +25,9 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
from modules.abstract_module import AbstractModule
|
||||
from lib.objects import Pgps
|
||||
from lib.objects.Items import Item
|
||||
from trackers.Tracker_Term import Tracker_Term
|
||||
from trackers.Tracker_Regex import Tracker_Regex
|
||||
from trackers.Tracker_Yara import Tracker_Yara
|
||||
|
||||
|
||||
class PgpDump(AbstractModule):
|
||||
|
|
@ -53,6 +56,10 @@ class PgpDump(AbstractModule):
|
|||
# Waiting time in seconds between to message processed
|
||||
self.pending_seconds = 1
|
||||
|
||||
self.tracker_term = Tracker_Term()
|
||||
self.tracker_regex = Tracker_Regex()
|
||||
self.tracker_yara = Tracker_Yara()
|
||||
|
||||
# init
|
||||
self.item_id = None
|
||||
self.keys = set()
|
||||
|
|
@ -209,10 +216,16 @@ class PgpDump(AbstractModule):
|
|||
pgp = Pgps.Pgp(name, 'name')
|
||||
pgp.add(date, self.item_id)
|
||||
print(f' name: {name}')
|
||||
self.tracker_term.compute(self.item_id, item_content=name)
|
||||
self.tracker_regex.compute(self.item_id, content=name)
|
||||
self.tracker_yara.compute(self.item_id, item_content=name)
|
||||
for mail in self.mails:
|
||||
pgp = Pgps.Pgp(mail, 'mail')
|
||||
pgp.add(date, self.item_id)
|
||||
print(f' mail: {mail}')
|
||||
self.tracker_term.compute(self.item_id, item_content=mail)
|
||||
self.tracker_regex.compute(self.item_id, content=mail)
|
||||
self.tracker_yara.compute(self.item_id, item_content=mail)
|
||||
|
||||
# Keys extracted from PGP PRIVATE KEY BLOCK
|
||||
for key in self.private_keys:
|
||||
|
|
|
|||
|
|
@ -43,6 +43,10 @@ subscribe = Redis_D4_client
|
|||
subscribe = Redis
|
||||
publish = Redis_Tags
|
||||
|
||||
[Tracker_Typo_Squatting]
|
||||
subscribe = Redis_Host
|
||||
publish = Redis_Tags
|
||||
|
||||
[Tracker_Term]
|
||||
subscribe = Redis_Global
|
||||
publish = Redis_Tags
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ class Tracker_Regex(AbstractModule):
|
|||
|
||||
self.redis_logger.info(f"Module: {self.module_name} Launched")
|
||||
|
||||
def compute(self, item_id):
|
||||
def compute(self, item_id, content=None):
|
||||
# refresh Tracked regex
|
||||
if self.last_refresh < Tracker.get_tracker_last_updated_by_type('regex'):
|
||||
self.dict_regex_tracked = Term.get_regex_tracked_words_dict()
|
||||
|
|
@ -58,6 +58,7 @@ class Tracker_Regex(AbstractModule):
|
|||
|
||||
item = Item(item_id)
|
||||
item_id = item.get_id()
|
||||
if not content:
|
||||
content = item.get_content()
|
||||
|
||||
for regex in self.dict_regex_tracked:
|
||||
|
|
@ -76,6 +77,7 @@ class Tracker_Regex(AbstractModule):
|
|||
# date = item.get_date()
|
||||
item_source = item.get_source()
|
||||
print(f'new tracked regex found: {tracker_name} in {item_id}')
|
||||
self.redis_logger.warning(f'new tracked regex found: {tracker_name} in {item_id}')
|
||||
|
||||
for tracker_uuid in uuid_list:
|
||||
tracker = Tracker.Tracker(tracker_uuid)
|
||||
|
|
|
|||
|
|
@ -61,7 +61,7 @@ class Tracker_Term(AbstractModule):
|
|||
|
||||
self.redis_logger.info(f"Module: {self.module_name} Launched")
|
||||
|
||||
def compute(self, item_id):
|
||||
def compute(self, item_id, item_content=None):
|
||||
# refresh Tracked term
|
||||
if self.last_refresh_word < Term.get_tracked_term_last_updated_by_type('word'):
|
||||
self.list_tracked_words = Term.get_tracked_words_list()
|
||||
|
|
@ -78,6 +78,7 @@ class Tracker_Term(AbstractModule):
|
|||
# Cast message as Item
|
||||
item = Item(item_id)
|
||||
item_date = item.get_date()
|
||||
if not item_content:
|
||||
item_content = item.get_content()
|
||||
|
||||
signal.alarm(self.max_execution_time)
|
||||
|
|
@ -120,7 +121,7 @@ class Tracker_Term(AbstractModule):
|
|||
item_id = item.get_id()
|
||||
item_date = item.get_date()
|
||||
item_source = item.get_source()
|
||||
self.redis_logger.info(f'new tracked term found: {term} in {item_id}')
|
||||
self.redis_logger.warning(f'new tracked term found: {term} in {item_id}')
|
||||
print(f'new tracked term found: {term} in {item_id}')
|
||||
for term_uuid in uuid_list:
|
||||
tracker_sources = Tracker.get_tracker_uuid_sources(term_uuid)
|
||||
|
|
|
|||
119
bin/trackers/Tracker_Typo_Squatting.py
Executable file
119
bin/trackers/Tracker_Typo_Squatting.py
Executable file
|
|
@ -0,0 +1,119 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
"""
|
||||
The Tracker_Typo_Squatting Module
|
||||
===================
|
||||
|
||||
"""
|
||||
|
||||
##################################
|
||||
# Import External packages
|
||||
##################################
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import requests
|
||||
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from modules.abstract_module import AbstractModule
|
||||
import NotificationHelper
|
||||
from packages.Item import Item
|
||||
from lib import Tracker
|
||||
|
||||
class Tracker_Typo_Squatting(AbstractModule):
|
||||
mail_body_template = "AIL Framework,\nNew occurrence for tracked Typo: {}\nitem id: {}\nurl: {}{}"
|
||||
|
||||
"""
|
||||
Tracker_Typo_Squatting module for AIL framework
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super(Tracker_Typo_Squatting, self).__init__()
|
||||
|
||||
self.pending_seconds = 5
|
||||
|
||||
self.full_item_url = self.process.config.get("Notifications", "ail_domain") + "/object/item?id="
|
||||
|
||||
# loads typosquatting
|
||||
self.typosquat_tracked_words_list = Tracker.get_typosquatting_tracked_words_list()
|
||||
self.last_refresh_typosquat = time.time()
|
||||
|
||||
self.redis_logger.info(f"Module: {self.module_name} Launched")
|
||||
|
||||
def compute(self, message):
|
||||
# refresh Tracked typo
|
||||
if self.last_refresh_typosquat < Tracker.get_tracker_last_updated_by_type('typosquatting'):
|
||||
self.typosquat_tracked_words_list = Tracker.get_typosquatting_tracked_words_list()
|
||||
self.last_refresh_typosquat = time.time()
|
||||
self.redis_logger.debug('Tracked typosquatting refreshed')
|
||||
print('Tracked typosquatting refreshed')
|
||||
|
||||
host, id = message.split()
|
||||
|
||||
# Cast message as Item
|
||||
for tracker in self.typosquat_tracked_words_list:
|
||||
if host in self.typosquat_tracked_words_list[tracker]:
|
||||
item = Item(id)
|
||||
self.new_tracker_found(tracker, 'typosquatting', item)
|
||||
|
||||
def new_tracker_found(self, tracker, tracker_type, item):
|
||||
item_id = item.get_id()
|
||||
item_date = item.get_date()
|
||||
item_source = item.get_source()
|
||||
#self.redis_logger.info(f'new tracked typo found: {tracker} in {item_id}')
|
||||
print(f'new tracked typosquatting found: {tracker} in {item_id}')
|
||||
self.redis_logger.warning(f'tracker typosquatting: {tracker} in {item_id}')
|
||||
|
||||
print(Tracker.get_tracker_uuid_list(tracker, tracker_type))
|
||||
for tracker_uuid in Tracker.get_tracker_uuid_list(tracker, tracker_type):
|
||||
# Source Filtering
|
||||
tracker_sources = Tracker.get_tracker_uuid_sources(tracker)
|
||||
if tracker_sources and item_source not in tracker_sources:
|
||||
continue
|
||||
|
||||
Tracker.add_tracked_item(tracker_uuid, item_id)
|
||||
|
||||
# Tags
|
||||
tags_to_add = Tracker.get_tracker_tags(tracker_uuid)
|
||||
for tag in tags_to_add:
|
||||
msg = f'{tag};{item_id}'
|
||||
self.send_message_to_queue(msg, 'Tags')
|
||||
|
||||
mail_to_notify = Tracker.get_tracker_mails(tracker_uuid)
|
||||
if mail_to_notify:
|
||||
mail_subject = Tracker.get_email_subject(tracker_uuid)
|
||||
mail_body = Tracker_Typo_Squatting.mail_body_template.format(tracker, item_id, self.full_item_url, item_id)
|
||||
for mail in mail_to_notify:
|
||||
NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body)
|
||||
|
||||
# Webhook
|
||||
webhook_to_post = Tracker.get_tracker_webhook(tracker_uuid)
|
||||
if webhook_to_post:
|
||||
json_request = {"trackerId": tracker_uuid,
|
||||
"itemId": item_id,
|
||||
"itemURL": self.full_item_url + item_id,
|
||||
"tracker": tracker,
|
||||
"itemSource": item_source,
|
||||
"itemDate": item_date,
|
||||
"tags": tags_to_add,
|
||||
"emailNotification": f'{mail_to_notify}',
|
||||
"trackerType": tracker_type
|
||||
}
|
||||
try:
|
||||
response = requests.post(webhook_to_post, json=json_request)
|
||||
if response.status_code >= 400:
|
||||
self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: {response.reason}")
|
||||
except:
|
||||
self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: Something went wrong")
|
||||
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
module = Tracker_Typo_Squatting()
|
||||
module.run()
|
||||
#module.compute('g00gle.com tests/2020/01/01/test.gz')
|
||||
|
|
@ -45,7 +45,7 @@ class Tracker_Yara(AbstractModule):
|
|||
|
||||
self.redis_logger.info(f"Module: {self.module_name} Launched")
|
||||
|
||||
def compute(self, item_id):
|
||||
def compute(self, item_id, item_content=None):
|
||||
# refresh YARA list
|
||||
if self.last_refresh < Tracker.get_tracker_last_updated_by_type('yara'):
|
||||
self.rules = Tracker.reload_yara_rules()
|
||||
|
|
@ -54,12 +54,14 @@ class Tracker_Yara(AbstractModule):
|
|||
print('Tracked set refreshed')
|
||||
|
||||
self.item = Item(item_id)
|
||||
if not item_content:
|
||||
item_content = self.item.get_content()
|
||||
|
||||
try:
|
||||
yara_match = self.rules.match(data=item_content, callback=self.yara_rules_match,
|
||||
which_callbacks=yara.CALLBACK_MATCHES, timeout=60)
|
||||
if yara_match:
|
||||
self.redis_logger.info(f'{self.item.get_id()}: {yara_match}')
|
||||
self.redis_logger.warning(f'tracker yara: new match {self.item.get_id()}: {yara_match}')
|
||||
print(f'{self.item.get_id()}: {yara_match}')
|
||||
except yara.TimeoutError as e:
|
||||
print(f'{self.item.get_id()}: yara scanning timed out')
|
||||
|
|
|
|||
|
|
@ -22,6 +22,18 @@ pystemonpath = /home/pystemon/pystemon/
|
|||
|
||||
sentiment_lexicon_file = sentiment/vader_lexicon.zip/vader_lexicon/vader_lexicon.txt
|
||||
|
||||
##### Logs ######
|
||||
[Logs]
|
||||
# activate syslog
|
||||
ail_logs_syslog = False
|
||||
ail_logs_syslog_server =
|
||||
# default=514
|
||||
ail_logs_syslog_port =
|
||||
# ['auth', 'authpriv', 'cron', 'daemon', 'ftp', 'kern', 'lpr', 'mail', 'news', 'syslog', 'user', 'uucp', 'local0', 'local1', 'local2', 'local3', 'local4', 'local5', 'local6', 'local7']
|
||||
ail_logs_syslog_facility =
|
||||
# ['DEBUG', 'INFO', 'NOTICE', 'WARNING', 'ERROR', 'CRITICAL']
|
||||
ail_logs_syslog_level =
|
||||
|
||||
##### Notifications ######
|
||||
[Notifications]
|
||||
ail_domain = https://localhost:7000
|
||||
|
|
@ -228,7 +240,7 @@ dns = 8.8.8.8
|
|||
dns = 8.8.8.8
|
||||
|
||||
[Web]
|
||||
dns = 149.13.33.69
|
||||
dns = 8.8.8.8
|
||||
|
||||
# Indexer configuration
|
||||
[Indexer]
|
||||
|
|
@ -247,7 +259,8 @@ maxDuplicateToPushToMISP=10
|
|||
# e.g.: tcp://127.0.0.1:5556,tcp://127.0.0.1:5557
|
||||
[ZMQ_Global]
|
||||
#address = tcp://crf.circl.lu:5556
|
||||
address = tcp://127.0.0.1:5556,tcp://crf.circl.lu:5556
|
||||
# address = tcp://127.0.0.1:5556,tcp://crf.circl.lu:5556
|
||||
address = tcp://127.0.0.1:5556
|
||||
channel = 102
|
||||
bind = tcp://127.0.0.1:5556
|
||||
|
||||
|
|
|
|||
|
|
@ -71,6 +71,8 @@ flask>=1.1.4
|
|||
flask-login
|
||||
bcrypt>3.1.6
|
||||
|
||||
# Ail typo squatting
|
||||
ail_typo_squatting
|
||||
|
||||
# Tests
|
||||
nose>=1.3.7
|
||||
|
|
|
|||
26
update/v4.2.1/Update.py
Executable file
26
update/v4.2.1/Update.py
Executable file
|
|
@ -0,0 +1,26 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import datetime
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin'))
|
||||
from ail_updater import AIL_Updater
|
||||
|
||||
class Updater(AIL_Updater):
|
||||
"""default Updater."""
|
||||
|
||||
def __init__(self, version):
|
||||
super(Updater, self).__init__(version)
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
updater = Updater('v4.2.1')
|
||||
updater.run_update()
|
||||
29
update/v4.2.1/Update.sh
Executable file
29
update/v4.2.1/Update.sh
Executable file
|
|
@ -0,0 +1,29 @@
|
|||
#!/bin/bash
|
||||
|
||||
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
|
||||
|
||||
export PATH=$AIL_HOME:$PATH
|
||||
export PATH=$AIL_REDIS:$PATH
|
||||
export PATH=$AIL_ARDB:$PATH
|
||||
export PATH=$AIL_BIN:$PATH
|
||||
export PATH=$AIL_FLASK:$PATH
|
||||
|
||||
GREEN="\\033[1;32m"
|
||||
DEFAULT="\\033[0;39m"
|
||||
|
||||
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
# SUBMODULES #
|
||||
git submodule update
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Updating pusblogger ..."$DEFAULT
|
||||
pip3 install -U pubsublogger
|
||||
|
||||
exit 0
|
||||
|
|
@ -85,6 +85,16 @@ def tracked_menu_yara():
|
|||
global_trackers = Tracker.get_global_trackers_metadata(tracker_type=tracker_type)
|
||||
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
|
||||
|
||||
@hunter.route("/trackers/typosquatting")
|
||||
@login_required
|
||||
@login_read_only
|
||||
def tracked_menu_typosquatting():
|
||||
filter_type = 'typosquatting'
|
||||
user_id = current_user.get_id()
|
||||
user_term = Term.get_all_user_tracked_terms(user_id, filter_type=filter_type)
|
||||
global_term = Term.get_all_global_tracked_terms(filter_type=filter_type)
|
||||
return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type)
|
||||
|
||||
|
||||
@hunter.route("/tracker/add", methods=['GET', 'POST'])
|
||||
@login_required
|
||||
|
|
@ -225,6 +235,13 @@ def show_tracker():
|
|||
else:
|
||||
yara_rule_content = None
|
||||
|
||||
if tracker_metadata['type'] == 'typosquatting':
|
||||
typo_squatting = list(Tracker.get_tracker_typosquatting_domains(tracker_uuid))
|
||||
typo_squatting.sort()
|
||||
else:
|
||||
typo_squatting = None
|
||||
|
||||
|
||||
if date_from:
|
||||
res = Term.parse_get_tracker_term_item({'uuid': tracker_uuid, 'date_from': date_from, 'date_to': date_to}, user_id)
|
||||
if res[1] !=200:
|
||||
|
|
@ -241,6 +258,7 @@ def show_tracker():
|
|||
|
||||
return render_template("showTracker.html", tracker_metadata=tracker_metadata,
|
||||
yara_rule_content=yara_rule_content,
|
||||
typo_squatting=typo_squatting,
|
||||
bootstrap_label=bootstrap_label)
|
||||
|
||||
@hunter.route("/tracker/update_tracker_description", methods=['POST'])
|
||||
|
|
|
|||
|
|
@ -104,6 +104,7 @@
|
|||
<option value="set">Set</option>
|
||||
<option value="regex">Regex</option>
|
||||
<option value="yara">YARA rule</option>
|
||||
<option value="typosquatting">Typo-squatting</option>
|
||||
</select>
|
||||
|
||||
<p id="tracker_desc">Terms to track (space separated)</p>
|
||||
|
|
@ -209,6 +210,12 @@ $(document).ready(function(){
|
|||
$("#tracker").hide();
|
||||
$("#nb_word").hide();
|
||||
$("#yara_rule").show();
|
||||
} else if (tracker_type=="typosquatting") {
|
||||
$("#tracker_desc").text("Generation of variation for domain name. Only one domain name at a time.");
|
||||
$("#tracker_desc").show();
|
||||
$("#tracker").show();
|
||||
$("#nb_word").hide();
|
||||
$("#yara_rule").hide();
|
||||
}
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -91,7 +91,25 @@
|
|||
<tbody>
|
||||
<tr>
|
||||
<td>{{ tracker_metadata['type'] }}</td>
|
||||
{% if tracker_metadata['type'] == 'typosquatting' %}
|
||||
<td>
|
||||
<a class="btn btn-primary" data-toggle="collapse" href="#collapseTypo" role="button" aria-expanded="false" aria-controls="collapseTypo">
|
||||
{{ tracker_metadata['tracker'].split(",")[0] }}
|
||||
</a>
|
||||
<div class="collapse" id="collapseTypo">
|
||||
<div class="card card-body">
|
||||
{% if typo_squatting %}
|
||||
{% for typo in typo_squatting %}
|
||||
{{typo}}
|
||||
<br/>
|
||||
{% endfor %}
|
||||
{%endif%}
|
||||
</div>
|
||||
</div>
|
||||
</td>
|
||||
{% else %}
|
||||
<td>{{ tracker_metadata['tracker'] }}</td>
|
||||
{% endif %}
|
||||
<td>{{ tracker_metadata['date'][0:4] }}/{{ tracker_metadata['date'][4:6] }}/{{ tracker_metadata['date'][6:8] }}</td>
|
||||
<td>
|
||||
{% if tracker_metadata['level'] == 0 %}
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from lib import crawlers
|
||||
from lib import Users
|
||||
from lib.objects.Items import Item
|
||||
from lib import Tag
|
||||
|
|
@ -46,8 +47,8 @@ restApi = Blueprint('restApi', __name__, template_folder='templates')
|
|||
|
||||
# ============ AUTH FUNCTIONS ============
|
||||
|
||||
def check_token_format(strg, search=re.compile(r'[^a-zA-Z0-9_-]').search):
|
||||
return not bool(search(strg))
|
||||
def check_token_format(token, search=re.compile(r'[^a-zA-Z0-9_-]').search):
|
||||
return not bool(search(token))
|
||||
|
||||
def verify_token(token):
|
||||
if len(token) != 41:
|
||||
|
|
@ -90,7 +91,7 @@ def get_auth_from_header():
|
|||
def authErrors(user_role):
|
||||
# Check auth
|
||||
if not request.headers.get('Authorization'):
|
||||
return ({'status': 'error', 'reason': 'Authentication needed'}, 401)
|
||||
return {'status': 'error', 'reason': 'Authentication needed'}, 401
|
||||
token = get_auth_from_header()
|
||||
data = None
|
||||
# verify token format
|
||||
|
|
@ -102,7 +103,7 @@ def authErrors(user_role):
|
|||
if login_failed_ip:
|
||||
login_failed_ip = int(login_failed_ip)
|
||||
if login_failed_ip >= 5:
|
||||
return ({'status': 'error', 'reason': 'Max Connection Attempts reached, Please wait {}s'.format(r_cache.ttl('failed_login_ip_api:{}'.format(current_ip)))}, 401)
|
||||
return {'status': 'error', 'reason': 'Max Connection Attempts reached, Please wait {}s'.format(r_cache.ttl('failed_login_ip_api:{}'.format(current_ip)))}, 401
|
||||
|
||||
try:
|
||||
authenticated = False
|
||||
|
|
@ -146,9 +147,6 @@ def is_valid_uuid_v4(header_uuid):
|
|||
except:
|
||||
return False
|
||||
|
||||
def one():
|
||||
return 1
|
||||
|
||||
# ============= ROUTES ==============
|
||||
|
||||
# @restApi.route("/api", methods=['GET'])
|
||||
|
|
@ -576,6 +574,20 @@ def get_crawled_domain_list():
|
|||
dict_res['domain_type'] = domain_type
|
||||
return create_json_response(dict_res, res[1])
|
||||
|
||||
# # TODO: ADD RESULT JSON Response
|
||||
@restApi.route("api/v1/add/crawler/task", methods=['POST'])
|
||||
@token_required('analyst')
|
||||
def add_crawler_task():
|
||||
data = request.get_json()
|
||||
user_token = get_auth_from_header()
|
||||
user_id = get_user_from_token(user_token)
|
||||
res = crawlers.api_add_crawler_task(data, user_id=user_id)
|
||||
if res:
|
||||
return create_json_response(res[0], res[1])
|
||||
|
||||
dict_res = {'url': data['url']}
|
||||
return create_json_response(dict_res, 200)
|
||||
|
||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||
# # # # # # # # # # # # # IMPORT # # # # # # # # # # # # # # # # # #
|
||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||
|
|
|
|||
|
|
@ -42,6 +42,12 @@
|
|||
<span class="bg-danger text-white font-weight-bold" style="font-size: 120%"> { </span>
|
||||
<span> YARA</span>
|
||||
</a>
|
||||
</li>
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href="{{url_for('hunter.tracked_menu_typosquatting')}}" id="nav_tracker_typosquatting">
|
||||
<i class="fa fa-clone"></i>
|
||||
<span>Typo-squatting</span>
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
<h5 class="d-flex text-muted w-100" id="nav_title_retro_hunt">
|
||||
|
|
|
|||
Loading…
Reference in a new issue