mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-14 02:28:23 +00:00
chg: [core] merge conflict
This commit is contained in:
commit
524a404dc8
21 changed files with 468 additions and 52 deletions
|
@ -117,16 +117,39 @@ function launching_kvrocks {
|
||||||
}
|
}
|
||||||
|
|
||||||
function launching_logs {
|
function launching_logs {
|
||||||
|
conf_dir="${AIL_HOME}/configs/"
|
||||||
|
syslog_cmd=""
|
||||||
|
syslog_enabled=`cat $conf_dir/core.cfg | grep 'ail_logs_syslog' | cut -d " " -f 3 `
|
||||||
|
if [ "$syslog_enabled" = "True" ]; then
|
||||||
|
syslog_cmd="--syslog"
|
||||||
|
fi
|
||||||
|
syslog_server=`cat $conf_dir/core.cfg | grep 'ail_logs_syslog_server' | cut -d " " -f 3 `
|
||||||
|
syslog_port=`cat $conf_dir/core.cfg | grep 'ail_logs_syslog_port' | cut -d " " -f 3 `
|
||||||
|
if [ ! -z "$syslog_server" -a "$str" != " " ]; then
|
||||||
|
syslog_cmd="${syslog_cmd} -ss ${syslog_server}"
|
||||||
|
if [ ! -z "$syslog_port" -a "$str" != " " ]; then
|
||||||
|
syslog_cmd="${syslog_cmd} -sp ${syslog_port}"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
syslog_facility=`cat $conf_dir/core.cfg | grep 'ail_logs_syslog_facility' | cut -d " " -f 3 `
|
||||||
|
if [ ! -z "$syslog_facility" -a "$str" != " " ]; then
|
||||||
|
syslog_cmd="${syslog_cmd} -sf ${syslog_facility}"
|
||||||
|
fi
|
||||||
|
syslog_level=`cat $conf_dir/core.cfg | grep 'ail_logs_syslog_level' | cut -d " " -f 3 `
|
||||||
|
if [ ! -z "$syslog_level" -a "$str" != " " ]; then
|
||||||
|
syslog_cmd="${syslog_cmd} -sl ${syslog_level}"
|
||||||
|
fi
|
||||||
|
|
||||||
screen -dmS "Logging_AIL"
|
screen -dmS "Logging_AIL"
|
||||||
sleep 0.1
|
sleep 0.1
|
||||||
echo -e $GREEN"\t* Launching logging process"$DEFAULT
|
echo -e $GREEN"\t* Launching logging process"$DEFAULT
|
||||||
screen -S "Logging_AIL" -X screen -t "LogQueue" bash -c "cd ${AIL_BIN}; ${AIL_VENV}/bin/log_subscriber -p 6380 -c Queuing -l ../logs/; read x"
|
screen -S "Logging_AIL" -X screen -t "LogQueue" bash -c "cd ${AIL_BIN}; ${AIL_VENV}/bin/log_subscriber -p 6380 -c Queuing -l ../logs/ ${syslog_cmd}; read x"
|
||||||
sleep 0.1
|
sleep 0.1
|
||||||
screen -S "Logging_AIL" -X screen -t "LogScript" bash -c "cd ${AIL_BIN}; ${AIL_VENV}/bin/log_subscriber -p 6380 -c Script -l ../logs/; read x"
|
screen -S "Logging_AIL" -X screen -t "LogScript" bash -c "cd ${AIL_BIN}; ${AIL_VENV}/bin/log_subscriber -p 6380 -c Script -l ../logs/ ${syslog_cmd}; read x"
|
||||||
sleep 0.1
|
sleep 0.1
|
||||||
screen -S "Logging_AIL" -X screen -t "LogScript" bash -c "cd ${AIL_BIN}; ${AIL_VENV}/bin/log_subscriber -p 6380 -c Sync -l ../logs/; read x"
|
screen -S "Logging_AIL" -X screen -t "LogSync" bash -c "cd ${AIL_BIN}; ${AIL_VENV}/bin/log_subscriber -p 6380 -c Sync -l ../logs/ ${syslog_cmd}; read x"
|
||||||
sleep 0.1
|
sleep 0.1
|
||||||
screen -S "Logging_AIL" -X screen -t "LogScript" bash -c "cd ${AIL_BIN}; ${AIL_VENV}/bin/log_subscriber -p 6380 -c Crawler -l ../logs/; read x"
|
screen -S "Logging_AIL" -X screen -t "LogCrawler" bash -c "cd ${AIL_BIN}; ${AIL_VENV}/bin/log_subscriber -p 6380 -c Crawler -l ../logs/ ${syslog_cmd}; read x"
|
||||||
}
|
}
|
||||||
|
|
||||||
function launching_queues {
|
function launching_queues {
|
||||||
|
@ -259,6 +282,8 @@ function launching_scripts {
|
||||||
##################################
|
##################################
|
||||||
# TRACKERS MODULES #
|
# TRACKERS MODULES #
|
||||||
##################################
|
##################################
|
||||||
|
screen -S "Script_AIL" -X screen -t "Tracker_Typo_Squatting" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Typo_Squatting.py; read x"
|
||||||
|
sleep 0.1
|
||||||
screen -S "Script_AIL" -X screen -t "Tracker_Term" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Term.py; read x"
|
screen -S "Script_AIL" -X screen -t "Tracker_Term" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Term.py; read x"
|
||||||
sleep 0.1
|
sleep 0.1
|
||||||
screen -S "Script_AIL" -X screen -t "Tracker_Regex" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Regex.py; read x"
|
screen -S "Script_AIL" -X screen -t "Tracker_Regex" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Regex.py; read x"
|
||||||
|
|
57
bin/import/ail_json_importer/Ail_bgp_monitor.py
Executable file
57
bin/import/ail_json_importer/Ail_bgp_monitor.py
Executable file
|
@ -0,0 +1,57 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*-coding:UTF-8 -*
|
||||||
|
"""
|
||||||
|
The JSON Receiver Module
|
||||||
|
================
|
||||||
|
|
||||||
|
Recieve Json Items (example: Twitter feeder)
|
||||||
|
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
import datetime
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
from packages import Tag
|
||||||
|
|
||||||
|
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
|
||||||
|
import item_basic
|
||||||
|
|
||||||
|
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'import', 'ail_json_importer'))
|
||||||
|
from Default_json import Default_json
|
||||||
|
|
||||||
|
class Ail_bgp_monitor(Default_json):
|
||||||
|
"""urlextract Feeder functions"""
|
||||||
|
|
||||||
|
def __init__(self, name, json_item):
|
||||||
|
super().__init__(name, json_item)
|
||||||
|
|
||||||
|
def get_feeder_name(self):
|
||||||
|
return 'bgp_monitor'
|
||||||
|
|
||||||
|
# # define item id
|
||||||
|
# def get_item_id(self):
|
||||||
|
# # use twitter timestamp ?
|
||||||
|
# item_date = datetime.date.today().strftime("%Y/%m/%d")
|
||||||
|
# item_id = str(self.json_item['meta']['twitter:url-extracted'])
|
||||||
|
# item_id = item_id.split('//')
|
||||||
|
# if len(item_id) > 1:
|
||||||
|
# item_id = ''.join(item_id[1:])
|
||||||
|
# else:
|
||||||
|
# item_id = item_id[0]
|
||||||
|
# item_id = item_id.replace('/', '_')
|
||||||
|
# if len(item_id) > 215:
|
||||||
|
# item_id = '{}{}.gz'.format(item_id[:215], str(uuid.uuid4()))
|
||||||
|
# else:
|
||||||
|
# item_id = '{}{}.gz'.format(item_id, str(uuid.uuid4()))
|
||||||
|
# return os.path.join('urlextract', item_date, item_id)
|
||||||
|
|
||||||
|
def process_json_meta(self, process, item_id):
|
||||||
|
'''
|
||||||
|
Process JSON meta filed.
|
||||||
|
'''
|
||||||
|
json_meta = self.get_json_meta()
|
||||||
|
|
||||||
|
tag = 'infoleak:automatic-detection=bgp_monitor'
|
||||||
|
Tag.add_tag('item', tag, item_id)
|
|
@ -10,6 +10,9 @@ import yara
|
||||||
import datetime
|
import datetime
|
||||||
import base64
|
import base64
|
||||||
|
|
||||||
|
from ail_typo_squatting import runAll
|
||||||
|
import math
|
||||||
|
|
||||||
|
|
||||||
from flask import escape
|
from flask import escape
|
||||||
|
|
||||||
|
@ -182,8 +185,8 @@ def get_all_tracker_type():
|
||||||
def get_all_tracker_uuid():
|
def get_all_tracker_uuid():
|
||||||
return r_serv_tracker.smembers(f'trackers:all')
|
return r_serv_tracker.smembers(f'trackers:all')
|
||||||
|
|
||||||
def get_all_tracker_by_type(tracker_type):
|
def get_all_tracker_uuid_by_type(tracker_type):
|
||||||
r_serv_tracker.smembers(f'trackers:all:{tracker_type}')
|
return r_serv_tracker.smembers(f'trackers:all:{tracker_type}')
|
||||||
|
|
||||||
# def get_all_tracker():
|
# def get_all_tracker():
|
||||||
# l_keys_name = []
|
# l_keys_name = []
|
||||||
|
@ -231,7 +234,7 @@ def get_tracker_mails(tracker_uuid):
|
||||||
return list(r_serv_tracker.smembers('tracker:mail:{}'.format(tracker_uuid)))
|
return list(r_serv_tracker.smembers('tracker:mail:{}'.format(tracker_uuid)))
|
||||||
|
|
||||||
def get_tracker_webhook(tracker_uuid):
|
def get_tracker_webhook(tracker_uuid):
|
||||||
return r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'webhook')
|
return r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'webhook')
|
||||||
|
|
||||||
def get_tracker_uuid_sources(tracker_uuid):
|
def get_tracker_uuid_sources(tracker_uuid):
|
||||||
return list(r_serv_tracker.smembers(f'tracker:sources:{tracker_uuid}'))
|
return list(r_serv_tracker.smembers(f'tracker:sources:{tracker_uuid}'))
|
||||||
|
@ -345,6 +348,20 @@ def get_tracker_items_by_daterange(tracker_uuid, date_from, date_to):
|
||||||
all_item_id |= r_serv_tracker.smembers(f'tracker:item:{tracker_uuid}:{date_day}')
|
all_item_id |= r_serv_tracker.smembers(f'tracker:item:{tracker_uuid}:{date_day}')
|
||||||
return all_item_id
|
return all_item_id
|
||||||
|
|
||||||
|
def get_tracker_typosquatting_domains(tracker_uuid):
|
||||||
|
return r_serv_tracker.smembers(f'tracker:typosquatting:{tracker_uuid}')
|
||||||
|
|
||||||
|
def get_typosquatting_tracked_words_list():
|
||||||
|
typosquattings = {}
|
||||||
|
typos_uuid = get_all_tracker_uuid_by_type("typosquatting")
|
||||||
|
|
||||||
|
for typo_uuid in typos_uuid:
|
||||||
|
tracker = get_tracker_by_uuid(typo_uuid)
|
||||||
|
typosquattings[tracker] = get_tracker_typosquatting_domains(typo_uuid)
|
||||||
|
|
||||||
|
return typosquattings
|
||||||
|
|
||||||
|
|
||||||
def add_tracked_item(tracker_uuid, item_id):
|
def add_tracked_item(tracker_uuid, item_id):
|
||||||
item_date = item_basic.get_item_date(item_id)
|
item_date = item_basic.get_item_date(item_id)
|
||||||
# track item
|
# track item
|
||||||
|
@ -416,14 +433,14 @@ def get_email_subject(tracker_uuid):
|
||||||
return 'AIL framework: {}'.format(tracker_description)
|
return 'AIL framework: {}'.format(tracker_description)
|
||||||
|
|
||||||
def get_tracker_last_updated_by_type(tracker_type):
|
def get_tracker_last_updated_by_type(tracker_type):
|
||||||
epoch_update = r_serv_tracker.get('tracker:refresh:{}'.format(tracker_type))
|
epoch_update = r_cache.get(f'tracker:refresh:{tracker_type}')
|
||||||
if not epoch_update:
|
if not epoch_update:
|
||||||
epoch_update = 0
|
epoch_update = 0
|
||||||
return float(epoch_update)
|
return float(epoch_update)
|
||||||
|
|
||||||
# # TODO: check type API
|
# # TODO: check type API
|
||||||
def trigger_trackers_refresh(tracker_type):
|
def trigger_trackers_refresh(tracker_type):
|
||||||
r_serv_tracker.set(f'tracker:refresh:{tracker_type}', time.time())
|
r_cache.set(f'tracker:refresh:{tracker_type}', time.time())
|
||||||
|
|
||||||
######################
|
######################
|
||||||
#### TRACKERS ACL ####
|
#### TRACKERS ACL ####
|
||||||
|
@ -542,6 +559,15 @@ def api_validate_tracker_to_add(tracker , tracker_type, nb_words=1):
|
||||||
|
|
||||||
tracker = ",".join(words_set)
|
tracker = ",".join(words_set)
|
||||||
tracker = "{};{}".format(tracker, nb_words)
|
tracker = "{};{}".format(tracker, nb_words)
|
||||||
|
elif tracker_type == 'typosquatting':
|
||||||
|
tracker = tracker.lower()
|
||||||
|
# Take only the first term
|
||||||
|
domain = tracker.split(" ")
|
||||||
|
if len(domain) > 1:
|
||||||
|
return {"status": "error", "reason": "Only one domain is accepted at a time"}, 400
|
||||||
|
if not "." in tracker:
|
||||||
|
return {"status": "error", "reason": "Invalid domain name"}, 400
|
||||||
|
|
||||||
|
|
||||||
elif tracker_type=='yara_custom':
|
elif tracker_type=='yara_custom':
|
||||||
if not is_valid_yara_rule(tracker):
|
if not is_valid_yara_rule(tracker):
|
||||||
|
@ -594,6 +620,12 @@ def create_tracker(tracker, tracker_type, user_id, level, tags, mails, descripti
|
||||||
tracker = save_yara_rule(tracker_type, tracker, tracker_uuid=tracker_uuid)
|
tracker = save_yara_rule(tracker_type, tracker, tracker_uuid=tracker_uuid)
|
||||||
tracker_type = 'yara'
|
tracker_type = 'yara'
|
||||||
|
|
||||||
|
elif tracker_type == 'typosquatting':
|
||||||
|
domain = tracker.split(" ")[0]
|
||||||
|
typo_generation = runAll(domain=domain, limit=math.inf, formatoutput="text", pathOutput="-", verbose=False)
|
||||||
|
for typo in typo_generation:
|
||||||
|
r_serv_tracker.sadd(f'tracker:typosquatting:{tracker_uuid}', typo)
|
||||||
|
|
||||||
# create metadata
|
# create metadata
|
||||||
r_serv_tracker.hset(f'tracker:{tracker_uuid}', 'tracked', tracker)
|
r_serv_tracker.hset(f'tracker:{tracker_uuid}', 'tracked', tracker)
|
||||||
r_serv_tracker.hset(f'tracker:{tracker_uuid}', 'type', tracker_type)
|
r_serv_tracker.hset(f'tracker:{tracker_uuid}', 'type', tracker_type)
|
||||||
|
@ -666,9 +698,9 @@ def create_tracker(tracker, tracker_type, user_id, level, tags, mails, descripti
|
||||||
# escape source ?
|
# escape source ?
|
||||||
r_serv_tracker.sadd(f'tracker:sources:{tracker_uuid}', escape(source))
|
r_serv_tracker.sadd(f'tracker:sources:{tracker_uuid}', escape(source))
|
||||||
# toggle refresh module tracker list/set
|
# toggle refresh module tracker list/set
|
||||||
r_serv_tracker.set('tracker:refresh:{}'.format(tracker_type), time.time())
|
trigger_trackers_refresh(tracker_type)
|
||||||
if tracker_type != old_type: # toggle old type refresh
|
if tracker_type != old_type: # toggle old type refresh
|
||||||
r_serv_tracker.set('tracker:refresh:{}'.format(old_type), time.time())
|
trigger_trackers_refresh(old_type)
|
||||||
return tracker_uuid
|
return tracker_uuid
|
||||||
|
|
||||||
def api_add_tracker(dict_input, user_id):
|
def api_add_tracker(dict_input, user_id):
|
||||||
|
|
|
@ -1091,7 +1091,6 @@ def get_captures_status():
|
||||||
|
|
||||||
##-- CRAWLER STATE --##
|
##-- CRAWLER STATE --##
|
||||||
|
|
||||||
#### CRAWLER TASKS ####
|
|
||||||
|
|
||||||
#### CRAWLER TASK ####
|
#### CRAWLER TASK ####
|
||||||
|
|
||||||
|
@ -1147,6 +1146,7 @@ class CrawlerTask:
|
||||||
def get_proxy(self):
|
def get_proxy(self):
|
||||||
return r_crawler.hget(f'crawler:task:{self.uuid}', 'proxy')
|
return r_crawler.hget(f'crawler:task:{self.uuid}', 'proxy')
|
||||||
|
|
||||||
|
<<<<<<< HEAD
|
||||||
def get_parent(self):
|
def get_parent(self):
|
||||||
return r_crawler.hget(f'crawler:task:{self.uuid}', 'parent')
|
return r_crawler.hget(f'crawler:task:{self.uuid}', 'parent')
|
||||||
|
|
||||||
|
@ -1316,6 +1316,22 @@ def create_task(url, depth=1, har=True, screenshot=True, header=None, cookiejar=
|
||||||
|
|
||||||
|
|
||||||
## -- CRAWLER TASK -- ##
|
## -- CRAWLER TASK -- ##
|
||||||
|
=======
|
||||||
|
def send_url_to_crawl_in_queue(crawler_mode, crawler_type, url):
|
||||||
|
print(f'{crawler_type}_crawler_priority_queue', f'{url};{crawler_mode}')
|
||||||
|
r_serv_onion.sadd(f'{crawler_type}_crawler_priority_queue', f'{url};{crawler_mode}')
|
||||||
|
# add auto crawled url for user UI
|
||||||
|
if crawler_mode == 'auto':
|
||||||
|
r_serv_onion.sadd(f'auto_crawler_url:{crawler_type}', url)
|
||||||
|
|
||||||
|
def add_url_to_crawl_in_queue(url, crawler_mode='manual'): # crawler_type
|
||||||
|
#print(f'{crawler_type}_crawler_priority_queue', f'{url};{crawler_mode}')
|
||||||
|
r_serv_onion.sadd(f'{crawler_type}_crawler_priority_queue', f'{url};{crawler_mode}')
|
||||||
|
# CURRENTLY DISABLED
|
||||||
|
# # add auto crawled url for user UI
|
||||||
|
# if crawler_mode == 'auto':
|
||||||
|
# r_serv_onion.sadd(f'auto_crawler_url:{crawler_type}', url)
|
||||||
|
>>>>>>> master
|
||||||
|
|
||||||
#### CRAWLER TASK API ####
|
#### CRAWLER TASK API ####
|
||||||
|
|
||||||
|
@ -1666,6 +1682,20 @@ def test_ail_crawlers():
|
||||||
|
|
||||||
#### ---- ####
|
#### ---- ####
|
||||||
|
|
||||||
|
# TODO CHECK MIGRATION - Rest API
|
||||||
|
|
||||||
|
# def add_auto_crawler_in_queue(domain, domain_type, port, epoch, delta, message):
|
||||||
|
# r_serv_onion.zadd('crawler_auto_queue', int(time.time() + delta) , f'{message};{domain_type}')
|
||||||
|
# # update list, last auto crawled domains
|
||||||
|
# r_serv_onion.lpush('last_auto_crawled', f'{domain}:{port};{epoch}')
|
||||||
|
# r_serv_onion.ltrim('last_auto_crawled', 0, 9)
|
||||||
|
|
||||||
|
# TODO MIGRATE ME
|
||||||
|
# def api_create_crawler_task(user_id, url, screenshot=True, har=True, depth_limit=1, max_pages=100, auto_crawler=False, crawler_delta=3600, crawler_type=None, cookiejar_uuid=None, user_agent=None):
|
||||||
|
# # validate url
|
||||||
|
# if url is None or url=='' or url=='\n':
|
||||||
|
# return ({'error':'invalid depth limit'}, 400)
|
||||||
|
|
||||||
|
|
||||||
# TODO MOVE ME IN CRAWLER OR FLASK
|
# TODO MOVE ME IN CRAWLER OR FLASK
|
||||||
load_blacklist()
|
load_blacklist()
|
||||||
|
|
|
@ -24,13 +24,11 @@ Requirements
|
||||||
# Import External packages
|
# Import External packages
|
||||||
##################################
|
##################################
|
||||||
import base64
|
import base64
|
||||||
import hashlib
|
|
||||||
import io
|
import io
|
||||||
import gzip
|
import gzip
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
import datetime
|
|
||||||
|
|
||||||
from hashlib import md5
|
from hashlib import md5
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
@ -40,7 +38,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
||||||
# Import Project packages
|
# Import Project packages
|
||||||
##################################
|
##################################
|
||||||
from modules.abstract_module import AbstractModule
|
from modules.abstract_module import AbstractModule
|
||||||
from lib.ConfigLoader import ConfigLoader
|
from lib.ail_core import get_ail_uuid
|
||||||
from lib.data_retention_engine import update_obj_date
|
from lib.data_retention_engine import update_obj_date
|
||||||
from lib import item_basic
|
from lib import item_basic
|
||||||
# from lib import Statistics
|
# from lib import Statistics
|
||||||
|
@ -67,6 +65,8 @@ class Global(AbstractModule):
|
||||||
|
|
||||||
# Send module state to logs
|
# Send module state to logs
|
||||||
self.redis_logger.info(f"Module {self.module_name} initialized")
|
self.redis_logger.info(f"Module {self.module_name} initialized")
|
||||||
|
# Send module state to logs
|
||||||
|
self.redis_logger.critical(f"AIL {get_ail_uuid()} started")
|
||||||
|
|
||||||
def computeNone(self):
|
def computeNone(self):
|
||||||
difftime = time.time() - self.time_last_stats
|
difftime = time.time() - self.time_last_stats
|
||||||
|
|
|
@ -25,6 +25,9 @@ sys.path.append(os.environ['AIL_BIN'])
|
||||||
from modules.abstract_module import AbstractModule
|
from modules.abstract_module import AbstractModule
|
||||||
from lib.objects import Pgps
|
from lib.objects import Pgps
|
||||||
from lib.objects.Items import Item
|
from lib.objects.Items import Item
|
||||||
|
from trackers.Tracker_Term import Tracker_Term
|
||||||
|
from trackers.Tracker_Regex import Tracker_Regex
|
||||||
|
from trackers.Tracker_Yara import Tracker_Yara
|
||||||
|
|
||||||
|
|
||||||
class PgpDump(AbstractModule):
|
class PgpDump(AbstractModule):
|
||||||
|
@ -53,6 +56,10 @@ class PgpDump(AbstractModule):
|
||||||
# Waiting time in seconds between to message processed
|
# Waiting time in seconds between to message processed
|
||||||
self.pending_seconds = 1
|
self.pending_seconds = 1
|
||||||
|
|
||||||
|
self.tracker_term = Tracker_Term()
|
||||||
|
self.tracker_regex = Tracker_Regex()
|
||||||
|
self.tracker_yara = Tracker_Yara()
|
||||||
|
|
||||||
# init
|
# init
|
||||||
self.item_id = None
|
self.item_id = None
|
||||||
self.keys = set()
|
self.keys = set()
|
||||||
|
@ -209,10 +216,16 @@ class PgpDump(AbstractModule):
|
||||||
pgp = Pgps.Pgp(name, 'name')
|
pgp = Pgps.Pgp(name, 'name')
|
||||||
pgp.add(date, self.item_id)
|
pgp.add(date, self.item_id)
|
||||||
print(f' name: {name}')
|
print(f' name: {name}')
|
||||||
|
self.tracker_term.compute(self.item_id, item_content=name)
|
||||||
|
self.tracker_regex.compute(self.item_id, content=name)
|
||||||
|
self.tracker_yara.compute(self.item_id, item_content=name)
|
||||||
for mail in self.mails:
|
for mail in self.mails:
|
||||||
pgp = Pgps.Pgp(mail, 'mail')
|
pgp = Pgps.Pgp(mail, 'mail')
|
||||||
pgp.add(date, self.item_id)
|
pgp.add(date, self.item_id)
|
||||||
print(f' mail: {mail}')
|
print(f' mail: {mail}')
|
||||||
|
self.tracker_term.compute(self.item_id, item_content=mail)
|
||||||
|
self.tracker_regex.compute(self.item_id, content=mail)
|
||||||
|
self.tracker_yara.compute(self.item_id, item_content=mail)
|
||||||
|
|
||||||
# Keys extracted from PGP PRIVATE KEY BLOCK
|
# Keys extracted from PGP PRIVATE KEY BLOCK
|
||||||
for key in self.private_keys:
|
for key in self.private_keys:
|
||||||
|
|
|
@ -43,6 +43,10 @@ subscribe = Redis_D4_client
|
||||||
subscribe = Redis
|
subscribe = Redis
|
||||||
publish = Redis_Tags
|
publish = Redis_Tags
|
||||||
|
|
||||||
|
[Tracker_Typo_Squatting]
|
||||||
|
subscribe = Redis_Host
|
||||||
|
publish = Redis_Tags
|
||||||
|
|
||||||
[Tracker_Term]
|
[Tracker_Term]
|
||||||
subscribe = Redis_Global
|
subscribe = Redis_Global
|
||||||
publish = Redis_Tags
|
publish = Redis_Tags
|
||||||
|
|
|
@ -48,7 +48,7 @@ class Tracker_Regex(AbstractModule):
|
||||||
|
|
||||||
self.redis_logger.info(f"Module: {self.module_name} Launched")
|
self.redis_logger.info(f"Module: {self.module_name} Launched")
|
||||||
|
|
||||||
def compute(self, item_id):
|
def compute(self, item_id, content=None):
|
||||||
# refresh Tracked regex
|
# refresh Tracked regex
|
||||||
if self.last_refresh < Tracker.get_tracker_last_updated_by_type('regex'):
|
if self.last_refresh < Tracker.get_tracker_last_updated_by_type('regex'):
|
||||||
self.dict_regex_tracked = Term.get_regex_tracked_words_dict()
|
self.dict_regex_tracked = Term.get_regex_tracked_words_dict()
|
||||||
|
@ -58,6 +58,7 @@ class Tracker_Regex(AbstractModule):
|
||||||
|
|
||||||
item = Item(item_id)
|
item = Item(item_id)
|
||||||
item_id = item.get_id()
|
item_id = item.get_id()
|
||||||
|
if not content:
|
||||||
content = item.get_content()
|
content = item.get_content()
|
||||||
|
|
||||||
for regex in self.dict_regex_tracked:
|
for regex in self.dict_regex_tracked:
|
||||||
|
@ -76,6 +77,7 @@ class Tracker_Regex(AbstractModule):
|
||||||
# date = item.get_date()
|
# date = item.get_date()
|
||||||
item_source = item.get_source()
|
item_source = item.get_source()
|
||||||
print(f'new tracked regex found: {tracker_name} in {item_id}')
|
print(f'new tracked regex found: {tracker_name} in {item_id}')
|
||||||
|
self.redis_logger.warning(f'new tracked regex found: {tracker_name} in {item_id}')
|
||||||
|
|
||||||
for tracker_uuid in uuid_list:
|
for tracker_uuid in uuid_list:
|
||||||
tracker = Tracker.Tracker(tracker_uuid)
|
tracker = Tracker.Tracker(tracker_uuid)
|
||||||
|
|
|
@ -61,7 +61,7 @@ class Tracker_Term(AbstractModule):
|
||||||
|
|
||||||
self.redis_logger.info(f"Module: {self.module_name} Launched")
|
self.redis_logger.info(f"Module: {self.module_name} Launched")
|
||||||
|
|
||||||
def compute(self, item_id):
|
def compute(self, item_id, item_content=None):
|
||||||
# refresh Tracked term
|
# refresh Tracked term
|
||||||
if self.last_refresh_word < Term.get_tracked_term_last_updated_by_type('word'):
|
if self.last_refresh_word < Term.get_tracked_term_last_updated_by_type('word'):
|
||||||
self.list_tracked_words = Term.get_tracked_words_list()
|
self.list_tracked_words = Term.get_tracked_words_list()
|
||||||
|
@ -78,6 +78,7 @@ class Tracker_Term(AbstractModule):
|
||||||
# Cast message as Item
|
# Cast message as Item
|
||||||
item = Item(item_id)
|
item = Item(item_id)
|
||||||
item_date = item.get_date()
|
item_date = item.get_date()
|
||||||
|
if not item_content:
|
||||||
item_content = item.get_content()
|
item_content = item.get_content()
|
||||||
|
|
||||||
signal.alarm(self.max_execution_time)
|
signal.alarm(self.max_execution_time)
|
||||||
|
@ -120,7 +121,7 @@ class Tracker_Term(AbstractModule):
|
||||||
item_id = item.get_id()
|
item_id = item.get_id()
|
||||||
item_date = item.get_date()
|
item_date = item.get_date()
|
||||||
item_source = item.get_source()
|
item_source = item.get_source()
|
||||||
self.redis_logger.info(f'new tracked term found: {term} in {item_id}')
|
self.redis_logger.warning(f'new tracked term found: {term} in {item_id}')
|
||||||
print(f'new tracked term found: {term} in {item_id}')
|
print(f'new tracked term found: {term} in {item_id}')
|
||||||
for term_uuid in uuid_list:
|
for term_uuid in uuid_list:
|
||||||
tracker_sources = Tracker.get_tracker_uuid_sources(term_uuid)
|
tracker_sources = Tracker.get_tracker_uuid_sources(term_uuid)
|
||||||
|
|
119
bin/trackers/Tracker_Typo_Squatting.py
Executable file
119
bin/trackers/Tracker_Typo_Squatting.py
Executable file
|
@ -0,0 +1,119 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*-coding:UTF-8 -*
|
||||||
|
|
||||||
|
"""
|
||||||
|
The Tracker_Typo_Squatting Module
|
||||||
|
===================
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
##################################
|
||||||
|
# Import External packages
|
||||||
|
##################################
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import requests
|
||||||
|
|
||||||
|
|
||||||
|
sys.path.append(os.environ['AIL_BIN'])
|
||||||
|
##################################
|
||||||
|
# Import Project packages
|
||||||
|
##################################
|
||||||
|
from modules.abstract_module import AbstractModule
|
||||||
|
import NotificationHelper
|
||||||
|
from packages.Item import Item
|
||||||
|
from lib import Tracker
|
||||||
|
|
||||||
|
class Tracker_Typo_Squatting(AbstractModule):
|
||||||
|
mail_body_template = "AIL Framework,\nNew occurrence for tracked Typo: {}\nitem id: {}\nurl: {}{}"
|
||||||
|
|
||||||
|
"""
|
||||||
|
Tracker_Typo_Squatting module for AIL framework
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
super(Tracker_Typo_Squatting, self).__init__()
|
||||||
|
|
||||||
|
self.pending_seconds = 5
|
||||||
|
|
||||||
|
self.full_item_url = self.process.config.get("Notifications", "ail_domain") + "/object/item?id="
|
||||||
|
|
||||||
|
# loads typosquatting
|
||||||
|
self.typosquat_tracked_words_list = Tracker.get_typosquatting_tracked_words_list()
|
||||||
|
self.last_refresh_typosquat = time.time()
|
||||||
|
|
||||||
|
self.redis_logger.info(f"Module: {self.module_name} Launched")
|
||||||
|
|
||||||
|
def compute(self, message):
|
||||||
|
# refresh Tracked typo
|
||||||
|
if self.last_refresh_typosquat < Tracker.get_tracker_last_updated_by_type('typosquatting'):
|
||||||
|
self.typosquat_tracked_words_list = Tracker.get_typosquatting_tracked_words_list()
|
||||||
|
self.last_refresh_typosquat = time.time()
|
||||||
|
self.redis_logger.debug('Tracked typosquatting refreshed')
|
||||||
|
print('Tracked typosquatting refreshed')
|
||||||
|
|
||||||
|
host, id = message.split()
|
||||||
|
|
||||||
|
# Cast message as Item
|
||||||
|
for tracker in self.typosquat_tracked_words_list:
|
||||||
|
if host in self.typosquat_tracked_words_list[tracker]:
|
||||||
|
item = Item(id)
|
||||||
|
self.new_tracker_found(tracker, 'typosquatting', item)
|
||||||
|
|
||||||
|
def new_tracker_found(self, tracker, tracker_type, item):
|
||||||
|
item_id = item.get_id()
|
||||||
|
item_date = item.get_date()
|
||||||
|
item_source = item.get_source()
|
||||||
|
#self.redis_logger.info(f'new tracked typo found: {tracker} in {item_id}')
|
||||||
|
print(f'new tracked typosquatting found: {tracker} in {item_id}')
|
||||||
|
self.redis_logger.warning(f'tracker typosquatting: {tracker} in {item_id}')
|
||||||
|
|
||||||
|
print(Tracker.get_tracker_uuid_list(tracker, tracker_type))
|
||||||
|
for tracker_uuid in Tracker.get_tracker_uuid_list(tracker, tracker_type):
|
||||||
|
# Source Filtering
|
||||||
|
tracker_sources = Tracker.get_tracker_uuid_sources(tracker)
|
||||||
|
if tracker_sources and item_source not in tracker_sources:
|
||||||
|
continue
|
||||||
|
|
||||||
|
Tracker.add_tracked_item(tracker_uuid, item_id)
|
||||||
|
|
||||||
|
# Tags
|
||||||
|
tags_to_add = Tracker.get_tracker_tags(tracker_uuid)
|
||||||
|
for tag in tags_to_add:
|
||||||
|
msg = f'{tag};{item_id}'
|
||||||
|
self.send_message_to_queue(msg, 'Tags')
|
||||||
|
|
||||||
|
mail_to_notify = Tracker.get_tracker_mails(tracker_uuid)
|
||||||
|
if mail_to_notify:
|
||||||
|
mail_subject = Tracker.get_email_subject(tracker_uuid)
|
||||||
|
mail_body = Tracker_Typo_Squatting.mail_body_template.format(tracker, item_id, self.full_item_url, item_id)
|
||||||
|
for mail in mail_to_notify:
|
||||||
|
NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body)
|
||||||
|
|
||||||
|
# Webhook
|
||||||
|
webhook_to_post = Tracker.get_tracker_webhook(tracker_uuid)
|
||||||
|
if webhook_to_post:
|
||||||
|
json_request = {"trackerId": tracker_uuid,
|
||||||
|
"itemId": item_id,
|
||||||
|
"itemURL": self.full_item_url + item_id,
|
||||||
|
"tracker": tracker,
|
||||||
|
"itemSource": item_source,
|
||||||
|
"itemDate": item_date,
|
||||||
|
"tags": tags_to_add,
|
||||||
|
"emailNotification": f'{mail_to_notify}',
|
||||||
|
"trackerType": tracker_type
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
response = requests.post(webhook_to_post, json=json_request)
|
||||||
|
if response.status_code >= 400:
|
||||||
|
self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: {response.reason}")
|
||||||
|
except:
|
||||||
|
self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: Something went wrong")
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
module = Tracker_Typo_Squatting()
|
||||||
|
module.run()
|
||||||
|
#module.compute('g00gle.com tests/2020/01/01/test.gz')
|
|
@ -45,7 +45,7 @@ class Tracker_Yara(AbstractModule):
|
||||||
|
|
||||||
self.redis_logger.info(f"Module: {self.module_name} Launched")
|
self.redis_logger.info(f"Module: {self.module_name} Launched")
|
||||||
|
|
||||||
def compute(self, item_id):
|
def compute(self, item_id, item_content=None):
|
||||||
# refresh YARA list
|
# refresh YARA list
|
||||||
if self.last_refresh < Tracker.get_tracker_last_updated_by_type('yara'):
|
if self.last_refresh < Tracker.get_tracker_last_updated_by_type('yara'):
|
||||||
self.rules = Tracker.reload_yara_rules()
|
self.rules = Tracker.reload_yara_rules()
|
||||||
|
@ -54,12 +54,14 @@ class Tracker_Yara(AbstractModule):
|
||||||
print('Tracked set refreshed')
|
print('Tracked set refreshed')
|
||||||
|
|
||||||
self.item = Item(item_id)
|
self.item = Item(item_id)
|
||||||
|
if not item_content:
|
||||||
item_content = self.item.get_content()
|
item_content = self.item.get_content()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
yara_match = self.rules.match(data=item_content, callback=self.yara_rules_match,
|
yara_match = self.rules.match(data=item_content, callback=self.yara_rules_match,
|
||||||
which_callbacks=yara.CALLBACK_MATCHES, timeout=60)
|
which_callbacks=yara.CALLBACK_MATCHES, timeout=60)
|
||||||
if yara_match:
|
if yara_match:
|
||||||
self.redis_logger.info(f'{self.item.get_id()}: {yara_match}')
|
self.redis_logger.warning(f'tracker yara: new match {self.item.get_id()}: {yara_match}')
|
||||||
print(f'{self.item.get_id()}: {yara_match}')
|
print(f'{self.item.get_id()}: {yara_match}')
|
||||||
except yara.TimeoutError as e:
|
except yara.TimeoutError as e:
|
||||||
print(f'{self.item.get_id()}: yara scanning timed out')
|
print(f'{self.item.get_id()}: yara scanning timed out')
|
||||||
|
|
|
@ -22,6 +22,18 @@ pystemonpath = /home/pystemon/pystemon/
|
||||||
|
|
||||||
sentiment_lexicon_file = sentiment/vader_lexicon.zip/vader_lexicon/vader_lexicon.txt
|
sentiment_lexicon_file = sentiment/vader_lexicon.zip/vader_lexicon/vader_lexicon.txt
|
||||||
|
|
||||||
|
##### Logs ######
|
||||||
|
[Logs]
|
||||||
|
# activate syslog
|
||||||
|
ail_logs_syslog = False
|
||||||
|
ail_logs_syslog_server =
|
||||||
|
# default=514
|
||||||
|
ail_logs_syslog_port =
|
||||||
|
# ['auth', 'authpriv', 'cron', 'daemon', 'ftp', 'kern', 'lpr', 'mail', 'news', 'syslog', 'user', 'uucp', 'local0', 'local1', 'local2', 'local3', 'local4', 'local5', 'local6', 'local7']
|
||||||
|
ail_logs_syslog_facility =
|
||||||
|
# ['DEBUG', 'INFO', 'NOTICE', 'WARNING', 'ERROR', 'CRITICAL']
|
||||||
|
ail_logs_syslog_level =
|
||||||
|
|
||||||
##### Notifications ######
|
##### Notifications ######
|
||||||
[Notifications]
|
[Notifications]
|
||||||
ail_domain = https://localhost:7000
|
ail_domain = https://localhost:7000
|
||||||
|
@ -228,7 +240,7 @@ dns = 8.8.8.8
|
||||||
dns = 8.8.8.8
|
dns = 8.8.8.8
|
||||||
|
|
||||||
[Web]
|
[Web]
|
||||||
dns = 149.13.33.69
|
dns = 8.8.8.8
|
||||||
|
|
||||||
# Indexer configuration
|
# Indexer configuration
|
||||||
[Indexer]
|
[Indexer]
|
||||||
|
@ -247,7 +259,8 @@ maxDuplicateToPushToMISP=10
|
||||||
# e.g.: tcp://127.0.0.1:5556,tcp://127.0.0.1:5557
|
# e.g.: tcp://127.0.0.1:5556,tcp://127.0.0.1:5557
|
||||||
[ZMQ_Global]
|
[ZMQ_Global]
|
||||||
#address = tcp://crf.circl.lu:5556
|
#address = tcp://crf.circl.lu:5556
|
||||||
address = tcp://127.0.0.1:5556,tcp://crf.circl.lu:5556
|
# address = tcp://127.0.0.1:5556,tcp://crf.circl.lu:5556
|
||||||
|
address = tcp://127.0.0.1:5556
|
||||||
channel = 102
|
channel = 102
|
||||||
bind = tcp://127.0.0.1:5556
|
bind = tcp://127.0.0.1:5556
|
||||||
|
|
||||||
|
|
|
@ -71,6 +71,8 @@ flask>=1.1.4
|
||||||
flask-login
|
flask-login
|
||||||
bcrypt>3.1.6
|
bcrypt>3.1.6
|
||||||
|
|
||||||
|
# Ail typo squatting
|
||||||
|
ail_typo_squatting
|
||||||
|
|
||||||
# Tests
|
# Tests
|
||||||
nose>=1.3.7
|
nose>=1.3.7
|
||||||
|
|
26
update/v4.2.1/Update.py
Executable file
26
update/v4.2.1/Update.py
Executable file
|
@ -0,0 +1,26 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*-coding:UTF-8 -*
|
||||||
|
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import redis
|
||||||
|
import datetime
|
||||||
|
|
||||||
|
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||||
|
import ConfigLoader
|
||||||
|
|
||||||
|
sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin'))
|
||||||
|
from ail_updater import AIL_Updater
|
||||||
|
|
||||||
|
class Updater(AIL_Updater):
|
||||||
|
"""default Updater."""
|
||||||
|
|
||||||
|
def __init__(self, version):
|
||||||
|
super(Updater, self).__init__(version)
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
|
||||||
|
updater = Updater('v4.2.1')
|
||||||
|
updater.run_update()
|
29
update/v4.2.1/Update.sh
Executable file
29
update/v4.2.1/Update.sh
Executable file
|
@ -0,0 +1,29 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
|
||||||
|
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
|
||||||
|
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||||
|
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||||
|
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
|
||||||
|
|
||||||
|
export PATH=$AIL_HOME:$PATH
|
||||||
|
export PATH=$AIL_REDIS:$PATH
|
||||||
|
export PATH=$AIL_ARDB:$PATH
|
||||||
|
export PATH=$AIL_BIN:$PATH
|
||||||
|
export PATH=$AIL_FLASK:$PATH
|
||||||
|
|
||||||
|
GREEN="\\033[1;32m"
|
||||||
|
DEFAULT="\\033[0;39m"
|
||||||
|
|
||||||
|
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
|
||||||
|
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||||
|
wait
|
||||||
|
|
||||||
|
# SUBMODULES #
|
||||||
|
git submodule update
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo -e $GREEN"Updating pusblogger ..."$DEFAULT
|
||||||
|
pip3 install -U pubsublogger
|
||||||
|
|
||||||
|
exit 0
|
|
@ -85,6 +85,16 @@ def tracked_menu_yara():
|
||||||
global_trackers = Tracker.get_global_trackers_metadata(tracker_type=tracker_type)
|
global_trackers = Tracker.get_global_trackers_metadata(tracker_type=tracker_type)
|
||||||
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
|
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
|
||||||
|
|
||||||
|
@hunter.route("/trackers/typosquatting")
|
||||||
|
@login_required
|
||||||
|
@login_read_only
|
||||||
|
def tracked_menu_typosquatting():
|
||||||
|
filter_type = 'typosquatting'
|
||||||
|
user_id = current_user.get_id()
|
||||||
|
user_term = Term.get_all_user_tracked_terms(user_id, filter_type=filter_type)
|
||||||
|
global_term = Term.get_all_global_tracked_terms(filter_type=filter_type)
|
||||||
|
return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type)
|
||||||
|
|
||||||
|
|
||||||
@hunter.route("/tracker/add", methods=['GET', 'POST'])
|
@hunter.route("/tracker/add", methods=['GET', 'POST'])
|
||||||
@login_required
|
@login_required
|
||||||
|
@ -225,6 +235,13 @@ def show_tracker():
|
||||||
else:
|
else:
|
||||||
yara_rule_content = None
|
yara_rule_content = None
|
||||||
|
|
||||||
|
if tracker_metadata['type'] == 'typosquatting':
|
||||||
|
typo_squatting = list(Tracker.get_tracker_typosquatting_domains(tracker_uuid))
|
||||||
|
typo_squatting.sort()
|
||||||
|
else:
|
||||||
|
typo_squatting = None
|
||||||
|
|
||||||
|
|
||||||
if date_from:
|
if date_from:
|
||||||
res = Term.parse_get_tracker_term_item({'uuid': tracker_uuid, 'date_from': date_from, 'date_to': date_to}, user_id)
|
res = Term.parse_get_tracker_term_item({'uuid': tracker_uuid, 'date_from': date_from, 'date_to': date_to}, user_id)
|
||||||
if res[1] !=200:
|
if res[1] !=200:
|
||||||
|
@ -241,6 +258,7 @@ def show_tracker():
|
||||||
|
|
||||||
return render_template("showTracker.html", tracker_metadata=tracker_metadata,
|
return render_template("showTracker.html", tracker_metadata=tracker_metadata,
|
||||||
yara_rule_content=yara_rule_content,
|
yara_rule_content=yara_rule_content,
|
||||||
|
typo_squatting=typo_squatting,
|
||||||
bootstrap_label=bootstrap_label)
|
bootstrap_label=bootstrap_label)
|
||||||
|
|
||||||
@hunter.route("/tracker/update_tracker_description", methods=['POST'])
|
@hunter.route("/tracker/update_tracker_description", methods=['POST'])
|
||||||
|
|
|
@ -104,6 +104,7 @@
|
||||||
<option value="set">Set</option>
|
<option value="set">Set</option>
|
||||||
<option value="regex">Regex</option>
|
<option value="regex">Regex</option>
|
||||||
<option value="yara">YARA rule</option>
|
<option value="yara">YARA rule</option>
|
||||||
|
<option value="typosquatting">Typo-squatting</option>
|
||||||
</select>
|
</select>
|
||||||
|
|
||||||
<p id="tracker_desc">Terms to track (space separated)</p>
|
<p id="tracker_desc">Terms to track (space separated)</p>
|
||||||
|
@ -209,6 +210,12 @@ $(document).ready(function(){
|
||||||
$("#tracker").hide();
|
$("#tracker").hide();
|
||||||
$("#nb_word").hide();
|
$("#nb_word").hide();
|
||||||
$("#yara_rule").show();
|
$("#yara_rule").show();
|
||||||
|
} else if (tracker_type=="typosquatting") {
|
||||||
|
$("#tracker_desc").text("Generation of variation for domain name. Only one domain name at a time.");
|
||||||
|
$("#tracker_desc").show();
|
||||||
|
$("#tracker").show();
|
||||||
|
$("#nb_word").hide();
|
||||||
|
$("#yara_rule").hide();
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
|
@ -91,7 +91,25 @@
|
||||||
<tbody>
|
<tbody>
|
||||||
<tr>
|
<tr>
|
||||||
<td>{{ tracker_metadata['type'] }}</td>
|
<td>{{ tracker_metadata['type'] }}</td>
|
||||||
|
{% if tracker_metadata['type'] == 'typosquatting' %}
|
||||||
|
<td>
|
||||||
|
<a class="btn btn-primary" data-toggle="collapse" href="#collapseTypo" role="button" aria-expanded="false" aria-controls="collapseTypo">
|
||||||
|
{{ tracker_metadata['tracker'].split(",")[0] }}
|
||||||
|
</a>
|
||||||
|
<div class="collapse" id="collapseTypo">
|
||||||
|
<div class="card card-body">
|
||||||
|
{% if typo_squatting %}
|
||||||
|
{% for typo in typo_squatting %}
|
||||||
|
{{typo}}
|
||||||
|
<br/>
|
||||||
|
{% endfor %}
|
||||||
|
{%endif%}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</td>
|
||||||
|
{% else %}
|
||||||
<td>{{ tracker_metadata['tracker'] }}</td>
|
<td>{{ tracker_metadata['tracker'] }}</td>
|
||||||
|
{% endif %}
|
||||||
<td>{{ tracker_metadata['date'][0:4] }}/{{ tracker_metadata['date'][4:6] }}/{{ tracker_metadata['date'][6:8] }}</td>
|
<td>{{ tracker_metadata['date'][0:4] }}/{{ tracker_metadata['date'][4:6] }}/{{ tracker_metadata['date'][6:8] }}</td>
|
||||||
<td>
|
<td>
|
||||||
{% if tracker_metadata['level'] == 0 %}
|
{% if tracker_metadata['level'] == 0 %}
|
||||||
|
|
|
@ -16,6 +16,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
||||||
##################################
|
##################################
|
||||||
# Import Project packages
|
# Import Project packages
|
||||||
##################################
|
##################################
|
||||||
|
from lib import crawlers
|
||||||
from lib import Users
|
from lib import Users
|
||||||
from lib.objects.Items import Item
|
from lib.objects.Items import Item
|
||||||
from lib import Tag
|
from lib import Tag
|
||||||
|
@ -46,8 +47,8 @@ restApi = Blueprint('restApi', __name__, template_folder='templates')
|
||||||
|
|
||||||
# ============ AUTH FUNCTIONS ============
|
# ============ AUTH FUNCTIONS ============
|
||||||
|
|
||||||
def check_token_format(strg, search=re.compile(r'[^a-zA-Z0-9_-]').search):
|
def check_token_format(token, search=re.compile(r'[^a-zA-Z0-9_-]').search):
|
||||||
return not bool(search(strg))
|
return not bool(search(token))
|
||||||
|
|
||||||
def verify_token(token):
|
def verify_token(token):
|
||||||
if len(token) != 41:
|
if len(token) != 41:
|
||||||
|
@ -90,7 +91,7 @@ def get_auth_from_header():
|
||||||
def authErrors(user_role):
|
def authErrors(user_role):
|
||||||
# Check auth
|
# Check auth
|
||||||
if not request.headers.get('Authorization'):
|
if not request.headers.get('Authorization'):
|
||||||
return ({'status': 'error', 'reason': 'Authentication needed'}, 401)
|
return {'status': 'error', 'reason': 'Authentication needed'}, 401
|
||||||
token = get_auth_from_header()
|
token = get_auth_from_header()
|
||||||
data = None
|
data = None
|
||||||
# verify token format
|
# verify token format
|
||||||
|
@ -102,7 +103,7 @@ def authErrors(user_role):
|
||||||
if login_failed_ip:
|
if login_failed_ip:
|
||||||
login_failed_ip = int(login_failed_ip)
|
login_failed_ip = int(login_failed_ip)
|
||||||
if login_failed_ip >= 5:
|
if login_failed_ip >= 5:
|
||||||
return ({'status': 'error', 'reason': 'Max Connection Attempts reached, Please wait {}s'.format(r_cache.ttl('failed_login_ip_api:{}'.format(current_ip)))}, 401)
|
return {'status': 'error', 'reason': 'Max Connection Attempts reached, Please wait {}s'.format(r_cache.ttl('failed_login_ip_api:{}'.format(current_ip)))}, 401
|
||||||
|
|
||||||
try:
|
try:
|
||||||
authenticated = False
|
authenticated = False
|
||||||
|
@ -146,9 +147,6 @@ def is_valid_uuid_v4(header_uuid):
|
||||||
except:
|
except:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def one():
|
|
||||||
return 1
|
|
||||||
|
|
||||||
# ============= ROUTES ==============
|
# ============= ROUTES ==============
|
||||||
|
|
||||||
# @restApi.route("/api", methods=['GET'])
|
# @restApi.route("/api", methods=['GET'])
|
||||||
|
@ -576,6 +574,20 @@ def get_crawled_domain_list():
|
||||||
dict_res['domain_type'] = domain_type
|
dict_res['domain_type'] = domain_type
|
||||||
return create_json_response(dict_res, res[1])
|
return create_json_response(dict_res, res[1])
|
||||||
|
|
||||||
|
# # TODO: ADD RESULT JSON Response
|
||||||
|
@restApi.route("api/v1/add/crawler/task", methods=['POST'])
|
||||||
|
@token_required('analyst')
|
||||||
|
def add_crawler_task():
|
||||||
|
data = request.get_json()
|
||||||
|
user_token = get_auth_from_header()
|
||||||
|
user_id = get_user_from_token(user_token)
|
||||||
|
res = crawlers.api_add_crawler_task(data, user_id=user_id)
|
||||||
|
if res:
|
||||||
|
return create_json_response(res[0], res[1])
|
||||||
|
|
||||||
|
dict_res = {'url': data['url']}
|
||||||
|
return create_json_response(dict_res, 200)
|
||||||
|
|
||||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||||
# # # # # # # # # # # # # IMPORT # # # # # # # # # # # # # # # # # #
|
# # # # # # # # # # # # # IMPORT # # # # # # # # # # # # # # # # # #
|
||||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||||
|
|
|
@ -42,6 +42,12 @@
|
||||||
<span class="bg-danger text-white font-weight-bold" style="font-size: 120%"> { </span>
|
<span class="bg-danger text-white font-weight-bold" style="font-size: 120%"> { </span>
|
||||||
<span> YARA</span>
|
<span> YARA</span>
|
||||||
</a>
|
</a>
|
||||||
|
</li>
|
||||||
|
<li class="nav-item">
|
||||||
|
<a class="nav-link" href="{{url_for('hunter.tracked_menu_typosquatting')}}" id="nav_tracker_typosquatting">
|
||||||
|
<i class="fa fa-clone"></i>
|
||||||
|
<span>Typo-squatting</span>
|
||||||
|
</a>
|
||||||
</li>
|
</li>
|
||||||
</ul>
|
</ul>
|
||||||
<h5 class="d-flex text-muted w-100" id="nav_title_retro_hunt">
|
<h5 class="d-flex text-muted w-100" id="nav_title_retro_hunt">
|
||||||
|
|
Loading…
Reference in a new issue