mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 08:38:28 +00:00
Merge branch 'master' into crawler_manager
This commit is contained in:
Terrtia
commit
8901ffe989
19 changed files with 445 additions and 33 deletions
3
.gitmodules
vendored
Normal file
3
.gitmodules
vendored
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
[submodule "bin/trackers/yara/ail-yara-rules"]
|
||||
path = bin/trackers/yara/ail-yara-rules
|
||||
url = https://github.com/ail-project/ail-yara-rules.git
|
||||
|
|
@ -184,6 +184,8 @@ function launching_scripts {
|
|||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "RegexTracker" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./RegexTracker.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Tracker_Yara" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Yara.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Indexer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Indexer.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Keys" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Keys.py; read x"
|
||||
|
|
|
|||
|
|
@ -231,6 +231,15 @@ def get_git_upper_tags_remote(current_tag, is_fork):
|
|||
aborting_update()
|
||||
sys.exit(0)
|
||||
|
||||
def update_submodules():
|
||||
print('{}git submodule update:{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
|
||||
process = subprocess.run(['git', 'submodule', 'update'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
if process.returncode == 0:
|
||||
print(process.stdout.decode())
|
||||
print()
|
||||
else:
|
||||
print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT))
|
||||
|
||||
def update_ail(current_tag, list_upper_tags_remote, current_version_path, is_fork):
|
||||
print('{}git checkout master:{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
|
||||
process = subprocess.run(['git', 'checkout', 'master'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
|
|
@ -238,6 +247,9 @@ def update_ail(current_tag, list_upper_tags_remote, current_version_path, is_for
|
|||
if process.returncode == 0:
|
||||
print(process.stdout.decode())
|
||||
print()
|
||||
|
||||
update_submodules()
|
||||
|
||||
print('{}git pull:{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
|
||||
process = subprocess.run(['git', 'pull'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import os
|
|||
import sys
|
||||
import time
|
||||
import redis
|
||||
import yara
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
|
|
@ -14,12 +15,117 @@ config_loader = ConfigLoader.ConfigLoader()
|
|||
r_serv_tracker = config_loader.get_redis_conn("ARDB_Tracker")
|
||||
config_loader = None
|
||||
|
||||
def get_tracker_uuid_list(tracker, tracker_type):
|
||||
return list(r_serv_tracker.smembers('all:tracker_uuid:{}:{}'.format(tracker_type, tracker)))
|
||||
|
||||
def get_tracker_tags(tracker_uuid):
|
||||
return list(r_serv_tracker.smembers('tracker:tags:{}'.format(tracker_uuid)))
|
||||
|
||||
def get_tracker_mails(tracker_uuid):
|
||||
return list(r_serv_tracker.smembers('tracker:mail:{}'.format(tracker_uuid)))
|
||||
|
||||
def get_tracker_description(tracker_uuid):
|
||||
return r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'description')
|
||||
|
||||
def add_tracked_item(tracker_uuid, item_id, item_date):
|
||||
# track item
|
||||
r_serv_tracker.sadd('tracker:item:{}:{}'.format(tracker_uuid, item_date), item_id)
|
||||
# track nb item by date
|
||||
r_serv_tracker.zadd('tracker:stat:{}'.format(tracker_uuid), item_date, int(item_date))
|
||||
|
||||
def get_email_subject(tracker_uuid):
|
||||
tracker_description = get_tracker_description(tracker_uuid)
|
||||
if not tracker_description:
|
||||
return "AIL framework: Tracker Alert"
|
||||
else:
|
||||
return 'AIL framework: {}'.format(tracker_description)
|
||||
|
||||
def get_tracker_last_updated_by_type(tracker_type):
|
||||
epoch_update = r_serv_tracker.get('tracker:refresh:{}'.format(term_type))
|
||||
if not epoch_update:
|
||||
epoch_update = 0
|
||||
return float(epoch_update)
|
||||
|
||||
#### YARA ####
|
||||
def get_yara_rules_dir():
|
||||
return os.path.join(os.environ['AIL_BIN'], 'trackers', 'yara')
|
||||
|
||||
def get_yara_rules_default_dir():
|
||||
return os.path.join(os.environ['AIL_BIN'], 'trackers', 'yara', 'ail-yara-rules', 'rules')
|
||||
|
||||
# # TODO: cache + update
|
||||
def get_all_default_yara_rules_types():
|
||||
yara_dir = get_yara_rules_default_dir()
|
||||
all_yara_types = next(os.walk(yara_dir))[1]
|
||||
# save in cache ?
|
||||
return all_yara_types
|
||||
|
||||
# # TODO: cache + update
|
||||
def get_all_default_yara_files():
|
||||
yara_dir = get_yara_rules_default_dir()
|
||||
all_default_yara_files = {}
|
||||
for rules_type in get_all_default_yara_rules_types():
|
||||
all_default_yara_files[rules_type] = os.listdir(os.path.join(yara_dir, rules_type))
|
||||
return all_default_yara_files
|
||||
|
||||
def get_all_default_yara_rules_by_type(yara_types):
|
||||
all_default_yara_files = get_all_default_yara_files()
|
||||
if yara_types in all_default_yara_files:
|
||||
return all_default_yara_files[yara_types]
|
||||
else:
|
||||
return []
|
||||
|
||||
def get_all_tracked_yara_files():
|
||||
yara_files = r_serv_tracker.smembers('all:tracker:yara')
|
||||
if not yara_files:
|
||||
yara_files = []
|
||||
return yara_files
|
||||
|
||||
def reload_yara_rules():
|
||||
yara_files = get_all_tracked_yara_files()
|
||||
# {uuid: filename}
|
||||
rule_dict = {}
|
||||
for yar_path in yara_files:
|
||||
l_tracker_uuid = get_tracker_uuid_list(yar_path, 'yara')
|
||||
for tracker_uuid in l_tracker_uuid:
|
||||
rule_dict[tracker_uuid] = os.path.join(get_yara_rules_dir(), yar_path)
|
||||
rules = yara.compile(filepaths=rule_dict)
|
||||
return rules
|
||||
|
||||
def is_valid_yara_rule(yara_rule):
|
||||
try:
|
||||
yara.compile(source=yara_rule)
|
||||
return True
|
||||
except:
|
||||
return False
|
||||
|
||||
def is_valid_default_yara_rule(yara_rule):
|
||||
yara_dir = get_yara_rules_default_dir()
|
||||
filename = os.path.join(yara_dir, yara_rule)
|
||||
filename = os.path.realpath(filename)
|
||||
|
||||
# incorrect filename
|
||||
if not os.path.commonprefix([filename, yara_dir]) == yara_dir:
|
||||
return False
|
||||
else:
|
||||
if os.path.isfile(filename):
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def save_yara_rule(yara_rule_type, yara_rule, tracker_uuid=None):
|
||||
if yara_rule_type == 'yara_custom':
|
||||
if not tracker_uuid:
|
||||
tracker_uuid = str(uuid.uuid4())
|
||||
filename = os.path.join('custom-rules', tracker_uuid + '.yar')
|
||||
with open(os.path.join(get_yara_rules_dir(), filename), 'w') as f:
|
||||
f.write(str(yara_rule))
|
||||
if yara_rule_type == 'yara_default':
|
||||
filename = os.path.join('ail-yara-rules', 'rules', yara_rule)
|
||||
return filename
|
||||
##-- YARA --##
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
res = is_valid_yara_rule('rule dummy { }')
|
||||
print(res)
|
||||
|
|
|
|||
|
|
@ -430,7 +430,7 @@ def save_crawler_config(crawler_mode, crawler_type, crawler_config, domain, url=
|
|||
if crawler_mode == 'manual':
|
||||
r_cache.set('crawler_config:{}:{}:{}'.format(crawler_mode, crawler_type, domain), json.dumps(crawler_config))
|
||||
elif crawler_mode == 'auto':
|
||||
r_serv_onion.set('crawler_config:{}:{}:{}:{}'.format(crawler_type, crawler_type, domain, url), json.dumps(crawler_config))
|
||||
r_serv_onion.set('crawler_config:{}:{}:{}:{}'.format(crawler_mode, crawler_type, domain, url), json.dumps(crawler_config))
|
||||
|
||||
def send_url_to_crawl_in_queue(crawler_mode, crawler_type, url):
|
||||
r_serv_onion.sadd('{}_crawler_priority_queue'.format(crawler_type), '{};{}'.format(url, crawler_mode))
|
||||
|
|
|
|||
|
|
@ -166,20 +166,20 @@ class SimpleCorrelation(object): #social_name
|
|||
else:
|
||||
return []
|
||||
|
||||
# def get_correlation_all_object(self, correlation_type, correlation_value, correlation_objects=[]):
|
||||
# if not correlation_objects:
|
||||
# correlation_objects = get_all_correlation_objects()
|
||||
# correlation_obj = {}
|
||||
# for correlation_object in correlation_objects:
|
||||
# if correlation_object == 'paste':
|
||||
# res = self._get_items(correlation_type, correlation_value)
|
||||
# elif correlation_object == 'domain':
|
||||
# res = self.get_correlation_obj_domain(correlation_value, correlation_type=correlation_type)
|
||||
# else:
|
||||
# res = None
|
||||
# if res:
|
||||
# correlation_obj[correlation_object] = res
|
||||
# return correlation_obj
|
||||
def get_correlation_all_object(self, correlation_value, correlation_objects=[]):
|
||||
if not correlation_objects:
|
||||
correlation_objects = get_all_correlation_objects()
|
||||
correlation_obj = {}
|
||||
for correlation_object in correlation_objects:
|
||||
if correlation_object == 'paste':
|
||||
res = self._get_items(correlation_value)
|
||||
elif correlation_object == 'domain':
|
||||
res = self.get_correlation_obj_domain(correlation_value)
|
||||
else:
|
||||
res = None
|
||||
if res:
|
||||
correlation_obj[correlation_object] = res
|
||||
return correlation_obj
|
||||
|
||||
def update_correlation_daterange(self, obj_id, date):
|
||||
date = int(date)
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from textblob import TextBlob
|
|||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
import Tracker
|
||||
|
||||
from flask import escape
|
||||
|
||||
|
|
@ -219,7 +220,12 @@ def parse_tracked_term_to_add(term , term_type, nb_words=1):
|
|||
|
||||
if nb_words > len(words_set):
|
||||
nb_words = len(words_set)
|
||||
|
||||
elif term_type=='yara_custom':
|
||||
if not Tracker.is_valid_yara_rule(term):
|
||||
return ({"status": "error", "reason": "Invalid custom Yara Rule"}, 400)
|
||||
elif term_type=='yara_default':
|
||||
if not Tracker.is_valid_default_yara_rule(term):
|
||||
return ({"status": "error", "reason": "The Yara Rule doesn't exist"}, 400)
|
||||
else:
|
||||
return ({"status": "error", "reason": "Incorrect type"}, 400)
|
||||
return ({"status": "success", "term": term, "type": term_type}, 200)
|
||||
|
|
@ -228,8 +234,13 @@ def add_tracked_term(term , term_type, user_id, level, tags, mails, description,
|
|||
|
||||
term_uuid = str(uuid.uuid4())
|
||||
|
||||
# YARA
|
||||
if term_type == 'yara_custom' or term_type == 'yara_default':
|
||||
term = Tracker.save_yara_rule(term_type, term, tracker_uuid=term_uuid)
|
||||
term_type = 'yara'
|
||||
|
||||
# create metadata
|
||||
r_serv_term.hset('tracker:{}'.format(term_uuid), 'tracked',term)
|
||||
r_serv_term.hset('tracker:{}'.format(term_uuid), 'tracked',term) # # TODO: use hash
|
||||
r_serv_term.hset('tracker:{}'.format(term_uuid), 'type', term_type)
|
||||
r_serv_term.hset('tracker:{}'.format(term_uuid), 'date', datetime.date.today().strftime("%Y%m%d"))
|
||||
r_serv_term.hset('tracker:{}'.format(term_uuid), 'user_id', user_id)
|
||||
|
|
@ -310,6 +321,10 @@ def delete_term(term_uuid):
|
|||
r_serv_term.delete('tracker:item:{}:{}'.format(term_uuid, date))
|
||||
r_serv_term.delete('tracker:stat:{}'.format(term_uuid))
|
||||
|
||||
if term_type == 'yara':
|
||||
# # TODO:
|
||||
pass
|
||||
|
||||
def replace_tracker_description(term_uuid, description):
|
||||
description = escape(description)
|
||||
r_serv_term.hset('tracker:{}'.format(term_uuid), 'description', description)
|
||||
|
|
|
|||
|
|
@ -34,6 +34,10 @@ publish = Redis_Tags
|
|||
subscribe = Redis_Global
|
||||
publish = Redis_Tags
|
||||
|
||||
[Tracker_Yara]
|
||||
subscribe = Redis_Global
|
||||
publish = Redis_Tags
|
||||
|
||||
[Tools]
|
||||
subscribe = Redis_Global
|
||||
publish = Redis_Tags
|
||||
|
|
|
|||
89
bin/trackers/Tracker_Yara.py
Executable file
89
bin/trackers/Tracker_Yara.py
Executable file
|
|
@ -0,0 +1,89 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
Yara trackers
|
||||
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import yara
|
||||
|
||||
from pubsublogger import publisher
|
||||
#
|
||||
# import NotificationHelper
|
||||
#
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
from Helper import Process
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
|
||||
import Term
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
|
||||
import Tracker
|
||||
import item_basic
|
||||
|
||||
|
||||
full_item_url = "/showsavedpaste/?paste="
|
||||
mail_body_template = "AIL Framework,\nNew YARA match: {}\nitem id: {}\nurl: {}{}"
|
||||
|
||||
last_refresh = time.time()
|
||||
|
||||
def yara_rules_match(data):
|
||||
#print(data)
|
||||
tracker_uuid = data['namespace']
|
||||
|
||||
item_date = item_basic.get_item_date(item_id)
|
||||
Tracker.add_tracked_item(tracker_uuid, item_id, item_date)
|
||||
|
||||
# Tags
|
||||
tags_to_add = Tracker.get_tracker_tags(tracker_uuid)
|
||||
for tag in tags_to_add:
|
||||
msg = '{};{}'.format(tag, item_id)
|
||||
p.populate_set_out(msg, 'Tags')
|
||||
|
||||
# Mails
|
||||
mail_to_notify = Tracker.get_tracker_mails(tracker_uuid)
|
||||
if mail_to_notify:
|
||||
mail_subject = Tracker.get_email_subject(tracker_uuid)
|
||||
mail_body = mail_body_template.format(term, item_id, full_item_url, item_id)
|
||||
for mail in mail_to_notify:
|
||||
NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body)
|
||||
|
||||
return yara.CALLBACK_CONTINUE
|
||||
|
||||
if __name__ == "__main__":
|
||||
publisher.port = 6380
|
||||
publisher.channel = "Script"
|
||||
publisher.info("Script Tracker_Yara started")
|
||||
|
||||
config_section = 'Tracker_Yara'
|
||||
module_name = "Tracker_Yara"
|
||||
p = Process(config_section)
|
||||
|
||||
full_item_url = p.config.get("Notifications", "ail_domain") + full_item_url
|
||||
|
||||
# Load Yara rules
|
||||
rules = Tracker.reload_yara_rules()
|
||||
|
||||
# Regex Frequency
|
||||
while True:
|
||||
item_id = p.get_from_set()
|
||||
if item_id is not None:
|
||||
item_content = item_basic.get_item_content(item_id)
|
||||
yara_match = rules.match(data=item_content, callback=yara_rules_match, which_callbacks=yara.CALLBACK_MATCHES, timeout=60)
|
||||
if yara_match:
|
||||
print(f'{item_id}: {yara_match}')
|
||||
|
||||
time.sleep(30)
|
||||
|
||||
else:
|
||||
time.sleep(5)
|
||||
|
||||
# refresh YARA list
|
||||
if last_refresh < Tracker.get_tracker_last_updated_by_type('yara'):
|
||||
rules = Tracker.reload_yara_rules()
|
||||
last_refresh = time.time()
|
||||
print('Tracked set refreshed')
|
||||
1
bin/trackers/yara/ail-yara-rules
Submodule
1
bin/trackers/yara/ail-yara-rules
Submodule
|
|
@ -0,0 +1 @@
|
|||
Subproject commit edc390c4a8d93a028e29938e92aacb399e270cc4
|
||||
14
bin/trackers/yara/custom-rules/test.yar
Normal file
14
bin/trackers/yara/custom-rules/test.yar
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
|
||||
/*
|
||||
Test Rule
|
||||
*/
|
||||
|
||||
rule certificatestest
|
||||
{
|
||||
strings:
|
||||
$ssh_priv = "BEGIN RSA PRIVATE KEY" wide ascii nocase
|
||||
$pem_cert = "BEGIN CERTIFICATE" wide ascii nocase
|
||||
|
||||
condition:
|
||||
any of them
|
||||
}
|
||||
|
|
@ -39,6 +39,10 @@ sudo apt-get install build-essential libffi-dev automake autoconf libtool -qq
|
|||
# sflock, gz requirement
|
||||
sudo apt-get install p7zip-full -qq
|
||||
|
||||
# SUBMODULES #
|
||||
git submodule init
|
||||
git submodule update
|
||||
|
||||
# REDIS #
|
||||
test ! -d redis/ && git clone https://github.com/antirez/redis.git
|
||||
pushd redis/
|
||||
|
|
@ -113,7 +117,7 @@ $AIL_HOME/doc/generate_modules_data_flow_graph.sh
|
|||
# init update version
|
||||
pushd ${AIL_HOME}
|
||||
# shallow clone
|
||||
git fetch --tags --prune --depth=10000
|
||||
git fetch --depth=500 --tags --prune
|
||||
git describe --abbrev=0 --tags | tr -d '\n' > ${AIL_HOME}/update/current_version
|
||||
echo "AIL current version:"
|
||||
git describe --abbrev=0 --tags
|
||||
|
|
|
|||
|
|
@ -22,6 +22,8 @@ textblob
|
|||
#Tokeniser
|
||||
nltk
|
||||
|
||||
yara-python
|
||||
|
||||
#Crawler
|
||||
scrapy
|
||||
scrapy-splash
|
||||
|
|
|
|||
39
update/v3.1.1/Update.py
Executable file
39
update/v3.1.1/Update.py
Executable file
|
|
@ -0,0 +1,39 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import argparse
|
||||
import datetime
|
||||
import configparser
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
|
||||
new_version = 'v3.1.1'
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
start_deb = time.time()
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
|
||||
config_loader = None
|
||||
|
||||
#### NEW EXPORTER
|
||||
# remove old tags errors
|
||||
#r_serv_db.delete('mess_not_saved_export')
|
||||
|
||||
# move solo tags to export in tags_db
|
||||
#all_misp_tags = r_serv_db.smembers('whitelist_misp')
|
||||
#all_hive_tags = r_serv_db.smembers('whitelist_hive')
|
||||
# # TODO: save them in tags db
|
||||
#### NEW EXPORTER
|
||||
|
||||
#Set current ail version
|
||||
r_serv_db.set('ail:version', new_version)
|
||||
|
||||
#Set current ail version
|
||||
r_serv_db.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d"))
|
||||
48
update/v3.1.1/Update.sh
Executable file
48
update/v3.1.1/Update.sh
Executable file
|
|
@ -0,0 +1,48 @@
|
|||
#!/bin/bash
|
||||
|
||||
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
|
||||
|
||||
export PATH=$AIL_HOME:$PATH
|
||||
export PATH=$AIL_REDIS:$PATH
|
||||
export PATH=$AIL_ARDB:$PATH
|
||||
export PATH=$AIL_BIN:$PATH
|
||||
export PATH=$AIL_FLASK:$PATH
|
||||
|
||||
GREEN="\\033[1;32m"
|
||||
DEFAULT="\\033[0;39m"
|
||||
|
||||
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ldbv &
|
||||
wait
|
||||
echo ""
|
||||
|
||||
echo -e $GREEN"Installing YARA ..."$DEFAULT
|
||||
pip3 install yara-python
|
||||
bash ${AIL_BIN}/LAUNCH.sh -t
|
||||
|
||||
# SUBMODULES #
|
||||
git submodule init
|
||||
git submodule update
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT
|
||||
echo ""
|
||||
python ${AIL_HOME}/update/v3.1.1/Update.py
|
||||
wait
|
||||
echo ""
|
||||
echo ""
|
||||
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
exit 0
|
||||
|
|
@ -4,6 +4,8 @@
|
|||
'''
|
||||
Flask functions and routes for tracked items
|
||||
'''
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import redis
|
||||
import datetime
|
||||
|
|
@ -14,14 +16,11 @@ from flask import Flask, render_template, jsonify, request, Blueprint, url_for,
|
|||
from Role_Manager import login_admin, login_analyst, login_read_only
|
||||
from flask_login import login_required, current_user
|
||||
|
||||
import re
|
||||
from pprint import pprint
|
||||
import Levenshtein
|
||||
|
||||
# ---------------------------------------------------------------
|
||||
|
||||
import Paste
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
|
||||
import Term
|
||||
import Tracker
|
||||
|
||||
# ============ VARIABLES ============
|
||||
import Flask_config
|
||||
|
|
@ -78,6 +77,16 @@ def tracked_menu_regex():
|
|||
global_term = Term.get_all_global_tracked_terms(filter_type=filter_type)
|
||||
return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type)
|
||||
|
||||
@hunter.route("/trackers/yara")
|
||||
@login_required
|
||||
@login_read_only
|
||||
def tracked_menu_yara():
|
||||
filter_type = 'yara'
|
||||
user_id = current_user.get_id()
|
||||
user_term = Term.get_all_user_tracked_terms(user_id, filter_type=filter_type)
|
||||
global_term = Term.get_all_global_tracked_terms(filter_type=filter_type)
|
||||
return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type)
|
||||
|
||||
|
||||
@hunter.route("/tracker/add", methods=['GET', 'POST'])
|
||||
@login_required
|
||||
|
|
@ -92,6 +101,18 @@ def add_tracked_menu():
|
|||
tags = request.form.get("tags", [])
|
||||
mails = request.form.get("mails", [])
|
||||
|
||||
# YARA #
|
||||
if term_type == 'yara':
|
||||
yara_default_rule = request.form.get("yara_default_rule")
|
||||
yara_custom_rule = request.form.get("yara_custom_rule")
|
||||
if yara_custom_rule:
|
||||
term = yara_custom_rule
|
||||
term_type='yara_custom'
|
||||
else:
|
||||
term = yara_default_rule
|
||||
term_type='yara_default'
|
||||
# #
|
||||
|
||||
if level == 'on':
|
||||
level = 1
|
||||
|
||||
|
|
@ -109,7 +130,8 @@ def add_tracked_menu():
|
|||
## TODO: use modal
|
||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
else:
|
||||
return render_template("Add_tracker.html")
|
||||
all_yara_files = Tracker.get_all_default_yara_files()
|
||||
return render_template("Add_tracker.html", all_yara_files=all_yara_files)
|
||||
|
||||
@hunter.route("/tracker/show_tracker")
|
||||
@login_required
|
||||
|
|
@ -225,5 +247,12 @@ def get_json_tracker_stats():
|
|||
res = Term.get_list_tracked_term_stats_by_day([tracker_uuid])
|
||||
return jsonify(res)
|
||||
|
||||
# @hunter.route("/tracker/get_all_default_yara_rules_by_type", methods=['GET'])
|
||||
# @login_required
|
||||
# @login_read_only
|
||||
# def get_all_default_yara_rules_by_type():
|
||||
# yara_types = request.args.get('yara_types')
|
||||
# get_all_default_yara_rules_by_types(yara_types)
|
||||
|
||||
# ========= REGISTRATION =========
|
||||
app.register_blueprint(hunter, url_prefix=baseUrl)
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@
|
|||
<h5 class="card-title">Create a new tracker</h5>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<p class="card-text">Enter a domain and choose what kind of data you want.</p>
|
||||
<p class="card-text">Select a tracker type.</p>
|
||||
|
||||
<form action="{{ url_for('hunter.add_tracked_menu') }}" method='post'>
|
||||
|
||||
|
|
@ -74,11 +74,12 @@
|
|||
<option value="word">Word</option>
|
||||
<option value="set">Set</option>
|
||||
<option value="regex">Regex</option>
|
||||
<option value="yara">YARA rule</option>
|
||||
</select>
|
||||
|
||||
<p id="tracker_desc">Terms to track (space separated)</p>
|
||||
|
||||
<div class="row">
|
||||
<div class="row" id="simple_input">
|
||||
<div class="col-12 col-lg-10">
|
||||
<input id="term" name="term" class="form-control" placeholder="Terms to track (space separated)" type="text">
|
||||
</div>
|
||||
|
|
@ -88,6 +89,24 @@
|
|||
</div>
|
||||
|
||||
|
||||
<div class="" id="yara_rule">
|
||||
<div class="" id="yara_default_rules">
|
||||
|
||||
<select class="custom-select w-100 mb-3" name="yara_default_rule">
|
||||
<option selected>Select a default rule</option>
|
||||
{% for yara_types in all_yara_files %}
|
||||
{% for yara_file in all_yara_files[yara_types] %}
|
||||
<option value="{{yara_types}}/{{yara_file}}">{{yara_types}} - {{yara_file}}</option>
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div class="row" id="textarea">
|
||||
<textarea class="form-control mx-3" id="text_input" name="yara_custom_rule" placeholder="Enter your own YARA rule" rows="5"></textarea>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<br>
|
||||
<button class="btn btn-success mt-2">
|
||||
<i class="fas fa-plus"></i> Add Tracker
|
||||
|
|
@ -97,8 +116,6 @@
|
|||
|
||||
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
|
@ -119,6 +136,7 @@ $(document).ready(function(){
|
|||
$("#tracker_desc").hide();
|
||||
$("#term").hide();
|
||||
$("#nb_word").hide();
|
||||
$("#yara_rule").hide();
|
||||
|
||||
$('#tracker_type').on('change', function() {
|
||||
var tracker_type = this.value;
|
||||
|
|
@ -127,16 +145,25 @@ $(document).ready(function(){
|
|||
$("#tracker_desc").show();
|
||||
$("#term").show();
|
||||
$("#nb_word").hide();
|
||||
$("#yara_rule").hide();
|
||||
} else if (tracker_type=="set") {
|
||||
$("#tracker_desc").text("Set of Terms to track (space separated). This tracker is used to check if an item contain one or more terms specified in a set. If an item contain NB unique terms (by default NB of unique keywords = 1), this tracker is triggered. You need to use a regex if you want to use one of the following special characters [<>~!?@#$%^&*|()_-+={}\":;,.\'\n\r\t]/\\ ");
|
||||
$("#tracker_desc").show();
|
||||
$("#term").show();
|
||||
$("#nb_word").show();
|
||||
} else {
|
||||
$("#yara_rule").hide();
|
||||
} else if (tracker_type=="regex") {
|
||||
$("#tracker_desc").text("Enter a valid Python regex");
|
||||
$("#tracker_desc").show();
|
||||
$("#term").show();
|
||||
$("#nb_word").hide();
|
||||
$("#yara_rule").hide();
|
||||
} else if (tracker_type=="yara") {
|
||||
$("#tracker_desc").text("Select a default yara rule or create your own rule:");
|
||||
$("#tracker_desc").show();
|
||||
$("#term").hide();
|
||||
$("#nb_word").hide();
|
||||
$("#yara_rule").show();
|
||||
}
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -22,19 +22,25 @@
|
|||
<li class="nav-item">
|
||||
<a class="nav-link" href="{{url_for('hunter.tracked_menu_word')}}" id="nav_tracker_word">
|
||||
<i class="fas fa-font"></i>
|
||||
<span>Tracked Words</span>
|
||||
<span>Words</span>
|
||||
</a>
|
||||
</li>
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href="{{url_for('hunter.tracked_menu_set')}}" id="nav_tracker_set">
|
||||
<i class="fas fa-layer-group"></i>
|
||||
<span>Tracked Set</span>
|
||||
<span>Set</span>
|
||||
</a>
|
||||
</li>
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href="{{url_for('hunter.tracked_menu_regex')}}" id="nav_tracker_regex">
|
||||
<i class="fas fa-ruler-vertical"></i>
|
||||
<span>Regex</span>
|
||||
</a>
|
||||
</li>
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href="{{url_for('hunter.tracked_menu_yara')}}" id="nav_tracker_yara">
|
||||
<i class="fas fa-ruler"></i>
|
||||
<span>Tracked Regex</span>
|
||||
<span>YARA</span>
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
|
|
|
|||
|
|
@ -2,6 +2,9 @@
|
|||
|
||||
set -e
|
||||
|
||||
# submodules
|
||||
git submodule update
|
||||
|
||||
wget -q http://dygraphs.com/dygraph-combined.js -O ./static/js/dygraph-combined.js
|
||||
|
||||
SBADMIN_VERSION='3.3.7'
|
||||
|
|
@ -90,13 +93,21 @@ wget -q https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb61
|
|||
wget -q http://omnipotent.net/jquery.sparkline/2.1.2/jquery.sparkline.min.js -O ./static/js/jquery.sparkline.min.js
|
||||
wget -q http://canvasjs.com/fdm/chart/ -O temp/canvasjs.zip
|
||||
unzip -qq temp/canvasjs.zip -d temp/
|
||||
mv temp/jquery.canvasjs.min.js ./static/js/jquery.canvasjs.min.js
|
||||
mv temp/canvasjs-2.3.2/Chart\ 2.3.2\ GA\ -\ Stable/jquery.canvasjs.min.js ./static/js/jquery.canvasjs.min.js
|
||||
|
||||
wget -q https://jqueryui.com/resources/download/jquery-ui-1.12.1.zip -O temp/jquery-ui.zip
|
||||
unzip -qq temp/jquery-ui.zip -d temp/
|
||||
mv temp/jquery-ui-1.12.1/jquery-ui.min.js ./static/js/jquery-ui.min.js
|
||||
mv temp/jquery-ui-1.12.1/jquery-ui.min.css ./static/css/jquery-ui.min.css
|
||||
|
||||
# INSTALL YARA
|
||||
YARA_VERSION="4.0.2"
|
||||
wget -q https://github.com/VirusTotal/yara/archive/v${YARA_VERSION}.zip -O temp/yara.zip
|
||||
unzip -qq temp/yara.zip -d temp/
|
||||
pushd temp/yara-${YARA_VERSION}
|
||||
./bootstrap.sh
|
||||
popd
|
||||
|
||||
|
||||
rm -rf temp
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue