chg: [trackers] add yara trackers

This commit is contained in:
Terrtia 2020-08-12 09:28:36 +02:00
parent a88b57498d
commit e70ae376c5
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
10 changed files with 302 additions and 11 deletions

View file

@ -5,6 +5,7 @@ import os
import sys import sys
import time import time
import redis import redis
import yara
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader import ConfigLoader
@ -14,12 +15,117 @@ config_loader = ConfigLoader.ConfigLoader()
r_serv_tracker = config_loader.get_redis_conn("ARDB_Tracker") r_serv_tracker = config_loader.get_redis_conn("ARDB_Tracker")
config_loader = None config_loader = None
def get_tracker_uuid_list(tracker, tracker_type):
return list(r_serv_tracker.smembers('all:tracker_uuid:{}:{}'.format(tracker_type, tracker)))
def get_tracker_tags(tracker_uuid):
return list(r_serv_tracker.smembers('tracker:tags:{}'.format(tracker_uuid)))
def get_tracker_mails(tracker_uuid):
return list(r_serv_tracker.smembers('tracker:mail:{}'.format(tracker_uuid)))
def get_tracker_description(tracker_uuid): def get_tracker_description(tracker_uuid):
return r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'description') return r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'description')
def add_tracked_item(tracker_uuid, item_id, item_date):
# track item
r_serv_tracker.sadd('tracker:item:{}:{}'.format(tracker_uuid, item_date), item_id)
# track nb item by date
r_serv_tracker.zadd('tracker:stat:{}'.format(tracker_uuid), item_date, int(item_date))
def get_email_subject(tracker_uuid): def get_email_subject(tracker_uuid):
tracker_description = get_tracker_description(tracker_uuid) tracker_description = get_tracker_description(tracker_uuid)
if not tracker_description: if not tracker_description:
return "AIL framework: Tracker Alert" return "AIL framework: Tracker Alert"
else: else:
return 'AIL framework: {}'.format(tracker_description) return 'AIL framework: {}'.format(tracker_description)
def get_tracker_last_updated_by_type(tracker_type):
epoch_update = r_serv_tracker.get('tracker:refresh:{}'.format(term_type))
if not epoch_update:
epoch_update = 0
return float(epoch_update)
#### YARA ####
def get_yara_rules_dir():
return os.path.join(os.environ['AIL_BIN'], 'trackers', 'yara')
def get_yara_rules_default_dir():
return os.path.join(os.environ['AIL_BIN'], 'trackers', 'yara', 'ail-yara-rules', 'rules')
# # TODO: cache + update
def get_all_default_yara_rules_types():
yara_dir = get_yara_rules_default_dir()
all_yara_types = next(os.walk(yara_dir))[1]
# save in cache ?
return all_yara_types
# # TODO: cache + update
def get_all_default_yara_files():
yara_dir = get_yara_rules_default_dir()
all_default_yara_files = {}
for rules_type in get_all_default_yara_rules_types():
all_default_yara_files[rules_type] = os.listdir(os.path.join(yara_dir, rules_type))
return all_default_yara_files
def get_all_default_yara_rules_by_type(yara_types):
all_default_yara_files = get_all_default_yara_files()
if yara_types in all_default_yara_files:
return all_default_yara_files[yara_types]
else:
return []
def get_all_tracked_yara_files():
yara_files = r_serv_tracker.smembers('all:tracker:yara')
if not yara_files:
yara_files = []
return yara_files
def reload_yara_rules():
yara_files = get_all_tracked_yara_files()
# {uuid: filename}
rule_dict = {}
for yar_path in yara_files:
l_tracker_uuid = get_tracker_uuid_list(yar_path, 'yara')
for tracker_uuid in l_tracker_uuid:
rule_dict[tracker_uuid] = os.path.join(get_yara_rules_dir(), yar_path)
rules = yara.compile(filepaths=rule_dict)
return rules
def is_valid_yara_rule(yara_rule):
try:
yara.compile(source=yara_rule)
return True
except:
return False
def is_valid_default_yara_rule(yara_rule):
yara_dir = get_yara_rules_default_dir()
filename = os.path.join(yara_dir, yara_rule)
filename = os.path.realpath(filename)
# incorrect filename
if not os.path.commonprefix([filename, yara_dir]) == yara_dir:
return False
else:
if os.path.isfile(filename):
return True
else:
return False
def save_yara_rule(yara_rule_type, yara_rule, tracker_uuid=None):
if yara_rule_type == 'yara_custom':
if not tracker_uuid:
tracker_uuid = str(uuid.uuid4())
filename = os.path.join('custom-rules', tracker_uuid + '.yar')
with open(os.path.join(get_yara_rules_dir(), filename), 'w') as f:
f.write(str(yara_rule))
if yara_rule_type == 'yara_default':
filename = os.path.join('ail-yara-rules', 'rules', yara_rule)
return filename
##-- YARA --##
if __name__ == '__main__':
res = is_valid_yara_rule('rule dummy { }')
print(res)

View file

@ -16,6 +16,7 @@ from textblob import TextBlob
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader import ConfigLoader
import Tracker
from flask import escape from flask import escape
@ -219,7 +220,12 @@ def parse_tracked_term_to_add(term , term_type, nb_words=1):
if nb_words > len(words_set): if nb_words > len(words_set):
nb_words = len(words_set) nb_words = len(words_set)
elif term_type=='yara_custom':
if not Tracker.is_valid_yara_rule(term):
return ({"status": "error", "reason": "Invalid custom Yara Rule"}, 400)
elif term_type=='yara_default':
if not Tracker.is_valid_default_yara_rule(term):
return ({"status": "error", "reason": "The Yara Rule doesn't exist"}, 400)
else: else:
return ({"status": "error", "reason": "Incorrect type"}, 400) return ({"status": "error", "reason": "Incorrect type"}, 400)
return ({"status": "success", "term": term, "type": term_type}, 200) return ({"status": "success", "term": term, "type": term_type}, 200)
@ -228,8 +234,13 @@ def add_tracked_term(term , term_type, user_id, level, tags, mails, description,
term_uuid = str(uuid.uuid4()) term_uuid = str(uuid.uuid4())
# YARA
if term_type == 'yara_custom' or term_type == 'yara_default':
term = Tracker.save_yara_rule(term_type, term, tracker_uuid=term_uuid)
term_type = 'yara'
# create metadata # create metadata
r_serv_term.hset('tracker:{}'.format(term_uuid), 'tracked',term) r_serv_term.hset('tracker:{}'.format(term_uuid), 'tracked',term) # # TODO: use hash
r_serv_term.hset('tracker:{}'.format(term_uuid), 'type', term_type) r_serv_term.hset('tracker:{}'.format(term_uuid), 'type', term_type)
r_serv_term.hset('tracker:{}'.format(term_uuid), 'date', datetime.date.today().strftime("%Y%m%d")) r_serv_term.hset('tracker:{}'.format(term_uuid), 'date', datetime.date.today().strftime("%Y%m%d"))
r_serv_term.hset('tracker:{}'.format(term_uuid), 'user_id', user_id) r_serv_term.hset('tracker:{}'.format(term_uuid), 'user_id', user_id)
@ -310,6 +321,10 @@ def delete_term(term_uuid):
r_serv_term.delete('tracker:item:{}:{}'.format(term_uuid, date)) r_serv_term.delete('tracker:item:{}:{}'.format(term_uuid, date))
r_serv_term.delete('tracker:stat:{}'.format(term_uuid)) r_serv_term.delete('tracker:stat:{}'.format(term_uuid))
if term_type == 'yara':
# # TODO:
pass
def replace_tracker_description(term_uuid, description): def replace_tracker_description(term_uuid, description):
description = escape(description) description = escape(description)
r_serv_term.hset('tracker:{}'.format(term_uuid), 'description', description) r_serv_term.hset('tracker:{}'.format(term_uuid), 'description', description)

View file

@ -34,6 +34,10 @@ publish = Redis_Tags
subscribe = Redis_Global subscribe = Redis_Global
publish = Redis_Tags publish = Redis_Tags
[Tracker_Yara]
subscribe = Redis_Global
publish = Redis_Tags
[Tools] [Tools]
subscribe = Redis_Global subscribe = Redis_Global
publish = Redis_Tags publish = Redis_Tags

90
bin/trackers/Tracker_Yara.py Executable file
View file

@ -0,0 +1,90 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
"""
Yara trackers
"""
import os
import re
import sys
import time
import yara
from pubsublogger import publisher
#
# import NotificationHelper
#
sys.path.append(os.environ['AIL_BIN'])
from Helper import Process
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
import Term
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
import Tracker
import item_basic
full_item_url = "/showsavedpaste/?paste="
mail_body_template = "AIL Framework,\nNew YARA match: {}\nitem id: {}\nurl: {}{}"
last_refresh = time.time()
def yara_rules_match(data):
#print(data)
tracker_uuid = data['namespace']
item_date = item_basic.get_item_date(item_id)
Tracker.add_tracked_item(tracker_uuid, item_id, item_date)
# Tags
tags_to_add = Tracker.get_tracker_tags(tracker_uuid)
for tag in tags_to_add:
msg = '{};{}'.format(tag, item_id)
p.populate_set_out(msg, 'Tags')
# Mails
mail_to_notify = Tracker.get_tracker_mails(tracker_uuid)
if mail_to_notify:
mail_subject = Tracker.get_email_subject(tracker_uuid)
mail_body = mail_body_template.format(term, item_id, full_item_url, item_id)
for mail in mail_to_notify:
NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body)
return yara.CALLBACK_CONTINUE
if __name__ == "__main__":
publisher.port = 6380
publisher.channel = "Script"
publisher.info("Script Tracker_Yara started")
config_section = 'Tracker_Yara'
module_name = "Tracker_Yara"
p = Process(config_section)
full_item_url = p.config.get("Notifications", "ail_domain") + full_item_url
# Load Yara rules
rules = Tracker.reload_yara_rules()
# Regex Frequency
while True:
item_id = p.get_from_set()
item_id = 'archive/pastebin.com_pro/2020/03/04/AnwFX3w2.gz'
if item_id is not None:
item_content = item_basic.get_item_content(item_id)
yara_match = rules.match(data=item_content, callback=yara_rules_match, which_callbacks=yara.CALLBACK_MATCHES, timeout=60)
if yara_match:
print(f'{item_id}: {yara_match}')
time.sleep(30)
else:
time.sleep(5)
# refresh YARA list
if last_refresh < Tracker.get_tracker_last_updated_by_type('yara'):
rules = Tracker.reload_yara_rules()
last_refresh = time.time()
print('Tracked set refreshed')

View file

@ -0,0 +1,14 @@
/*
Test Rule
*/
rule certificatestest
{
strings:
$ssh_priv = "BEGIN RSA PRIVATE KEY" wide ascii nocase
$pem_cert = "BEGIN CERTIFICATE" wide ascii nocase
condition:
any of them
}

View file

@ -22,6 +22,8 @@ textblob
#Tokeniser #Tokeniser
nltk nltk
yara-python
#Crawler #Crawler
scrapy scrapy
scrapy-splash scrapy-splash

View file

@ -4,6 +4,8 @@
''' '''
Flask functions and routes for tracked items Flask functions and routes for tracked items
''' '''
import os
import sys
import json import json
import redis import redis
import datetime import datetime
@ -14,14 +16,11 @@ from flask import Flask, render_template, jsonify, request, Blueprint, url_for,
from Role_Manager import login_admin, login_analyst, login_read_only from Role_Manager import login_admin, login_analyst, login_read_only
from flask_login import login_required, current_user from flask_login import login_required, current_user
import re
from pprint import pprint
import Levenshtein
# --------------------------------------------------------------- # ---------------------------------------------------------------
import Paste sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
import Term import Term
import Tracker
# ============ VARIABLES ============ # ============ VARIABLES ============
import Flask_config import Flask_config
@ -78,6 +77,16 @@ def tracked_menu_regex():
global_term = Term.get_all_global_tracked_terms(filter_type=filter_type) global_term = Term.get_all_global_tracked_terms(filter_type=filter_type)
return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type) return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type)
@hunter.route("/trackers/yara")
@login_required
@login_read_only
def tracked_menu_yara():
filter_type = 'yara'
user_id = current_user.get_id()
user_term = Term.get_all_user_tracked_terms(user_id, filter_type=filter_type)
global_term = Term.get_all_global_tracked_terms(filter_type=filter_type)
return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type)
@hunter.route("/tracker/add", methods=['GET', 'POST']) @hunter.route("/tracker/add", methods=['GET', 'POST'])
@login_required @login_required
@ -92,6 +101,18 @@ def add_tracked_menu():
tags = request.form.get("tags", []) tags = request.form.get("tags", [])
mails = request.form.get("mails", []) mails = request.form.get("mails", [])
# YARA #
if term_type == 'yara':
yara_default_rule = request.form.get("yara_default_rule")
yara_custom_rule = request.form.get("yara_custom_rule")
if yara_custom_rule:
term = yara_custom_rule
term_type='yara_custom'
else:
term = yara_default_rule
term_type='yara_default'
# #
if level == 'on': if level == 'on':
level = 1 level = 1
@ -109,7 +130,8 @@ def add_tracked_menu():
## TODO: use modal ## TODO: use modal
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
else: else:
return render_template("Add_tracker.html") all_yara_files = Tracker.get_all_default_yara_files()
return render_template("Add_tracker.html", all_yara_files=all_yara_files)
@hunter.route("/tracker/show_tracker") @hunter.route("/tracker/show_tracker")
@login_required @login_required
@ -225,5 +247,12 @@ def get_json_tracker_stats():
res = Term.get_list_tracked_term_stats_by_day([tracker_uuid]) res = Term.get_list_tracked_term_stats_by_day([tracker_uuid])
return jsonify(res) return jsonify(res)
# @hunter.route("/tracker/get_all_default_yara_rules_by_type", methods=['GET'])
# @login_required
# @login_read_only
# def get_all_default_yara_rules_by_type():
# yara_types = request.args.get('yara_types')
# get_all_default_yara_rules_by_types(yara_types)
# ========= REGISTRATION ========= # ========= REGISTRATION =========
app.register_blueprint(hunter, url_prefix=baseUrl) app.register_blueprint(hunter, url_prefix=baseUrl)

View file

@ -74,11 +74,12 @@
<option value="word">Word</option> <option value="word">Word</option>
<option value="set">Set</option> <option value="set">Set</option>
<option value="regex">Regex</option> <option value="regex">Regex</option>
<option value="yara">YARA rule</option>
</select> </select>
<p id="tracker_desc">Terms to track (space separated)</p> <p id="tracker_desc">Terms to track (space separated)</p>
<div class="row"> <div class="row" id="simple_input">
<div class="col-12 col-lg-10"> <div class="col-12 col-lg-10">
<input id="term" name="term" class="form-control" placeholder="Terms to track (space separated)" type="text"> <input id="term" name="term" class="form-control" placeholder="Terms to track (space separated)" type="text">
</div> </div>
@ -88,6 +89,24 @@
</div> </div>
<div class="" id="yara_rule">
<div class="" id="yara_default_rules">
<select class="custom-select w-100 mb-3" name="yara_default_rule">
<option selected>Select a default rule</option>
{% for yara_types in all_yara_files %}
{% for yara_file in all_yara_files[yara_types] %}
<option value="{{yara_types}}/{{yara_file}}">{{yara_types}} - {{yara_file}}</option>
{% endfor %}
{% endfor %}
</select>
</div>
<div class="row" id="textarea">
<textarea class="form-control" id="text_input" name="yara_custom_rule" rows="5"></textarea>
</div>
</div>
<br> <br>
<button class="btn btn-success mt-2"> <button class="btn btn-success mt-2">
<i class="fas fa-plus"></i> Add Tracker <i class="fas fa-plus"></i> Add Tracker
@ -97,8 +116,6 @@
</div> </div>
</div> </div>

View file

@ -37,6 +37,12 @@
<span>Tracked Regex</span> <span>Tracked Regex</span>
</a> </a>
</li> </li>
<li class="nav-item">
<a class="nav-link" href="{{url_for('hunter.tracked_menu_yara')}}" id="nav_tracker_yara">
<i class="fas fa-ruler"></i>
<span>YARA</span>
</a>
</li>
</ul> </ul>
</nav> </nav>
</div> </div>

View file

@ -97,6 +97,14 @@ unzip -qq temp/jquery-ui.zip -d temp/
mv temp/jquery-ui-1.12.1/jquery-ui.min.js ./static/js/jquery-ui.min.js mv temp/jquery-ui-1.12.1/jquery-ui.min.js ./static/js/jquery-ui.min.js
mv temp/jquery-ui-1.12.1/jquery-ui.min.css ./static/css/jquery-ui.min.css mv temp/jquery-ui-1.12.1/jquery-ui.min.css ./static/css/jquery-ui.min.css
# INSTALL YARA
YARA_VERSION="4.0.2"
wget -q https://github.com/VirusTotal/yara/archive/v${YARA_VERSION}.zip -O temp/yara.zip
unzip -qq temp/yara.zip -d temp/
pushd temp/yara-${YARA_VERSION}
./bootstrap.sh
popd
rm -rf temp rm -rf temp