chg: [trackers] add yara trackers

This commit is contained in:
Terrtia 2020-08-12 09:28:36 +02:00
parent a88b57498d
commit e70ae376c5
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
10 changed files with 302 additions and 11 deletions

View file

@ -5,6 +5,7 @@ import os
import sys
import time
import redis
import yara
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
@ -14,12 +15,117 @@ config_loader = ConfigLoader.ConfigLoader()
r_serv_tracker = config_loader.get_redis_conn("ARDB_Tracker")
config_loader = None
def get_tracker_uuid_list(tracker, tracker_type):
return list(r_serv_tracker.smembers('all:tracker_uuid:{}:{}'.format(tracker_type, tracker)))
def get_tracker_tags(tracker_uuid):
return list(r_serv_tracker.smembers('tracker:tags:{}'.format(tracker_uuid)))
def get_tracker_mails(tracker_uuid):
return list(r_serv_tracker.smembers('tracker:mail:{}'.format(tracker_uuid)))
def get_tracker_description(tracker_uuid):
return r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'description')
def add_tracked_item(tracker_uuid, item_id, item_date):
# track item
r_serv_tracker.sadd('tracker:item:{}:{}'.format(tracker_uuid, item_date), item_id)
# track nb item by date
r_serv_tracker.zadd('tracker:stat:{}'.format(tracker_uuid), item_date, int(item_date))
def get_email_subject(tracker_uuid):
tracker_description = get_tracker_description(tracker_uuid)
if not tracker_description:
return "AIL framework: Tracker Alert"
else:
return 'AIL framework: {}'.format(tracker_description)
def get_tracker_last_updated_by_type(tracker_type):
epoch_update = r_serv_tracker.get('tracker:refresh:{}'.format(term_type))
if not epoch_update:
epoch_update = 0
return float(epoch_update)
#### YARA ####
def get_yara_rules_dir():
return os.path.join(os.environ['AIL_BIN'], 'trackers', 'yara')
def get_yara_rules_default_dir():
return os.path.join(os.environ['AIL_BIN'], 'trackers', 'yara', 'ail-yara-rules', 'rules')
# # TODO: cache + update
def get_all_default_yara_rules_types():
yara_dir = get_yara_rules_default_dir()
all_yara_types = next(os.walk(yara_dir))[1]
# save in cache ?
return all_yara_types
# # TODO: cache + update
def get_all_default_yara_files():
yara_dir = get_yara_rules_default_dir()
all_default_yara_files = {}
for rules_type in get_all_default_yara_rules_types():
all_default_yara_files[rules_type] = os.listdir(os.path.join(yara_dir, rules_type))
return all_default_yara_files
def get_all_default_yara_rules_by_type(yara_types):
all_default_yara_files = get_all_default_yara_files()
if yara_types in all_default_yara_files:
return all_default_yara_files[yara_types]
else:
return []
def get_all_tracked_yara_files():
yara_files = r_serv_tracker.smembers('all:tracker:yara')
if not yara_files:
yara_files = []
return yara_files
def reload_yara_rules():
yara_files = get_all_tracked_yara_files()
# {uuid: filename}
rule_dict = {}
for yar_path in yara_files:
l_tracker_uuid = get_tracker_uuid_list(yar_path, 'yara')
for tracker_uuid in l_tracker_uuid:
rule_dict[tracker_uuid] = os.path.join(get_yara_rules_dir(), yar_path)
rules = yara.compile(filepaths=rule_dict)
return rules
def is_valid_yara_rule(yara_rule):
try:
yara.compile(source=yara_rule)
return True
except:
return False
def is_valid_default_yara_rule(yara_rule):
yara_dir = get_yara_rules_default_dir()
filename = os.path.join(yara_dir, yara_rule)
filename = os.path.realpath(filename)
# incorrect filename
if not os.path.commonprefix([filename, yara_dir]) == yara_dir:
return False
else:
if os.path.isfile(filename):
return True
else:
return False
def save_yara_rule(yara_rule_type, yara_rule, tracker_uuid=None):
if yara_rule_type == 'yara_custom':
if not tracker_uuid:
tracker_uuid = str(uuid.uuid4())
filename = os.path.join('custom-rules', tracker_uuid + '.yar')
with open(os.path.join(get_yara_rules_dir(), filename), 'w') as f:
f.write(str(yara_rule))
if yara_rule_type == 'yara_default':
filename = os.path.join('ail-yara-rules', 'rules', yara_rule)
return filename
##-- YARA --##
if __name__ == '__main__':
res = is_valid_yara_rule('rule dummy { }')
print(res)

View file

@ -16,6 +16,7 @@ from textblob import TextBlob
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
import Tracker
from flask import escape
@ -219,7 +220,12 @@ def parse_tracked_term_to_add(term , term_type, nb_words=1):
if nb_words > len(words_set):
nb_words = len(words_set)
elif term_type=='yara_custom':
if not Tracker.is_valid_yara_rule(term):
return ({"status": "error", "reason": "Invalid custom Yara Rule"}, 400)
elif term_type=='yara_default':
if not Tracker.is_valid_default_yara_rule(term):
return ({"status": "error", "reason": "The Yara Rule doesn't exist"}, 400)
else:
return ({"status": "error", "reason": "Incorrect type"}, 400)
return ({"status": "success", "term": term, "type": term_type}, 200)
@ -228,8 +234,13 @@ def add_tracked_term(term , term_type, user_id, level, tags, mails, description,
term_uuid = str(uuid.uuid4())
# YARA
if term_type == 'yara_custom' or term_type == 'yara_default':
term = Tracker.save_yara_rule(term_type, term, tracker_uuid=term_uuid)
term_type = 'yara'
# create metadata
r_serv_term.hset('tracker:{}'.format(term_uuid), 'tracked',term)
r_serv_term.hset('tracker:{}'.format(term_uuid), 'tracked',term) # # TODO: use hash
r_serv_term.hset('tracker:{}'.format(term_uuid), 'type', term_type)
r_serv_term.hset('tracker:{}'.format(term_uuid), 'date', datetime.date.today().strftime("%Y%m%d"))
r_serv_term.hset('tracker:{}'.format(term_uuid), 'user_id', user_id)
@ -310,6 +321,10 @@ def delete_term(term_uuid):
r_serv_term.delete('tracker:item:{}:{}'.format(term_uuid, date))
r_serv_term.delete('tracker:stat:{}'.format(term_uuid))
if term_type == 'yara':
# # TODO:
pass
def replace_tracker_description(term_uuid, description):
description = escape(description)
r_serv_term.hset('tracker:{}'.format(term_uuid), 'description', description)

View file

@ -34,6 +34,10 @@ publish = Redis_Tags
subscribe = Redis_Global
publish = Redis_Tags
[Tracker_Yara]
subscribe = Redis_Global
publish = Redis_Tags
[Tools]
subscribe = Redis_Global
publish = Redis_Tags

90
bin/trackers/Tracker_Yara.py Executable file
View file

@ -0,0 +1,90 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
"""
Yara trackers
"""
import os
import re
import sys
import time
import yara
from pubsublogger import publisher
#
# import NotificationHelper
#
sys.path.append(os.environ['AIL_BIN'])
from Helper import Process
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
import Term
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
import Tracker
import item_basic
full_item_url = "/showsavedpaste/?paste="
mail_body_template = "AIL Framework,\nNew YARA match: {}\nitem id: {}\nurl: {}{}"
last_refresh = time.time()
def yara_rules_match(data):
#print(data)
tracker_uuid = data['namespace']
item_date = item_basic.get_item_date(item_id)
Tracker.add_tracked_item(tracker_uuid, item_id, item_date)
# Tags
tags_to_add = Tracker.get_tracker_tags(tracker_uuid)
for tag in tags_to_add:
msg = '{};{}'.format(tag, item_id)
p.populate_set_out(msg, 'Tags')
# Mails
mail_to_notify = Tracker.get_tracker_mails(tracker_uuid)
if mail_to_notify:
mail_subject = Tracker.get_email_subject(tracker_uuid)
mail_body = mail_body_template.format(term, item_id, full_item_url, item_id)
for mail in mail_to_notify:
NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body)
return yara.CALLBACK_CONTINUE
if __name__ == "__main__":
publisher.port = 6380
publisher.channel = "Script"
publisher.info("Script Tracker_Yara started")
config_section = 'Tracker_Yara'
module_name = "Tracker_Yara"
p = Process(config_section)
full_item_url = p.config.get("Notifications", "ail_domain") + full_item_url
# Load Yara rules
rules = Tracker.reload_yara_rules()
# Regex Frequency
while True:
item_id = p.get_from_set()
item_id = 'archive/pastebin.com_pro/2020/03/04/AnwFX3w2.gz'
if item_id is not None:
item_content = item_basic.get_item_content(item_id)
yara_match = rules.match(data=item_content, callback=yara_rules_match, which_callbacks=yara.CALLBACK_MATCHES, timeout=60)
if yara_match:
print(f'{item_id}: {yara_match}')
time.sleep(30)
else:
time.sleep(5)
# refresh YARA list
if last_refresh < Tracker.get_tracker_last_updated_by_type('yara'):
rules = Tracker.reload_yara_rules()
last_refresh = time.time()
print('Tracked set refreshed')

View file

@ -0,0 +1,14 @@
/*
Test Rule
*/
rule certificatestest
{
strings:
$ssh_priv = "BEGIN RSA PRIVATE KEY" wide ascii nocase
$pem_cert = "BEGIN CERTIFICATE" wide ascii nocase
condition:
any of them
}

View file

@ -22,6 +22,8 @@ textblob
#Tokeniser
nltk
yara-python
#Crawler
scrapy
scrapy-splash

View file

@ -4,6 +4,8 @@
'''
Flask functions and routes for tracked items
'''
import os
import sys
import json
import redis
import datetime
@ -14,14 +16,11 @@ from flask import Flask, render_template, jsonify, request, Blueprint, url_for,
from Role_Manager import login_admin, login_analyst, login_read_only
from flask_login import login_required, current_user
import re
from pprint import pprint
import Levenshtein
# ---------------------------------------------------------------
import Paste
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
import Term
import Tracker
# ============ VARIABLES ============
import Flask_config
@ -78,6 +77,16 @@ def tracked_menu_regex():
global_term = Term.get_all_global_tracked_terms(filter_type=filter_type)
return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type)
@hunter.route("/trackers/yara")
@login_required
@login_read_only
def tracked_menu_yara():
filter_type = 'yara'
user_id = current_user.get_id()
user_term = Term.get_all_user_tracked_terms(user_id, filter_type=filter_type)
global_term = Term.get_all_global_tracked_terms(filter_type=filter_type)
return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type)
@hunter.route("/tracker/add", methods=['GET', 'POST'])
@login_required
@ -92,6 +101,18 @@ def add_tracked_menu():
tags = request.form.get("tags", [])
mails = request.form.get("mails", [])
# YARA #
if term_type == 'yara':
yara_default_rule = request.form.get("yara_default_rule")
yara_custom_rule = request.form.get("yara_custom_rule")
if yara_custom_rule:
term = yara_custom_rule
term_type='yara_custom'
else:
term = yara_default_rule
term_type='yara_default'
# #
if level == 'on':
level = 1
@ -109,7 +130,8 @@ def add_tracked_menu():
## TODO: use modal
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
else:
return render_template("Add_tracker.html")
all_yara_files = Tracker.get_all_default_yara_files()
return render_template("Add_tracker.html", all_yara_files=all_yara_files)
@hunter.route("/tracker/show_tracker")
@login_required
@ -225,5 +247,12 @@ def get_json_tracker_stats():
res = Term.get_list_tracked_term_stats_by_day([tracker_uuid])
return jsonify(res)
# @hunter.route("/tracker/get_all_default_yara_rules_by_type", methods=['GET'])
# @login_required
# @login_read_only
# def get_all_default_yara_rules_by_type():
# yara_types = request.args.get('yara_types')
# get_all_default_yara_rules_by_types(yara_types)
# ========= REGISTRATION =========
app.register_blueprint(hunter, url_prefix=baseUrl)

View file

@ -74,11 +74,12 @@
<option value="word">Word</option>
<option value="set">Set</option>
<option value="regex">Regex</option>
<option value="yara">YARA rule</option>
</select>
<p id="tracker_desc">Terms to track (space separated)</p>
<div class="row">
<div class="row" id="simple_input">
<div class="col-12 col-lg-10">
<input id="term" name="term" class="form-control" placeholder="Terms to track (space separated)" type="text">
</div>
@ -88,6 +89,24 @@
</div>
<div class="" id="yara_rule">
<div class="" id="yara_default_rules">
<select class="custom-select w-100 mb-3" name="yara_default_rule">
<option selected>Select a default rule</option>
{% for yara_types in all_yara_files %}
{% for yara_file in all_yara_files[yara_types] %}
<option value="{{yara_types}}/{{yara_file}}">{{yara_types}} - {{yara_file}}</option>
{% endfor %}
{% endfor %}
</select>
</div>
<div class="row" id="textarea">
<textarea class="form-control" id="text_input" name="yara_custom_rule" rows="5"></textarea>
</div>
</div>
<br>
<button class="btn btn-success mt-2">
<i class="fas fa-plus"></i> Add Tracker
@ -97,8 +116,6 @@
</div>
</div>

View file

@ -37,6 +37,12 @@
<span>Tracked Regex</span>
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="{{url_for('hunter.tracked_menu_yara')}}" id="nav_tracker_yara">
<i class="fas fa-ruler"></i>
<span>YARA</span>
</a>
</li>
</ul>
</nav>
</div>

View file

@ -97,6 +97,14 @@ unzip -qq temp/jquery-ui.zip -d temp/
mv temp/jquery-ui-1.12.1/jquery-ui.min.js ./static/js/jquery-ui.min.js
mv temp/jquery-ui-1.12.1/jquery-ui.min.css ./static/css/jquery-ui.min.css
# INSTALL YARA
YARA_VERSION="4.0.2"
wget -q https://github.com/VirusTotal/yara/archive/v${YARA_VERSION}.zip -O temp/yara.zip
unzip -qq temp/yara.zip -d temp/
pushd temp/yara-${YARA_VERSION}
./bootstrap.sh
popd
rm -rf temp