chg: [Tracker_Yara module] create module class

This commit is contained in:
Terrtia 2021-06-02 16:04:52 +02:00
parent ed37232a2d
commit 5acb583701
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
10 changed files with 189 additions and 189 deletions

View file

@ -147,7 +147,9 @@ function launching_scripts {
sleep 0.1
echo -e $GREEN"\t* Launching scripts"$DEFAULT
# LAUNCH CORE MODULE
##################################
# CORE MODULES #
##################################
screen -S "Script_AIL" -X screen -t "JSON_importer" bash -c "cd ${AIL_BIN}/import; ${ENV_PY} ./JSON_importer.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Crawler_manager" bash -c "cd ${AIL_BIN}/core; ${ENV_PY} ./Crawler_manager.py; read x"
@ -157,57 +159,77 @@ function launching_scripts {
screen -S "Script_AIL" -X screen -t "DbCleaner" bash -c "cd ${AIL_BIN}/core; ${ENV_PY} ./DbCleaner.py; read x"
sleep 0.1
##################################
# MODULES #
##################################
screen -S "Script_AIL" -X screen -t "Global" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Global.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Categ" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Categ.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Indexer" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Indexer.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Tags" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Tags.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./submit_paste.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "ApiKey" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./ApiKey.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Credential" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Credential.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "CreditCards" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./CreditCards.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Decoder" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Decoder.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./DomClassifier.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Keys" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Keys.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Onion" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Onion.py; read x"
sleep 0.1
##################################
# TRACKERS MODULES #
##################################
screen -S "Script_AIL" -X screen -t "Tracker_Term" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Term.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Tracker_Yara" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Yara.py; read x"
sleep 0.1
##################################
# DISABLED MODULES #
##################################
#screen -S "Script_AIL" -X screen -t "Phone" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Phone.py; read x"
#sleep 0.1
##################################
# #
##################################
screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModulesInformationV2.py -k 0 -c 1; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Mixer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mixer.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Global" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Global.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Duplicates" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Duplicates.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./DomClassifier.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Categ" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Categ.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "CreditCards" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./CreditCards.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "BankAccount" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./BankAccount.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Onion" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Onion.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Mail" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mail.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "ApiKey" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ApiKey.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Urls" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Urls.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Credential" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Credential.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "TermTrackerMod" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./TermTrackerMod.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "RegexTracker" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./RegexTracker.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Tracker_Yara" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Yara.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Indexer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Indexer.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Keys" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Keys.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "PgpDump" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./PgpDump.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Decoder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Decoder.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Cryptocurrency" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Cryptocurrencies.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Telegram" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Telegram.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Tools" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Tools.py; read x"
sleep 0.1
#screen -S "Script_AIL" -X screen -t "Phone" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Phone.py; read x"
#sleep 0.1
#screen -S "Script_AIL" -X screen -t "Release" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Release.py; read x"
#sleep 0.1
screen -S "Script_AIL" -X screen -t "Cve" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Cve.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModuleStats.py; read x"
@ -218,48 +240,18 @@ function launching_scripts {
sleep 0.1
screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./MISP_The_Hive_feeder.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Tags" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Tags.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Languages" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Languages.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./SentimentAnalysis.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "UpdateBackground" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./update-background.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./submit_paste.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "IPAddress" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./IPAddress.py; read x"
}
#screen -S "Script_AIL" -X screen -t "Release" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Release.py; read x"
#sleep 0.1
# function launching_crawler {
# if [[ ! $iscrawler ]]; then
# CONFIG=$AIL_HOME/configs/core.cfg
# lport=$(awk '/^\[Crawler\]/{f=1} f==1&&/^splash_port/{print $3;exit}' "${CONFIG}")
#
# IFS='-' read -ra PORTS <<< "$lport"
# if [ ${#PORTS[@]} -eq 1 ]
# then
# first_port=${PORTS[0]}
# last_port=${PORTS[0]}
# else
# first_port=${PORTS[0]}
# last_port=${PORTS[1]}
# fi
#
# screen -dmS "Crawler_AIL"
# sleep 0.1
#
# for ((i=first_port;i<=last_port;i++)); do
# screen -S "Crawler_AIL" -X screen -t "onion_crawler:$i" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Crawler.py $i; read x"
# sleep 0.1
# done
#
# echo -e $GREEN"\t* Launching Crawler_AIL scripts"$DEFAULT
# else
# echo -e $RED"\t* A screen is already launched"$DEFAULT
# fi
# }
}
function shutting_down_redis {
redis_dir=${AIL_HOME}/redis/src/
@ -491,7 +483,7 @@ function update_thirdparty {
function launch_tests() {
tests_dir=${AIL_HOME}/tests
bin_dir=${AIL_BIN}
python3 `which nosetests` -w $tests_dir --with-coverage --cover-package=$bin_dir -d #--cover-erase
python3 `which nosetests` -w $tests_dir --with-coverage --cover-package=$bin_dir -d --cover-erase
}
function reset_password() {

View file

@ -7,27 +7,27 @@ The Phone Module
This module is consuming the Redis-list created by the Categ module.
It apply phone number regexes on paste content and warn if above a threshold.
It apply phone number regexes on item content and warn if above a threshold.
"""
##################################
# Import External packages
##################################
import time
import os
import re
import sys
import time
import phonenumbers
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from module.abstract_module import AbstractModule
from packages import Paste
from pubsublogger import publisher
from Helper import Process
from modules.abstract_module import AbstractModule
from packages.Item import Item
# # TODO: # FIXME: improve regex / filter false positives
class Phone(AbstractModule):
"""
Phone module for AIL framework
@ -46,21 +46,21 @@ class Phone(AbstractModule):
def compute(self, message):
paste = Paste.Paste(message)
content = paste.get_p_content()
# List of the regex results in the Paste, may be null
item = Item(message)
content = item.get_content()
# List of the regex results in the Item, may be null
results = self.REG_PHONE.findall(content)
# If the list is greater than 4, we consider the Paste may contain a list of phone numbers
# If the list is greater than 4, we consider the Item may contain a list of phone numbers
if len(results) > 4:
self.redis_logger.debug(results)
self.redis_logger.warning(f'{paste.p_name} contains PID (phone numbers)')
self.redis_logger.warning(f'{item.get_id()} contains PID (phone numbers)')
msg = f'infoleak:automatic-detection="phone-number";{message}'
self.process.populate_set_out(msg, 'Tags')
msg = f'infoleak:automatic-detection="phone-number";{item.get_id()}'
self.send_message_to_queue(msg, 'Tags')
# Send to duplicate
self.process.populate_set_out(message, 'Duplicate')
self.send_message_to_queue(item.get_id(), 'Duplicate')
stats = {}
for phone_number in results:
@ -75,10 +75,10 @@ class Phone(AbstractModule):
pass
for country_code in stats:
if stats[country_code] > 4:
self.redis_logger.warning(f'{paste.p_name} contains Phone numbers with country code {country_code}')
self.redis_logger.warning(f'{item.get_id()} contains Phone numbers with country code {country_code}')
if __name__ == '__main__':
module = Phone()
module.run()

View file

@ -26,16 +26,14 @@ import calendar
import redis
import json
import signal
from pubsublogger import publisher
from nltk.sentiment.vader import SentimentIntensityAnalyzer
from nltk import tokenize, download
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from module.abstract_module import AbstractModule
from Helper import Process
from modules.abstract_module import AbstractModule
from packages import Paste
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
@ -49,13 +47,13 @@ def timeout_handler(signum, frame):
signal.signal(signal.SIGALRM, timeout_handler)
## TODO: REFACTOR MODULE + CLEAN HISTORY
class SentimentAnalysis(AbstractModule):
"""
SentimentAnalysis module for AIL framework
"""
# Config Variables
accepted_Mime_type = ['text/plain']
line_max_length_threshold = 1000

View file

@ -12,11 +12,14 @@ This module add tags to an item.
##################################
# Import External packages
##################################
import os
import sys
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from module.abstract_module import AbstractModule
from modules.abstract_module import AbstractModule
from packages.Item import Item
from packages import Tag

View file

@ -23,21 +23,18 @@ import time
# from sflock.main import unpack
# import sflock
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from module.abstract_module import AbstractModule
from Helper import Process
from pubsublogger import publisher
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
import Tag
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
from modules.abstract_module import AbstractModule
from packages import Tag
from lib import ConfigLoader
class SubmitPaste(AbstractModule):
"""
Company Credentials module for AIL framework
SubmitPaste module for AIL framework
"""
expire_time = 120

View file

@ -20,6 +20,7 @@ import Tracker
from flask import escape
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
import Date
import Item

View file

@ -26,7 +26,7 @@ publish = Redis_D4_client
[D4_client]
subscribe = Redis_D4_client
[TermTrackerMod]
[Tracker_Term]
subscribe = Redis_Global
publish = Redis_Tags

View file

@ -1,7 +1,7 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
"""
The TermTracker Module
The Tracker_Term Module
===================
"""
@ -14,12 +14,11 @@ import sys
import time
import signal
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from Helper import Process
from pubsublogger import publisher
from module.abstract_module import AbstractModule
from modules.abstract_module import AbstractModule
import NotificationHelper
from packages import Item
from packages import Term
@ -33,19 +32,19 @@ def timeout_handler(signum, frame):
signal.signal(signal.SIGALRM, timeout_handler)
class TermTrackerMod(AbstractModule):
class Tracker_Term(AbstractModule):
mail_body_template = "AIL Framework,\nNew occurrence for term tracked term: {}\nitem id: {}\nurl: {}{}"
mail_body_template = "AIL Framework,\nNew occurrence for tracked term: {}\nitem id: {}\nurl: {}{}"
"""
TermTrackerMod module for AIL framework
Tracker_Term module for AIL framework
"""
def __init__(self):
super(TermTrackerMod, self).__init__()
super(Tracker_Term, self).__init__()
self.pending_seconds = 5
self.max_execution_time = self.process.config.getint('TermTrackerMod', "max_execution_time")
self.max_execution_time = self.process.config.getint('Tracker_Term', "max_execution_time")
self.full_item_url = self.process.config.get("Notifications", "ail_domain") + "/object/item?id="
@ -55,8 +54,7 @@ class TermTrackerMod(AbstractModule):
self.set_tracked_words_list = Term.get_set_tracked_words_list()
self.last_refresh_set = time.time()
# Send module state to logs
self.redis_logger.info("Module %s initialized"%(self._module_name()))
self.redis_logger.info(f"Module: {self.module_name} Launched")
def compute(self, item_id):
@ -72,8 +70,9 @@ class TermTrackerMod(AbstractModule):
self.redis_logger.debug('Tracked set refreshed')
# Cast message as Item
item_date = Item.get_item_date(item_id)
item_content = Item.get_item_content(item_id)
item = Item(item_id)
item_date = item.get_date()
item_content = item.get_content()
signal.alarm(self.max_execution_time)
@ -81,7 +80,7 @@ class TermTrackerMod(AbstractModule):
try:
dict_words_freq = Term.get_text_word_frequency(item_content)
except TimeoutException:
self.redis_logger.warning("{0} processing timeout".format(item_id))
self.redis_logger.warning(f"{item.get_id()} processing timeout")
else:
signal.alarm(0)
@ -93,7 +92,7 @@ class TermTrackerMod(AbstractModule):
# check solo words
for word in self.list_tracked_words:
if word in dict_words_freq:
self.new_term_found(word, 'word', item_id, item_date)
self.new_term_found(word, 'word', item.get_id(), item_date)
# check words set
for elem in self.set_tracked_words_list:
@ -106,11 +105,12 @@ class TermTrackerMod(AbstractModule):
if word in dict_words_freq:
nb_uniq_word += 1
if nb_uniq_word >= nb_words_threshold:
self.new_term_found(word_set, 'set', item_id, item_date)
self.new_term_found(word_set, 'set', item.get_id(), item_date)
def new_term_found(self, term, term_type, item_id, item_date):
uuid_list = Term.get_term_uuid_list(term, term_type)
self.redis_logger.info('new tracked term found: {} in {}'.format(term, item_id))
self.redis_logger.info(f'new tracked term found: {term} in {item_id}')
print(f'new tracked term found: {term} in {item_id}')
for term_uuid in uuid_list:
Term.add_tracked_item(term_uuid, item_id, item_date)
@ -118,18 +118,19 @@ class TermTrackerMod(AbstractModule):
tags_to_add = Term.get_term_tags(term_uuid)
for tag in tags_to_add:
msg = '{};{}'.format(tag, item_id)
self.process.populate_set_out(msg, 'Tags')
self.send_message_to_queue(msg, 'Tags')
mail_to_notify = Term.get_term_mails(term_uuid)
if mail_to_notify:
mail_subject = Tracker.get_email_subject(term_uuid)
mail_body = TermTrackerMod.mail_body_template.format(term, item_id, self.full_item_url, item_id)
mail_body = Tracker_Term.mail_body_template.format(term, item_id, self.full_item_url, item_id)
for mail in mail_to_notify:
self.redis_logger.debug('Send Mail {}'.format(mail_subject))
self.redis_logger.debug(f'Send Mail {mail_subject}')
print(f'Send Mail {mail_subject}')
NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body)
if __name__ == '__main__':
module = TermTrackerMod()
module = Tracker_Term()
module.run()

View file

@ -1,87 +1,99 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
"""
Yara trackers
The Tracker_Yara trackers module
===================
"""
##################################
# Import External packages
##################################
import os
import re
import sys
import time
import yara
from pubsublogger import publisher
sys.path.append(os.environ['AIL_BIN'])
from Helper import Process
##################################
# Import Project packages
##################################
from modules.abstract_module import AbstractModule
from packages import Term
from packages.Item import Item
from lib import Tracker
import NotificationHelper # # TODO: refractor
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
import Term
class Tracker_Yara(AbstractModule):
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
import Tracker
import item_basic
mail_body_template = "AIL Framework,\nNew YARA match: {}\nitem id: {}\nurl: {}{}"
"""
Tracker_Yara module for AIL framework
"""
def __init__(self):
super(Tracker_Yara, self).__init__()
self.pending_seconds = 5
self.full_item_url = self.process.config.get("Notifications", "ail_domain") + "/object/item?id="
# Load Yara rules
self.rules = Tracker.reload_yara_rules()
self.last_refresh = time.time()
self.item = None
self.redis_logger.info(f"Module: {self.module_name} Launched")
full_item_url = "/object/item?id="
mail_body_template = "AIL Framework,\nNew YARA match: {}\nitem id: {}\nurl: {}{}"
last_refresh = time.time()
def yara_rules_match(data):
#print(data)
tracker_uuid = data['namespace']
item_date = item_basic.get_item_date(item_id)
Tracker.add_tracked_item(tracker_uuid, item_id, item_date)
# Tags
tags_to_add = Tracker.get_tracker_tags(tracker_uuid)
for tag in tags_to_add:
msg = '{};{}'.format(tag, item_id)
p.populate_set_out(msg, 'Tags')
# Mails
mail_to_notify = Tracker.get_tracker_mails(tracker_uuid)
if mail_to_notify:
mail_subject = Tracker.get_email_subject(tracker_uuid)
mail_body = mail_body_template.format(data['rule'], item_id, full_item_url, item_id)
for mail in mail_to_notify:
NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body)
return yara.CALLBACK_CONTINUE
if __name__ == "__main__":
publisher.port = 6380
publisher.channel = "Script"
publisher.info("Script Tracker_Yara started")
config_section = 'Tracker_Yara'
module_name = "Tracker_Yara"
p = Process(config_section)
full_item_url = p.config.get("Notifications", "ail_domain") + full_item_url
# Load Yara rules
rules = Tracker.reload_yara_rules()
# Regex Frequency
while True:
item_id = p.get_from_set()
if item_id is not None:
item_content = item_basic.get_item_content(item_id)
try:
yara_match = rules.match(data=item_content, callback=yara_rules_match, which_callbacks=yara.CALLBACK_MATCHES, timeout=60)
if yara_match:
print(f'{item_id}: {yara_match}')
except yara.TimeoutError as e:
print(f'{item_id}: yara scanning timed out')
else:
time.sleep(5)
def compute(self, item_id):
# refresh YARA list
if last_refresh < Tracker.get_tracker_last_updated_by_type('yara'):
rules = Tracker.reload_yara_rules()
last_refresh = time.time()
if self.last_refresh < Tracker.get_tracker_last_updated_by_type('yara'):
self.rules = Tracker.reload_yara_rules()
self.last_refresh = time.time()
self.redis_logger.debug('Tracked set refreshed')
print('Tracked set refreshed')
self.item = Item(item_id)
item_content = self.item.get_content()
try:
yara_match = self.rules.match(data=item_content, callback=self.yara_rules_match, which_callbacks=yara.CALLBACK_MATCHES, timeout=60)
if yara_match:
self.redis_logger.info(f'{self.item.get_id()}: {yara_match}')
print(f'{self.item.get_id()}: {yara_match}')
except yara.TimeoutError as e:
print(f'{self.item.get_id()}: yara scanning timed out')
self.redis_logger.info(f'{self.item.get_id()}: yara scanning timed out')
def yara_rules_match(self, data):
tracker_uuid = data['namespace']
item_id = self.item.get_id()
item_date = self.item.get_date()
Tracker.add_tracked_item(tracker_uuid, item_id, item_date)
# Tags
tags_to_add = Tracker.get_tracker_tags(tracker_uuid)
for tag in tags_to_add:
msg = '{};{}'.format(tag, item_id)
self.send_message_to_queue(msg, 'Tags')
# Mails
mail_to_notify = Tracker.get_tracker_mails(tracker_uuid)
if mail_to_notify:
mail_subject = Tracker.get_email_subject(tracker_uuid)
mail_body = Tracker_Yara.mail_body_template.format(data['rule'], item_id, self.full_item_url, item_id)
for mail in mail_to_notify:
self.redis_logger.debug(f'Send Mail {mail_subject}')
print(f'Send Mail {mail_subject}')
NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body)
return yara.CALLBACK_CONTINUE
if __name__ == '__main__':
module = Tracker_Yara()
module.run()

View file

@ -116,7 +116,7 @@ operation_mode = 3
ttl_duplicate = 86400
default_unnamed_feed_name = unnamed_feeder
[TermTrackerMod]
[Tracker_Term]
max_execution_time = 120
[RegexTracker]
@ -253,10 +253,6 @@ address = tcp://127.0.0.1:5556,tcp://crf.circl.lu:5556
channel = 102
bind = tcp://127.0.0.1:5556
[ZMQ_Url]
address = tcp://127.0.0.1:5004
channel = urls
[ZMQ_FetchedOnion]
address = tcp://127.0.0.1:5005
channel = FetchedOnion