From 5acb58370144faa622a83ea6d4d5290c112a96a8 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 2 Jun 2021 16:04:52 +0200 Subject: [PATCH] chg: [Tracker_Yara module] create module class --- bin/LAUNCH.sh | 114 +++++++++++------------- bin/modules/Phone.py | 36 ++++---- bin/modules/SentimentAnalysis.py | 10 +-- bin/modules/Tags.py | 5 +- bin/modules/submit_paste.py | 13 ++- bin/packages/Term.py | 1 + bin/packages/modules.cfg | 2 +- bin/trackers/Tracker_Term.py | 43 ++++----- bin/trackers/Tracker_Yara.py | 148 +++++++++++++++++-------------- configs/core.cfg.sample | 6 +- 10 files changed, 189 insertions(+), 189 deletions(-) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index a08f3b79..5cbf9344 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -147,7 +147,9 @@ function launching_scripts { sleep 0.1 echo -e $GREEN"\t* Launching scripts"$DEFAULT - # LAUNCH CORE MODULE + ################################## + # CORE MODULES # + ################################## screen -S "Script_AIL" -X screen -t "JSON_importer" bash -c "cd ${AIL_BIN}/import; ${ENV_PY} ./JSON_importer.py; read x" sleep 0.1 screen -S "Script_AIL" -X screen -t "Crawler_manager" bash -c "cd ${AIL_BIN}/core; ${ENV_PY} ./Crawler_manager.py; read x" @@ -157,57 +159,77 @@ function launching_scripts { screen -S "Script_AIL" -X screen -t "DbCleaner" bash -c "cd ${AIL_BIN}/core; ${ENV_PY} ./DbCleaner.py; read x" sleep 0.1 + ################################## + # MODULES # + ################################## + screen -S "Script_AIL" -X screen -t "Global" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Global.py; read x" + sleep 0.1 + screen -S "Script_AIL" -X screen -t "Categ" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Categ.py; read x" + sleep 0.1 + screen -S "Script_AIL" -X screen -t "Indexer" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Indexer.py; read x" + sleep 0.1 + screen -S "Script_AIL" -X screen -t "Tags" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Tags.py; read x" + sleep 0.1 + screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./submit_paste.py; read x" + sleep 0.1 + screen -S "Script_AIL" -X screen -t "ApiKey" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./ApiKey.py; read x" + sleep 0.1 + screen -S "Script_AIL" -X screen -t "Credential" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Credential.py; read x" + sleep 0.1 + screen -S "Script_AIL" -X screen -t "CreditCards" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./CreditCards.py; read x" + sleep 0.1 + screen -S "Script_AIL" -X screen -t "Decoder" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Decoder.py; read x" + sleep 0.1 + screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./DomClassifier.py; read x" + sleep 0.1 + screen -S "Script_AIL" -X screen -t "Keys" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Keys.py; read x" + sleep 0.1 + screen -S "Script_AIL" -X screen -t "Onion" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Onion.py; read x" + sleep 0.1 + + ################################## + # TRACKERS MODULES # + ################################## + screen -S "Script_AIL" -X screen -t "Tracker_Term" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Term.py; read x" + sleep 0.1 + + screen -S "Script_AIL" -X screen -t "Tracker_Yara" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Yara.py; read x" + sleep 0.1 + + ################################## + # DISABLED MODULES # + ################################## + #screen -S "Script_AIL" -X screen -t "Phone" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Phone.py; read x" + #sleep 0.1 + + ################################## + # # + ################################## screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModulesInformationV2.py -k 0 -c 1; read x" sleep 0.1 screen -S "Script_AIL" -X screen -t "Mixer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mixer.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Global" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Global.py; read x" - sleep 0.1 screen -S "Script_AIL" -X screen -t "Duplicates" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Duplicates.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./DomClassifier.py; read x" - sleep 0.1 - screen -S "Script_AIL" -X screen -t "Categ" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Categ.py; read x" - sleep 0.1 screen -S "Script_AIL" -X screen -t "CreditCards" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./CreditCards.py; read x" sleep 0.1 screen -S "Script_AIL" -X screen -t "BankAccount" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./BankAccount.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Onion" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Onion.py; read x" - sleep 0.1 screen -S "Script_AIL" -X screen -t "Mail" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mail.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "ApiKey" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ApiKey.py; read x" - sleep 0.1 screen -S "Script_AIL" -X screen -t "Urls" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Urls.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Credential" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Credential.py; read x" - sleep 0.1 - screen -S "Script_AIL" -X screen -t "TermTrackerMod" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./TermTrackerMod.py; read x" - sleep 0.1 screen -S "Script_AIL" -X screen -t "RegexTracker" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./RegexTracker.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Tracker_Yara" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Yara.py; read x" - sleep 0.1 - screen -S "Script_AIL" -X screen -t "Indexer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Indexer.py; read x" - sleep 0.1 - screen -S "Script_AIL" -X screen -t "Keys" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Keys.py; read x" - sleep 0.1 screen -S "Script_AIL" -X screen -t "PgpDump" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./PgpDump.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Decoder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Decoder.py; read x" - sleep 0.1 screen -S "Script_AIL" -X screen -t "Cryptocurrency" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Cryptocurrencies.py; read x" sleep 0.1 screen -S "Script_AIL" -X screen -t "Telegram" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Telegram.py; read x" sleep 0.1 screen -S "Script_AIL" -X screen -t "Tools" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Tools.py; read x" sleep 0.1 - #screen -S "Script_AIL" -X screen -t "Phone" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Phone.py; read x" - #sleep 0.1 - #screen -S "Script_AIL" -X screen -t "Release" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Release.py; read x" - #sleep 0.1 screen -S "Script_AIL" -X screen -t "Cve" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Cve.py; read x" sleep 0.1 screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModuleStats.py; read x" @@ -218,48 +240,18 @@ function launching_scripts { sleep 0.1 screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./MISP_The_Hive_feeder.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Tags" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Tags.py; read x" - sleep 0.1 screen -S "Script_AIL" -X screen -t "Languages" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Languages.py; read x" sleep 0.1 screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./SentimentAnalysis.py; read x" sleep 0.1 screen -S "Script_AIL" -X screen -t "UpdateBackground" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./update-background.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./submit_paste.py; read x" - sleep 0.1 screen -S "Script_AIL" -X screen -t "IPAddress" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./IPAddress.py; read x" -} + #screen -S "Script_AIL" -X screen -t "Release" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Release.py; read x" + #sleep 0.1 -# function launching_crawler { -# if [[ ! $iscrawler ]]; then -# CONFIG=$AIL_HOME/configs/core.cfg -# lport=$(awk '/^\[Crawler\]/{f=1} f==1&&/^splash_port/{print $3;exit}' "${CONFIG}") -# -# IFS='-' read -ra PORTS <<< "$lport" -# if [ ${#PORTS[@]} -eq 1 ] -# then -# first_port=${PORTS[0]} -# last_port=${PORTS[0]} -# else -# first_port=${PORTS[0]} -# last_port=${PORTS[1]} -# fi -# -# screen -dmS "Crawler_AIL" -# sleep 0.1 -# -# for ((i=first_port;i<=last_port;i++)); do -# screen -S "Crawler_AIL" -X screen -t "onion_crawler:$i" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Crawler.py $i; read x" -# sleep 0.1 -# done -# -# echo -e $GREEN"\t* Launching Crawler_AIL scripts"$DEFAULT -# else -# echo -e $RED"\t* A screen is already launched"$DEFAULT -# fi -# } +} function shutting_down_redis { redis_dir=${AIL_HOME}/redis/src/ @@ -491,7 +483,7 @@ function update_thirdparty { function launch_tests() { tests_dir=${AIL_HOME}/tests bin_dir=${AIL_BIN} - python3 `which nosetests` -w $tests_dir --with-coverage --cover-package=$bin_dir -d #--cover-erase + python3 `which nosetests` -w $tests_dir --with-coverage --cover-package=$bin_dir -d --cover-erase } function reset_password() { diff --git a/bin/modules/Phone.py b/bin/modules/Phone.py index 74a5a7cc..b918481a 100755 --- a/bin/modules/Phone.py +++ b/bin/modules/Phone.py @@ -7,27 +7,27 @@ The Phone Module This module is consuming the Redis-list created by the Categ module. -It apply phone number regexes on paste content and warn if above a threshold. +It apply phone number regexes on item content and warn if above a threshold. """ ################################## # Import External packages ################################## -import time +import os import re +import sys +import time import phonenumbers - +sys.path.append(os.environ['AIL_BIN']) ################################## # Import Project packages ################################## -from module.abstract_module import AbstractModule -from packages import Paste -from pubsublogger import publisher -from Helper import Process - +from modules.abstract_module import AbstractModule +from packages.Item import Item +# # TODO: # FIXME: improve regex / filter false positives class Phone(AbstractModule): """ Phone module for AIL framework @@ -46,21 +46,21 @@ class Phone(AbstractModule): def compute(self, message): - paste = Paste.Paste(message) - content = paste.get_p_content() - # List of the regex results in the Paste, may be null + item = Item(message) + content = item.get_content() + # List of the regex results in the Item, may be null results = self.REG_PHONE.findall(content) - # If the list is greater than 4, we consider the Paste may contain a list of phone numbers + # If the list is greater than 4, we consider the Item may contain a list of phone numbers if len(results) > 4: self.redis_logger.debug(results) - self.redis_logger.warning(f'{paste.p_name} contains PID (phone numbers)') + self.redis_logger.warning(f'{item.get_id()} contains PID (phone numbers)') - msg = f'infoleak:automatic-detection="phone-number";{message}' - self.process.populate_set_out(msg, 'Tags') + msg = f'infoleak:automatic-detection="phone-number";{item.get_id()}' + self.send_message_to_queue(msg, 'Tags') # Send to duplicate - self.process.populate_set_out(message, 'Duplicate') + self.send_message_to_queue(item.get_id(), 'Duplicate') stats = {} for phone_number in results: @@ -75,10 +75,10 @@ class Phone(AbstractModule): pass for country_code in stats: if stats[country_code] > 4: - self.redis_logger.warning(f'{paste.p_name} contains Phone numbers with country code {country_code}') + self.redis_logger.warning(f'{item.get_id()} contains Phone numbers with country code {country_code}') if __name__ == '__main__': - + module = Phone() module.run() diff --git a/bin/modules/SentimentAnalysis.py b/bin/modules/SentimentAnalysis.py index dcaebf00..ba8032a7 100755 --- a/bin/modules/SentimentAnalysis.py +++ b/bin/modules/SentimentAnalysis.py @@ -26,16 +26,14 @@ import calendar import redis import json import signal -from pubsublogger import publisher from nltk.sentiment.vader import SentimentIntensityAnalyzer from nltk import tokenize, download - +sys.path.append(os.environ['AIL_BIN']) ################################## # Import Project packages ################################## -from module.abstract_module import AbstractModule -from Helper import Process +from modules.abstract_module import AbstractModule from packages import Paste sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) import ConfigLoader @@ -49,13 +47,13 @@ def timeout_handler(signum, frame): signal.signal(signal.SIGALRM, timeout_handler) - +## TODO: REFACTOR MODULE + CLEAN HISTORY class SentimentAnalysis(AbstractModule): """ SentimentAnalysis module for AIL framework """ - + # Config Variables accepted_Mime_type = ['text/plain'] line_max_length_threshold = 1000 diff --git a/bin/modules/Tags.py b/bin/modules/Tags.py index 5bc68188..f21fa858 100755 --- a/bin/modules/Tags.py +++ b/bin/modules/Tags.py @@ -12,11 +12,14 @@ This module add tags to an item. ################################## # Import External packages ################################## +import os +import sys +sys.path.append(os.environ['AIL_BIN']) ################################## # Import Project packages ################################## -from module.abstract_module import AbstractModule +from modules.abstract_module import AbstractModule from packages.Item import Item from packages import Tag diff --git a/bin/modules/submit_paste.py b/bin/modules/submit_paste.py index ab688740..ddd76f4a 100755 --- a/bin/modules/submit_paste.py +++ b/bin/modules/submit_paste.py @@ -23,21 +23,18 @@ import time # from sflock.main import unpack # import sflock +sys.path.append(os.environ['AIL_BIN']) ################################## # Import Project packages ################################## -from module.abstract_module import AbstractModule -from Helper import Process -from pubsublogger import publisher -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) -import Tag -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import ConfigLoader +from modules.abstract_module import AbstractModule +from packages import Tag +from lib import ConfigLoader class SubmitPaste(AbstractModule): """ - Company Credentials module for AIL framework + SubmitPaste module for AIL framework """ expire_time = 120 diff --git a/bin/packages/Term.py b/bin/packages/Term.py index 773310c9..45b8d639 100755 --- a/bin/packages/Term.py +++ b/bin/packages/Term.py @@ -20,6 +20,7 @@ import Tracker from flask import escape +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) import Date import Item diff --git a/bin/packages/modules.cfg b/bin/packages/modules.cfg index c29ff926..83ced250 100644 --- a/bin/packages/modules.cfg +++ b/bin/packages/modules.cfg @@ -26,7 +26,7 @@ publish = Redis_D4_client [D4_client] subscribe = Redis_D4_client -[TermTrackerMod] +[Tracker_Term] subscribe = Redis_Global publish = Redis_Tags diff --git a/bin/trackers/Tracker_Term.py b/bin/trackers/Tracker_Term.py index ef00a6c8..88a53a0a 100755 --- a/bin/trackers/Tracker_Term.py +++ b/bin/trackers/Tracker_Term.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*-coding:UTF-8 -* """ -The TermTracker Module +The Tracker_Term Module =================== """ @@ -14,12 +14,11 @@ import sys import time import signal +sys.path.append(os.environ['AIL_BIN']) ################################## # Import Project packages ################################## -from Helper import Process -from pubsublogger import publisher -from module.abstract_module import AbstractModule +from modules.abstract_module import AbstractModule import NotificationHelper from packages import Item from packages import Term @@ -33,19 +32,19 @@ def timeout_handler(signum, frame): signal.signal(signal.SIGALRM, timeout_handler) -class TermTrackerMod(AbstractModule): +class Tracker_Term(AbstractModule): - mail_body_template = "AIL Framework,\nNew occurrence for term tracked term: {}\nitem id: {}\nurl: {}{}" + mail_body_template = "AIL Framework,\nNew occurrence for tracked term: {}\nitem id: {}\nurl: {}{}" """ - TermTrackerMod module for AIL framework + Tracker_Term module for AIL framework """ def __init__(self): - super(TermTrackerMod, self).__init__() + super(Tracker_Term, self).__init__() self.pending_seconds = 5 - self.max_execution_time = self.process.config.getint('TermTrackerMod', "max_execution_time") + self.max_execution_time = self.process.config.getint('Tracker_Term', "max_execution_time") self.full_item_url = self.process.config.get("Notifications", "ail_domain") + "/object/item?id=" @@ -55,8 +54,7 @@ class TermTrackerMod(AbstractModule): self.set_tracked_words_list = Term.get_set_tracked_words_list() self.last_refresh_set = time.time() - # Send module state to logs - self.redis_logger.info("Module %s initialized"%(self._module_name())) + self.redis_logger.info(f"Module: {self.module_name} Launched") def compute(self, item_id): @@ -72,8 +70,9 @@ class TermTrackerMod(AbstractModule): self.redis_logger.debug('Tracked set refreshed') # Cast message as Item - item_date = Item.get_item_date(item_id) - item_content = Item.get_item_content(item_id) + item = Item(item_id) + item_date = item.get_date() + item_content = item.get_content() signal.alarm(self.max_execution_time) @@ -81,7 +80,7 @@ class TermTrackerMod(AbstractModule): try: dict_words_freq = Term.get_text_word_frequency(item_content) except TimeoutException: - self.redis_logger.warning("{0} processing timeout".format(item_id)) + self.redis_logger.warning(f"{item.get_id()} processing timeout") else: signal.alarm(0) @@ -93,7 +92,7 @@ class TermTrackerMod(AbstractModule): # check solo words for word in self.list_tracked_words: if word in dict_words_freq: - self.new_term_found(word, 'word', item_id, item_date) + self.new_term_found(word, 'word', item.get_id(), item_date) # check words set for elem in self.set_tracked_words_list: @@ -106,11 +105,12 @@ class TermTrackerMod(AbstractModule): if word in dict_words_freq: nb_uniq_word += 1 if nb_uniq_word >= nb_words_threshold: - self.new_term_found(word_set, 'set', item_id, item_date) + self.new_term_found(word_set, 'set', item.get_id(), item_date) def new_term_found(self, term, term_type, item_id, item_date): uuid_list = Term.get_term_uuid_list(term, term_type) - self.redis_logger.info('new tracked term found: {} in {}'.format(term, item_id)) + self.redis_logger.info(f'new tracked term found: {term} in {item_id}') + print(f'new tracked term found: {term} in {item_id}') for term_uuid in uuid_list: Term.add_tracked_item(term_uuid, item_id, item_date) @@ -118,18 +118,19 @@ class TermTrackerMod(AbstractModule): tags_to_add = Term.get_term_tags(term_uuid) for tag in tags_to_add: msg = '{};{}'.format(tag, item_id) - self.process.populate_set_out(msg, 'Tags') + self.send_message_to_queue(msg, 'Tags') mail_to_notify = Term.get_term_mails(term_uuid) if mail_to_notify: mail_subject = Tracker.get_email_subject(term_uuid) - mail_body = TermTrackerMod.mail_body_template.format(term, item_id, self.full_item_url, item_id) + mail_body = Tracker_Term.mail_body_template.format(term, item_id, self.full_item_url, item_id) for mail in mail_to_notify: - self.redis_logger.debug('Send Mail {}'.format(mail_subject)) + self.redis_logger.debug(f'Send Mail {mail_subject}') + print(f'Send Mail {mail_subject}') NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body) if __name__ == '__main__': - module = TermTrackerMod() + module = Tracker_Term() module.run() diff --git a/bin/trackers/Tracker_Yara.py b/bin/trackers/Tracker_Yara.py index b0356b55..8aa4269c 100755 --- a/bin/trackers/Tracker_Yara.py +++ b/bin/trackers/Tracker_Yara.py @@ -1,87 +1,99 @@ #!/usr/bin/env python3 # -*-coding:UTF-8 -* """ -Yara trackers +The Tracker_Yara trackers module +=================== """ + +################################## +# Import External packages +################################## import os import re import sys import time import yara -from pubsublogger import publisher - sys.path.append(os.environ['AIL_BIN']) -from Helper import Process +################################## +# Import Project packages +################################## +from modules.abstract_module import AbstractModule +from packages import Term +from packages.Item import Item +from lib import Tracker + import NotificationHelper # # TODO: refractor -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages')) -import Term +class Tracker_Yara(AbstractModule): -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) -import Tracker -import item_basic + mail_body_template = "AIL Framework,\nNew YARA match: {}\nitem id: {}\nurl: {}{}" + + """ + Tracker_Yara module for AIL framework + """ + def __init__(self): + super(Tracker_Yara, self).__init__() + self.pending_seconds = 5 + + self.full_item_url = self.process.config.get("Notifications", "ail_domain") + "/object/item?id=" + + # Load Yara rules + self.rules = Tracker.reload_yara_rules() + self.last_refresh = time.time() + + self.item = None + + self.redis_logger.info(f"Module: {self.module_name} Launched") -full_item_url = "/object/item?id=" -mail_body_template = "AIL Framework,\nNew YARA match: {}\nitem id: {}\nurl: {}{}" - -last_refresh = time.time() - -def yara_rules_match(data): - #print(data) - tracker_uuid = data['namespace'] - - item_date = item_basic.get_item_date(item_id) - Tracker.add_tracked_item(tracker_uuid, item_id, item_date) - - # Tags - tags_to_add = Tracker.get_tracker_tags(tracker_uuid) - for tag in tags_to_add: - msg = '{};{}'.format(tag, item_id) - p.populate_set_out(msg, 'Tags') - - # Mails - mail_to_notify = Tracker.get_tracker_mails(tracker_uuid) - if mail_to_notify: - mail_subject = Tracker.get_email_subject(tracker_uuid) - mail_body = mail_body_template.format(data['rule'], item_id, full_item_url, item_id) - for mail in mail_to_notify: - NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body) - - return yara.CALLBACK_CONTINUE - -if __name__ == "__main__": - publisher.port = 6380 - publisher.channel = "Script" - publisher.info("Script Tracker_Yara started") - - config_section = 'Tracker_Yara' - module_name = "Tracker_Yara" - p = Process(config_section) - - full_item_url = p.config.get("Notifications", "ail_domain") + full_item_url - - # Load Yara rules - rules = Tracker.reload_yara_rules() - - # Regex Frequency - while True: - item_id = p.get_from_set() - if item_id is not None: - item_content = item_basic.get_item_content(item_id) - try: - yara_match = rules.match(data=item_content, callback=yara_rules_match, which_callbacks=yara.CALLBACK_MATCHES, timeout=60) - if yara_match: - print(f'{item_id}: {yara_match}') - except yara.TimeoutError as e: - print(f'{item_id}: yara scanning timed out') - else: - time.sleep(5) - + def compute(self, item_id): # refresh YARA list - if last_refresh < Tracker.get_tracker_last_updated_by_type('yara'): - rules = Tracker.reload_yara_rules() - last_refresh = time.time() + if self.last_refresh < Tracker.get_tracker_last_updated_by_type('yara'): + self.rules = Tracker.reload_yara_rules() + self.last_refresh = time.time() + self.redis_logger.debug('Tracked set refreshed') print('Tracked set refreshed') + + self.item = Item(item_id) + item_content = self.item.get_content() + try: + yara_match = self.rules.match(data=item_content, callback=self.yara_rules_match, which_callbacks=yara.CALLBACK_MATCHES, timeout=60) + if yara_match: + self.redis_logger.info(f'{self.item.get_id()}: {yara_match}') + print(f'{self.item.get_id()}: {yara_match}') + except yara.TimeoutError as e: + print(f'{self.item.get_id()}: yara scanning timed out') + self.redis_logger.info(f'{self.item.get_id()}: yara scanning timed out') + + def yara_rules_match(self, data): + tracker_uuid = data['namespace'] + + item_id = self.item.get_id() + item_date = self.item.get_date() + Tracker.add_tracked_item(tracker_uuid, item_id, item_date) + + # Tags + tags_to_add = Tracker.get_tracker_tags(tracker_uuid) + for tag in tags_to_add: + msg = '{};{}'.format(tag, item_id) + self.send_message_to_queue(msg, 'Tags') + + # Mails + mail_to_notify = Tracker.get_tracker_mails(tracker_uuid) + if mail_to_notify: + mail_subject = Tracker.get_email_subject(tracker_uuid) + mail_body = Tracker_Yara.mail_body_template.format(data['rule'], item_id, self.full_item_url, item_id) + for mail in mail_to_notify: + self.redis_logger.debug(f'Send Mail {mail_subject}') + print(f'Send Mail {mail_subject}') + NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body) + + return yara.CALLBACK_CONTINUE + + +if __name__ == '__main__': + + module = Tracker_Yara() + module.run() diff --git a/configs/core.cfg.sample b/configs/core.cfg.sample index 658e63a9..89d6d22d 100644 --- a/configs/core.cfg.sample +++ b/configs/core.cfg.sample @@ -116,7 +116,7 @@ operation_mode = 3 ttl_duplicate = 86400 default_unnamed_feed_name = unnamed_feeder -[TermTrackerMod] +[Tracker_Term] max_execution_time = 120 [RegexTracker] @@ -253,10 +253,6 @@ address = tcp://127.0.0.1:5556,tcp://crf.circl.lu:5556 channel = 102 bind = tcp://127.0.0.1:5556 -[ZMQ_Url] -address = tcp://127.0.0.1:5004 -channel = urls - [ZMQ_FetchedOnion] address = tcp://127.0.0.1:5005 channel = FetchedOnion