From 56e670077acf53038638b1f69080fdf9e04f24b2 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 17 Jun 2021 14:48:26 +0200 Subject: [PATCH] chg: [Trackers regex + yara] filter by item source --- bin/lib/Tracker.py | 20 +++++++++------- bin/trackers/Tracker_Regex.py | 44 ++++++++++++++++++++++------------- bin/trackers/Tracker_Yara.py | 14 +++++++++-- 3 files changed, 52 insertions(+), 26 deletions(-) diff --git a/bin/lib/Tracker.py b/bin/lib/Tracker.py index 1e722dd8..44221474 100755 --- a/bin/lib/Tracker.py +++ b/bin/lib/Tracker.py @@ -140,7 +140,7 @@ def add_tracked_item(tracker_uuid, item_id, item_date): # track nb item by date if res == 1: r_serv_tracker.zadd('tracker:stat:{}'.format(tracker_uuid), item_date, int(item_date)) -bin/lib/Tracker.py + def get_email_subject(tracker_uuid): tracker_description = get_tracker_description(tracker_uuid) if not tracker_description: @@ -553,11 +553,15 @@ if __name__ == '__main__': #res = is_valid_yara_rule('rule dummy { }') # res = create_tracker('test', 'word', 'admin@admin.test', 1, [], [], None, sources=['crawled', 'pastebin.com', 'rt/pastebin.com']) - res = create_tracker('test', 'word', 'admin@admin.test', 1, [], [], None) - # print(res) - - t_uuid = '1c2d35b0-9330-4feb-b454-da13007aa9f7' - res = get_tracker_sources('test', 'word') - - + res = create_tracker('circl\.lu', 'regex', 'admin@admin.test', 1, [], [], None, sources=['crawled','pastebin.com']) print(res) + + #t_uuid = '1c2d35b0-9330-4feb-b454-da13007aa9f7' + #res = get_tracker_sources('ail-yara-rules/rules/crypto/certificate.yar', 'yara') + + # sys.path.append(os.environ['AIL_BIN']) + # from packages import Term + # Term.delete_term('074ab4be-6049-45b5-a20e-8125a4e4f500') + + + #print(res) diff --git a/bin/trackers/Tracker_Regex.py b/bin/trackers/Tracker_Regex.py index 82343661..a5b047f3 100755 --- a/bin/trackers/Tracker_Regex.py +++ b/bin/trackers/Tracker_Regex.py @@ -27,7 +27,7 @@ import NotificationHelper class Tracker_Regex(AbstractModule): - mail_body_template = "AIL Framework,\nNew occurrence for term tracked regex: {}\nitem id: {}\nurl: {}{}" + mail_body_template = "AIL Framework,\nNew occurrence for tracked regex: {}\nitem id: {}\nurl: {}{}" """ Tracker_Regex module for AIL framework @@ -43,7 +43,7 @@ class Tracker_Regex(AbstractModule): self.redis_cache_key = regex_helper.generate_redis_cache_key(self.module_name) - # refresh Tracked term + # refresh Tracked Regex self.dict_regex_tracked = Term.get_regex_tracked_words_dict() self.last_refresh = time.time() @@ -51,42 +51,54 @@ class Tracker_Regex(AbstractModule): def compute(self, item_id): # refresh Tracked regex - if self.last_refresh < Term.get_tracked_term_last_updated_by_type('regex'): + if self.last_refresh < Tracker.get_tracker_last_updated_by_type('regex'): self.dict_regex_tracked = Term.get_regex_tracked_words_dict() self.last_refresh = time.time() - self.redis_logger.debug('Tracked word refreshed') - print('Tracked set refreshed') + self.redis_logger.debug('Tracked regex refreshed') + print('Tracked regex refreshed') item = Item(item_id) item_id = item.get_id() - item_date = item.get_date() item_content = item.get_content() for regex in self.dict_regex_tracked: matched = regex_helper.regex_search(self.module_name, self.redis_cache_key, self.dict_regex_tracked[regex], item_id, item_content, max_time=self.max_execution_time) if matched: - self.new_term_found(regex, 'regex', item_id, item_date) + self.new_tracker_found(regex, 'regex', item) - def new_term_found(self, term, tracker_type, item_id, item_date): - uuid_list = Term.get_term_uuid_list(term, tracker_type) - print('new tracked regex found: {} in {}'.format(term, item_id)) + def new_tracker_found(self, tracker, tracker_type, item): + uuid_list = Tracker.get_tracker_uuid_list(tracker, tracker_type) + + item_id = item.get_id() + print(f'new tracked regex found: {tracker} in {item_id}') for tracker_uuid in uuid_list: - Term.add_tracked_item(tracker_uuid, item_id, item_date) + # Source Filtering + item_source = item.get_source() + tracker_sources = Tracker.get_tracker_uuid_sources(tracker_uuid) + if tracker_sources and item_source not in tracker_sources: + continue - tags_to_add = Term.get_term_tags(tracker_uuid) + item_date = item.get_date() + + Tracker.add_tracked_item(tracker_uuid, item_id, item_date) + + tags_to_add = Tracker.get_tracker_tags(tracker_uuid) for tag in tags_to_add: - msg = '{};{}'.format(tag, item_id) + msg = f'{tag};{item_id}' self.send_message_to_queue(msg, 'Tags') - mail_to_notify = Term.get_term_mails(tracker_uuid) + mail_to_notify = Tracker.get_tracker_mails(tracker_uuid) if mail_to_notify: mail_subject = Tracker.get_email_subject(tracker_uuid) - mail_body = Tracker_Regex.mail_body_template.format(term, item_id, self.full_item_url, item_id) + mail_body = Tracker_Regex.mail_body_template.format(tracker, item_id, self.full_item_url, item_id) for mail in mail_to_notify: NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body) if __name__ == "__main__": module = Tracker_Regex() - module.run() + #module.run() + + id = 'submitted/2020/06/29/516c4161-e305-4a89-978f-729f2ec05df8.gz' + module.compute(id) diff --git a/bin/trackers/Tracker_Yara.py b/bin/trackers/Tracker_Yara.py index 8aa4269c..af16e2a0 100755 --- a/bin/trackers/Tracker_Yara.py +++ b/bin/trackers/Tracker_Yara.py @@ -69,8 +69,15 @@ class Tracker_Yara(AbstractModule): def yara_rules_match(self, data): tracker_uuid = data['namespace'] - item_id = self.item.get_id() + item_source = self.item.get_source() + + # Source Filtering + tracker_sources = Tracker.get_tracker_uuid_sources(tracker_uuid) + if tracker_sources and item_source not in tracker_sources: + print(f'Source Filtering: {data["rule"]}') + return yara.CALLBACK_CONTINUE + item_date = self.item.get_date() Tracker.add_tracked_item(tracker_uuid, item_id, item_date) @@ -96,4 +103,7 @@ class Tracker_Yara(AbstractModule): if __name__ == '__main__': module = Tracker_Yara() - module.run() + #module.run() + + id = 'crawled/2020/09/14/circl.lu9bde82e5-a4de-487c-bc29-7601f0922b46' + module.compute(id)