chg: [Trackers regex + yara] filter by item source

This commit is contained in:
Terrtia 2021-06-17 14:48:26 +02:00
parent 2abe5217aa
commit 56e670077a
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
3 changed files with 52 additions and 26 deletions

View file

@ -140,7 +140,7 @@ def add_tracked_item(tracker_uuid, item_id, item_date):
# track nb item by date
if res == 1:
r_serv_tracker.zadd('tracker:stat:{}'.format(tracker_uuid), item_date, int(item_date))
bin/lib/Tracker.py
def get_email_subject(tracker_uuid):
tracker_description = get_tracker_description(tracker_uuid)
if not tracker_description:
@ -553,11 +553,15 @@ if __name__ == '__main__':
#res = is_valid_yara_rule('rule dummy { }')
# res = create_tracker('test', 'word', 'admin@admin.test', 1, [], [], None, sources=['crawled', 'pastebin.com', 'rt/pastebin.com'])
res = create_tracker('test', 'word', 'admin@admin.test', 1, [], [], None)
# print(res)
t_uuid = '1c2d35b0-9330-4feb-b454-da13007aa9f7'
res = get_tracker_sources('test', 'word')
res = create_tracker('circl\.lu', 'regex', 'admin@admin.test', 1, [], [], None, sources=['crawled','pastebin.com'])
print(res)
#t_uuid = '1c2d35b0-9330-4feb-b454-da13007aa9f7'
#res = get_tracker_sources('ail-yara-rules/rules/crypto/certificate.yar', 'yara')
# sys.path.append(os.environ['AIL_BIN'])
# from packages import Term
# Term.delete_term('074ab4be-6049-45b5-a20e-8125a4e4f500')
#print(res)

View file

@ -27,7 +27,7 @@ import NotificationHelper
class Tracker_Regex(AbstractModule):
mail_body_template = "AIL Framework,\nNew occurrence for term tracked regex: {}\nitem id: {}\nurl: {}{}"
mail_body_template = "AIL Framework,\nNew occurrence for tracked regex: {}\nitem id: {}\nurl: {}{}"
"""
Tracker_Regex module for AIL framework
@ -43,7 +43,7 @@ class Tracker_Regex(AbstractModule):
self.redis_cache_key = regex_helper.generate_redis_cache_key(self.module_name)
# refresh Tracked term
# refresh Tracked Regex
self.dict_regex_tracked = Term.get_regex_tracked_words_dict()
self.last_refresh = time.time()
@ -51,42 +51,54 @@ class Tracker_Regex(AbstractModule):
def compute(self, item_id):
# refresh Tracked regex
if self.last_refresh < Term.get_tracked_term_last_updated_by_type('regex'):
if self.last_refresh < Tracker.get_tracker_last_updated_by_type('regex'):
self.dict_regex_tracked = Term.get_regex_tracked_words_dict()
self.last_refresh = time.time()
self.redis_logger.debug('Tracked word refreshed')
print('Tracked set refreshed')
self.redis_logger.debug('Tracked regex refreshed')
print('Tracked regex refreshed')
item = Item(item_id)
item_id = item.get_id()
item_date = item.get_date()
item_content = item.get_content()
for regex in self.dict_regex_tracked:
matched = regex_helper.regex_search(self.module_name, self.redis_cache_key, self.dict_regex_tracked[regex], item_id, item_content, max_time=self.max_execution_time)
if matched:
self.new_term_found(regex, 'regex', item_id, item_date)
self.new_tracker_found(regex, 'regex', item)
def new_term_found(self, term, tracker_type, item_id, item_date):
uuid_list = Term.get_term_uuid_list(term, tracker_type)
print('new tracked regex found: {} in {}'.format(term, item_id))
def new_tracker_found(self, tracker, tracker_type, item):
uuid_list = Tracker.get_tracker_uuid_list(tracker, tracker_type)
item_id = item.get_id()
print(f'new tracked regex found: {tracker} in {item_id}')
for tracker_uuid in uuid_list:
Term.add_tracked_item(tracker_uuid, item_id, item_date)
# Source Filtering
item_source = item.get_source()
tracker_sources = Tracker.get_tracker_uuid_sources(tracker_uuid)
if tracker_sources and item_source not in tracker_sources:
continue
tags_to_add = Term.get_term_tags(tracker_uuid)
item_date = item.get_date()
Tracker.add_tracked_item(tracker_uuid, item_id, item_date)
tags_to_add = Tracker.get_tracker_tags(tracker_uuid)
for tag in tags_to_add:
msg = '{};{}'.format(tag, item_id)
msg = f'{tag};{item_id}'
self.send_message_to_queue(msg, 'Tags')
mail_to_notify = Term.get_term_mails(tracker_uuid)
mail_to_notify = Tracker.get_tracker_mails(tracker_uuid)
if mail_to_notify:
mail_subject = Tracker.get_email_subject(tracker_uuid)
mail_body = Tracker_Regex.mail_body_template.format(term, item_id, self.full_item_url, item_id)
mail_body = Tracker_Regex.mail_body_template.format(tracker, item_id, self.full_item_url, item_id)
for mail in mail_to_notify:
NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body)
if __name__ == "__main__":
module = Tracker_Regex()
module.run()
#module.run()
id = 'submitted/2020/06/29/516c4161-e305-4a89-978f-729f2ec05df8.gz'
module.compute(id)

View file

@ -69,8 +69,15 @@ class Tracker_Yara(AbstractModule):
def yara_rules_match(self, data):
tracker_uuid = data['namespace']
item_id = self.item.get_id()
item_source = self.item.get_source()
# Source Filtering
tracker_sources = Tracker.get_tracker_uuid_sources(tracker_uuid)
if tracker_sources and item_source not in tracker_sources:
print(f'Source Filtering: {data["rule"]}')
return yara.CALLBACK_CONTINUE
item_date = self.item.get_date()
Tracker.add_tracked_item(tracker_uuid, item_id, item_date)
@ -96,4 +103,7 @@ class Tracker_Yara(AbstractModule):
if __name__ == '__main__':
module = Tracker_Yara()
module.run()
#module.run()
id = 'crawled/2020/09/14/circl.lu9bde82e5-a4de-487c-bc29-7601f0922b46'
module.compute(id)