mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 00:28:22 +00:00
chg: [Trackers regex + yara] filter by item source
This commit is contained in:
parent
2abe5217aa
commit
56e670077a
3 changed files with 52 additions and 26 deletions
|
@ -140,7 +140,7 @@ def add_tracked_item(tracker_uuid, item_id, item_date):
|
|||
# track nb item by date
|
||||
if res == 1:
|
||||
r_serv_tracker.zadd('tracker:stat:{}'.format(tracker_uuid), item_date, int(item_date))
|
||||
bin/lib/Tracker.py
|
||||
|
||||
def get_email_subject(tracker_uuid):
|
||||
tracker_description = get_tracker_description(tracker_uuid)
|
||||
if not tracker_description:
|
||||
|
@ -553,11 +553,15 @@ if __name__ == '__main__':
|
|||
#res = is_valid_yara_rule('rule dummy { }')
|
||||
|
||||
# res = create_tracker('test', 'word', 'admin@admin.test', 1, [], [], None, sources=['crawled', 'pastebin.com', 'rt/pastebin.com'])
|
||||
res = create_tracker('test', 'word', 'admin@admin.test', 1, [], [], None)
|
||||
# print(res)
|
||||
|
||||
t_uuid = '1c2d35b0-9330-4feb-b454-da13007aa9f7'
|
||||
res = get_tracker_sources('test', 'word')
|
||||
|
||||
|
||||
res = create_tracker('circl\.lu', 'regex', 'admin@admin.test', 1, [], [], None, sources=['crawled','pastebin.com'])
|
||||
print(res)
|
||||
|
||||
#t_uuid = '1c2d35b0-9330-4feb-b454-da13007aa9f7'
|
||||
#res = get_tracker_sources('ail-yara-rules/rules/crypto/certificate.yar', 'yara')
|
||||
|
||||
# sys.path.append(os.environ['AIL_BIN'])
|
||||
# from packages import Term
|
||||
# Term.delete_term('074ab4be-6049-45b5-a20e-8125a4e4f500')
|
||||
|
||||
|
||||
#print(res)
|
||||
|
|
|
@ -27,7 +27,7 @@ import NotificationHelper
|
|||
|
||||
class Tracker_Regex(AbstractModule):
|
||||
|
||||
mail_body_template = "AIL Framework,\nNew occurrence for term tracked regex: {}\nitem id: {}\nurl: {}{}"
|
||||
mail_body_template = "AIL Framework,\nNew occurrence for tracked regex: {}\nitem id: {}\nurl: {}{}"
|
||||
|
||||
"""
|
||||
Tracker_Regex module for AIL framework
|
||||
|
@ -43,7 +43,7 @@ class Tracker_Regex(AbstractModule):
|
|||
|
||||
self.redis_cache_key = regex_helper.generate_redis_cache_key(self.module_name)
|
||||
|
||||
# refresh Tracked term
|
||||
# refresh Tracked Regex
|
||||
self.dict_regex_tracked = Term.get_regex_tracked_words_dict()
|
||||
self.last_refresh = time.time()
|
||||
|
||||
|
@ -51,42 +51,54 @@ class Tracker_Regex(AbstractModule):
|
|||
|
||||
def compute(self, item_id):
|
||||
# refresh Tracked regex
|
||||
if self.last_refresh < Term.get_tracked_term_last_updated_by_type('regex'):
|
||||
if self.last_refresh < Tracker.get_tracker_last_updated_by_type('regex'):
|
||||
self.dict_regex_tracked = Term.get_regex_tracked_words_dict()
|
||||
self.last_refresh = time.time()
|
||||
self.redis_logger.debug('Tracked word refreshed')
|
||||
print('Tracked set refreshed')
|
||||
self.redis_logger.debug('Tracked regex refreshed')
|
||||
print('Tracked regex refreshed')
|
||||
|
||||
item = Item(item_id)
|
||||
item_id = item.get_id()
|
||||
item_date = item.get_date()
|
||||
item_content = item.get_content()
|
||||
|
||||
for regex in self.dict_regex_tracked:
|
||||
matched = regex_helper.regex_search(self.module_name, self.redis_cache_key, self.dict_regex_tracked[regex], item_id, item_content, max_time=self.max_execution_time)
|
||||
if matched:
|
||||
self.new_term_found(regex, 'regex', item_id, item_date)
|
||||
self.new_tracker_found(regex, 'regex', item)
|
||||
|
||||
def new_term_found(self, term, tracker_type, item_id, item_date):
|
||||
uuid_list = Term.get_term_uuid_list(term, tracker_type)
|
||||
print('new tracked regex found: {} in {}'.format(term, item_id))
|
||||
def new_tracker_found(self, tracker, tracker_type, item):
|
||||
uuid_list = Tracker.get_tracker_uuid_list(tracker, tracker_type)
|
||||
|
||||
item_id = item.get_id()
|
||||
print(f'new tracked regex found: {tracker} in {item_id}')
|
||||
|
||||
for tracker_uuid in uuid_list:
|
||||
Term.add_tracked_item(tracker_uuid, item_id, item_date)
|
||||
# Source Filtering
|
||||
item_source = item.get_source()
|
||||
tracker_sources = Tracker.get_tracker_uuid_sources(tracker_uuid)
|
||||
if tracker_sources and item_source not in tracker_sources:
|
||||
continue
|
||||
|
||||
tags_to_add = Term.get_term_tags(tracker_uuid)
|
||||
item_date = item.get_date()
|
||||
|
||||
Tracker.add_tracked_item(tracker_uuid, item_id, item_date)
|
||||
|
||||
tags_to_add = Tracker.get_tracker_tags(tracker_uuid)
|
||||
for tag in tags_to_add:
|
||||
msg = '{};{}'.format(tag, item_id)
|
||||
msg = f'{tag};{item_id}'
|
||||
self.send_message_to_queue(msg, 'Tags')
|
||||
|
||||
mail_to_notify = Term.get_term_mails(tracker_uuid)
|
||||
mail_to_notify = Tracker.get_tracker_mails(tracker_uuid)
|
||||
if mail_to_notify:
|
||||
mail_subject = Tracker.get_email_subject(tracker_uuid)
|
||||
mail_body = Tracker_Regex.mail_body_template.format(term, item_id, self.full_item_url, item_id)
|
||||
mail_body = Tracker_Regex.mail_body_template.format(tracker, item_id, self.full_item_url, item_id)
|
||||
for mail in mail_to_notify:
|
||||
NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body)
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
module = Tracker_Regex()
|
||||
module.run()
|
||||
#module.run()
|
||||
|
||||
id = 'submitted/2020/06/29/516c4161-e305-4a89-978f-729f2ec05df8.gz'
|
||||
module.compute(id)
|
||||
|
|
|
@ -69,8 +69,15 @@ class Tracker_Yara(AbstractModule):
|
|||
|
||||
def yara_rules_match(self, data):
|
||||
tracker_uuid = data['namespace']
|
||||
|
||||
item_id = self.item.get_id()
|
||||
item_source = self.item.get_source()
|
||||
|
||||
# Source Filtering
|
||||
tracker_sources = Tracker.get_tracker_uuid_sources(tracker_uuid)
|
||||
if tracker_sources and item_source not in tracker_sources:
|
||||
print(f'Source Filtering: {data["rule"]}')
|
||||
return yara.CALLBACK_CONTINUE
|
||||
|
||||
item_date = self.item.get_date()
|
||||
Tracker.add_tracked_item(tracker_uuid, item_id, item_date)
|
||||
|
||||
|
@ -96,4 +103,7 @@ class Tracker_Yara(AbstractModule):
|
|||
if __name__ == '__main__':
|
||||
|
||||
module = Tracker_Yara()
|
||||
module.run()
|
||||
#module.run()
|
||||
|
||||
id = 'crawled/2020/09/14/circl.lu9bde82e5-a4de-487c-bc29-7601f0922b46'
|
||||
module.compute(id)
|
||||
|
|
Loading…
Reference in a new issue