chg: [Trackers regex + yara] filter by item source

bin/lib/Tracker.py

@@ -140,7 +140,7 @@ def add_tracked_item(tracker_uuid, item_id, item_date):
     # track nb item by date
     if res == 1:
         r_serv_tracker.zadd('tracker:stat:{}'.format(tracker_uuid), item_date, int(item_date))
-bin/lib/Tracker.py
+
 def get_email_subject(tracker_uuid):
     tracker_description = get_tracker_description(tracker_uuid)
     if not tracker_description:
@@ -553,11 +553,15 @@ if __name__ == '__main__':
     #res = is_valid_yara_rule('rule dummy {  }')
 
     # res = create_tracker('test', 'word', 'admin@admin.test', 1, [], [], None, sources=['crawled', 'pastebin.com', 'rt/pastebin.com'])
-    res = create_tracker('test', 'word', 'admin@admin.test', 1, [], [], None)
+    res = create_tracker('circl\.lu', 'regex', 'admin@admin.test', 1, [], [], None, sources=['crawled','pastebin.com'])
-    # print(res)
-
-    t_uuid = '1c2d35b0-9330-4feb-b454-da13007aa9f7'
-    res = get_tracker_sources('test', 'word')
-
-
     print(res)
+
+    #t_uuid = '1c2d35b0-9330-4feb-b454-da13007aa9f7'
+    #res = get_tracker_sources('ail-yara-rules/rules/crypto/certificate.yar', 'yara')
+
+    # sys.path.append(os.environ['AIL_BIN'])
+    # from packages import Term
+    # Term.delete_term('074ab4be-6049-45b5-a20e-8125a4e4f500')
+
+
+    #print(res)

bin/trackers/Tracker_Regex.py

@@ -27,7 +27,7 @@ import NotificationHelper
 
 class Tracker_Regex(AbstractModule):
 
-    mail_body_template = "AIL Framework,\nNew occurrence for term tracked regex: {}\nitem id: {}\nurl: {}{}"
+    mail_body_template = "AIL Framework,\nNew occurrence for tracked regex: {}\nitem id: {}\nurl: {}{}"
 
     """
     Tracker_Regex module for AIL framework
@@ -43,7 +43,7 @@ class Tracker_Regex(AbstractModule):
 
         self.redis_cache_key = regex_helper.generate_redis_cache_key(self.module_name)
 
-        # refresh Tracked term
+        # refresh Tracked Regex
         self.dict_regex_tracked = Term.get_regex_tracked_words_dict()
         self.last_refresh = time.time()
 
@@ -51,42 +51,54 @@ class Tracker_Regex(AbstractModule):
 
     def compute(self, item_id):
         # refresh Tracked regex
-        if self.last_refresh < Term.get_tracked_term_last_updated_by_type('regex'):
+        if self.last_refresh < Tracker.get_tracker_last_updated_by_type('regex'):
             self.dict_regex_tracked = Term.get_regex_tracked_words_dict()
             self.last_refresh = time.time()
-            self.redis_logger.debug('Tracked word refreshed')
+            self.redis_logger.debug('Tracked regex refreshed')
-            print('Tracked set refreshed')
+            print('Tracked regex refreshed')
 
         item = Item(item_id)
         item_id = item.get_id()
-        item_date = item.get_date()
         item_content = item.get_content()
 
         for regex in self.dict_regex_tracked:
             matched = regex_helper.regex_search(self.module_name, self.redis_cache_key, self.dict_regex_tracked[regex], item_id, item_content, max_time=self.max_execution_time)
             if matched:
-                self.new_term_found(regex, 'regex', item_id, item_date)
+                self.new_tracker_found(regex, 'regex', item)
 
-    def new_term_found(self, term, tracker_type, item_id, item_date):
+    def new_tracker_found(self, tracker, tracker_type, item):
-        uuid_list = Term.get_term_uuid_list(term, tracker_type)
+        uuid_list = Tracker.get_tracker_uuid_list(tracker, tracker_type)
-        print('new tracked regex found: {} in {}'.format(term, item_id))
+
+        item_id = item.get_id()
+        print(f'new tracked regex found: {tracker} in {item_id}')
 
         for tracker_uuid in uuid_list:
-            Term.add_tracked_item(tracker_uuid, item_id, item_date)
+            # Source Filtering
+            item_source =  item.get_source()
+            tracker_sources = Tracker.get_tracker_uuid_sources(tracker_uuid)
+            if tracker_sources and item_source not in tracker_sources:
+                continue
 
-            tags_to_add = Term.get_term_tags(tracker_uuid)
+            item_date = item.get_date()
+
+            Tracker.add_tracked_item(tracker_uuid, item_id, item_date)
+
+            tags_to_add = Tracker.get_tracker_tags(tracker_uuid)
             for tag in tags_to_add:
-                msg = '{};{}'.format(tag, item_id)
+                msg = f'{tag};{item_id}'
                 self.send_message_to_queue(msg, 'Tags')
 
-            mail_to_notify = Term.get_term_mails(tracker_uuid)
+            mail_to_notify = Tracker.get_tracker_mails(tracker_uuid)
             if mail_to_notify:
                 mail_subject = Tracker.get_email_subject(tracker_uuid)
-                mail_body = Tracker_Regex.mail_body_template.format(term, item_id, self.full_item_url, item_id)
+                mail_body = Tracker_Regex.mail_body_template.format(tracker, item_id, self.full_item_url, item_id)
             for mail in mail_to_notify:
                 NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body)
 
 if __name__ == "__main__":
 
     module = Tracker_Regex()
-    module.run()
+    #module.run()
+
+    id = 'submitted/2020/06/29/516c4161-e305-4a89-978f-729f2ec05df8.gz'
+    module.compute(id)

bin/trackers/Tracker_Yara.py

@@ -69,8 +69,15 @@ class Tracker_Yara(AbstractModule):
 
     def yara_rules_match(self, data):
         tracker_uuid = data['namespace']
-
         item_id = self.item.get_id()
+        item_source = self.item.get_source()
+
+        # Source Filtering
+        tracker_sources = Tracker.get_tracker_uuid_sources(tracker_uuid)
+        if tracker_sources and item_source not in tracker_sources:
+            print(f'Source Filtering: {data["rule"]}')
+            return yara.CALLBACK_CONTINUE
+
         item_date = self.item.get_date()
         Tracker.add_tracked_item(tracker_uuid, item_id, item_date)
 
@@ -96,4 +103,7 @@ class Tracker_Yara(AbstractModule):
 if __name__ == '__main__':
 
     module = Tracker_Yara()
-    module.run()
+    #module.run()
+
+    id = 'crawled/2020/09/14/circl.lu9bde82e5-a4de-487c-bc29-7601f0922b46'
+    module.compute(id)