chg: [Tracker_Yara module] create module class

bin/LAUNCH.sh

@@ -147,7 +147,9 @@ function launching_scripts {
     sleep 0.1
     echo -e $GREEN"\t* Launching scripts"$DEFAULT
 
-    # LAUNCH CORE MODULE
+    ##################################
+    #         CORE MODULES           #
+    ##################################
     screen -S "Script_AIL" -X screen -t "JSON_importer" bash -c "cd ${AIL_BIN}/import; ${ENV_PY} ./JSON_importer.py; read x"
     sleep 0.1
     screen -S "Script_AIL" -X screen -t "Crawler_manager" bash -c "cd ${AIL_BIN}/core; ${ENV_PY} ./Crawler_manager.py; read x"
@@ -157,57 +159,77 @@ function launching_scripts {
     screen -S "Script_AIL" -X screen -t "DbCleaner" bash -c "cd ${AIL_BIN}/core; ${ENV_PY} ./DbCleaner.py; read x"
     sleep 0.1
 
+    ##################################
+    #           MODULES              #
+    ##################################
+    screen -S "Script_AIL" -X screen -t "Global" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Global.py; read x"
+    sleep 0.1
+    screen -S "Script_AIL" -X screen -t "Categ" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Categ.py; read x"
+    sleep 0.1
+    screen -S "Script_AIL" -X screen -t "Indexer" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Indexer.py; read x"
+    sleep 0.1
+    screen -S "Script_AIL" -X screen -t "Tags" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Tags.py; read x"
+    sleep 0.1
+    screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./submit_paste.py; read x"
+    sleep 0.1
 
+    screen -S "Script_AIL" -X screen -t "ApiKey" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./ApiKey.py; read x"
+    sleep 0.1
+    screen -S "Script_AIL" -X screen -t "Credential" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Credential.py; read x"
+    sleep 0.1
+    screen -S "Script_AIL" -X screen -t "CreditCards" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./CreditCards.py; read x"
+    sleep 0.1
+    screen -S "Script_AIL" -X screen -t "Decoder" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Decoder.py; read x"
+    sleep 0.1
+    screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./DomClassifier.py; read x"
+    sleep 0.1
+    screen -S "Script_AIL" -X screen -t "Keys" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Keys.py; read x"
+    sleep 0.1
+    screen -S "Script_AIL" -X screen -t "Onion" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Onion.py; read x"
+    sleep 0.1
+
+    ##################################
+    #       TRACKERS MODULES         #
+    ##################################
+    screen -S "Script_AIL" -X screen -t "Tracker_Term" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Term.py; read x"
+    sleep 0.1
+
+    screen -S "Script_AIL" -X screen -t "Tracker_Yara" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Yara.py; read x"
+    sleep 0.1
+
+    ##################################
+    #       DISABLED MODULES         #
+    ##################################
+    #screen -S "Script_AIL" -X screen -t "Phone" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Phone.py; read x"
+    #sleep 0.1
+
+    ##################################
+    #                                #
+    ##################################
     screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModulesInformationV2.py -k 0 -c 1; read x"
     sleep 0.1
     screen -S "Script_AIL" -X screen -t "Mixer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mixer.py; read x"
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Global" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Global.py; read x"
-    sleep 0.1
     screen -S "Script_AIL" -X screen -t "Duplicates" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Duplicates.py; read x"
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./DomClassifier.py; read x"
-    sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Categ" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Categ.py; read x"
-    sleep 0.1
     screen -S "Script_AIL" -X screen -t "CreditCards" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./CreditCards.py; read x"
     sleep 0.1
     screen -S "Script_AIL" -X screen -t "BankAccount" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./BankAccount.py; read x"
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Onion" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Onion.py; read x"
-    sleep 0.1
     screen -S "Script_AIL" -X screen -t "Mail" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mail.py; read x"
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "ApiKey" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ApiKey.py; read x"
-    sleep 0.1
     screen -S "Script_AIL" -X screen -t "Urls" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Urls.py; read x"
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Credential" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Credential.py; read x"
-    sleep 0.1
-    screen -S "Script_AIL" -X screen -t "TermTrackerMod" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./TermTrackerMod.py; read x"
-    sleep 0.1
     screen -S "Script_AIL" -X screen -t "RegexTracker" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./RegexTracker.py; read x"
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Tracker_Yara" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Yara.py; read x"
-    sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Indexer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Indexer.py; read x"
-    sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Keys" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Keys.py; read x"
-    sleep 0.1
     screen -S "Script_AIL" -X screen -t "PgpDump" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./PgpDump.py; read x"
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Decoder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Decoder.py; read x"
-    sleep 0.1
     screen -S "Script_AIL" -X screen -t "Cryptocurrency" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Cryptocurrencies.py; read x"
     sleep 0.1
     screen -S "Script_AIL" -X screen -t "Telegram" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Telegram.py; read x"
     sleep 0.1
     screen -S "Script_AIL" -X screen -t "Tools" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Tools.py; read x"
     sleep 0.1
-    #screen -S "Script_AIL" -X screen -t "Phone" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Phone.py; read x"
-    #sleep 0.1
-    #screen -S "Script_AIL" -X screen -t "Release" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Release.py; read x"
-    #sleep 0.1
     screen -S "Script_AIL" -X screen -t "Cve" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Cve.py; read x"
     sleep 0.1
     screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModuleStats.py; read x"
@@ -218,48 +240,18 @@ function launching_scripts {
     sleep 0.1
     screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./MISP_The_Hive_feeder.py; read x"
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Tags" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Tags.py; read x"
-    sleep 0.1
     screen -S "Script_AIL" -X screen -t "Languages" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Languages.py; read x"
     sleep 0.1
     screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./SentimentAnalysis.py; read x"
     sleep 0.1
     screen -S "Script_AIL" -X screen -t "UpdateBackground" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./update-background.py; read x"
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./submit_paste.py; read x"
-    sleep 0.1
     screen -S "Script_AIL" -X screen -t "IPAddress" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./IPAddress.py; read x"
 
-}
+    #screen -S "Script_AIL" -X screen -t "Release" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Release.py; read x"
+    #sleep 0.1
 
-# function launching_crawler {
+}
-#     if [[ ! $iscrawler ]]; then
-#         CONFIG=$AIL_HOME/configs/core.cfg
-#         lport=$(awk '/^\[Crawler\]/{f=1} f==1&&/^splash_port/{print $3;exit}' "${CONFIG}")
-#
-#         IFS='-' read -ra PORTS <<< "$lport"
-#         if [ ${#PORTS[@]} -eq 1 ]
-#         then
-#             first_port=${PORTS[0]}
-#             last_port=${PORTS[0]}
-#         else
-#             first_port=${PORTS[0]}
-#             last_port=${PORTS[1]}
-#         fi
-#
-#         screen -dmS "Crawler_AIL"
-#         sleep 0.1
-#
-#         for ((i=first_port;i<=last_port;i++)); do
-#             screen -S "Crawler_AIL" -X screen -t "onion_crawler:$i" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Crawler.py $i; read x"
-#             sleep 0.1
-#         done
-#
-#         echo -e $GREEN"\t* Launching Crawler_AIL scripts"$DEFAULT
-#     else
-#         echo -e $RED"\t* A screen is already launched"$DEFAULT
-#     fi
-# }
 
 function shutting_down_redis {
     redis_dir=${AIL_HOME}/redis/src/
@@ -491,7 +483,7 @@ function update_thirdparty {
 function launch_tests() {
   tests_dir=${AIL_HOME}/tests
   bin_dir=${AIL_BIN}
-  python3 `which nosetests` -w $tests_dir --with-coverage --cover-package=$bin_dir -d #--cover-erase
+  python3 `which nosetests` -w $tests_dir --with-coverage --cover-package=$bin_dir -d --cover-erase
 }
 
 function reset_password() {

bin/modules/Phone.py

@@ -7,27 +7,27 @@ The Phone Module
 
 This module is consuming the Redis-list created by the Categ module.
 
-It apply phone number regexes on paste content and warn if above a threshold.
+It apply phone number regexes on item content and warn if above a threshold.
 
 """
 
 ##################################
 # Import External packages
 ##################################
-import time
+import os
 import re
+import sys
+import time
 import phonenumbers
 
-
+sys.path.append(os.environ['AIL_BIN'])
 ##################################
 # Import Project packages
 ##################################
-from module.abstract_module import AbstractModule
+from modules.abstract_module import AbstractModule
-from packages import Paste
+from packages.Item import Item
-from pubsublogger import publisher
-from Helper import Process
-
 
+# # TODO: # FIXME:  improve regex / filter false positives
 class Phone(AbstractModule):
     """
     Phone module for AIL framework
@@ -46,21 +46,21 @@ class Phone(AbstractModule):
 
 
     def compute(self, message):
-        paste = Paste.Paste(message)
+        item = Item(message)
-        content = paste.get_p_content()
+        content = item.get_content()
-        # List of the regex results in the Paste, may be null
+        # List of the regex results in the Item, may be null
         results = self.REG_PHONE.findall(content)
 
-        # If the list is greater than 4, we consider the Paste may contain a list of phone numbers
+        # If the list is greater than 4, we consider the Item may contain a list of phone numbers
         if len(results) > 4:
             self.redis_logger.debug(results)
-            self.redis_logger.warning(f'{paste.p_name} contains PID (phone numbers)')
+            self.redis_logger.warning(f'{item.get_id()} contains PID (phone numbers)')
 
-            msg = f'infoleak:automatic-detection="phone-number";{message}'
+            msg = f'infoleak:automatic-detection="phone-number";{item.get_id()}'
-            self.process.populate_set_out(msg, 'Tags')
+            self.send_message_to_queue(msg, 'Tags')
 
             # Send to duplicate
-            self.process.populate_set_out(message, 'Duplicate')
+            self.send_message_to_queue(item.get_id(), 'Duplicate')
 
             stats = {}
             for phone_number in results:
@@ -75,7 +75,7 @@ class Phone(AbstractModule):
                     pass
             for country_code in stats:
                 if stats[country_code] > 4:
-                    self.redis_logger.warning(f'{paste.p_name} contains Phone numbers with country code {country_code}')
+                    self.redis_logger.warning(f'{item.get_id()} contains Phone numbers with country code {country_code}')
 
 
 if __name__ == '__main__':

bin/modules/SentimentAnalysis.py

@@ -26,16 +26,14 @@ import calendar
 import redis
 import json
 import signal
-from pubsublogger import publisher
 from nltk.sentiment.vader import SentimentIntensityAnalyzer
 from nltk import tokenize, download
 
-
+sys.path.append(os.environ['AIL_BIN'])
 ##################################
 # Import Project packages
 ##################################
-from module.abstract_module import AbstractModule
+from modules.abstract_module import AbstractModule
-from Helper import Process
 from packages import Paste
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
 import ConfigLoader
@@ -49,7 +47,7 @@ def timeout_handler(signum, frame):
 
 signal.signal(signal.SIGALRM, timeout_handler)
 
-
+## TODO: REFACTOR MODULE + CLEAN HISTORY
 class SentimentAnalysis(AbstractModule):
     """
     SentimentAnalysis module for AIL framework

bin/modules/Tags.py

@@ -12,11 +12,14 @@ This module add tags to an item.
 ##################################
 # Import External packages
 ##################################
+import os
+import sys
 
+sys.path.append(os.environ['AIL_BIN'])
 ##################################
 # Import Project packages
 ##################################
-from module.abstract_module import AbstractModule
+from modules.abstract_module import AbstractModule
 from packages.Item import Item
 from packages import Tag
 

bin/modules/submit_paste.py

@@ -23,21 +23,18 @@ import time
 # from sflock.main import unpack
 # import sflock
 
+sys.path.append(os.environ['AIL_BIN'])
 ##################################
 # Import Project packages
 ##################################
-from module.abstract_module import AbstractModule
+from modules.abstract_module import AbstractModule
-from Helper import Process
+from packages import Tag
-from pubsublogger import publisher
+from lib import ConfigLoader
-sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
-import Tag
-sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
-import ConfigLoader
 
 
 class SubmitPaste(AbstractModule):
     """
-    Company Credentials module for AIL framework
+    SubmitPaste module for AIL framework
     """
 
     expire_time = 120

bin/packages/Term.py

@@ -20,6 +20,7 @@ import Tracker
 
 from flask import escape
 
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
 import Date
 import Item
 

bin/packages/modules.cfg

@@ -26,7 +26,7 @@ publish = Redis_D4_client
 [D4_client]
 subscribe = Redis_D4_client
 
-[TermTrackerMod]
+[Tracker_Term]
 subscribe = Redis_Global
 publish = Redis_Tags
 

bin/trackers/Tracker_Term.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # -*-coding:UTF-8 -*
 """
-The TermTracker Module
+The Tracker_Term Module
 ===================
 
 """
@@ -14,12 +14,11 @@ import sys
 import time
 import signal
 
+sys.path.append(os.environ['AIL_BIN'])
 ##################################
 # Import Project packages
 ##################################
-from Helper import Process
+from modules.abstract_module import AbstractModule
-from pubsublogger import publisher
-from module.abstract_module import AbstractModule
 import NotificationHelper
 from packages import Item
 from packages import Term
@@ -33,19 +32,19 @@ def timeout_handler(signum, frame):
 signal.signal(signal.SIGALRM, timeout_handler)
 
 
-class TermTrackerMod(AbstractModule):
+class Tracker_Term(AbstractModule):
 
-    mail_body_template = "AIL Framework,\nNew occurrence for term tracked term: {}\nitem id: {}\nurl: {}{}"
+    mail_body_template = "AIL Framework,\nNew occurrence for tracked term: {}\nitem id: {}\nurl: {}{}"
 
     """
-    TermTrackerMod module for AIL framework
+    Tracker_Term module for AIL framework
     """
     def __init__(self):
-        super(TermTrackerMod, self).__init__()
+        super(Tracker_Term, self).__init__()
 
         self.pending_seconds = 5
 
-        self.max_execution_time = self.process.config.getint('TermTrackerMod', "max_execution_time")
+        self.max_execution_time = self.process.config.getint('Tracker_Term', "max_execution_time")
 
         self.full_item_url = self.process.config.get("Notifications", "ail_domain") + "/object/item?id="
 
@@ -55,8 +54,7 @@ class TermTrackerMod(AbstractModule):
         self.set_tracked_words_list = Term.get_set_tracked_words_list()
         self.last_refresh_set = time.time()
 
-        # Send module state to logs
+        self.redis_logger.info(f"Module: {self.module_name} Launched")
-        self.redis_logger.info("Module %s initialized"%(self._module_name()))
 
 
     def compute(self, item_id):
@@ -72,8 +70,9 @@ class TermTrackerMod(AbstractModule):
             self.redis_logger.debug('Tracked set refreshed')
 
         # Cast message as Item
-        item_date = Item.get_item_date(item_id)
+        item = Item(item_id)
-        item_content = Item.get_item_content(item_id)
+        item_date = item.get_date()
+        item_content = item.get_content()
 
         signal.alarm(self.max_execution_time)
 
@@ -81,7 +80,7 @@ class TermTrackerMod(AbstractModule):
         try:
             dict_words_freq = Term.get_text_word_frequency(item_content)
         except TimeoutException:
-            self.redis_logger.warning("{0} processing timeout".format(item_id))
+            self.redis_logger.warning(f"{item.get_id()} processing timeout")
         else:
             signal.alarm(0)
 
@@ -93,7 +92,7 @@ class TermTrackerMod(AbstractModule):
             # check solo words
             for word in self.list_tracked_words:
                 if word in dict_words_freq:
-                    self.new_term_found(word, 'word', item_id, item_date)
+                    self.new_term_found(word, 'word', item.get_id(), item_date)
 
                 # check words set
                 for elem in self.set_tracked_words_list:
@@ -106,11 +105,12 @@ class TermTrackerMod(AbstractModule):
                         if word in dict_words_freq:
                             nb_uniq_word += 1
                     if nb_uniq_word >= nb_words_threshold:
-                        self.new_term_found(word_set, 'set', item_id, item_date)
+                        self.new_term_found(word_set, 'set', item.get_id(), item_date)
 
     def new_term_found(self, term, term_type, item_id, item_date):
         uuid_list = Term.get_term_uuid_list(term, term_type)
-        self.redis_logger.info('new tracked term found: {} in {}'.format(term, item_id))
+        self.redis_logger.info(f'new tracked term found: {term} in {item_id}')
+        print(f'new tracked term found: {term} in {item_id}')
 
         for term_uuid in uuid_list:
             Term.add_tracked_item(term_uuid, item_id, item_date)
@@ -118,18 +118,19 @@ class TermTrackerMod(AbstractModule):
             tags_to_add = Term.get_term_tags(term_uuid)
             for tag in tags_to_add:
                 msg = '{};{}'.format(tag, item_id)
-                self.process.populate_set_out(msg, 'Tags')
+                self.send_message_to_queue(msg, 'Tags')
 
             mail_to_notify = Term.get_term_mails(term_uuid)
             if mail_to_notify:
                 mail_subject = Tracker.get_email_subject(term_uuid)
-                mail_body = TermTrackerMod.mail_body_template.format(term, item_id, self.full_item_url, item_id)
+                mail_body = Tracker_Term.mail_body_template.format(term, item_id, self.full_item_url, item_id)
             for mail in mail_to_notify:
-                self.redis_logger.debug('Send Mail {}'.format(mail_subject))
+                self.redis_logger.debug(f'Send Mail {mail_subject}')
+                print(f'Send Mail {mail_subject}')
                 NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body)
 
 
 if __name__ == '__main__':
 
-    module = TermTrackerMod()
+    module = Tracker_Term()
     module.run()

bin/trackers/Tracker_Yara.py

@@ -1,87 +1,99 @@
 #!/usr/bin/env python3
 # -*-coding:UTF-8 -*
 """
-Yara trackers
+The Tracker_Yara trackers module
+===================
 
 """
+
+##################################
+# Import External packages
+##################################
 import os
 import re
 import sys
 import time
 import yara
 
-from pubsublogger import publisher
-
 sys.path.append(os.environ['AIL_BIN'])
-from Helper import Process
+##################################
+# Import Project packages
+##################################
+from modules.abstract_module import AbstractModule
+from packages import Term
+from packages.Item import Item
+from lib import Tracker
+
 import NotificationHelper # # TODO: refractor
 
-sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
+class Tracker_Yara(AbstractModule):
-import Term
 
-sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
+    mail_body_template = "AIL Framework,\nNew YARA match: {}\nitem id: {}\nurl: {}{}"
-import Tracker
+
-import item_basic
+    """
+    Tracker_Yara module for AIL framework
+    """
+    def __init__(self):
+        super(Tracker_Yara, self).__init__()
+        self.pending_seconds = 5
+
+        self.full_item_url = self.process.config.get("Notifications", "ail_domain") + "/object/item?id="
+
+        # Load Yara rules
+        self.rules = Tracker.reload_yara_rules()
+        self.last_refresh = time.time()
+
+        self.item = None
+
+        self.redis_logger.info(f"Module: {self.module_name} Launched")
 
 
-full_item_url = "/object/item?id="
+    def compute(self, item_id):
-mail_body_template = "AIL Framework,\nNew YARA match: {}\nitem id: {}\nurl: {}{}"
+        # refresh YARA list
+        if self.last_refresh < Tracker.get_tracker_last_updated_by_type('yara'):
+            self.rules = Tracker.reload_yara_rules()
+            self.last_refresh = time.time()
+            self.redis_logger.debug('Tracked set refreshed')
+            print('Tracked set refreshed')
 
-last_refresh = time.time()
+        self.item = Item(item_id)
+        item_content = self.item.get_content()
+        try:
+            yara_match = self.rules.match(data=item_content, callback=self.yara_rules_match, which_callbacks=yara.CALLBACK_MATCHES, timeout=60)
+            if yara_match:
+                self.redis_logger.info(f'{self.item.get_id()}: {yara_match}')
+                print(f'{self.item.get_id()}: {yara_match}')
+        except yara.TimeoutError as e:
+            print(f'{self.item.get_id()}: yara scanning timed out')
+            self.redis_logger.info(f'{self.item.get_id()}: yara scanning timed out')
 
-def yara_rules_match(data):
+    def yara_rules_match(self, data):
-    #print(data)
         tracker_uuid = data['namespace']
 
-    item_date = item_basic.get_item_date(item_id)
+        item_id = self.item.get_id()
+        item_date = self.item.get_date()
         Tracker.add_tracked_item(tracker_uuid, item_id, item_date)
 
         # Tags
         tags_to_add = Tracker.get_tracker_tags(tracker_uuid)
         for tag in tags_to_add:
             msg = '{};{}'.format(tag, item_id)
-        p.populate_set_out(msg, 'Tags')
+            self.send_message_to_queue(msg, 'Tags')
 
         # Mails
         mail_to_notify = Tracker.get_tracker_mails(tracker_uuid)
         if mail_to_notify:
             mail_subject = Tracker.get_email_subject(tracker_uuid)
-        mail_body = mail_body_template.format(data['rule'], item_id, full_item_url, item_id)
+            mail_body = Tracker_Yara.mail_body_template.format(data['rule'], item_id, self.full_item_url, item_id)
         for mail in mail_to_notify:
+            self.redis_logger.debug(f'Send Mail {mail_subject}')
+            print(f'Send Mail {mail_subject}')
             NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body)
 
         return yara.CALLBACK_CONTINUE
 
-if __name__ == "__main__":
-    publisher.port = 6380
-    publisher.channel = "Script"
-    publisher.info("Script Tracker_Yara started")
 
-    config_section = 'Tracker_Yara'
+if __name__ == '__main__':
-    module_name = "Tracker_Yara"
-    p = Process(config_section)
 
-    full_item_url = p.config.get("Notifications", "ail_domain") + full_item_url
+    module = Tracker_Yara()
-
+    module.run()
-    # Load Yara rules
-    rules = Tracker.reload_yara_rules()
-
-    # Regex Frequency
-    while True:
-        item_id = p.get_from_set()
-        if item_id is not None:
-            item_content = item_basic.get_item_content(item_id)
-            try:
-                yara_match = rules.match(data=item_content, callback=yara_rules_match, which_callbacks=yara.CALLBACK_MATCHES, timeout=60)
-                if yara_match:
-                    print(f'{item_id}: {yara_match}')
-            except yara.TimeoutError as e:
-                print(f'{item_id}: yara scanning timed out')
-        else:
-            time.sleep(5)
-
-        # refresh YARA list
-        if last_refresh < Tracker.get_tracker_last_updated_by_type('yara'):
-            rules = Tracker.reload_yara_rules()
-            last_refresh = time.time()
-            print('Tracked set refreshed')

configs/core.cfg.sample

@@ -116,7 +116,7 @@ operation_mode = 3
 ttl_duplicate = 86400
 default_unnamed_feed_name = unnamed_feeder
 
-[TermTrackerMod]
+[Tracker_Term]
 max_execution_time = 120
 
 [RegexTracker]
@@ -253,10 +253,6 @@ address = tcp://127.0.0.1:5556,tcp://crf.circl.lu:5556
 channel = 102
 bind = tcp://127.0.0.1:5556
 
-[ZMQ_Url]
-address = tcp://127.0.0.1:5004
-channel = urls
-
 [ZMQ_FetchedOnion]
 address = tcp://127.0.0.1:5005
 channel = FetchedOnion