mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 08:38:28 +00:00
chg: [Tracker_Regex module] create module class + mv module
This commit is contained in:
parent
5acb583701
commit
2f52bdb9af
6 changed files with 98 additions and 94 deletions
|
@ -193,7 +193,8 @@ function launching_scripts {
|
||||||
##################################
|
##################################
|
||||||
screen -S "Script_AIL" -X screen -t "Tracker_Term" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Term.py; read x"
|
screen -S "Script_AIL" -X screen -t "Tracker_Term" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Term.py; read x"
|
||||||
sleep 0.1
|
sleep 0.1
|
||||||
|
screen -S "Script_AIL" -X screen -t "Tracker_Regex" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Regex.py; read x"
|
||||||
|
sleep 0.1
|
||||||
screen -S "Script_AIL" -X screen -t "Tracker_Yara" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Yara.py; read x"
|
screen -S "Script_AIL" -X screen -t "Tracker_Yara" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Yara.py; read x"
|
||||||
sleep 0.1
|
sleep 0.1
|
||||||
|
|
||||||
|
@ -212,16 +213,12 @@ function launching_scripts {
|
||||||
sleep 0.1
|
sleep 0.1
|
||||||
screen -S "Script_AIL" -X screen -t "Duplicates" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Duplicates.py; read x"
|
screen -S "Script_AIL" -X screen -t "Duplicates" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Duplicates.py; read x"
|
||||||
sleep 0.1
|
sleep 0.1
|
||||||
screen -S "Script_AIL" -X screen -t "CreditCards" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./CreditCards.py; read x"
|
|
||||||
sleep 0.1
|
|
||||||
screen -S "Script_AIL" -X screen -t "BankAccount" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./BankAccount.py; read x"
|
screen -S "Script_AIL" -X screen -t "BankAccount" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./BankAccount.py; read x"
|
||||||
sleep 0.1
|
sleep 0.1
|
||||||
screen -S "Script_AIL" -X screen -t "Mail" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mail.py; read x"
|
screen -S "Script_AIL" -X screen -t "Mail" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mail.py; read x"
|
||||||
sleep 0.1
|
sleep 0.1
|
||||||
screen -S "Script_AIL" -X screen -t "Urls" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Urls.py; read x"
|
screen -S "Script_AIL" -X screen -t "Urls" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Urls.py; read x"
|
||||||
sleep 0.1
|
sleep 0.1
|
||||||
screen -S "Script_AIL" -X screen -t "RegexTracker" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./RegexTracker.py; read x"
|
|
||||||
sleep 0.1
|
|
||||||
screen -S "Script_AIL" -X screen -t "PgpDump" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./PgpDump.py; read x"
|
screen -S "Script_AIL" -X screen -t "PgpDump" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./PgpDump.py; read x"
|
||||||
sleep 0.1
|
sleep 0.1
|
||||||
screen -S "Script_AIL" -X screen -t "Cryptocurrency" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Cryptocurrencies.py; read x"
|
screen -S "Script_AIL" -X screen -t "Cryptocurrency" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Cryptocurrencies.py; read x"
|
||||||
|
|
|
@ -1,87 +0,0 @@
|
||||||
#!/usr/bin/env python3
|
|
||||||
# -*-coding:UTF-8 -*
|
|
||||||
"""
|
|
||||||
This Module is used for regex tracking.
|
|
||||||
It processes every paste coming from the global module and test the regexs
|
|
||||||
supplied in the term webpage.
|
|
||||||
|
|
||||||
"""
|
|
||||||
import os
|
|
||||||
import re
|
|
||||||
import sys
|
|
||||||
import time
|
|
||||||
|
|
||||||
from Helper import Process
|
|
||||||
from pubsublogger import publisher
|
|
||||||
|
|
||||||
import NotificationHelper
|
|
||||||
|
|
||||||
from packages import Item
|
|
||||||
from packages import Term
|
|
||||||
|
|
||||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
|
|
||||||
import Tracker
|
|
||||||
import regex_helper
|
|
||||||
|
|
||||||
full_item_url = "/object/item?id="
|
|
||||||
mail_body_template = "AIL Framework,\nNew occurrence for term tracked regex: {}\nitem id: {}\nurl: {}{}"
|
|
||||||
|
|
||||||
dict_regex_tracked = Term.get_regex_tracked_words_dict()
|
|
||||||
last_refresh = time.time()
|
|
||||||
|
|
||||||
def new_term_found(term, term_type, item_id, item_date):
|
|
||||||
uuid_list = Term.get_term_uuid_list(term, 'regex')
|
|
||||||
print('new tracked term found: {} in {}'.format(term, item_id))
|
|
||||||
|
|
||||||
for term_uuid in uuid_list:
|
|
||||||
Term.add_tracked_item(term_uuid, item_id, item_date)
|
|
||||||
|
|
||||||
tags_to_add = Term.get_term_tags(term_uuid)
|
|
||||||
for tag in tags_to_add:
|
|
||||||
msg = '{};{}'.format(tag, item_id)
|
|
||||||
p.populate_set_out(msg, 'Tags')
|
|
||||||
|
|
||||||
mail_to_notify = Term.get_term_mails(term_uuid)
|
|
||||||
if mail_to_notify:
|
|
||||||
mail_subject = Tracker.get_email_subject(term_uuid)
|
|
||||||
mail_body = mail_body_template.format(term, item_id, full_item_url, item_id)
|
|
||||||
for mail in mail_to_notify:
|
|
||||||
NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body)
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
publisher.port = 6380
|
|
||||||
publisher.channel = "Script"
|
|
||||||
publisher.info("Script RegexTracker started")
|
|
||||||
|
|
||||||
config_section = 'RegexTracker'
|
|
||||||
module_name = "RegexTracker"
|
|
||||||
p = Process(config_section)
|
|
||||||
max_execution_time = p.config.getint(config_section, "max_execution_time")
|
|
||||||
|
|
||||||
full_item_url = p.config.get("Notifications", "ail_domain") + full_item_url
|
|
||||||
|
|
||||||
redis_cache_key = regex_helper.generate_redis_cache_key(module_name)
|
|
||||||
|
|
||||||
# Regex Frequency
|
|
||||||
while True:
|
|
||||||
|
|
||||||
item_id = p.get_from_set()
|
|
||||||
|
|
||||||
if item_id is not None:
|
|
||||||
|
|
||||||
item_date = Item.get_item_date(item_id)
|
|
||||||
item_content = Item.get_item_content(item_id)
|
|
||||||
|
|
||||||
for regex in dict_regex_tracked:
|
|
||||||
matched = regex_helper.regex_search(module_name, redis_cache_key, dict_regex_tracked[regex], item_id, item_content, max_time=max_execution_time)
|
|
||||||
if matched:
|
|
||||||
new_term_found(regex, 'regex', item_id, item_date)
|
|
||||||
|
|
||||||
else:
|
|
||||||
time.sleep(5)
|
|
||||||
|
|
||||||
# refresh Tracked term
|
|
||||||
if last_refresh < Term.get_tracked_term_last_updated_by_type('regex'):
|
|
||||||
dict_regex_tracked = Term.get_regex_tracked_words_dict()
|
|
||||||
last_refresh = time.time()
|
|
||||||
print('Tracked set refreshed')
|
|
|
@ -30,7 +30,7 @@ subscribe = Redis_D4_client
|
||||||
subscribe = Redis_Global
|
subscribe = Redis_Global
|
||||||
publish = Redis_Tags
|
publish = Redis_Tags
|
||||||
|
|
||||||
[RegexTracker]
|
[Tracker_Regex]
|
||||||
subscribe = Redis_Global
|
subscribe = Redis_Global
|
||||||
publish = Redis_Tags
|
publish = Redis_Tags
|
||||||
|
|
||||||
|
|
92
bin/trackers/Tracker_Regex.py
Executable file
92
bin/trackers/Tracker_Regex.py
Executable file
|
@ -0,0 +1,92 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*-coding:UTF-8 -*
|
||||||
|
"""
|
||||||
|
The Tracker_Regex trackers module
|
||||||
|
===================
|
||||||
|
|
||||||
|
This Module is used for regex tracking.
|
||||||
|
It processes every item coming from the global module and test the regexs
|
||||||
|
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
|
||||||
|
sys.path.append(os.environ['AIL_BIN'])
|
||||||
|
##################################
|
||||||
|
# Import Project packages
|
||||||
|
##################################
|
||||||
|
from modules.abstract_module import AbstractModule
|
||||||
|
from packages.Item import Item
|
||||||
|
from packages import Term
|
||||||
|
from lib import Tracker
|
||||||
|
from lib import regex_helper
|
||||||
|
|
||||||
|
import NotificationHelper
|
||||||
|
|
||||||
|
class Tracker_Regex(AbstractModule):
|
||||||
|
|
||||||
|
mail_body_template = "AIL Framework,\nNew occurrence for term tracked regex: {}\nitem id: {}\nurl: {}{}"
|
||||||
|
|
||||||
|
"""
|
||||||
|
Tracker_Regex module for AIL framework
|
||||||
|
"""
|
||||||
|
def __init__(self):
|
||||||
|
super(Tracker_Regex, self).__init__()
|
||||||
|
|
||||||
|
self.pending_seconds = 5
|
||||||
|
|
||||||
|
self.max_execution_time = self.process.config.getint(self.module_name, "max_execution_time")
|
||||||
|
|
||||||
|
self.full_item_url = self.process.config.get("Notifications", "ail_domain") + "/object/item?id="
|
||||||
|
|
||||||
|
self.redis_cache_key = regex_helper.generate_redis_cache_key(self.module_name)
|
||||||
|
|
||||||
|
# refresh Tracked term
|
||||||
|
self.dict_regex_tracked = Term.get_regex_tracked_words_dict()
|
||||||
|
self.last_refresh = time.time()
|
||||||
|
|
||||||
|
self.redis_logger.info(f"Module: {self.module_name} Launched")
|
||||||
|
|
||||||
|
def compute(self, item_id):
|
||||||
|
# refresh Tracked regex
|
||||||
|
if self.last_refresh < Term.get_tracked_term_last_updated_by_type('regex'):
|
||||||
|
self.dict_regex_tracked = Term.get_regex_tracked_words_dict()
|
||||||
|
self.last_refresh = time.time()
|
||||||
|
self.redis_logger.debug('Tracked word refreshed')
|
||||||
|
print('Tracked set refreshed')
|
||||||
|
|
||||||
|
item = Item(item_id)
|
||||||
|
item_id = item.get_id()
|
||||||
|
item_date = item.get_date()
|
||||||
|
item_content = item.get_content()
|
||||||
|
|
||||||
|
for regex in self.dict_regex_tracked:
|
||||||
|
matched = regex_helper.regex_search(self.module_name, self.redis_cache_key, self.dict_regex_tracked[regex], item_id, item_content, max_time=self.max_execution_time)
|
||||||
|
if matched:
|
||||||
|
self.new_term_found(regex, 'regex', item_id, item_date)
|
||||||
|
|
||||||
|
def new_term_found(self, term, tracker_type, item_id, item_date):
|
||||||
|
uuid_list = Term.get_term_uuid_list(term, tracker_type)
|
||||||
|
print('new tracked regex found: {} in {}'.format(term, item_id))
|
||||||
|
|
||||||
|
for tracker_uuid in uuid_list:
|
||||||
|
Term.add_tracked_item(tracker_uuid, item_id, item_date)
|
||||||
|
|
||||||
|
tags_to_add = Term.get_term_tags(tracker_uuid)
|
||||||
|
for tag in tags_to_add:
|
||||||
|
msg = '{};{}'.format(tag, item_id)
|
||||||
|
self.send_message_to_queue(msg, 'Tags')
|
||||||
|
|
||||||
|
mail_to_notify = Term.get_term_mails(tracker_uuid)
|
||||||
|
if mail_to_notify:
|
||||||
|
mail_subject = Tracker.get_email_subject(tracker_uuid)
|
||||||
|
mail_body = Tracker_Regex.mail_body_template.format(term, item_id, self.full_item_url, item_id)
|
||||||
|
for mail in mail_to_notify:
|
||||||
|
NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body)
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
|
||||||
|
module = Tracker_Regex()
|
||||||
|
module.run()
|
|
@ -63,11 +63,13 @@ class Tracker_Term(AbstractModule):
|
||||||
self.list_tracked_words = Term.get_tracked_words_list()
|
self.list_tracked_words = Term.get_tracked_words_list()
|
||||||
self.last_refresh_word = time.time()
|
self.last_refresh_word = time.time()
|
||||||
self.redis_logger.debug('Tracked word refreshed')
|
self.redis_logger.debug('Tracked word refreshed')
|
||||||
|
print('Tracked word refreshed')
|
||||||
|
|
||||||
if self.last_refresh_set < Term.get_tracked_term_last_updated_by_type('set'):
|
if self.last_refresh_set < Term.get_tracked_term_last_updated_by_type('set'):
|
||||||
self.set_tracked_words_list = Term.get_set_tracked_words_list()
|
self.set_tracked_words_list = Term.get_set_tracked_words_list()
|
||||||
self.last_refresh_set = time.time()
|
self.last_refresh_set = time.time()
|
||||||
self.redis_logger.debug('Tracked set refreshed')
|
self.redis_logger.debug('Tracked set refreshed')
|
||||||
|
print('Tracked set refreshed')
|
||||||
|
|
||||||
# Cast message as Item
|
# Cast message as Item
|
||||||
item = Item(item_id)
|
item = Item(item_id)
|
||||||
|
|
|
@ -119,7 +119,7 @@ default_unnamed_feed_name = unnamed_feeder
|
||||||
[Tracker_Term]
|
[Tracker_Term]
|
||||||
max_execution_time = 120
|
max_execution_time = 120
|
||||||
|
|
||||||
[RegexTracker]
|
[Tracker_Regex]
|
||||||
max_execution_time = 60
|
max_execution_time = 60
|
||||||
|
|
||||||
##### Redis #####
|
##### Redis #####
|
||||||
|
|
Loading…
Reference in a new issue