AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-10 08:38:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

RegexpForTerm timeout from config

Browse source 
Critical log removed
This commit is contained in:
kovacsbalu 2018-10-09 09:32:32 +02:00
parent 6d199f0150
commit 272b0fe960
2 changed files with 8 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
bin/RegexForTermsFrequency.py
View file

@ -28,10 +28,8 @@ def timeout_handler(signum, frame):
signal.signal(signal.SIGALRM, timeout_handler) signal.signal(signal.SIGALRM, timeout_handler)
# Config Variables # Config Variables
DICO_REFRESH_TIME = 60 # s DICO_REFRESH_TIME = 60 # s
PROCESS_TIMEOUT = 60
BlackListTermsSet_Name = "BlackListSetTermSet" BlackListTermsSet_Name = "BlackListSetTermSet"
TrackedTermsSet_Name = "TrackedSetTermSet" TrackedTermsSet_Name = "TrackedSetTermSet"
@ -63,6 +61,7 @@ if __name__ == "__main__":
config_section = 'RegexForTermsFrequency' config_section = 'RegexForTermsFrequency'
p = Process(config_section) p = Process(config_section)
max_execution_time = p.config.getint("BankAccount", "max_execution_time")
# REDIS # # REDIS #
server_term = redis.StrictRedis( server_term = redis.StrictRedis(
@ -97,18 +96,17 @@ if __name__ == "__main__":
timestamp = calendar.timegm((int(temp[-4]), int(temp[-3]), int(temp[-2]), 0, 0, 0)) timestamp = calendar.timegm((int(temp[-4]), int(temp[-3]), int(temp[-2]), 0, 0, 0))
curr_set = top_termFreq_setName_day[0] + str(timestamp) curr_set = top_termFreq_setName_day[0] + str(timestamp)
content = Paste.Paste(filename).get_p_content() paste = Paste.Paste(filename)
content = paste.get_p_content()
# iterate the word with the regex # iterate the word with the regex
for regex_str, compiled_regex in dico_regex.items(): for regex_str, compiled_regex in dico_regex.items():
signal.alarm(PROCESS_TIMEOUT) signal.alarm(max_execution_time)
try: try:
matched = compiled_regex.search(content) matched = compiled_regex.search(content)
except TimeoutException: except TimeoutException:
log_msg = "{0} processing timeout".format(filename) print ("{0} processing timeout".format(paste.p_path))
print (log_msg)
publisher.critical(log_msg)
continue continue
else: else:
signal.alarm(0) signal.alarm(0)

3
bin/packages/config.cfg.sample
View file

@ -96,6 +96,9 @@ operation_mode = 3
#Define the time that a paste will be considerate duplicate. in seconds (1day = 86400) #Define the time that a paste will be considerate duplicate. in seconds (1day = 86400)
ttl_duplicate = 86400 ttl_duplicate = 86400
[RegexForTermsFrequency]
max_execution_time = 60
##### Redis ##### ##### Redis #####
[Redis_Cache] [Redis_Cache]
host = localhost host = localhost