AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-22 22:27:17 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

chg: [queues] timeout obj after 2 days

Browse source
This commit is contained in:
terrtia 2024-01-08 13:50:56 +01:00
parent c05f4d7833
commit 0af5ea9d48
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
3 changed files with 35 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
bin/core/Sync_module.py
View file

@ -22,13 +22,13 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages
##################################
from core import ail_2_ail
from lib.ail_queues import get_processed_end_obj
from lib.ail_queues import get_processed_end_obj, timeout_processed_objs
from lib.exceptions import ModuleQueueError
from lib.objects import ail_objects
from modules.abstract_module import AbstractModule
class Sync_module(AbstractModule): # TODO KEEP A QUEUE ???????????????????????????????????????????????
class Sync_module(AbstractModule):
"""
Sync_module module for AIL framework
"""
@ -81,6 +81,10 @@ class Sync_module(AbstractModule): # TODO KEEP A QUEUE ?????????????????????????
# Endless loop processing messages from the input queue
while self.proceed:
# Timeout queues
timeout_processed_objs()
# Get one message (paste) from the QueueIn (copy of Redis_Global publish)
global_id = get_processed_end_obj()
if global_id:

3
bin/lib/ail_core.py
View file

@ -57,6 +57,9 @@ def get_object_all_subtypes(obj_type): # TODO Dynamic subtype
return r_object.smembers(f'all_chat:subtypes')
return []
def get_obj_queued():
return ['item', 'image']
def get_objects_tracked():
return ['decoded', 'item', 'pgp', 'title']

25
bin/lib/ail_queues.py
View file

@ -14,10 +14,12 @@ sys.path.append(os.environ['AIL_BIN'])
##################################
from lib.exceptions import ModuleQueueError
from lib.ConfigLoader import ConfigLoader
from lib import ail_core
config_loader = ConfigLoader()
r_queues = config_loader.get_redis_conn("Redis_Queues")
r_obj_process = config_loader.get_redis_conn("Redis_Process")
timeout_queue_obj = 172800
config_loader = None
MODULES_FILE = os.path.join(os.environ['AIL_HOME'], 'configs', 'modules.cfg')
@ -248,6 +250,29 @@ def rename_processed_obj(new_id, old_id):
r_obj_process.srem(f'objs:process', old_id)
add_processed_obj(new_id, x_hash, module=module)
def timeout_process_obj(obj_global_id):
for q in get_processed_obj_queues(obj_global_id):
queue, x_hash = q.split(':', 1)
r_obj_process.zrem(f'obj:queues:{obj_global_id}', f'{queue}:{x_hash}')
for m in get_processed_obj_modules(obj_global_id):
module, x_hash = m.split(':', 1)
r_obj_process.zrem(f'obj:modules:{obj_global_id}', f'{module}:{x_hash}')
obj_type = obj_global_id.split(':', 1)[0]
r_obj_process.zrem(f'objs:process:{obj_type}', obj_global_id)
r_obj_process.srem(f'objs:process', obj_global_id)
r_obj_process.sadd(f'objs:processed', obj_global_id)
print(f'timeout: {obj_global_id}')
def timeout_processed_objs():
curr_time = int(time.time())
time_limit = curr_time - timeout_queue_obj
for obj_type in ail_core.get_obj_queued():
for obj_global_id in r_obj_process.zrangebyscore(f'objs:process:{obj_type}', 0, time_limit):
timeout_process_obj(obj_global_id)
def delete_processed_obj(obj_global_id):
for q in get_processed_obj_queues(obj_global_id):
queue, x_hash = q.split(':', 1)