AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-23 06:37:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Redis cache added fix #5

Browse source 
The paste will be add in Redis during 5min and also saved on disk.
Now if a module want to get the paste for further processing, it will first try to get it in the cache
instead of getting it directly on the disk and wasting I/O.
This commit is contained in:
Starow 2014-08-08 16:48:02 +02:00
parent af827a3402
commit 44addf1afe
1 changed files with 19 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
bin/packages/Paste.py
View file

@ -16,7 +16,7 @@ Conditions to fulfill to be able to use this class correctly:
""" """
import os, magic, gzip, langid, pprint, redis, operator, string, re, json import os, magic, gzip, langid, pprint, redis, operator, string, re, json, ConfigParser
from Date import Date from Date import Date
from Hash import Hash from Hash import Hash
@ -30,6 +30,10 @@ from lib_refine import *
clean = lambda dirty: ''.join(filter(string.printable.__contains__, dirty)) clean = lambda dirty: ''.join(filter(string.printable.__contains__, dirty))
"""It filters out non-printable characters from the string it receives.""" """It filters out non-printable characters from the string it receives."""
configfile = './config.cfg'
cfg = ConfigParser.ConfigParser()
cfg.read(configfile)
class Paste(object): class Paste(object):
""" """
This class representing a Paste as an object. This class representing a Paste as an object.
@ -72,6 +76,11 @@ class Paste(object):
self.p_source = var[-5] self.p_source = var[-5]
self.cache = redis.StrictRedis(
host = cfg.get("Redis_Queues", "host"),
port = cfg.getint("Redis_Queues", "port"),
db = cfg.getint("Redis_Queues", "db"))
def get_p_content(self): def get_p_content(self):
""" """
@ -82,8 +91,15 @@ class Paste(object):
PST.get_p_content() PST.get_p_content()
""" """
with gzip.open(self.p_path, 'rb') as F: r_serv = self.cache
return F.read()
if r_serv.exist(self.p_path):
paste = r_serv.get(self.p_path)
else:
with gzip.open(self.p_path, 'rb') as F:
paste = r_serv.getset(self.p_path, F.read())
r_serv.expire(self.p_path, 300)
return paste
def get_lines_info(self): def get_lines_info(self):
""" """