Redis cache added fix #5

The paste will be add in Redis during 5min and also saved on disk.
Now if a module want to get the paste for further processing, it will first try to get it in the cache
instead of getting it directly on the disk and wasting I/O.
This commit is contained in:
Starow 2014-08-08 16:48:02 +02:00
parent af827a3402
commit 44addf1afe

View file

@ -16,7 +16,7 @@ Conditions to fulfill to be able to use this class correctly:
"""
import os, magic, gzip, langid, pprint, redis, operator, string, re, json
import os, magic, gzip, langid, pprint, redis, operator, string, re, json, ConfigParser
from Date import Date
from Hash import Hash
@ -30,6 +30,10 @@ from lib_refine import *
clean = lambda dirty: ''.join(filter(string.printable.__contains__, dirty))
"""It filters out non-printable characters from the string it receives."""
configfile = './config.cfg'
cfg = ConfigParser.ConfigParser()
cfg.read(configfile)
class Paste(object):
"""
This class representing a Paste as an object.
@ -72,6 +76,11 @@ class Paste(object):
self.p_source = var[-5]
self.cache = redis.StrictRedis(
host = cfg.get("Redis_Queues", "host"),
port = cfg.getint("Redis_Queues", "port"),
db = cfg.getint("Redis_Queues", "db"))
def get_p_content(self):
"""
@ -82,8 +91,15 @@ class Paste(object):
PST.get_p_content()
"""
with gzip.open(self.p_path, 'rb') as F:
return F.read()
r_serv = self.cache
if r_serv.exist(self.p_path):
paste = r_serv.get(self.p_path)
else:
with gzip.open(self.p_path, 'rb') as F:
paste = r_serv.getset(self.p_path, F.read())
r_serv.expire(self.p_path, 300)
return paste
def get_lines_info(self):
"""