AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-23 06:37:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Redis cache added fix #5

Browse source 
The paste will be add in Redis during 5min and also saved on disk.
Now if a module want to get the paste for further processing, it will first try to get it in the cache
instead of getting it directly on the disk and wasting I/O.
This commit is contained in:
Starow 2014-08-08 16:48:02 +02:00
parent af827a3402
commit 44addf1afe
1 changed files with 19 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
bin/packages/Paste.py
View file

@ -16,7 +16,7 @@ Conditions to fulfill to be able to use this class correctly:
"""
import os, magic, gzip, langid, pprint, redis, operator, string, re, json
import os, magic, gzip, langid, pprint, redis, operator, string, re, json, ConfigParser
from Date import Date
from Hash import Hash
@ -30,6 +30,10 @@ from lib_refine import *
clean = lambda dirty: ''.join(filter(string.printable.__contains__, dirty))
"""It filters out non-printable characters from the string it receives."""
configfile = './config.cfg'
cfg = ConfigParser.ConfigParser()
cfg.read(configfile)
class Paste(object):
"""
This class representing a Paste as an object.
@ -72,6 +76,11 @@ class Paste(object):
self.p_source = var[-5]
self.cache = redis.StrictRedis(
host = cfg.get("Redis_Queues", "host"),
port = cfg.getint("Redis_Queues", "port"),
db = cfg.getint("Redis_Queues", "db"))
def get_p_content(self):
"""
@ -82,8 +91,15 @@ class Paste(object):
PST.get_p_content()
"""
with gzip.open(self.p_path, 'rb') as F:
return F.read()
r_serv = self.cache
if r_serv.exist(self.p_path):
paste = r_serv.get(self.p_path)
else:
with gzip.open(self.p_path, 'rb') as F:
paste = r_serv.getset(self.p_path, F.read())
r_serv.expire(self.p_path, 300)
return paste
def get_lines_info(self):
"""