mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-23 06:37:15 +00:00
Redis cache added fix #5
The paste will be add in Redis during 5min and also saved on disk. Now if a module want to get the paste for further processing, it will first try to get it in the cache instead of getting it directly on the disk and wasting I/O.
This commit is contained in:
parent
af827a3402
commit
44addf1afe
1 changed files with 19 additions and 3 deletions
|
@ -16,7 +16,7 @@ Conditions to fulfill to be able to use this class correctly:
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import os, magic, gzip, langid, pprint, redis, operator, string, re, json
|
import os, magic, gzip, langid, pprint, redis, operator, string, re, json, ConfigParser
|
||||||
from Date import Date
|
from Date import Date
|
||||||
from Hash import Hash
|
from Hash import Hash
|
||||||
|
|
||||||
|
@ -30,6 +30,10 @@ from lib_refine import *
|
||||||
clean = lambda dirty: ''.join(filter(string.printable.__contains__, dirty))
|
clean = lambda dirty: ''.join(filter(string.printable.__contains__, dirty))
|
||||||
"""It filters out non-printable characters from the string it receives."""
|
"""It filters out non-printable characters from the string it receives."""
|
||||||
|
|
||||||
|
configfile = './config.cfg'
|
||||||
|
cfg = ConfigParser.ConfigParser()
|
||||||
|
cfg.read(configfile)
|
||||||
|
|
||||||
class Paste(object):
|
class Paste(object):
|
||||||
"""
|
"""
|
||||||
This class representing a Paste as an object.
|
This class representing a Paste as an object.
|
||||||
|
@ -72,6 +76,11 @@ class Paste(object):
|
||||||
|
|
||||||
self.p_source = var[-5]
|
self.p_source = var[-5]
|
||||||
|
|
||||||
|
self.cache = redis.StrictRedis(
|
||||||
|
host = cfg.get("Redis_Queues", "host"),
|
||||||
|
port = cfg.getint("Redis_Queues", "port"),
|
||||||
|
db = cfg.getint("Redis_Queues", "db"))
|
||||||
|
|
||||||
|
|
||||||
def get_p_content(self):
|
def get_p_content(self):
|
||||||
"""
|
"""
|
||||||
|
@ -82,8 +91,15 @@ class Paste(object):
|
||||||
PST.get_p_content()
|
PST.get_p_content()
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
r_serv = self.cache
|
||||||
|
|
||||||
|
if r_serv.exist(self.p_path):
|
||||||
|
paste = r_serv.get(self.p_path)
|
||||||
|
else:
|
||||||
with gzip.open(self.p_path, 'rb') as F:
|
with gzip.open(self.p_path, 'rb') as F:
|
||||||
return F.read()
|
paste = r_serv.getset(self.p_path, F.read())
|
||||||
|
r_serv.expire(self.p_path, 300)
|
||||||
|
return paste
|
||||||
|
|
||||||
def get_lines_info(self):
|
def get_lines_info(self):
|
||||||
"""
|
"""
|
||||||
|
|
Loading…
Reference in a new issue