AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-09-20 07:58:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

stop killing the disk when creating the word curve

Browse source
This commit is contained in:
Raphaël Vinot 2014-09-02 18:20:28 +02:00
parent 7542eaf739
commit 9e8611a42d
2 changed files with 42 additions and 63 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
bin/Curve.py
View file

@ -24,10 +24,10 @@ Requirements
"""
import redis
import time
from packages import Paste
from pubsublogger import publisher
from packages import lib_words
import os
import datetime
from Helper import Process
@ -57,23 +57,31 @@ if __name__ == "__main__":
prec_filename = None
while True:
if message is not None:
generate_new_graph = True
filename, word, score = message.split()
if prec_filename is None or filename != prec_filename:
PST = Paste.Paste(filename)
lib_words.create_curve_with_word_file(
r_serv1, csv_path, wordfile_path, int(PST.p_date.year),
int(PST.p_date.month))
temp = filename.split('/')
date = temp[-4] + temp[-3] + temp[-2]
prec_filename = filename
prev_score = r_serv1.hget(word.lower(), PST.p_date)
low_word = word.lower()
prev_score = r_serv1.hget(low_word, date)
if prev_score is not None:
r_serv1.hset(word.lower(), PST.p_date,
int(prev_score) + int(score))
r_serv1.hset(low_word, date, int(prev_score) + int(score))
else:
r_serv1.hset(word.lower(), PST.p_date, score)
r_serv1.hset(low_word, date, score)
else:
if generate_new_graph:
generate_new_graph = False
print 'Building graph'
today = datetime.date.today()
year = today.year
month = today.month
lib_words.create_curve_with_word_file(r_serv1, csv_path,
wordfile_path, year,
month)
publisher.debug("Script Curve is Idling")
print "sleeping"
time.sleep(1)
time.sleep(10)
message = p.get_from_set()

71
bin/packages/lib_words.py
View file

@ -6,6 +6,7 @@ from pubsublogger import publisher
import calendar
from datetime import date
from dateutil.rrule import rrule, DAILY
import csv
def listdirectory(path):
@ -80,60 +81,30 @@ def create_curve_with_word_file(r_serv, csvfilename, feederfilename, year, month
to keep the timeline of the curve correct.
"""
a = date(year, month, 01)
b = date(year, month, calendar.monthrange(year, month)[1])
days = {}
first_day = date(year, month, 01)
last_day = date(year, month, calendar.monthrange(year, month)[1])
words = []
with open(feederfilename, 'rb') as F:
with open(feederfilename, 'rb') as f:
# words of the files
for word in F:
# list of words (sorted as in the file)
words.append(word[:-1])
words = sorted([word.strip() for word in f])
headers = ['Date'] + words
with open(csvfilename+'.csv', 'wb') as f:
writer = csv.writer(f)
writer.writerow(headers)
# for each days
for dt in rrule(DAILY, dtstart=a, until=b):
mot = []
mot1 = []
mot2 = []
days[dt.strftime("%Y%m%d")] = ''
for dt in rrule(DAILY, dtstart=first_day, until=last_day):
row = []
curdate = dt.strftime("%Y%m%d")
row.append(curdate)
# from the 1srt day to the last of the list
for word in sorted(words):
# if the word have a value for the day
if r_serv.hexists(word, dt.strftime("%Y%m%d")):
mot1.append(str(word))
mot2.append(r_serv.hget(word, dt.strftime("%Y%m%d")))
mot = zip(mot1, mot2)
days[dt.strftime("%Y%m%d")] = mot
for word in words:
value = r_serv.hget(word, curdate)
if value is None:
row.append(0)
else:
mot1.append(str(word))
mot2.append(0)
mot = zip(mot1, mot2)
days[dt.strftime("%Y%m%d")] = mot
with open(csvfilename+".csv", 'wb') as F:
F.write("Date," + ",".join(sorted(words)) + '\n')
for x, s in days.items():
val = []
for y in s:
val.append(y[1])
F.write(x + ',' + str(val) + '\n')
with open(csvfilename+".csv", 'rb') as F:
h = F.read()
h = h.replace("[", "")
h = h.replace("]", "")
h = h.replace('\'', "")
with open(csvfilename+".csv", 'wb') as F:
F.write(h)
# if the word have a value for the day
row.append(value)
writer.writerow(row)