#!/usr/bin/env python2 # -*-coding:UTF-8 -* """ Template for new modules """ import time import datetime import redis import os from packages import lib_words from packages.Date import Date from pubsublogger import publisher from Helper import Process from packages import Paste # Config Var max_set_cardinality = 8 def get_date_range(num_day): curr_date = datetime.date.today() date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2)) date_list = [] for i in range(0, num_day+1): date_list.append(date.substract_day(i)) return date_list def compute_most_posted(server, message): module, num, keyword, paste_date = message.split(';') redis_progression_name_set = 'top_'+ module +'_set' # Add/Update in Redis prev_score = server.hget(paste_date, module+'-'+keyword) if prev_score is not None: ok = server.hset(paste_date, module+'-'+keyword, int(prev_score) + int(num)) else: ok = server.hset(paste_date, module+'-'+keyword, int(num)) # Compute Most Posted date = get_date_range(0)[0] # check if this keyword is eligible for progression keyword_total_sum = 0 curr_value = server.hget(date, module+'-'+keyword) keyword_total_sum += int(curr_value) if curr_value is not None else 0 if keyword in server.smembers(redis_progression_name_set): # if it is already in the set return if (server.scard(redis_progression_name_set) < max_set_cardinality): server.sadd(redis_progression_name_set, keyword) else: #not in the set #Check value for all members member_set = [] for keyw in server.smembers(redis_progression_name_set): keyw_value = server.hget(paste_date, module+'-'+keyw) if keyw_value is not None: member_set.append((keyw, int(keyw_value))) else: #No data for this set for today member_set.append((keyw, int(0))) member_set.sort(key=lambda tup: tup[1]) if len(member_set) > 0: if member_set[0][1] < keyword_total_sum: #remove min from set and add the new one print module + ': adding ' +keyword+ '(' +str(keyword_total_sum)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')' server.srem(redis_progression_name_set, member_set[0][0]) server.sadd(redis_progression_name_set, keyword) def compute_provider_info(server, path): redis_avg_size_name_set = 'top_size_set' redis_providers_name_set = 'providers_set' paste = Paste.Paste(path) paste_size = paste._get_p_size() paste_provider = paste.p_source paste_date = paste._get_p_date() new_avg = paste_size # Add/Update in Redis prev_num_paste = server.hget(paste_provider+'_num', paste_date) if prev_num_paste is not None: ok = server.hset(paste_provider+'_num', paste_date, int(prev_num_paste)+1) prev_sum_size = server.hget(paste_provider+'_size', paste_date) if prev_sum_size is not None: ok = server.hset(paste_provider+'_size', paste_date, float(prev_sum_size)+paste_size) new_avg = (float(prev_sum_size)+paste_size) / (int(prev_num_paste)+1) else: ok = server.hset(paste_provider+'_size', paste_date, paste_size) else: ok = server.hset(paste_provider+'_num', paste_date, 1) prev_num_paste = 0 # # Compute Most Posted # # Size if paste_provider not in server.smembers(redis_avg_size_name_set): # if it is already in the set if (server.scard(redis_avg_size_name_set) < max_set_cardinality): server.sadd(redis_avg_size_name_set, paste_provider) else: #set full capacity #Check value for all members member_set = [] for provider in server.smembers(redis_avg_size_name_set): curr_avg = 0.0 curr_size = server.hget(provider+'_size', paste_date) curr_num = server.hget(provider+'_num', paste_date) if (curr_size is not None) and (curr_num is not None): curr_avg = float(curr_size) / float(curr_num) member_set.append((provider, curr_avg)) member_set.sort(key=lambda tup: tup[1]) if member_set[0][1] < new_avg: #remove min from set and add the new one print 'Size - adding ' +paste_provider+ '(' +str(new_avg)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')' server.srem(redis_avg_size_name_set, member_set[0][0]) server.sadd(redis_avg_size_name_set, paste_provider) # Num if paste_provider not in server.smembers(redis_providers_name_set): # if it is already in the set if (server.scard(redis_providers_name_set) < max_set_cardinality): server.sadd(redis_providers_name_set, paste_provider) else: #set full capacity #Check value for all members member_set = [] for provider in server.smembers(redis_providers_name_set): curr_num = 0 curr_num = server.hget(provider+'_num', paste_date) if curr_num is not None: member_set.append((provider, int(curr_num))) member_set.sort(key=lambda tup: tup[1]) if len(member_set) > 0: if member_set[0][1] < int(prev_num_paste)+1: #remove min from set and add the new one print 'Num - adding ' +paste_provider+ '(' +str(int(prev_num_paste)+1)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')' server.srem(redis_providers_name_set, member_set[0][0]) server.sadd(redis_providers_name_set, paste_provider) if __name__ == '__main__': # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh) # Port of the redis instance used by pubsublogger publisher.port = 6380 # Script is the default channel used for the modules. publisher.channel = 'Script' # Section name in bin/packages/modules.cfg config_section = 'ModuleStats' # Setup the I/O queues p = Process(config_section) # Sent to the logging a description of the module publisher.info("Makes statistics about valid URL") # REDIS # r_serv_trend = redis.StrictRedis( host=p.config.get("Redis_Level_DB_Trending", "host"), port=p.config.get("Redis_Level_DB_Trending", "port"), db=p.config.get("Redis_Level_DB_Trending", "db")) # Endless loop getting messages from the input queue while True: # Get one message from the input queue message = p.get_from_set() if message is None: publisher.debug("{} queue is empty, waiting".format(config_section)) print 'sleeping' time.sleep(20) continue else: # Do something with the message from the queue if len(message.split(';')) > 1: compute_most_posted(r_serv_trend, message) else: compute_provider_info(r_serv_trend, message)