From 240bdd9cca69126ac101c75eec7ef2e61a6fbab1 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Wed, 27 Jul 2016 09:27:32 +0200 Subject: [PATCH] Added number of paste by provider in module stats --- bin/ModuleStats.py | 20 ++++++------ var/www/Flask_server.py | 32 +++++++++++++++++++ var/www/templates/Moduletrending.html | 3 +- .../trending_graphs/Moduletrending.html | 8 ++++- 4 files changed, 52 insertions(+), 11 deletions(-) diff --git a/bin/ModuleStats.py b/bin/ModuleStats.py index ce26c791..cfafedd0 100755 --- a/bin/ModuleStats.py +++ b/bin/ModuleStats.py @@ -68,9 +68,10 @@ def compute_most_posted(server, message, num_day): server.sadd(redis_progression_name_set, keyword) -def compute_provider_size(server, path, num_day_to_look): +def compute_provider_info(server, path, num_day_to_look): - redis_progression_name_set = 'top_size_set' + redis_avg_size_name_set = 'top_size_set' + redis_providers_name_set = 'providers_set' paste = Paste.Paste(path) paste_size = paste._get_p_size() @@ -79,6 +80,7 @@ def compute_provider_size(server, path, num_day_to_look): new_avg = paste_size # Add/Update in Redis + server.sadd(redis_providers_name_set, paste_provider) prev_num_paste = server.hget(paste_provider+'_num', paste_date) if prev_num_paste is not None: server.hset(paste_provider+'_num', paste_date, int(prev_num_paste)+1) @@ -96,16 +98,16 @@ def compute_provider_size(server, path, num_day_to_look): # Compute Most Posted # check if this keyword is eligible for progression - if paste_provider in server.smembers(redis_progression_name_set): # if it is already in the set + if paste_provider in server.smembers(redis_avg_size_name_set): # if it is already in the set return - elif (server.scard(redis_progression_name_set) < max_set_cardinality): - server.sadd(redis_progression_name_set, paste_provider) + elif (server.scard(redis_avg_size_name_set) < max_set_cardinality): + server.sadd(redis_avg_size_name_set, paste_provider) else: #set full capacity #Check value for all members member_set = [] - for provider in server.smembers(redis_progression_name_set): + for provider in server.smembers(redis_avg_size_name_set): curr_avg = 0.0 curr_size = server.hget(provider+'_size', paste_date) curr_num = server.hget(provider+'_num', paste_date) @@ -116,8 +118,8 @@ def compute_provider_size(server, path, num_day_to_look): if member_set[0][1] < new_avg: #remove min from set and add the new one print 'Adding ' +paste_provider+ '(' +str(new_avg)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')' - server.srem(redis_progression_name_set, member_set[0][0]) - server.sadd(redis_progression_name_set, paste_provider) + server.srem(redis_avg_size_name_set, member_set[0][0]) + server.sadd(redis_avg_size_name_set, paste_provider) @@ -159,4 +161,4 @@ if __name__ == '__main__': if len(message.split(';')) > 1: compute_most_posted(r_serv_trend, message, num_day_to_look) else: - compute_provider_size(r_serv_trend, message, num_day_to_look) + compute_provider_info(r_serv_trend, message, num_day_to_look) diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 22388cfb..446d051a 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -237,6 +237,38 @@ def sizeCharts(): return jsonify(member_set) +@app.route("/_numberChart", methods=['GET']) +def numberChart(): + #To be used later + keyword_name = request.args.get('keywordName') + module_name = request.args.get('moduleName') + bar_requested = True if request.args.get('bar') == "true" else False + + if (bar_requested): + num_day = int(request.args.get('days')) + bar_values = [] + + date_range = get_date_range(num_day) + # Retreive all data from the last num_day + for date in date_range: + curr_value = r_serv_charts.hget(keyword_name+'_num', date) + bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], int(curr_value if curr_value is not None else 0)]) + return jsonify(bar_values) + + else: + redis_provider_name_set = 'providers_set' + + member_set = [] + for keyw in r_serv_charts.smembers(redis_provider_name_set): + redis_provider_name = keyw+'_num' + keyw_value = r_serv_charts.hget(redis_provider_name, get_date_range(0)[0]) + keyw_value = keyw_value if keyw_value is not None else 0 + member_set.append((keyw, int(keyw_value))) + member_set.sort(key=lambda tup: tup[1], reverse=True) + if len(member_set) == 0: + member_set.append(("No relevant data", int(100))) + return jsonify(member_set) + @app.route("/search", methods=['POST']) def search(): diff --git a/var/www/templates/Moduletrending.html b/var/www/templates/Moduletrending.html index ef8da227..6a1f38ee 100644 --- a/var/www/templates/Moduletrending.html +++ b/var/www/templates/Moduletrending.html @@ -65,6 +65,7 @@ plot_top_graph("credential"); plot_top_graph("mail"); plot_top_graph("size"); + plot_top_graph("number"); }); @@ -105,7 +106,7 @@ function labelFormatter(label, series) { }; - var moduleCharts = "size" == module_name ? "sizeCharts" : "moduleCharts"; + var moduleCharts = "size" == module_name ? "sizeCharts" : ("number" == module_name ? "numberChart" : "moduleCharts"); var tot_sum = 0; var data_other = []; // Graph1 diff --git a/var/www/templates/trending_graphs/Moduletrending.html b/var/www/templates/trending_graphs/Moduletrending.html index 5ef90a23..339676dd 100644 --- a/var/www/templates/trending_graphs/Moduletrending.html +++ b/var/www/templates/trending_graphs/Moduletrending.html @@ -54,13 +54,19 @@
- Provider - average paste size + Provider Today
+

Average paste size by provider

Click on a part
+ +

Number of paste by provider

+
+
Click on a part
+