From 8daa72789e09549ba567688b528b7cd85ded52a7 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Fri, 9 Dec 2016 08:46:37 +0100 Subject: [PATCH] Splitted Flask_server into module related to website sections --- var/www/Flask_browsepastes.py | 100 ++++ var/www/Flask_config.py | 65 +++ var/www/Flask_dashboard.py | 65 +++ var/www/Flask_search.py | 124 +++++ var/www/Flask_sentiment.py | 137 ++++++ var/www/Flask_server.py | 821 +------------------------------ var/www/Flask_showpaste.py | 114 +++++ var/www/Flask_terms.py | 240 +++++++++ var/www/Flask_trendingcharts.py | 73 +++ var/www/Flask_trendingmodules.py | 111 +++++ 10 files changed, 1045 insertions(+), 805 deletions(-) create mode 100644 var/www/Flask_browsepastes.py create mode 100644 var/www/Flask_config.py create mode 100644 var/www/Flask_dashboard.py create mode 100644 var/www/Flask_search.py create mode 100644 var/www/Flask_sentiment.py create mode 100644 var/www/Flask_showpaste.py create mode 100644 var/www/Flask_terms.py create mode 100644 var/www/Flask_trendingcharts.py create mode 100644 var/www/Flask_trendingmodules.py diff --git a/var/www/Flask_browsepastes.py b/var/www/Flask_browsepastes.py new file mode 100644 index 00000000..e5aa5b84 --- /dev/null +++ b/var/www/Flask_browsepastes.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python2 +# -*-coding:UTF-8 -* + +''' + Flask functions and routes for the trending modules page +''' +import redis +import json +import flask +from flask import Flask, render_template, jsonify, request + +import Paste + +# ============ VARIABLES ============ +import Flask_config + +app = Flask_config.app +cfg = Flask_config.cfg +max_preview_char = Flask_config.max_preview_char +max_preview_modal = Flask_config.max_preview_modal +r_serv_db = Flask_config.r_serv_db +# ============ FUNCTIONS ============ + +def getPastebyType(server, module_name): + all_path = [] + for path in server.smembers('WARNING_'+module_name): + all_path.append(path) + return all_path + + +def event_stream_getImportantPasteByModule(module_name): + index = 0 + all_pastes_list = getPastebyType(r_serv_db, module_name) + for path in all_pastes_list: + index += 1 + paste = Paste.Paste(path) + content = paste.get_p_content().decode('utf8', 'ignore') + content_range = max_preview_char if len(content)>max_preview_char else len(content)-1 + curr_date = str(paste._get_p_date()) + curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:] + data = {} + data["module"] = module_name + data["index"] = index + data["path"] = path + data["content"] = content[0:content_range] + data["linenum"] = paste.get_lines_info()[0] + data["date"] = curr_date + data["char_to_display"] = max_preview_modal + data["finished"] = True if index == len(all_pastes_list) else False + yield 'retry: 100000\ndata: %s\n\n' % json.dumps(data) #retry to avoid reconnection of the browser + +# ============ ROUTES ============ + +@app.route("/browseImportantPaste/", methods=['GET']) +def browseImportantPaste(): + module_name = request.args.get('moduleName') + return render_template("browse_important_paste.html") + + +@app.route("/importantPasteByModule/", methods=['GET']) +def importantPasteByModule(): + module_name = request.args.get('moduleName') + + all_content = [] + paste_date = [] + paste_linenum = [] + all_path = [] + allPastes = getPastebyType(r_serv_db, module_name) + + for path in allPastes[0:10]: + all_path.append(path) + paste = Paste.Paste(path) + content = paste.get_p_content().decode('utf8', 'ignore') + content_range = max_preview_char if len(content)>max_preview_char else len(content)-1 + all_content.append(content[0:content_range].replace("\"", "\'").replace("\r", " ").replace("\n", " ")) + curr_date = str(paste._get_p_date()) + curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:] + paste_date.append(curr_date) + paste_linenum.append(paste.get_lines_info()[0]) + + if len(allPastes) > 10: + finished = "" + else: + finished = "display: none;" + + return render_template("important_paste_by_module.html", + moduleName=module_name, + all_path=all_path, + content=all_content, + paste_date=paste_date, + paste_linenum=paste_linenum, + char_to_display=max_preview_modal, + finished=finished) + +@app.route("/_getImportantPasteByModule") +def getImportantPasteByModule(): + module_name = request.args.get('moduleName') + return flask.Response(event_stream_getImportantPasteByModule(module_name), mimetype="text/event-stream") + + diff --git a/var/www/Flask_config.py b/var/www/Flask_config.py new file mode 100644 index 00000000..c15e4dca --- /dev/null +++ b/var/www/Flask_config.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python2 +# -*-coding:UTF-8 -* + +''' + Flask global variables shared accross modules +''' +import ConfigParser +import redis +import os + +# FLASK # +app = None + +# CONFIG # +configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') +if not os.path.exists(configfile): + raise Exception('Unable to find the configuration file. \ + Did you set environment variables? \ + Or activate the virtualenv.') + +cfg = ConfigParser.ConfigParser() +cfg.read(configfile) + + +# REDIS # +r_serv = redis.StrictRedis( + host=cfg.get("Redis_Queues", "host"), + port=cfg.getint("Redis_Queues", "port"), + db=cfg.getint("Redis_Queues", "db")) + +r_serv_log = redis.StrictRedis( + host=cfg.get("Redis_Log", "host"), + port=cfg.getint("Redis_Log", "port"), + db=cfg.getint("Redis_Log", "db")) + +r_serv_charts = redis.StrictRedis( + host=cfg.get("Redis_Level_DB_Trending", "host"), + port=cfg.getint("Redis_Level_DB_Trending", "port"), + db=cfg.getint("Redis_Level_DB_Trending", "db")) + +r_serv_db = redis.StrictRedis( + host=cfg.get("Redis_Level_DB", "host"), + port=cfg.getint("Redis_Level_DB", "port"), + db=cfg.getint("Redis_Level_DB", "db")) + +r_serv_sentiment = redis.StrictRedis( + host=cfg.get("Redis_Level_DB_Sentiment", "host"), + port=cfg.getint("Redis_Level_DB_Sentiment", "port"), + db=cfg.getint("Redis_Level_DB_Sentiment", "db")) + +r_serv_term = redis.StrictRedis( + host=cfg.get("Redis_Level_DB_TermFreq", "host"), + port=cfg.getint("Redis_Level_DB_TermFreq", "port"), + db=cfg.getint("Redis_Level_DB_TermFreq", "db")) + +r_serv_pasteName = redis.StrictRedis( + host=cfg.get("Redis_Paste_Name", "host"), + port=cfg.getint("Redis_Paste_Name", "port"), + db=cfg.getint("Redis_Paste_Name", "db")) + +# VARIABLES # +max_preview_char = int(cfg.get("Flask", "max_preview_char")) # Maximum number of character to display in the tooltip +max_preview_modal = int(cfg.get("Flask", "max_preview_modal")) # Maximum number of character to display in the modal + +tlsh_to_percent = 1000.0 #Use to display the estimated percentage instead of a raw value diff --git a/var/www/Flask_dashboard.py b/var/www/Flask_dashboard.py new file mode 100644 index 00000000..b6bcb219 --- /dev/null +++ b/var/www/Flask_dashboard.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python2 +# -*-coding:UTF-8 -* + +''' + Flask functions and routes for the dashboard page +''' +import flask +from flask import Flask, render_template, jsonify, request + +# ============ VARIABLES ============ +import Flask_config + +app = Flask_config.app +cfg = Flask_config.cfg +r_serv = Flask_config.r_serv +r_serv_log = Flask_config.r_serv_log +# ============ FUNCTIONS ============ + +def event_stream(): + pubsub = r_serv_log.pubsub() + pubsub.psubscribe("Script" + '.*') + for msg in pubsub.listen(): + level = msg['channel'].split('.')[1] + if msg['type'] == 'pmessage' and level != "DEBUG": + yield 'data: %s\n\n' % json.dumps(msg) + +def get_queues(r): + # We may want to put the llen in a pipeline to do only one query. + newData = [] + for queue, card in r.hgetall("queues").iteritems(): + key = "MODULE_" + queue + "_" + keySet = "MODULE_TYPE_" + queue + + for moduleNum in r.smembers(keySet): + + value = r.get(key + str(moduleNum)) + if value is not None: + timestamp, path = value.split(", ") + if timestamp is not None: + startTime_readable = datetime.datetime.fromtimestamp(int(timestamp)) + processed_time_readable = str((datetime.datetime.now() - startTime_readable)).split('.')[0] + seconds = int((datetime.datetime.now() - startTime_readable).total_seconds()) + newData.append( (queue, card, seconds, moduleNum) ) + else: + newData.append( (queue, cards, 0, moduleNum) ) + + return newData + +# ============ ROUTES ============ + +@app.route("/_logs") +def logs(): + return flask.Response(event_stream(), mimetype="text/event-stream") + + +@app.route("/_stuff", methods=['GET']) +def stuff(): + return jsonify(row1=get_queues(r_serv)) + + +@app.route("/") +def index(): + default_minute = cfg.get("Flask", "minute_processed_paste") + threshold_stucked_module = cfg.getint("Module_ModuleInformation", "threshold_stucked_module") + return render_template("index.html", default_minute = default_minute, threshold_stucked_module=threshold_stucked_module) diff --git a/var/www/Flask_search.py b/var/www/Flask_search.py new file mode 100644 index 00000000..b5c60898 --- /dev/null +++ b/var/www/Flask_search.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python2 +# -*-coding:UTF-8 -* + +''' + Flask functions and routes for the trending modules page +''' +import redis +import json +import os +import flask +from flask import Flask, render_template, jsonify, request + +import Paste + +# ============ VARIABLES ============ +import Flask_config + +app = Flask_config.app +cfg = Flask_config.cfg +r_serv_pasteName = Flask_config.r_serv_pasteName +max_preview_char = Flask_config.max_preview_char +max_preview_modal = Flask_config.max_preview_modal +# ============ FUNCTIONS ============ + + +# ============ ROUTES ============ + +@app.route("/search", methods=['POST']) +def search(): + query = request.form['query'] + q = [] + q.append(query) + r = [] #complete path + c = [] #preview of the paste content + paste_date = [] + paste_size = [] + num_elem_to_get = 50 + + # Search filename + for path in r_serv_pasteName.smembers(q[0]): + r.append(path) + paste = Paste.Paste(path) + content = paste.get_p_content().decode('utf8', 'ignore') + content_range = max_preview_char if len(content)>max_preview_char else len(content)-1 + c.append(content[0:content_range]) + curr_date = str(paste._get_p_date()) + curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:] + paste_date.append(curr_date) + paste_size.append(paste._get_p_size()) + + # Search full line + from whoosh import index + from whoosh.fields import Schema, TEXT, ID + schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT) + + indexpath = os.path.join(os.environ['AIL_HOME'], cfg.get("Indexer", "path")) + ix = index.open_dir(indexpath) + from whoosh.qparser import QueryParser + with ix.searcher() as searcher: + query = QueryParser("content", ix.schema).parse(" ".join(q)) + results = searcher.search_page(query, 1, pagelen=num_elem_to_get) + for x in results: + r.append(x.items()[0][1]) + paste = Paste.Paste(x.items()[0][1]) + content = paste.get_p_content().decode('utf8', 'ignore') + content_range = max_preview_char if len(content)>max_preview_char else len(content)-1 + c.append(content[0:content_range]) + curr_date = str(paste._get_p_date()) + curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:] + paste_date.append(curr_date) + paste_size.append(paste._get_p_size()) + results = searcher.search(query) + num_res = len(results) + + return render_template("search.html", r=r, c=c, query=request.form['query'], paste_date=paste_date, paste_size=paste_size, char_to_display=max_preview_modal, num_res=num_res) + + +@app.route("/get_more_search_result", methods=['POST']) +def get_more_search_result(): + query = request.form['query'] + q = [] + q.append(query) + page_offset = int(request.form['page_offset']) + num_elem_to_get = 50 + + path_array = [] + preview_array = [] + date_array = [] + size_array = [] + + from whoosh import index + from whoosh.fields import Schema, TEXT, ID + schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT) + + indexpath = os.path.join(os.environ['AIL_HOME'], cfg.get("Indexer", "path")) + ix = index.open_dir(indexpath) + from whoosh.qparser import QueryParser + with ix.searcher() as searcher: + query = QueryParser("content", ix.schema).parse(" ".join(q)) + results = searcher.search_page(query, page_offset, num_elem_to_get) + for x in results: + path_array.append(x.items()[0][1]) + paste = Paste.Paste(x.items()[0][1]) + content = paste.get_p_content().decode('utf8', 'ignore') + content_range = max_preview_char if len(content)>max_preview_char else len(content)-1 + preview_array.append(content[0:content_range]) + curr_date = str(paste._get_p_date()) + curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:] + date_array.append(curr_date) + size_array.append(paste._get_p_size()) + to_return = {} + to_return["path_array"] = path_array + to_return["preview_array"] = preview_array + to_return["date_array"] = date_array + to_return["size_array"] = size_array + print "len(path_array)="+str(len(path_array)) + if len(path_array) < num_elem_to_get: #pagelength + to_return["moreData"] = False + else: + to_return["moreData"] = True + + return jsonify(to_return) + + diff --git a/var/www/Flask_sentiment.py b/var/www/Flask_sentiment.py new file mode 100644 index 00000000..275cce39 --- /dev/null +++ b/var/www/Flask_sentiment.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python2 +# -*-coding:UTF-8 -* + +''' + Flask functions and routes for the trending modules page +''' +import redis +import datetime +import calendar +from Date import Date +import flask +from flask import Flask, render_template, jsonify, request + +import Paste + +# ============ VARIABLES ============ +import Flask_config + +app = Flask_config.app +cfg = Flask_config.cfg +r_serv_charts = Flask_config.r_serv_charts +r_serv_sentiment = Flask_config.r_serv_sentiment +# ============ FUNCTIONS ============ + +def get_date_range(num_day): + curr_date = datetime.date.today() + date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2)) + date_list = [] + + for i in range(0, num_day+1): + date_list.append(date.substract_day(i)) + return date_list + + +# ============ ROUTES ============ + +@app.route("/sentiment_analysis_trending/") +def sentiment_analysis_trending(): + return render_template("sentiment_analysis_trending.html") + + +@app.route("/sentiment_analysis_getplotdata/", methods=['GET']) +def sentiment_analysis_getplotdata(): + # Get the top providers based on number of pastes + oneHour = 60*60 + sevenDays = oneHour*24*7 + dateStart = datetime.datetime.now() + dateStart = dateStart.replace(minute=0, second=0, microsecond=0) + dateStart_timestamp = calendar.timegm(dateStart.timetuple()) + + getAllProviders = request.args.get('getProviders') + provider = request.args.get('provider') + allProvider = request.args.get('all') + if getAllProviders == 'True': + if allProvider == "True": + range_providers = r_serv_charts.smembers('all_provider_set') + return jsonify(list(range_providers)) + else: + range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(0)[0], '+inf', '-inf', start=0, num=8) + # if empty, get yesterday top providers + range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(1)[1], '+inf', '-inf', start=0, num=8) if range_providers == [] else range_providers + # if still empty, takes from all providers + if range_providers == []: + print 'today provider empty' + range_providers = r_serv_charts.smembers('all_provider_set') + return jsonify(list(range_providers)) + + elif provider is not None: + to_return = {} + + cur_provider_name = provider + '_' + list_date = {} + for cur_timestamp in range(int(dateStart_timestamp), int(dateStart_timestamp)-sevenDays-oneHour, -oneHour): + cur_set_name = cur_provider_name + str(cur_timestamp) + + list_value = [] + for cur_id in r_serv_sentiment.smembers(cur_set_name): + cur_value = r_serv_sentiment.get(cur_id) + list_value.append(cur_value) + list_date[cur_timestamp] = list_value + to_return[provider] = list_date + + return jsonify(to_return) + return "Bad request" + + + +@app.route("/sentiment_analysis_plot_tool/") +def sentiment_analysis_plot_tool(): + return render_template("sentiment_analysis_plot_tool.html") + + + +@app.route("/sentiment_analysis_plot_tool_getdata/", methods=['GET']) +def sentiment_analysis_plot_tool_getdata(): + getProviders = request.args.get('getProviders') + + if getProviders == 'True': + providers = [] + for cur_provider in r_serv_charts.smembers('all_provider_set'): + providers.append(cur_provider) + return jsonify(providers) + + else: + query = request.args.get('query') + query = query.split(',') + Qdate = request.args.get('Qdate') + + date1 = (Qdate.split('-')[0]).split('.') + date1 = datetime.date(int(date1[2]), int(date1[1]), int(date1[0])) + + date2 = (Qdate.split('-')[1]).split('.') + date2 = datetime.date(int(date2[2]), int(date2[1]), int(date2[0])) + + timestamp1 = calendar.timegm(date1.timetuple()) + timestamp2 = calendar.timegm(date2.timetuple()) + + oneHour = 60*60 + oneDay = oneHour*24 + + to_return = {} + for cur_provider in query: + list_date = {} + cur_provider_name = cur_provider + '_' + for cur_timestamp in range(int(timestamp1), int(timestamp2)+oneDay, oneHour): + cur_set_name = cur_provider_name + str(cur_timestamp) + + list_value = [] + for cur_id in r_serv_sentiment.smembers(cur_set_name): + cur_value = r_serv_sentiment.get(cur_id) + list_value.append(cur_value) + list_date[cur_timestamp] = list_value + to_return[cur_provider] = list_date + + return jsonify(to_return) + + diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 0eb04b47..4ece0c75 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -15,231 +15,30 @@ sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) import Paste from Date import Date +# Import config +import Flask_config + # CONFIG # -tlsh_to_percent = 1000.0 #Use to display the estimated percentage instead of a raw value +cfg = Flask_config.cfg -configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') -if not os.path.exists(configfile): - raise Exception('Unable to find the configuration file. \ - Did you set environment variables? \ - Or activate the virtualenv.') - -cfg = ConfigParser.ConfigParser() -cfg.read(configfile) - -max_preview_char = int(cfg.get("Flask", "max_preview_char")) # Maximum number of character to display in the tooltip -max_preview_modal = int(cfg.get("Flask", "max_preview_modal")) # Maximum number of character to display in the modal - -# REDIS # -r_serv = redis.StrictRedis( - host=cfg.get("Redis_Queues", "host"), - port=cfg.getint("Redis_Queues", "port"), - db=cfg.getint("Redis_Queues", "db")) - -r_serv_log = redis.StrictRedis( - host=cfg.get("Redis_Log", "host"), - port=cfg.getint("Redis_Log", "port"), - db=cfg.getint("Redis_Log", "db")) - -r_serv_charts = redis.StrictRedis( - host=cfg.get("Redis_Level_DB_Trending", "host"), - port=cfg.getint("Redis_Level_DB_Trending", "port"), - db=cfg.getint("Redis_Level_DB_Trending", "db")) - -r_serv_db = redis.StrictRedis( - host=cfg.get("Redis_Level_DB", "host"), - port=cfg.getint("Redis_Level_DB", "port"), - db=cfg.getint("Redis_Level_DB", "db")) - -r_serv_sentiment = redis.StrictRedis( - host=cfg.get("Redis_Level_DB_Sentiment", "host"), - port=cfg.getint("Redis_Level_DB_Sentiment", "port"), - db=cfg.getint("Redis_Level_DB_Sentiment", "db")) - -r_serv_term = redis.StrictRedis( - host=cfg.get("Redis_Level_DB_TermFreq", "host"), - port=cfg.getint("Redis_Level_DB_TermFreq", "port"), - db=cfg.getint("Redis_Level_DB_TermFreq", "db")) - -r_serv_pasteName = redis.StrictRedis( - host=cfg.get("Redis_Paste_Name", "host"), - port=cfg.getint("Redis_Paste_Name", "port"), - db=cfg.getint("Redis_Paste_Name", "db")) - - -app = Flask(__name__, static_url_path='/static/') - - -def event_stream(): - pubsub = r_serv_log.pubsub() - pubsub.psubscribe("Script" + '.*') - for msg in pubsub.listen(): - level = msg['channel'].split('.')[1] - if msg['type'] == 'pmessage' and level != "DEBUG": - yield 'data: %s\n\n' % json.dumps(msg) - -def event_stream_getImportantPasteByModule(module_name): - index = 0 - all_pastes_list = getPastebyType(r_serv_db, module_name) - for path in all_pastes_list: - index += 1 - paste = Paste.Paste(path) - content = paste.get_p_content().decode('utf8', 'ignore') - content_range = max_preview_char if len(content)>max_preview_char else len(content)-1 - curr_date = str(paste._get_p_date()) - curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:] - data = {} - data["module"] = module_name - data["index"] = index - data["path"] = path - data["content"] = content[0:content_range] - data["linenum"] = paste.get_lines_info()[0] - data["date"] = curr_date - data["char_to_display"] = max_preview_modal - data["finished"] = True if index == len(all_pastes_list) else False - yield 'retry: 100000\ndata: %s\n\n' % json.dumps(data) #retry to avoid reconnection of the browser - - -def get_queues(r): - # We may want to put the llen in a pipeline to do only one query. - newData = [] - for queue, card in r.hgetall("queues").iteritems(): - key = "MODULE_" + queue + "_" - keySet = "MODULE_TYPE_" + queue - - for moduleNum in r.smembers(keySet): - - value = r.get(key + str(moduleNum)) - if value is not None: - timestamp, path = value.split(", ") - if timestamp is not None: - startTime_readable = datetime.datetime.fromtimestamp(int(timestamp)) - processed_time_readable = str((datetime.datetime.now() - startTime_readable)).split('.')[0] - seconds = int((datetime.datetime.now() - startTime_readable).total_seconds()) - newData.append( (queue, card, seconds, moduleNum) ) - else: - newData.append( (queue, cards, 0, moduleNum) ) - - return newData +Flask_config.app = Flask(__name__, static_url_path='/static/') +app = Flask_config.app +# import routes and functions from modules +import Flask_dashboard +import Flask_trendingcharts +import Flask_trendingmodules +import Flask_browsepastes +import Flask_sentiment +import Flask_terms +import Flask_search +import Flask_showpaste def list_len(s): return len(s) app.jinja_env.filters['list_len'] = list_len -def showpaste(content_range): - requested_path = request.args.get('paste', '') - paste = Paste.Paste(requested_path) - p_date = str(paste._get_p_date()) - p_date = p_date[6:]+'/'+p_date[4:6]+'/'+p_date[0:4] - p_source = paste.p_source - p_encoding = paste._get_p_encoding() - p_language = paste._get_p_language() - p_size = paste.p_size - p_mime = paste.p_mime - p_lineinfo = paste.get_lines_info() - p_content = paste.get_p_content().decode('utf-8', 'ignore') - p_duplicate_full_list = json.loads(paste._get_p_duplicate()) - p_duplicate_list = [] - p_simil_list = [] - p_date_list = [] - p_hashtype_list = [] - - - for dup_list in p_duplicate_full_list: - if dup_list[0] == "tlsh": - dup_list[2] = int(((tlsh_to_percent - float(dup_list[2])) / tlsh_to_percent)*100) - else: - dup_list[2] = int(dup_list[2]) - - p_duplicate_full_list.sort(lambda x,y: cmp(x[2], y[2]), reverse=True) - - # Combine multiple duplicate paste name and format for display - new_dup_list = [] - dup_list_removed = [] - for dup_list_index in range(0, len(p_duplicate_full_list)): - if dup_list_index in dup_list_removed: - continue - indices = [i for i, x in enumerate(p_duplicate_full_list) if x[1] == p_duplicate_full_list[dup_list_index][1]] - hash_types = [] - comp_vals = [] - for i in indices: - hash_types.append(p_duplicate_full_list[i][0].encode('utf8')) - comp_vals.append(p_duplicate_full_list[i][2]) - dup_list_removed.append(i) - - hash_types = str(hash_types).replace("[","").replace("]","") if len(hash_types)==1 else str(hash_types) - comp_vals = str(comp_vals).replace("[","").replace("]","") if len(comp_vals)==1 else str(comp_vals) - if len(p_duplicate_full_list[dup_list_index]) > 3: - try: - date_paste = str(int(p_duplicate_full_list[dup_list_index][3])) - date_paste = date_paste[0:4]+"-"+date_paste[4:6]+"-"+date_paste[6:8] - except ValueError: - date_paste = str(p_duplicate_full_list[dup_list_index][3]) - else: - date_paste = "No date available" - new_dup_list.append([hash_types.replace("'", ""), p_duplicate_full_list[dup_list_index][1], comp_vals, date_paste]) - - # Create the list to pass to the webpage - for dup_list in new_dup_list: - hash_type, path, simil_percent, date_paste = dup_list - p_duplicate_list.append(path) - p_simil_list.append(simil_percent) - p_hashtype_list.append(hash_type) - p_date_list.append(date_paste) - - if content_range != 0: - p_content = p_content[0:content_range] - - - return render_template("show_saved_paste.html", date=p_date, source=p_source, encoding=p_encoding, language=p_language, size=p_size, mime=p_mime, lineinfo=p_lineinfo, content=p_content, initsize=len(p_content), duplicate_list = p_duplicate_list, simil_list = p_simil_list, hashtype_list = p_hashtype_list, date_list=p_date_list) - -def getPastebyType(server, module_name): - all_path = [] - for path in server.smembers('WARNING_'+module_name): - all_path.append(path) - return all_path - - -def get_date_range(num_day): - curr_date = datetime.date.today() - date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2)) - date_list = [] - - for i in range(0, num_day+1): - date_list.append(date.substract_day(i)) - return date_list - -# Iterate over elements in the module provided and return the today data or the last data -# return format: [('passed_days', num_of_passed_days), ('elem_name1', elem_value1), ('elem_name2', elem_value2)]] -def get_top_relevant_data(server, module_name): - days = 0 - for date in get_date_range(15): - redis_progression_name_set = 'top_'+ module_name +'_set_' + date - member_set = server.zrevrangebyscore(redis_progression_name_set, '+inf', '-inf', withscores=True) - if len(member_set) == 0: #No data for this date - days += 1 - else: - member_set.insert(0, ("passed_days", days)) - return member_set - - -def Term_getValueOverRange(word, startDate, num_day): - passed_days = 0 - oneDay = 60*60*24 - to_return = [] - curr_to_return = 0 - for timestamp in range(startDate, startDate - max(num_day)*oneDay, -oneDay): - value = r_serv_term.hget(timestamp, word) - curr_to_return += int(value) if value is not None else 0 - for i in num_day: - if passed_days == i-1: - to_return.append(curr_to_return) - passed_days += 1 - return to_return - - # ========= CACHE CONTROL ======== @app.after_request def add_header(response): @@ -251,595 +50,7 @@ def add_header(response): response.headers['Cache-Control'] = 'public, max-age=0' return response -# ============ ROUTES ============ - -@app.route("/_logs") -def logs(): - return flask.Response(event_stream(), mimetype="text/event-stream") - - -@app.route("/_stuff", methods=['GET']) -def stuff(): - return jsonify(row1=get_queues(r_serv)) - -@app.route("/_progressionCharts", methods=['GET']) -def progressionCharts(): - attribute_name = request.args.get('attributeName') - trending_name = request.args.get('trendingName') - bar_requested = True if request.args.get('bar') == "true" else False - - if (bar_requested): - num_day = int(request.args.get('days')) - bar_values = [] - - date_range = get_date_range(num_day) - # Retreive all data from the last num_day - for date in date_range: - curr_value = r_serv_charts.hget(attribute_name, date) - bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], int(curr_value if curr_value is not None else 0)]) - bar_values.insert(0, attribute_name) - return jsonify(bar_values) - - else: - redis_progression_name = "z_top_progression_" + trending_name - keyw_value = r_serv_charts.zrevrangebyscore(redis_progression_name, '+inf', '-inf', withscores=True, start=0, num=10) - return jsonify(keyw_value) - -@app.route("/_moduleCharts", methods=['GET']) -def modulesCharts(): - keyword_name = request.args.get('keywordName') - module_name = request.args.get('moduleName') - bar_requested = True if request.args.get('bar') == "true" else False - - if (bar_requested): - num_day = int(request.args.get('days')) - bar_values = [] - - date_range = get_date_range(num_day) - # Retreive all data from the last num_day - for date in date_range: - curr_value = r_serv_charts.hget(date, module_name+'-'+keyword_name) - bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], int(curr_value if curr_value is not None else 0)]) - bar_values.insert(0, keyword_name) - return jsonify(bar_values) - - else: - member_set = get_top_relevant_data(r_serv_charts, module_name) - if len(member_set) == 0: - member_set.append(("No relevant data", int(100))) - return jsonify(member_set) - - -@app.route("/_providersChart", methods=['GET']) -def providersChart(): - keyword_name = request.args.get('keywordName') - module_name = request.args.get('moduleName') - bar_requested = True if request.args.get('bar') == "true" else False - - if (bar_requested): - num_day = int(request.args.get('days')) - bar_values = [] - - date_range = get_date_range(num_day) - # Retreive all data from the last num_day - for date in date_range: - curr_value_size = r_serv_charts.hget(keyword_name+'_'+'size', date) - curr_value_num = r_serv_charts.hget(keyword_name+'_'+'num', date) - curr_value_size_avg = r_serv_charts.hget(keyword_name+'_'+'avg', date) - if module_name == "size": - curr_value = float(curr_value_size_avg if curr_value_size_avg is not None else 0) - else: - curr_value = float(curr_value_num if curr_value_num is not None else 0.0) - - bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], curr_value]) - bar_values.insert(0, keyword_name) - return jsonify(bar_values) - - else: - #redis_provider_name_set = 'top_size_set' if module_name == "size" else 'providers_set' - redis_provider_name_set = 'top_avg_size_set_' if module_name == "size" else 'providers_set_' - redis_provider_name_set = redis_provider_name_set + get_date_range(0)[0] - - member_set = r_serv_charts.zrevrangebyscore(redis_provider_name_set, '+inf', '-inf', withscores=True, start=0, num=8) - # Member set is a list of (value, score) pairs - if len(member_set) == 0: - member_set.append(("No relevant data", float(100))) - return jsonify(member_set) - - - -@app.route("/search", methods=['POST']) -def search(): - query = request.form['query'] - q = [] - q.append(query) - r = [] #complete path - c = [] #preview of the paste content - paste_date = [] - paste_size = [] - num_elem_to_get = 50 - - # Search filename - for path in r_serv_pasteName.smembers(q[0]): - r.append(path) - paste = Paste.Paste(path) - content = paste.get_p_content().decode('utf8', 'ignore') - content_range = max_preview_char if len(content)>max_preview_char else len(content)-1 - c.append(content[0:content_range]) - curr_date = str(paste._get_p_date()) - curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:] - paste_date.append(curr_date) - paste_size.append(paste._get_p_size()) - - # Search full line - from whoosh import index - from whoosh.fields import Schema, TEXT, ID - schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT) - - indexpath = os.path.join(os.environ['AIL_HOME'], cfg.get("Indexer", "path")) - ix = index.open_dir(indexpath) - from whoosh.qparser import QueryParser - with ix.searcher() as searcher: - query = QueryParser("content", ix.schema).parse(" ".join(q)) - results = searcher.search_page(query, 1, pagelen=num_elem_to_get) - for x in results: - r.append(x.items()[0][1]) - paste = Paste.Paste(x.items()[0][1]) - content = paste.get_p_content().decode('utf8', 'ignore') - content_range = max_preview_char if len(content)>max_preview_char else len(content)-1 - c.append(content[0:content_range]) - curr_date = str(paste._get_p_date()) - curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:] - paste_date.append(curr_date) - paste_size.append(paste._get_p_size()) - results = searcher.search(query) - num_res = len(results) - - return render_template("search.html", r=r, c=c, query=request.form['query'], paste_date=paste_date, paste_size=paste_size, char_to_display=max_preview_modal, num_res=num_res) - - -@app.route("/get_more_search_result", methods=['POST']) -def get_more_search_result(): - query = request.form['query'] - q = [] - q.append(query) - page_offset = int(request.form['page_offset']) - num_elem_to_get = 50 - - path_array = [] - preview_array = [] - date_array = [] - size_array = [] - - from whoosh import index - from whoosh.fields import Schema, TEXT, ID - schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT) - - indexpath = os.path.join(os.environ['AIL_HOME'], cfg.get("Indexer", "path")) - ix = index.open_dir(indexpath) - from whoosh.qparser import QueryParser - with ix.searcher() as searcher: - query = QueryParser("content", ix.schema).parse(" ".join(q)) - results = searcher.search_page(query, page_offset, num_elem_to_get) - for x in results: - path_array.append(x.items()[0][1]) - paste = Paste.Paste(x.items()[0][1]) - content = paste.get_p_content().decode('utf8', 'ignore') - content_range = max_preview_char if len(content)>max_preview_char else len(content)-1 - preview_array.append(content[0:content_range]) - curr_date = str(paste._get_p_date()) - curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:] - date_array.append(curr_date) - size_array.append(paste._get_p_size()) - to_return = {} - to_return["path_array"] = path_array - to_return["preview_array"] = preview_array - to_return["date_array"] = date_array - to_return["size_array"] = size_array - print "len(path_array)="+str(len(path_array)) - if len(path_array) < num_elem_to_get: #pagelength - to_return["moreData"] = False - else: - to_return["moreData"] = True - - return jsonify(to_return) - - -@app.route("/") -def index(): - default_minute = cfg.get("Flask", "minute_processed_paste") - threshold_stucked_module = cfg.getint("Module_ModuleInformation", "threshold_stucked_module") - return render_template("index.html", default_minute = default_minute, threshold_stucked_module=threshold_stucked_module) - - -@app.route("/monitoring/") -def monitoring(): - for queue in r_serv.smembers("queues"): - return render_template("Queue_live_Monitoring.html", last_value=queue) - - -@app.route("/wordstrending/") -def wordstrending(): - default_display = cfg.get("Flask", "default_display") - return render_template("Wordstrending.html", default_display = default_display) - - -@app.route("/protocolstrending/") -def protocolstrending(): - default_display = cfg.get("Flask", "default_display") - return render_template("Protocolstrending.html", default_display = default_display) - - -@app.route("/trending/") -def trending(): - default_display = cfg.get("Flask", "default_display") - return render_template("Trending.html", default_display = default_display) - -@app.route("/browseImportantPaste/", methods=['GET']) -def browseImportantPaste(): - module_name = request.args.get('moduleName') - return render_template("browse_important_paste.html") - - -@app.route("/importantPasteByModule/", methods=['GET']) -def importantPasteByModule(): - module_name = request.args.get('moduleName') - - all_content = [] - paste_date = [] - paste_linenum = [] - all_path = [] - allPastes = getPastebyType(r_serv_db, module_name) - - for path in allPastes[0:10]: - all_path.append(path) - paste = Paste.Paste(path) - content = paste.get_p_content().decode('utf8', 'ignore') - content_range = max_preview_char if len(content)>max_preview_char else len(content)-1 - all_content.append(content[0:content_range].replace("\"", "\'").replace("\r", " ").replace("\n", " ")) - curr_date = str(paste._get_p_date()) - curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:] - paste_date.append(curr_date) - paste_linenum.append(paste.get_lines_info()[0]) - - if len(allPastes) > 10: - finished = "" - else: - finished = "display: none;" - - return render_template("important_paste_by_module.html", moduleName=module_name, all_path=all_path, content=all_content, paste_date=paste_date, paste_linenum=paste_linenum, char_to_display=max_preview_modal, finished=finished) - -@app.route("/_getImportantPasteByModule") -def getImportantPasteByModule(): - module_name = request.args.get('moduleName') - return flask.Response(event_stream_getImportantPasteByModule(module_name), mimetype="text/event-stream") - -@app.route("/moduletrending/") -def moduletrending(): - return render_template("Moduletrending.html") - -@app.route("/sentiment_analysis_trending/") -def sentiment_analysis_trending(): - return render_template("sentiment_analysis_trending.html") - - -@app.route("/sentiment_analysis_getplotdata/", methods=['GET']) -def sentiment_analysis_getplotdata(): - # Get the top providers based on number of pastes - oneHour = 60*60 - sevenDays = oneHour*24*7 - dateStart = datetime.datetime.now() - dateStart = dateStart.replace(minute=0, second=0, microsecond=0) - dateStart_timestamp = calendar.timegm(dateStart.timetuple()) - - getAllProviders = request.args.get('getProviders') - provider = request.args.get('provider') - allProvider = request.args.get('all') - if getAllProviders == 'True': - if allProvider == "True": - range_providers = r_serv_charts.smembers('all_provider_set') - return jsonify(list(range_providers)) - else: - range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(0)[0], '+inf', '-inf', start=0, num=8) - # if empty, get yesterday top providers - range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(1)[1], '+inf', '-inf', start=0, num=8) if range_providers == [] else range_providers - # if still empty, takes from all providers - if range_providers == []: - print 'today provider empty' - range_providers = r_serv_charts.smembers('all_provider_set') - return jsonify(range_providers) - - elif provider is not None: - to_return = {} - - cur_provider_name = provider + '_' - list_date = {} - for cur_timestamp in range(int(dateStart_timestamp), int(dateStart_timestamp)-sevenDays-oneHour, -oneHour): - cur_set_name = cur_provider_name + str(cur_timestamp) - - list_value = [] - for cur_id in r_serv_sentiment.smembers(cur_set_name): - cur_value = r_serv_sentiment.get(cur_id) - list_value.append(cur_value) - list_date[cur_timestamp] = list_value - to_return[provider] = list_date - - return jsonify(to_return) - return "Bad request" - - - -@app.route("/sentiment_analysis_plot_tool/") -def sentiment_analysis_plot_tool(): - return render_template("sentiment_analysis_plot_tool.html") - - - -@app.route("/sentiment_analysis_plot_tool_getdata/", methods=['GET']) -def sentiment_analysis_plot_tool_getdata(): - getProviders = request.args.get('getProviders') - - if getProviders == 'True': - providers = [] - for cur_provider in r_serv_charts.smembers('all_provider_set'): - providers.append(cur_provider) - return jsonify(providers) - - else: - query = request.args.get('query') - query = query.split(',') - Qdate = request.args.get('Qdate') - - date1 = (Qdate.split('-')[0]).split('.') - date1 = datetime.date(int(date1[2]), int(date1[1]), int(date1[0])) - - date2 = (Qdate.split('-')[1]).split('.') - date2 = datetime.date(int(date2[2]), int(date2[1]), int(date2[0])) - - timestamp1 = calendar.timegm(date1.timetuple()) - timestamp2 = calendar.timegm(date2.timetuple()) - - oneHour = 60*60 - oneDay = oneHour*24 - - to_return = {} - for cur_provider in query: - list_date = {} - cur_provider_name = cur_provider + '_' - for cur_timestamp in range(int(timestamp1), int(timestamp2)+oneDay, oneHour): - cur_set_name = cur_provider_name + str(cur_timestamp) - - list_value = [] - for cur_id in r_serv_sentiment.smembers(cur_set_name): - cur_value = r_serv_sentiment.get(cur_id) - list_value.append(cur_value) - list_date[cur_timestamp] = list_value - to_return[cur_provider] = list_date - - return jsonify(to_return) - - -@app.route("/terms_management/") -def terms_management(): - TrackedTermsSet_Name = "TrackedSetTermSet" - BlackListTermsSet_Name = "BlackListSetTermSet" - TrackedTermsDate_Name = "TrackedTermDate" - BlackListTermsDate_Name = "BlackListTermDate" - - today = datetime.datetime.now() - today = today.replace(hour=0, minute=0, second=0, microsecond=0) - today_timestamp = calendar.timegm(today.timetuple()) - - track_list = [] - track_list_values = [] - track_list_num_of_paste = [] - for tracked_term in r_serv_term.smembers(TrackedTermsSet_Name): - track_list.append(tracked_term) - value_range = Term_getValueOverRange(tracked_term, today_timestamp, [1, 7, 31]) - - term_date = r_serv_term.hget(TrackedTermsDate_Name, tracked_term) - - set_paste_name = "tracked_" + tracked_term - track_list_num_of_paste.append(r_serv_term.scard(set_paste_name)) - term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded" - value_range.append(term_date) - track_list_values.append(value_range) - - - black_list = [] - for blacked_term in r_serv_term.smembers(BlackListTermsSet_Name): - term_date = r_serv_term.hget(BlackListTermsDate_Name, blacked_term) - term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded" - black_list.append([blacked_term, term_date]) - - return render_template("terms_management.html", black_list=black_list, track_list=track_list, track_list_values=track_list_values, track_list_num_of_paste=track_list_num_of_paste) - - -@app.route("/terms_management_query_paste/") -def terms_management_query_paste(): - term = request.args.get('term') - TrackedTermsSet_Name = "TrackedSetTermSet" - paste_info = [] - - set_paste_name = "tracked_" + term - track_list_path = r_serv_term.smembers(set_paste_name) - - for path in track_list_path: - paste = Paste.Paste(path) - p_date = str(paste._get_p_date()) - p_date = p_date[6:]+'/'+p_date[4:6]+'/'+p_date[0:4] - p_source = paste.p_source - p_encoding = paste._get_p_encoding() - p_size = paste.p_size - p_mime = paste.p_mime - p_lineinfo = paste.get_lines_info() - p_content = paste.get_p_content().decode('utf-8', 'ignore') - if p_content != 0: - p_content = p_content[0:400] - paste_info.append({"path": path, "date": p_date, "source": p_source, "encoding": p_encoding, "size": p_size, "mime": p_mime, "lineinfo": p_lineinfo, "content": p_content}) - - return jsonify(paste_info) - - -@app.route("/terms_management_query/") -def terms_management_query(): - TrackedTermsDate_Name = "TrackedTermDate" - BlackListTermsDate_Name = "BlackListTermDate" - term = request.args.get('term') - section = request.args.get('section') - - today = datetime.datetime.now() - today = today.replace(hour=0, minute=0, second=0, microsecond=0) - today_timestamp = calendar.timegm(today.timetuple()) - value_range = Term_getValueOverRange(term, today_timestamp, [1, 7, 31]) - - if section == "followTerm": - term_date = r_serv_term.hget(TrackedTermsDate_Name, term) - elif section == "blacklistTerm": - term_date = r_serv_term.hget(BlackListTermsDate_Name, term) - - term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded" - value_range.append(str(term_date)) - return jsonify(value_range) - - -@app.route("/terms_management_action/", methods=['GET']) -def terms_management_action(): - TrackedTermsSet_Name = "TrackedSetTermSet" - TrackedTermsDate_Name = "TrackedTermDate" - BlackListTermsDate_Name = "BlackListTermDate" - BlackListTermsSet_Name = "BlackListSetTermSet" - - today = datetime.datetime.now() - today = today.replace(microsecond=0) - today_timestamp = calendar.timegm(today.timetuple()) - - - section = request.args.get('section') - action = request.args.get('action') - term = request.args.get('term') - if action is None or term is None: - return "None" - else: - if section == "followTerm": - if action == "add": - r_serv_term.sadd(TrackedTermsSet_Name, term.lower()) - r_serv_term.hset(TrackedTermsDate_Name, term, today_timestamp) - else: - r_serv_term.srem(TrackedTermsSet_Name, term.lower()) - elif section == "blacklistTerm": - if action == "add": - r_serv_term.sadd(BlackListTermsSet_Name, term.lower()) - r_serv_term.hset(BlackListTermsDate_Name, term, today_timestamp) - else: - r_serv_term.srem(BlackListTermsSet_Name, term.lower()) - else: - return "None" - - to_return = {} - to_return["section"] = section - to_return["action"] = action - to_return["term"] = term - return jsonify(to_return) - - - -@app.route("/terms_plot_tool/") -def terms_plot_tool(): - term = request.args.get('term') - if term is not None: - return render_template("terms_plot_tool.html", term=term) - else: - return render_template("terms_plot_tool.html", term="") - - -@app.route("/terms_plot_tool_data/") -def terms_plot_tool_data(): - oneDay = 60*60*24 - range_start = datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_start')))) if request.args.get('range_start') is not None else 0; - range_start = range_start.replace(hour=0, minute=0, second=0, microsecond=0) - range_start = calendar.timegm(range_start.timetuple()) - range_end = datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_end')))) if request.args.get('range_end') is not None else 0; - range_end = range_end.replace(hour=0, minute=0, second=0, microsecond=0) - range_end = calendar.timegm(range_end.timetuple()) - term = request.args.get('term') - - if term is None: - return "None" - else: - value_range = [] - for timestamp in range(range_start, range_end+oneDay, oneDay): - value = r_serv_term.hget(timestamp, term) - curr_value_range = int(value) if value is not None else 0 - value_range.append([timestamp, curr_value_range]) - value_range.insert(0,term) - return jsonify(value_range) - - -@app.route("/terms_plot_top/") -def terms_plot_top(): - return render_template("terms_plot_top.html") - - -@app.route("/terms_plot_top_data/") -def terms_plot_top_data(): - oneDay = 60*60*24 - today = datetime.datetime.now() - today = today.replace(hour=0, minute=0, second=0, microsecond=0) - today_timestamp = calendar.timegm(today.timetuple()) - - set_day = "TopTermFreq_set_day_" + str(today_timestamp) - set_week = "TopTermFreq_set_week"; - set_month = "TopTermFreq_set_month"; - - the_set = request.args.get('set') - num_day = int(request.args.get('num_day')) - if the_set is None: - return "None" - else: - to_return = [] - if the_set == "TopTermFreq_set_day": - the_set += "_" + str(today_timestamp) - - for term, tot_value in r_serv_term.zrevrangebyscore(the_set, '+inf', '-inf', withscores=True, start=0, num=20): - position = {} - position['day'] = r_serv_term.zrevrank(set_day, term) - position['day'] = position['day']+1 if position['day'] is not None else "<20" - position['week'] = r_serv_term.zrevrank(set_week, term) - position['week'] = position['week']+1 if position['week'] is not None else "<20" - position['month'] = r_serv_term.zrevrank(set_month, term) - position['month'] = position['month']+1 if position['month'] is not None else "<20" - value_range = [] - for timestamp in range(today_timestamp, today_timestamp - num_day*oneDay, -oneDay): - value = r_serv_term.hget(timestamp, term) - curr_value_range = int(value) if value is not None else 0 - value_range.append([timestamp, curr_value_range]) - - to_return.append([term, value_range, tot_value, position]) - - return jsonify(to_return) - - - -@app.route("/showsavedpaste/") #completely shows the paste in a new tab -def showsavedpaste(): - return showpaste(0) - - -@app.route("/showpreviewpaste/") -def showpreviewpaste(): - num = request.args.get('num', '') - return "|num|"+num+"|num|"+showpaste(max_preview_modal) - - -@app.route("/getmoredata/") -def getmoredata(): - requested_path = request.args.get('paste', '') - paste = Paste.Paste(requested_path) - p_content = paste.get_p_content().decode('utf-8', 'ignore') - to_return = p_content[max_preview_modal-1:] - return to_return - +# ============ MAIN ============ if __name__ == "__main__": app.run(host='0.0.0.0', port=7000, threaded=True) diff --git a/var/www/Flask_showpaste.py b/var/www/Flask_showpaste.py new file mode 100644 index 00000000..71e2a4e2 --- /dev/null +++ b/var/www/Flask_showpaste.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python2 +# -*-coding:UTF-8 -* + +''' + Flask functions and routes for the trending modules page +''' +import redis +import json +import flask +from flask import Flask, render_template, jsonify, request + +import Paste + +# ============ VARIABLES ============ +import Flask_config + +app = Flask_config.app +cfg = Flask_config.cfg +r_serv_pasteName = Flask_config.r_serv_pasteName +max_preview_char = Flask_config.max_preview_char +max_preview_modal = Flask_config.max_preview_modal +tlsh_to_percent = Flask_config.tlsh_to_percent +# ============ FUNCTIONS ============ + +def showpaste(content_range): + requested_path = request.args.get('paste', '') + paste = Paste.Paste(requested_path) + p_date = str(paste._get_p_date()) + p_date = p_date[6:]+'/'+p_date[4:6]+'/'+p_date[0:4] + p_source = paste.p_source + p_encoding = paste._get_p_encoding() + p_language = paste._get_p_language() + p_size = paste.p_size + p_mime = paste.p_mime + p_lineinfo = paste.get_lines_info() + p_content = paste.get_p_content().decode('utf-8', 'ignore') + p_duplicate_full_list = json.loads(paste._get_p_duplicate()) + p_duplicate_list = [] + p_simil_list = [] + p_date_list = [] + p_hashtype_list = [] + + + for dup_list in p_duplicate_full_list: + if dup_list[0] == "tlsh": + dup_list[2] = int(((tlsh_to_percent - float(dup_list[2])) / tlsh_to_percent)*100) + else: + dup_list[2] = int(dup_list[2]) + + p_duplicate_full_list.sort(lambda x,y: cmp(x[2], y[2]), reverse=True) + + # Combine multiple duplicate paste name and format for display + new_dup_list = [] + dup_list_removed = [] + for dup_list_index in range(0, len(p_duplicate_full_list)): + if dup_list_index in dup_list_removed: + continue + indices = [i for i, x in enumerate(p_duplicate_full_list) if x[1] == p_duplicate_full_list[dup_list_index][1]] + hash_types = [] + comp_vals = [] + for i in indices: + hash_types.append(p_duplicate_full_list[i][0].encode('utf8')) + comp_vals.append(p_duplicate_full_list[i][2]) + dup_list_removed.append(i) + + hash_types = str(hash_types).replace("[","").replace("]","") if len(hash_types)==1 else str(hash_types) + comp_vals = str(comp_vals).replace("[","").replace("]","") if len(comp_vals)==1 else str(comp_vals) + if len(p_duplicate_full_list[dup_list_index]) > 3: + try: + date_paste = str(int(p_duplicate_full_list[dup_list_index][3])) + date_paste = date_paste[0:4]+"-"+date_paste[4:6]+"-"+date_paste[6:8] + except ValueError: + date_paste = str(p_duplicate_full_list[dup_list_index][3]) + else: + date_paste = "No date available" + new_dup_list.append([hash_types.replace("'", ""), p_duplicate_full_list[dup_list_index][1], comp_vals, date_paste]) + + # Create the list to pass to the webpage + for dup_list in new_dup_list: + hash_type, path, simil_percent, date_paste = dup_list + p_duplicate_list.append(path) + p_simil_list.append(simil_percent) + p_hashtype_list.append(hash_type) + p_date_list.append(date_paste) + + if content_range != 0: + p_content = p_content[0:content_range] + + + return render_template("show_saved_paste.html", date=p_date, source=p_source, encoding=p_encoding, language=p_language, size=p_size, mime=p_mime, lineinfo=p_lineinfo, content=p_content, initsize=len(p_content), duplicate_list = p_duplicate_list, simil_list = p_simil_list, hashtype_list = p_hashtype_list, date_list=p_date_list) + + + +# ============ ROUTES ============ + +@app.route("/showsavedpaste/") #completely shows the paste in a new tab +def showsavedpaste(): + return showpaste(0) + + +@app.route("/showpreviewpaste/") +def showpreviewpaste(): + num = request.args.get('num', '') + return "|num|"+num+"|num|"+showpaste(max_preview_modal) + + +@app.route("/getmoredata/") +def getmoredata(): + requested_path = request.args.get('paste', '') + paste = Paste.Paste(requested_path) + p_content = paste.get_p_content().decode('utf-8', 'ignore') + to_return = p_content[max_preview_modal-1:] + return to_return + diff --git a/var/www/Flask_terms.py b/var/www/Flask_terms.py new file mode 100644 index 00000000..f5416ddc --- /dev/null +++ b/var/www/Flask_terms.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python2 +# -*-coding:UTF-8 -* + +''' + Flask functions and routes for the trending modules page +''' +import redis +import datetime +import calendar +import flask +from flask import Flask, render_template, jsonify, request + +import Paste + +# ============ VARIABLES ============ +import Flask_config + +app = Flask_config.app +cfg = Flask_config.cfg +r_serv_term = Flask_config.r_serv_term +# ============ FUNCTIONS ============ + +def Term_getValueOverRange(word, startDate, num_day): + passed_days = 0 + oneDay = 60*60*24 + to_return = [] + curr_to_return = 0 + for timestamp in range(startDate, startDate - max(num_day)*oneDay, -oneDay): + value = r_serv_term.hget(timestamp, word) + curr_to_return += int(value) if value is not None else 0 + for i in num_day: + if passed_days == i-1: + to_return.append(curr_to_return) + passed_days += 1 + return to_return + + +# ============ ROUTES ============ + +@app.route("/terms_management/") +def terms_management(): + TrackedTermsSet_Name = "TrackedSetTermSet" + BlackListTermsSet_Name = "BlackListSetTermSet" + TrackedTermsDate_Name = "TrackedTermDate" + BlackListTermsDate_Name = "BlackListTermDate" + + today = datetime.datetime.now() + today = today.replace(hour=0, minute=0, second=0, microsecond=0) + today_timestamp = calendar.timegm(today.timetuple()) + + track_list = [] + track_list_values = [] + track_list_num_of_paste = [] + for tracked_term in r_serv_term.smembers(TrackedTermsSet_Name): + track_list.append(tracked_term) + value_range = Term_getValueOverRange(tracked_term, today_timestamp, [1, 7, 31]) + + term_date = r_serv_term.hget(TrackedTermsDate_Name, tracked_term) + + set_paste_name = "tracked_" + tracked_term + track_list_num_of_paste.append(r_serv_term.scard(set_paste_name)) + term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded" + value_range.append(term_date) + track_list_values.append(value_range) + + + black_list = [] + for blacked_term in r_serv_term.smembers(BlackListTermsSet_Name): + term_date = r_serv_term.hget(BlackListTermsDate_Name, blacked_term) + term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded" + black_list.append([blacked_term, term_date]) + + return render_template("terms_management.html", black_list=black_list, track_list=track_list, track_list_values=track_list_values, track_list_num_of_paste=track_list_num_of_paste) + + +@app.route("/terms_management_query_paste/") +def terms_management_query_paste(): + term = request.args.get('term') + TrackedTermsSet_Name = "TrackedSetTermSet" + paste_info = [] + + set_paste_name = "tracked_" + term + track_list_path = r_serv_term.smembers(set_paste_name) + + for path in track_list_path: + paste = Paste.Paste(path) + p_date = str(paste._get_p_date()) + p_date = p_date[6:]+'/'+p_date[4:6]+'/'+p_date[0:4] + p_source = paste.p_source + p_encoding = paste._get_p_encoding() + p_size = paste.p_size + p_mime = paste.p_mime + p_lineinfo = paste.get_lines_info() + p_content = paste.get_p_content().decode('utf-8', 'ignore') + if p_content != 0: + p_content = p_content[0:400] + paste_info.append({"path": path, "date": p_date, "source": p_source, "encoding": p_encoding, "size": p_size, "mime": p_mime, "lineinfo": p_lineinfo, "content": p_content}) + + return jsonify(paste_info) + + +@app.route("/terms_management_query/") +def terms_management_query(): + TrackedTermsDate_Name = "TrackedTermDate" + BlackListTermsDate_Name = "BlackListTermDate" + term = request.args.get('term') + section = request.args.get('section') + + today = datetime.datetime.now() + today = today.replace(hour=0, minute=0, second=0, microsecond=0) + today_timestamp = calendar.timegm(today.timetuple()) + value_range = Term_getValueOverRange(term, today_timestamp, [1, 7, 31]) + + if section == "followTerm": + term_date = r_serv_term.hget(TrackedTermsDate_Name, term) + elif section == "blacklistTerm": + term_date = r_serv_term.hget(BlackListTermsDate_Name, term) + + term_date = datetime.datetime.utcfromtimestamp(int(term_date)) if term_date is not None else "No date recorded" + value_range.append(str(term_date)) + return jsonify(value_range) + + +@app.route("/terms_management_action/", methods=['GET']) +def terms_management_action(): + TrackedTermsSet_Name = "TrackedSetTermSet" + TrackedTermsDate_Name = "TrackedTermDate" + BlackListTermsDate_Name = "BlackListTermDate" + BlackListTermsSet_Name = "BlackListSetTermSet" + + today = datetime.datetime.now() + today = today.replace(microsecond=0) + today_timestamp = calendar.timegm(today.timetuple()) + + + section = request.args.get('section') + action = request.args.get('action') + term = request.args.get('term') + if action is None or term is None: + return "None" + else: + if section == "followTerm": + if action == "add": + r_serv_term.sadd(TrackedTermsSet_Name, term.lower()) + r_serv_term.hset(TrackedTermsDate_Name, term, today_timestamp) + else: + r_serv_term.srem(TrackedTermsSet_Name, term.lower()) + elif section == "blacklistTerm": + if action == "add": + r_serv_term.sadd(BlackListTermsSet_Name, term.lower()) + r_serv_term.hset(BlackListTermsDate_Name, term, today_timestamp) + else: + r_serv_term.srem(BlackListTermsSet_Name, term.lower()) + else: + return "None" + + to_return = {} + to_return["section"] = section + to_return["action"] = action + to_return["term"] = term + return jsonify(to_return) + + + +@app.route("/terms_plot_tool/") +def terms_plot_tool(): + term = request.args.get('term') + if term is not None: + return render_template("terms_plot_tool.html", term=term) + else: + return render_template("terms_plot_tool.html", term="") + + +@app.route("/terms_plot_tool_data/") +def terms_plot_tool_data(): + oneDay = 60*60*24 + range_start = datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_start')))) if request.args.get('range_start') is not None else 0; + range_start = range_start.replace(hour=0, minute=0, second=0, microsecond=0) + range_start = calendar.timegm(range_start.timetuple()) + range_end = datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_end')))) if request.args.get('range_end') is not None else 0; + range_end = range_end.replace(hour=0, minute=0, second=0, microsecond=0) + range_end = calendar.timegm(range_end.timetuple()) + term = request.args.get('term') + + if term is None: + return "None" + else: + value_range = [] + for timestamp in range(range_start, range_end+oneDay, oneDay): + value = r_serv_term.hget(timestamp, term) + curr_value_range = int(value) if value is not None else 0 + value_range.append([timestamp, curr_value_range]) + value_range.insert(0,term) + return jsonify(value_range) + + +@app.route("/terms_plot_top/") +def terms_plot_top(): + return render_template("terms_plot_top.html") + + +@app.route("/terms_plot_top_data/") +def terms_plot_top_data(): + oneDay = 60*60*24 + today = datetime.datetime.now() + today = today.replace(hour=0, minute=0, second=0, microsecond=0) + today_timestamp = calendar.timegm(today.timetuple()) + + set_day = "TopTermFreq_set_day_" + str(today_timestamp) + set_week = "TopTermFreq_set_week"; + set_month = "TopTermFreq_set_month"; + + the_set = request.args.get('set') + num_day = int(request.args.get('num_day')) + if the_set is None: + return "None" + else: + to_return = [] + if the_set == "TopTermFreq_set_day": + the_set += "_" + str(today_timestamp) + + for term, tot_value in r_serv_term.zrevrangebyscore(the_set, '+inf', '-inf', withscores=True, start=0, num=20): + position = {} + position['day'] = r_serv_term.zrevrank(set_day, term) + position['day'] = position['day']+1 if position['day'] is not None else "<20" + position['week'] = r_serv_term.zrevrank(set_week, term) + position['week'] = position['week']+1 if position['week'] is not None else "<20" + position['month'] = r_serv_term.zrevrank(set_month, term) + position['month'] = position['month']+1 if position['month'] is not None else "<20" + value_range = [] + for timestamp in range(today_timestamp, today_timestamp - num_day*oneDay, -oneDay): + value = r_serv_term.hget(timestamp, term) + curr_value_range = int(value) if value is not None else 0 + value_range.append([timestamp, curr_value_range]) + + to_return.append([term, value_range, tot_value, position]) + + return jsonify(to_return) + + diff --git a/var/www/Flask_trendingcharts.py b/var/www/Flask_trendingcharts.py new file mode 100644 index 00000000..fdb1a3d4 --- /dev/null +++ b/var/www/Flask_trendingcharts.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python2 +# -*-coding:UTF-8 -* + +''' + Flask functions and routes for the trending charts page +''' +import redis +import datetime +from Date import Date +import flask +from flask import Flask, render_template, jsonify, request + +# ============ VARIABLES ============ +import Flask_config + +app = Flask_config.app +cfg = Flask_config.cfg +r_serv_charts = Flask_config.r_serv_charts +# ============ FUNCTIONS ============ + +def get_date_range(num_day): + curr_date = datetime.date.today() + date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2)) + date_list = [] + + for i in range(0, num_day+1): + date_list.append(date.substract_day(i)) + return date_list + + +# ============ ROUTES ============ + +@app.route("/_progressionCharts", methods=['GET']) +def progressionCharts(): + attribute_name = request.args.get('attributeName') + trending_name = request.args.get('trendingName') + bar_requested = True if request.args.get('bar') == "true" else False + + if (bar_requested): + num_day = int(request.args.get('days')) + bar_values = [] + + date_range = get_date_range(num_day) + # Retreive all data from the last num_day + for date in date_range: + curr_value = r_serv_charts.hget(attribute_name, date) + bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], int(curr_value if curr_value is not None else 0)]) + bar_values.insert(0, attribute_name) + return jsonify(bar_values) + + else: + redis_progression_name = "z_top_progression_" + trending_name + keyw_value = r_serv_charts.zrevrangebyscore(redis_progression_name, '+inf', '-inf', withscores=True, start=0, num=10) + return jsonify(keyw_value) + +@app.route("/wordstrending/") +def wordstrending(): + default_display = cfg.get("Flask", "default_display") + return render_template("Wordstrending.html", default_display = default_display) + + +@app.route("/protocolstrending/") +def protocolstrending(): + default_display = cfg.get("Flask", "default_display") + return render_template("Protocolstrending.html", default_display = default_display) + + +@app.route("/trending/") +def trending(): + default_display = cfg.get("Flask", "default_display") + return render_template("Trending.html", default_display = default_display) + + diff --git a/var/www/Flask_trendingmodules.py b/var/www/Flask_trendingmodules.py new file mode 100644 index 00000000..73cef7f5 --- /dev/null +++ b/var/www/Flask_trendingmodules.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python2 +# -*-coding:UTF-8 -* + +''' + Flask functions and routes for the trending modules page +''' +import redis +import datetime +import flask +from flask import Flask, render_template, jsonify, request + +# ============ VARIABLES ============ +import Flask_config + +app = Flask_config.app +cfg = Flask_config.cfg +r_serv_charts = Flask_config.r_serv_charts +# ============ FUNCTIONS ============ + +# Iterate over elements in the module provided and return the today data or the last data +# return format: [('passed_days', num_of_passed_days), ('elem_name1', elem_value1), ('elem_name2', elem_value2)]] +def get_top_relevant_data(server, module_name): + days = 0 + for date in get_date_range(15): + redis_progression_name_set = 'top_'+ module_name +'_set_' + date + member_set = server.zrevrangebyscore(redis_progression_name_set, '+inf', '-inf', withscores=True) + if len(member_set) == 0: #No data for this date + days += 1 + else: + member_set.insert(0, ("passed_days", days)) + return member_set + + +def get_date_range(num_day): + curr_date = datetime.date.today() + date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2)) + date_list = [] + + for i in range(0, num_day+1): + date_list.append(date.substract_day(i)) + return date_list + +# ============ ROUTES ============ + +@app.route("/_moduleCharts", methods=['GET']) +def modulesCharts(): + keyword_name = request.args.get('keywordName') + module_name = request.args.get('moduleName') + bar_requested = True if request.args.get('bar') == "true" else False + + if (bar_requested): + num_day = int(request.args.get('days')) + bar_values = [] + + date_range = get_date_range(num_day) + # Retreive all data from the last num_day + for date in date_range: + curr_value = r_serv_charts.hget(date, module_name+'-'+keyword_name) + bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], int(curr_value if curr_value is not None else 0)]) + bar_values.insert(0, keyword_name) + return jsonify(bar_values) + + else: + member_set = get_top_relevant_data(r_serv_charts, module_name) + if len(member_set) == 0: + member_set.append(("No relevant data", int(100))) + return jsonify(member_set) + + +@app.route("/_providersChart", methods=['GET']) +def providersChart(): + keyword_name = request.args.get('keywordName') + module_name = request.args.get('moduleName') + bar_requested = True if request.args.get('bar') == "true" else False + + if (bar_requested): + num_day = int(request.args.get('days')) + bar_values = [] + + date_range = get_date_range(num_day) + # Retreive all data from the last num_day + for date in date_range: + curr_value_size = r_serv_charts.hget(keyword_name+'_'+'size', date) + curr_value_num = r_serv_charts.hget(keyword_name+'_'+'num', date) + curr_value_size_avg = r_serv_charts.hget(keyword_name+'_'+'avg', date) + if module_name == "size": + curr_value = float(curr_value_size_avg if curr_value_size_avg is not None else 0) + else: + curr_value = float(curr_value_num if curr_value_num is not None else 0.0) + + bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], curr_value]) + bar_values.insert(0, keyword_name) + return jsonify(bar_values) + + else: + #redis_provider_name_set = 'top_size_set' if module_name == "size" else 'providers_set' + redis_provider_name_set = 'top_avg_size_set_' if module_name == "size" else 'providers_set_' + redis_provider_name_set = redis_provider_name_set + get_date_range(0)[0] + + member_set = r_serv_charts.zrevrangebyscore(redis_provider_name_set, '+inf', '-inf', withscores=True, start=0, num=8) + # Member set is a list of (value, score) pairs + if len(member_set) == 0: + member_set.append(("No relevant data", float(100))) + return jsonify(member_set) + + +@app.route("/moduletrending/") +def moduletrending(): + return render_template("Moduletrending.html") + +