diff --git a/var/www/Flask_base_template.py b/var/www/Flask_base_template.py new file mode 100644 index 00000000..cb7070a3 --- /dev/null +++ b/var/www/Flask_base_template.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python2 +# -*-coding:UTF-8 -* + +''' + Flask functions and routes for the trending modules page +''' +import redis +from flask import Flask, render_template, jsonify, request, Blueprint + +# ============ VARIABLES ============ +import Flask_config + +app = Flask_config.app +cfg = Flask_config.cfg + +MODULENAME = Blueprint('MODULENAME', __name__, template_folder='templates') + +# ============ FUNCTIONS ============ +def one(): + return 1 + +# ============= ROUTES ============== + +@MODULENAME.route("/MODULENAME/", methods=['GET']) +def MODULENAME_page(): + return render_template("MODULENAME.html") + + +# ========= REGISTRATION ========= +app.register_blueprint(MODULENAME) diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 2ee00295..914c4f33 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -9,10 +9,12 @@ import time import calendar from flask import Flask, render_template, jsonify, request import flask +import importlib import os +from os.path import join import sys sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) -sys.path.append('./Flasks/') +sys.path.append('./modules/') import Paste from Date import Date @@ -25,16 +27,70 @@ cfg = Flask_config.cfg Flask_config.app = Flask(__name__, static_url_path='/static/') app = Flask_config.app -# import routes and functions from modules -import Flask_dashboard -import Flask_trendingcharts -import Flask_trendingmodules -import Flask_browsepastes -import Flask_sentiment -import Flask_terms -import Flask_search -import Flask_showpaste +# ========= HEADER GENERATION ======== +# Get headers items that should be ignored (not displayed) +toIgnoreModule = set() +try: + with open('templates/ignored_modules.txt', 'r') as f: + lines = f.read().splitlines() + for line in lines: + toIgnoreModule.add(line) + +except IOError: + f = open('templates/ignored_modules.txt', 'w') + f.close() + + +# Dynamically import routes and functions from modules +# Also, prepare header.html +to_add_to_header_dico = {} +for root, dirs, files in os.walk('modules/'): + sys.path.append(join(root)) + + # Ignore the module + curr_dir = root.split('/')[1] + if curr_dir in toIgnoreModule: + continue + + for name in files: + module_name = root.split('/')[-2] + if name.startswith('Flask_') and name.endswith('.py'): + if name == 'Flask_config.py': + continue + name = name.strip('.py') + #print('importing {}'.format(name)) + importlib.import_module(name) + elif name == 'header_{}.html'.format(module_name): + with open(join(root, name), 'r') as f: + to_add_to_header_dico[module_name] = f.read() + +#create header.html +complete_header = "" +with open('templates/header_base.html', 'r') as f: + complete_header = f.read() +modified_header = complete_header + +#Add the header in the supplied order +for module_name, txt in to_add_to_header_dico.items(): + to_replace = ''.format(module_name) + if to_replace in complete_header: + modified_header = modified_header.replace(to_replace, txt) + del to_add_to_header_dico[module_name] + +#Add the header for no-supplied order +to_add_to_header = [] +for module_name, txt in to_add_to_header_dico.items(): + to_add_to_header.append(txt) + +modified_header = modified_header.replace('', '\n'.join(to_add_to_header)) + +#Write the header.html file +with open('templates/header.html', 'w') as f: + f.write(modified_header) + + +# ========= JINJA2 FUNCTIONS ======== def list_len(s): return len(s) app.jinja_env.filters['list_len'] = list_len @@ -51,6 +107,12 @@ def add_header(response): response.headers['Cache-Control'] = 'public, max-age=0' return response +# ========== ROUTES ============ +@app.route('/searchbox/') +def searchbox(): + return render_template("searchbox.html") + + # ============ MAIN ============ if __name__ == "__main__": diff --git a/var/www/Flasks/Flask_corpus.py b/var/www/Flasks/Flask_corpus.py deleted file mode 100644 index 7805e66e..00000000 --- a/var/www/Flasks/Flask_corpus.py +++ /dev/null @@ -1,219 +0,0 @@ -#!/usr/bin/env python2 -# -*-coding:UTF-8 -* - -''' - Flask functions and routes for the trending modules page -''' -import redis -import datetime -import calendar -import flask -from flask import Flask, render_template, jsonify, request - -import Paste - -# ============ VARIABLES ============ -import Flask_config - -app = Flask_config.app -cfg = Flask_config.cfg -r_serv_corpus = Flask_config.r_serv_corpus -# ============ FUNCTIONS ============ - -def Corpus_getValueOverRange(word, startDate, num_day): - passed_days = 0 - oneDay = 60*60*24 - to_return = [] - curr_to_return = 0 - for timestamp in range(startDate, startDate - max(num_day)*oneDay, -oneDay): - value = r_serv_corpus.hget(timestamp, word) - curr_to_return += int(value) if value is not None else 0 - for i in num_day: - if passed_days == i-1: - to_return.append(curr_to_return) - passed_days += 1 - return to_return - - -# ============ ROUTES ============ - -@app.route("/corpus_management/") -def corpus_management(): - TrackedCorpusSet_Name = "TrackedSetCorpusSet" - TrackedCorpusDate_Name = "TrackedCorpusDate" - - today = datetime.datetime.now() - today = today.replace(hour=0, minute=0, second=0, microsecond=0) - today_timestamp = calendar.timegm(today.timetuple()) - - track_list = [] - track_list_values = [] - track_list_num_of_paste = [] - for tracked_corpus in r_serv_corpus.smembers(TrackedCorpusSet_Name): - track_list.append(tracked_corpus) - value_range = Corpus_getValueOverRange(tracked_corpus, today_timestamp, [1, 7, 31]) - - corpus_date = r_serv_corpus.hget(TrackedCorpusDate_Name, tracked_corpus) - - set_paste_name = "tracked_" + tracked_corpus - track_list_num_of_paste.append(r_serv_corpus.scard(set_paste_name)) - corpus_date = datetime.datetime.utcfromtimestamp(int(corpus_date)) if corpus_date is not None else "No date recorded" - value_range.append(corpus_date) - track_list_values.append(value_range) - - - return render_template("corpus_management.html", black_list=black_list, track_list=track_list, track_list_values=track_list_values, track_list_num_of_paste=track_list_num_of_paste) - - -@app.route("/corpus_management_query_paste/") -def corpus_management_query_paste(): - corpus = request.args.get('corpus') - TrackedCorpusSet_Name = "TrackedSetCorpusSet" - paste_info = [] - - set_paste_name = "tracked_" + corpus - track_list_path = r_serv_corpus.smembers(set_paste_name) - - for path in track_list_path: - paste = Paste.Paste(path) - p_date = str(paste._get_p_date()) - p_date = p_date[6:]+'/'+p_date[4:6]+'/'+p_date[0:4] - p_source = paste.p_source - p_encoding = paste._get_p_encoding() - p_size = paste.p_size - p_mime = paste.p_mime - p_lineinfo = paste.get_lines_info() - p_content = paste.get_p_content().decode('utf-8', 'ignore') - if p_content != 0: - p_content = p_content[0:400] - paste_info.append({"path": path, "date": p_date, "source": p_source, "encoding": p_encoding, "size": p_size, "mime": p_mime, "lineinfo": p_lineinfo, "content": p_content}) - - return jsonify(paste_info) - - -@app.route("/corpus_management_query/") -def corpus_management_query(): - TrackedCorpusDate_Name = "TrackedCorpusDate" - corpus = request.args.get('corpus') - - today = datetime.datetime.now() - today = today.replace(hour=0, minute=0, second=0, microsecond=0) - today_timestamp = calendar.timegm(today.timetuple()) - value_range = corpus_getValueOverRange(corpus, today_timestamp, [1, 7, 31]) - - corpus_date = r_serv_corpus.hget(TrackedCorpusDate_Name, corpus) - - corpus_date = datetime.datetime.utcfromtimestamp(int(corpus_date)) if corpus_date is not None else "No date recorded" - value_range.append(str(corpus_date)) - return jsonify(value_range) - - -@app.route("/corpus_management_action/", methods=['GET']) -def corpus_management_action(): - TrackedCorpusSet_Name = "TrackedSetCorpusSet" - TrackedCorpusDate_Name = "TrackedCorpusDate" - - today = datetime.datetime.now() - today = today.replace(microsecond=0) - today_timestamp = calendar.timegm(today.timetuple()) - - - section = request.args.get('section') - action = request.args.get('action') - corpus = request.args.get('corpus') - if action is None or corpus is None: - return "None" - else: - if section == "followCorpus": - if action == "add": - r_serv_corpus.sadd(TrackedCorpusSet_Name, corpus.lower()) - r_serv_corpus.hset(TrackedCorpusDate_Name, corpus, today_timestamp) - else: - r_serv_corpus.srem(TrackedCorpusSet_Name, corpus.lower()) - else: - return "None" - - to_return = {} - to_return["section"] = section - to_return["action"] = action - to_return["corpus"] = corpus - return jsonify(to_return) - - - -@app.route("/corpus_plot_tool/") -def corpus_plot_tool(): - corpus = request.args.get('corpus') - if corpus is not None: - return render_template("corpus_plot_tool.html", corpus=corpus) - else: - return render_template("corpus_plot_tool.html", corpus="") - - -@app.route("/corpus_plot_tool_data/") -def corpus_plot_tool_data(): - oneDay = 60*60*24 - range_start = datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_start')))) if request.args.get('range_start') is not None else 0; - range_start = range_start.replace(hour=0, minute=0, second=0, microsecond=0) - range_start = calendar.timegm(range_start.timetuple()) - range_end = datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_end')))) if request.args.get('range_end') is not None else 0; - range_end = range_end.replace(hour=0, minute=0, second=0, microsecond=0) - range_end = calendar.timegm(range_end.timetuple()) - corpus = request.args.get('corpus') - - if corpus is None: - return "None" - else: - value_range = [] - for timestamp in range(range_start, range_end+oneDay, oneDay): - value = r_serv_corpus.hget(timestamp, corpus) - curr_value_range = int(value) if value is not None else 0 - value_range.append([timestamp, curr_value_range]) - value_range.insert(0,corpus) - return jsonify(value_range) - - -@app.route("/corpus_plot_top/") -def corpus_plot_top(): - return render_template("corpus_plot_top.html") - - -@app.route("/corpus_plot_top_data/") -def corpus_plot_top_data(): - oneDay = 60*60*24 - today = datetime.datetime.now() - today = today.replace(hour=0, minute=0, second=0, microsecond=0) - today_timestamp = calendar.timegm(today.timetuple()) - - set_day = "TopCorpusFreq_set_day_" + str(today_timestamp) - set_week = "TopCorpusFreq_set_week"; - set_month = "TopCorpusFreq_set_month"; - - the_set = request.args.get('set') - num_day = int(request.args.get('num_day')) - if the_set is None: - return "None" - else: - to_return = [] - if the_set == "TopCorpusFreq_set_day": - the_set += "_" + str(today_timestamp) - - for corpus, tot_value in r_serv_corpus.zrevrangebyscore(the_set, '+inf', '-inf', withscores=True, start=0, num=20): - position = {} - position['day'] = r_serv_corpus.zrevrank(set_day, corpus) - position['day'] = position['day']+1 if position['day'] is not None else "<20" - position['week'] = r_serv_corpus.zrevrank(set_week, corpus) - position['week'] = position['week']+1 if position['week'] is not None else "<20" - position['month'] = r_serv_corpus.zrevrank(set_month, corpus) - position['month'] = position['month']+1 if position['month'] is not None else "<20" - value_range = [] - for timestamp in range(today_timestamp, today_timestamp - num_day*oneDay, -oneDay): - value = r_serv_corpus.hget(timestamp, corpus) - curr_value_range = int(value) if value is not None else 0 - value_range.append([timestamp, curr_value_range]) - - to_return.append([corpus, value_range, tot_value, position]) - - return jsonify(to_return) - - diff --git a/var/www/create_new_web_module.py b/var/www/create_new_web_module.py new file mode 100755 index 00000000..42db6c7f --- /dev/null +++ b/var/www/create_new_web_module.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python2 +# -*-coding:UTF-8 -* + +"Hepler to create a new webpage associated with a module." + +import os + +def createModuleFolder(modulename): + path_module = os.path.join('modules', modulename) + os.mkdir(path_module) + + # create html template + with open('templates/base_template.html', 'r') as templateFile: + template = templateFile.read() + template = template.replace('MODULENAME', modulename) + + os.mkdir(os.path.join(path_module, 'templates')) + with open(os.path.join(os.path.join(path_module, 'templates'), modulename+'.html'), 'w') as toWriteTemplate: + toWriteTemplate.write(template) + + # create html header template + with open('templates/header_base_template.html', 'r') as header_templateFile: + header = header_templateFile.read() + header = header.replace('MODULENAME', modulename) + + with open(os.path.join(os.path.join(path_module, 'templates'), 'header_{}.html'.format(modulename) ), 'w') as toWriteHeader: + toWriteHeader.write(header) + + + #create flask template + with open('Flask_base_template.py', 'r') as flaskFile: + flask = flaskFile.read() + flask = flask.replace('MODULENAME', modulename) + + with open(os.path.join(path_module, 'Flask_{}.py'.format(modulename)), 'w') as toWriteFlask: + toWriteFlask.write(flask) + + +def main(): + rep1 = raw_input('New module name: ') + createModuleFolder(rep1) + +if __name__ == '__main__': + main() diff --git a/var/www/Flasks/Flask_config.py b/var/www/modules/Flask_config.py similarity index 100% rename from var/www/Flasks/Flask_config.py rename to var/www/modules/Flask_config.py diff --git a/var/www/Flasks/Flask_browsepastes.py b/var/www/modules/browsepastes/Flask_browsepastes.py similarity index 88% rename from var/www/Flasks/Flask_browsepastes.py rename to var/www/modules/browsepastes/Flask_browsepastes.py index b393ab9e..d3b1bb4d 100644 --- a/var/www/Flasks/Flask_browsepastes.py +++ b/var/www/modules/browsepastes/Flask_browsepastes.py @@ -7,7 +7,7 @@ import redis import json import flask -from flask import Flask, render_template, jsonify, request +from flask import Flask, render_template, jsonify, request, Blueprint import Paste @@ -19,6 +19,9 @@ cfg = Flask_config.cfg max_preview_char = Flask_config.max_preview_char max_preview_modal = Flask_config.max_preview_modal r_serv_db = Flask_config.r_serv_db + +browsepastes = Blueprint('browsepastes', __name__, template_folder='templates') + # ============ FUNCTIONS ============ def getPastebyType(server, module_name): @@ -51,13 +54,13 @@ def event_stream_getImportantPasteByModule(module_name): # ============ ROUTES ============ -@app.route("/browseImportantPaste/", methods=['GET']) +@browsepastes.route("/browseImportantPaste/", methods=['GET']) def browseImportantPaste(): module_name = request.args.get('moduleName') return render_template("browse_important_paste.html") -@app.route("/importantPasteByModule/", methods=['GET']) +@browsepastes.route("/importantPasteByModule/", methods=['GET']) def importantPasteByModule(): module_name = request.args.get('moduleName') @@ -92,9 +95,11 @@ def importantPasteByModule(): char_to_display=max_preview_modal, finished=finished) -@app.route("/_getImportantPasteByModule") +@browsepastes.route("/_getImportantPasteByModule") def getImportantPasteByModule(): module_name = request.args.get('moduleName') return flask.Response(event_stream_getImportantPasteByModule(module_name), mimetype="text/event-stream") +# ========= REGISTRATION ========= +app.register_blueprint(browsepastes) diff --git a/var/www/templates/browse_important_paste.html b/var/www/modules/browsepastes/templates/browse_important_paste.html similarity index 85% rename from var/www/templates/browse_important_paste.html rename to var/www/modules/browsepastes/templates/browse_important_paste.html index 61015a15..ee3503fd 100644 --- a/var/www/templates/browse_important_paste.html +++ b/var/www/modules/browsepastes/templates/browse_important_paste.html @@ -35,24 +35,7 @@ -
- + {% include 'navbar.html' %} @@ -176,7 +176,7 @@
- + + + + + + + + + + {% include 'navbar.html' %} + +
+
+
+

rawSkeleton

+
+ +
+ +
+ + + + + + + diff --git a/var/www/Flasks/Flask_search.py b/var/www/modules/search/Flask_search.py similarity index 95% rename from var/www/Flasks/Flask_search.py rename to var/www/modules/search/Flask_search.py index a89f011c..18dd89ec 100644 --- a/var/www/Flasks/Flask_search.py +++ b/var/www/modules/search/Flask_search.py @@ -9,7 +9,7 @@ import json import os import datetime import flask -from flask import Flask, render_template, jsonify, request +from flask import Flask, render_template, jsonify, request, Blueprint import Paste from whoosh import index @@ -30,6 +30,8 @@ baseindexpath = os.path.join(os.environ['AIL_HOME'], cfg.get("Indexer", "path")) indexRegister_path = os.path.join(os.environ['AIL_HOME'], cfg.get("Indexer", "register")) +searches = Blueprint('searches', __name__, template_folder='templates') + # ============ FUNCTIONS ============ def get_current_index(): with open(indexRegister_path, "r") as f: @@ -84,7 +86,7 @@ def to_iso_date(timestamp): # ============ ROUTES ============ -@app.route("/search", methods=['POST']) +@searches.route("/search", methods=['POST']) def search(): query = request.form['query'] q = [] @@ -144,7 +146,7 @@ def search(): ) -@app.route("/get_more_search_result", methods=['POST']) +@searches.route("/get_more_search_result", methods=['POST']) def get_more_search_result(): query = request.form['query'] q = [] @@ -193,3 +195,5 @@ def get_more_search_result(): return jsonify(to_return) +# ========= REGISTRATION ========= +app.register_blueprint(searches) diff --git a/var/www/templates/search.html b/var/www/modules/search/templates/search.html similarity index 89% rename from var/www/templates/search.html rename to var/www/modules/search/templates/search.html index 727d30a3..43895a9f 100644 --- a/var/www/templates/search.html +++ b/var/www/modules/search/templates/search.html @@ -37,23 +37,7 @@ -
- - + {% include 'navbar.html' %}