From 64ec44085d9d26fbd80284be7ba42572d802d47d Mon Sep 17 00:00:00 2001 From: = Date: Tue, 22 Aug 2017 17:52:15 +0200 Subject: [PATCH 1/6] Dynamically choose correct port for LevelDB in script --- bin/BrowseWarningPaste.py | 4 +++- bin/LAUNCH.sh | 1 - bin/Repartition_graph.py | 5 ++++- bin/packages/config.cfg.sample | 2 -- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/bin/BrowseWarningPaste.py b/bin/BrowseWarningPaste.py index 01704902..4f49f56b 100755 --- a/bin/BrowseWarningPaste.py +++ b/bin/BrowseWarningPaste.py @@ -28,9 +28,11 @@ if __name__ == "__main__": p = Process(config_section) + # port generated automatically depending on the date + curYear = datetime.now().year server = redis.StrictRedis( host=p.config.get("Redis_Level_DB", "host"), - port=p.config.get("Redis_Level_DB", "port"), + port=curYear, db=p.config.get("Redis_Level_DB", "db")) # FUNCTIONS # diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index f33af903..05c408b0 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -66,7 +66,6 @@ function launching_redis { function launching_lvldb { #Want to launch more level_db? - #FIXME update the date in config.cfg lvdbhost='127.0.0.1' lvdbdir="${AIL_HOME}/LEVEL_DB_DATA/" db1_y='2016' diff --git a/bin/Repartition_graph.py b/bin/Repartition_graph.py index f1955a0e..a8094bb0 100755 --- a/bin/Repartition_graph.py +++ b/bin/Repartition_graph.py @@ -4,6 +4,7 @@ import redis import argparse import ConfigParser +from datetime import datetime from pubsublogger import publisher import matplotlib.pyplot as plt @@ -28,9 +29,11 @@ def main(): args = parser.parse_args() # REDIS # + # port generated automatically depending on the date + curYear = datetime.now().year r_serv = redis.StrictRedis( host=cfg.get("Redis_Level_DB_Hashs", "host"), - port=cfg.getint("Redis_Level_DB_Hashs", "port"), + port=curYear, db=cfg.getint("Redis_Level_DB_Hashs", "db")) # LOGGING # diff --git a/bin/packages/config.cfg.sample b/bin/packages/config.cfg.sample index 2e2d246d..1adaf04d 100644 --- a/bin/packages/config.cfg.sample +++ b/bin/packages/config.cfg.sample @@ -104,7 +104,6 @@ db = 5 [Redis_Level_DB] host = localhost -port = 2016 db = 0 [Redis_Level_DB_Trending] @@ -114,7 +113,6 @@ db = 3 [Redis_Level_DB_Hashs] host = localhost -port = 2016 db = 1 [Url] From d6323357601713dc5fd9abbc57dc6ae8d03e2c7f Mon Sep 17 00:00:00 2001 From: = Date: Wed, 23 Aug 2017 10:16:56 +0200 Subject: [PATCH 2/6] Automatically start redis_levelDB with the correct date --- bin/LAUNCH.sh | 34 ++++++++++++--------------------- bin/Repartition_graph.py | 3 ++- var/www/modules/Flask_config.py | 6 +++++- 3 files changed, 19 insertions(+), 24 deletions(-) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 05c408b0..227d3a9b 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -65,36 +65,26 @@ function launching_redis { } function launching_lvldb { - #Want to launch more level_db? lvdbhost='127.0.0.1' lvdbdir="${AIL_HOME}/LEVEL_DB_DATA/" - db1_y='2016' - db2_y='2017' - dbn_y=`date +%Y` - - dbC1_y='3016' - dbCn_y=30`date +%y` nb_db=13 + db_y=`date +%Y` + #Verify that a dir with the correct year exists, create it otherwise + if [ ! -d "$lvdbdir$db_y" ]; then + mkdir -p "$db_y" + fi + screen -dmS "LevelDB" sleep 0.1 echo -e $GREEN"\t* Launching Levels DB servers"$DEFAULT - #Add lines here with appropriates options. - sleep 0.1 - screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db1_y' -M '$nb_db'; read x' - sleep 0.1 - screen -S "LevelDB" -X screen -t "2017" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2017/ -P '$db2_y' -M '$nb_db'; read x' - sleep 0.1 - screen -S "LevelDB" -X screen -t "$dbn_y" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir$dbn_y'/ -P '$dbn_y' -M '$nb_db'; read x' - - # For Curve - sleep 0.1 - screen -S "LevelDB" -X screen -t "3016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'3016/ -P '$dbC1_y' -M '$nb_db'; read x' - sleep 0.1 - screen -S "LevelDB" -X screen -t "3017" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'3017/ -P '$dbC1_y' -M '$nb_db'; read x' - sleep 0.1 - screen -S "LevelDB" -X screen -t "$dbCn_y" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir$dbCn_y'/ -P '$dbCn_y' -M '$nb_db'; read x' + #Launch a DB for each dir + for pathDir in $lvdbdir*/ ; do + yDir=$(basename "$pathDir") + sleep 0.1 + screen -S "LevelDB" -X screen -t "$yDir" bash -c 'redis-leveldb -H '$lvdbhost' -D '$pathDir'/ -P '$yDir' -M '$nb_db'; read x' + done } function launching_logs { diff --git a/bin/Repartition_graph.py b/bin/Repartition_graph.py index a8094bb0..38c5e1b6 100755 --- a/bin/Repartition_graph.py +++ b/bin/Repartition_graph.py @@ -25,12 +25,13 @@ def main(): parser.add_argument('-f', type=str, metavar="filename", default="figure", help='The absolute path name of the "figure.png"', action='store') + parser.add_argument('-y', '--year', type=int, required=False, default=None, help='The date related to the DB') args = parser.parse_args() # REDIS # # port generated automatically depending on the date - curYear = datetime.now().year + curYear = datetime.now().year if args.year is None else args.year r_serv = redis.StrictRedis( host=cfg.get("Redis_Level_DB_Hashs", "host"), port=curYear, diff --git a/var/www/modules/Flask_config.py b/var/www/modules/Flask_config.py index 161e6130..14468fa1 100644 --- a/var/www/modules/Flask_config.py +++ b/var/www/modules/Flask_config.py @@ -7,9 +7,11 @@ import ConfigParser import redis import os +from datetime import datetime # FLASK # app = None +curYear = None #can be set to fit the needs, Correspond to the level-DB year to be used # CONFIG # configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') @@ -38,9 +40,11 @@ r_serv_charts = redis.StrictRedis( port=cfg.getint("Redis_Level_DB_Trending", "port"), db=cfg.getint("Redis_Level_DB_Trending", "db")) +# port generated automatically depending on the date +curYear = datetime.now().year if curYear is None else curYear r_serv_db = redis.StrictRedis( host=cfg.get("Redis_Level_DB", "host"), - port=cfg.getint("Redis_Level_DB", "port"), + port=curYear, db=cfg.getint("Redis_Level_DB", "db")) r_serv_sentiment = redis.StrictRedis( From 3b3f3aa89cd5ccee94c2cbdc51192f2cdbafdd2d Mon Sep 17 00:00:00 2001 From: = Date: Wed, 23 Aug 2017 12:20:39 +0200 Subject: [PATCH 3/6] Flask: Added support of LevelDB database depending of the year --- var/www/modules/Flask_config.py | 8 ---- .../browsepastes/Flask_browsepastes.py | 37 +++++++++++++++---- .../templates/browse_important_paste.html | 32 ++++++++++++++-- .../templates/important_paste_by_module.html | 3 +- 4 files changed, 61 insertions(+), 19 deletions(-) diff --git a/var/www/modules/Flask_config.py b/var/www/modules/Flask_config.py index 14468fa1..8b6b01de 100644 --- a/var/www/modules/Flask_config.py +++ b/var/www/modules/Flask_config.py @@ -11,7 +11,6 @@ from datetime import datetime # FLASK # app = None -curYear = None #can be set to fit the needs, Correspond to the level-DB year to be used # CONFIG # configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') @@ -40,13 +39,6 @@ r_serv_charts = redis.StrictRedis( port=cfg.getint("Redis_Level_DB_Trending", "port"), db=cfg.getint("Redis_Level_DB_Trending", "db")) -# port generated automatically depending on the date -curYear = datetime.now().year if curYear is None else curYear -r_serv_db = redis.StrictRedis( - host=cfg.get("Redis_Level_DB", "host"), - port=curYear, - db=cfg.getint("Redis_Level_DB", "db")) - r_serv_sentiment = redis.StrictRedis( host=cfg.get("Redis_Level_DB_Sentiment", "host"), port=cfg.getint("Redis_Level_DB_Sentiment", "port"), diff --git a/var/www/modules/browsepastes/Flask_browsepastes.py b/var/www/modules/browsepastes/Flask_browsepastes.py index d3b1bb4d..30853c38 100644 --- a/var/www/modules/browsepastes/Flask_browsepastes.py +++ b/var/www/modules/browsepastes/Flask_browsepastes.py @@ -7,6 +7,8 @@ import redis import json import flask +import os +from datetime import datetime from flask import Flask, render_template, jsonify, request, Blueprint import Paste @@ -18,7 +20,25 @@ app = Flask_config.app cfg = Flask_config.cfg max_preview_char = Flask_config.max_preview_char max_preview_modal = Flask_config.max_preview_modal -r_serv_db = Flask_config.r_serv_db + +#init all lvlDB servers +curYear = datetime.now().year +r_serv_db = {} +# port generated automatically depending on available levelDB date +yearList = [] +lvdbdir= os.path.join(os.environ['AIL_HOME'], "LEVEL_DB_DATA/") +for year in os.listdir(lvdbdir): + try: + intYear = int(year) + except: + continue + + yearList.append([year, intYear, int(curYear) == intYear]) + r_serv_db[intYear] = redis.StrictRedis( + host=cfg.get("Redis_Level_DB", "host"), + port=intYear, + db=cfg.getint("Redis_Level_DB", "db")) +yearList.sort(reverse=True) browsepastes = Blueprint('browsepastes', __name__, template_folder='templates') @@ -31,9 +51,9 @@ def getPastebyType(server, module_name): return all_path -def event_stream_getImportantPasteByModule(module_name): +def event_stream_getImportantPasteByModule(module_name, year): index = 0 - all_pastes_list = getPastebyType(r_serv_db, module_name) + all_pastes_list = getPastebyType(r_serv_db[year], module_name) for path in all_pastes_list: index += 1 paste = Paste.Paste(path) @@ -57,18 +77,19 @@ def event_stream_getImportantPasteByModule(module_name): @browsepastes.route("/browseImportantPaste/", methods=['GET']) def browseImportantPaste(): module_name = request.args.get('moduleName') - return render_template("browse_important_paste.html") + return render_template("browse_important_paste.html", year_list=yearList, selected_year=curYear) @browsepastes.route("/importantPasteByModule/", methods=['GET']) def importantPasteByModule(): module_name = request.args.get('moduleName') + currentSelectYear = int(request.args.get('year')) all_content = [] paste_date = [] paste_linenum = [] all_path = [] - allPastes = getPastebyType(r_serv_db, module_name) + allPastes = getPastebyType(r_serv_db[currentSelectYear], module_name) for path in allPastes[0:10]: all_path.append(path) @@ -88,6 +109,7 @@ def importantPasteByModule(): return render_template("important_paste_by_module.html", moduleName=module_name, + year=currentSelectYear, all_path=all_path, content=all_content, paste_date=paste_date, @@ -95,10 +117,11 @@ def importantPasteByModule(): char_to_display=max_preview_modal, finished=finished) -@browsepastes.route("/_getImportantPasteByModule") +@browsepastes.route("/_getImportantPasteByModule", methods=['GET']) def getImportantPasteByModule(): module_name = request.args.get('moduleName') - return flask.Response(event_stream_getImportantPasteByModule(module_name), mimetype="text/event-stream") + currentSelectYear = int(request.args.get('year')) + return flask.Response(event_stream_getImportantPasteByModule(module_name, currentSelectYear), mimetype="text/event-stream") # ========= REGISTRATION ========= diff --git a/var/www/modules/browsepastes/templates/browse_important_paste.html b/var/www/modules/browsepastes/templates/browse_important_paste.html index 5d7d84e3..89fc8aaa 100644 --- a/var/www/modules/browsepastes/templates/browse_important_paste.html +++ b/var/www/modules/browsepastes/templates/browse_important_paste.html @@ -63,6 +63,20 @@ + +
+
+ Year: + +
+
+
+ +
@@ -117,9 +131,10 @@ $("#"+activePage).addClass("active"); var dataPath = 'credential'; - $.get("{{ url_for('browsepastes.importantPasteByModule') }}"+"?moduleName="+dataPath, function(data, status){ + $.get("{{ url_for('browsepastes.importantPasteByModule') }}"+"?moduleName="+dataPath+"&year="+currentSelectYear, function(data, status){ $('#'+dataPath+'-tab').html(data); }); + }); @@ -127,12 +142,23 @@