From e225090e07659d7359968b358669846e38a0bad1 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Wed, 15 Mar 2017 16:36:51 +0100 Subject: [PATCH 1/5] Added script that checks configuration and may auto-fix it + added missing field in config.cfg.sample --- bin/LAUNCH.sh | 5 ++- bin/Update-conf.py | 79 ++++++++++++++++++++++++++++++++++ bin/packages/config.cfg.sample | 1 + 3 files changed, 84 insertions(+), 1 deletion(-) create mode 100755 bin/Update-conf.py diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index e082b7f0..3e8814c0 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -183,7 +183,7 @@ islogged=`screen -ls | awk '/\.Logging\t/ {print strtonum($1)}'` isqueued=`screen -ls | awk '/\.Queue\t/ {print strtonum($1)}'` isscripted=`screen -ls | awk '/\.Script\t/ {print strtonum($1)}'` -options=("Redis" "LevelDB" "Logs" "Queues" "Scripts" "Killall" "Shutdown") +options=("Redis" "LevelDB" "Logs" "Queues" "Scripts" "Killall" "Shutdown" "Update-config") menu() { echo "What do you want to Launch?:" @@ -252,6 +252,9 @@ for i in ${!options[@]}; do Shutdown) bash -c "./Shutdown.py" ;; + Update-config) + bash -c "./Update-conf.py" + ;; esac fi done diff --git a/bin/Update-conf.py b/bin/Update-conf.py new file mode 100755 index 00000000..2b234db1 --- /dev/null +++ b/bin/Update-conf.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python2 +# -*-coding:UTF-8 -* + +import ConfigParser +from ConfigParser import RawConfigParser +import os + + +def main(): + + configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') + if not os.path.exists(configfile): + raise Exception('Unable to find the configuration file. \ + Did you set environment variables? \ + Or activate the virtualenv.') + configfileSample = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg.sample') + + cfg = ConfigParser.ConfigParser() + cfg.read(configfile) + cfgSample = ConfigParser.ConfigParser() + cfgSample.read(configfileSample) + + sections = RawConfigParser.sections(cfg) + sectionsSample = RawConfigParser.sections(cfgSample) + + missingSection = [] + dicoMissingSection = {} + missingItem = [] + dicoMissingItem = {} + + for sec in sectionsSample: + if sec not in sections: + missingSection += [sec] + dicoMissingSection[sec] = RawConfigParser.items(cfgSample, sec) + else: + setSample = set(RawConfigParser.options(cfgSample, sec)) + setNormal = set(RawConfigParser.options(cfg, sec)) + if setSample != setNormal: + missing_items = list(setSample.difference(setNormal)) + missingItem += [sec] + list_items = [] + for i in missing_items: + list_items.append( (i, cfgSample.get(sec, i)) ) + dicoMissingItem[sec] = list_items + + if len(missingSection) == 0 and len(missingItem) == 0: + print("Configuration up-to-date") + return + print("/!\\ Configuration not complete. Missing following configuration: /!\\") + print("+--------------------------------------------------------------------+") + for section in missingSection: + print("["+section+"]") + for item in dicoMissingSection[section]: + print(" - "+item[0]) + for section in missingItem: + print("["+section+"]") + for item in dicoMissingItem[section]: + print(" - "+item[0]) + print("+--------------------------------------------------------------------+") + resp = raw_input("Do you want to auto fix it? [y/n] ") + + if resp != 'y': + return + else: + for section in missingSection: + cfg.add_section(section) + for item, value in dicoMissingSection[section]: + cfg.set(section, item, value) + for section in missingItem: + for item, value in dicoMissingItem[section]: + cfg.set(section, item, value) + + with open(configfile, 'w') as f: + cfg.write(f) + + +if __name__ == "__main__": + main() + diff --git a/bin/packages/config.cfg.sample b/bin/packages/config.cfg.sample index a634e4f1..0e91a993 100644 --- a/bin/packages/config.cfg.sample +++ b/bin/packages/config.cfg.sample @@ -123,6 +123,7 @@ cc_tld = r'\.de$' [Indexer] type = whoosh path = indexdir +register = indexdir/all_index.txt #size in Mb index_max_size = 2000 From c79d4d65c8ef7f39960ce409cf82b951320d3f86 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Tue, 28 Mar 2017 09:54:24 +0200 Subject: [PATCH 2/5] Keep original section order while updating --- bin/Update-conf.py | 41 +++++++++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/bin/Update-conf.py b/bin/Update-conf.py index 2b234db1..b945dabe 100755 --- a/bin/Update-conf.py +++ b/bin/Update-conf.py @@ -2,8 +2,9 @@ # -*-coding:UTF-8 -* import ConfigParser -from ConfigParser import RawConfigParser +from ConfigParser import ConfigParser as cfgP import os +from collections import OrderedDict def main(): @@ -20,8 +21,8 @@ def main(): cfgSample = ConfigParser.ConfigParser() cfgSample.read(configfileSample) - sections = RawConfigParser.sections(cfg) - sectionsSample = RawConfigParser.sections(cfgSample) + sections = cfgP.sections(cfg) + sectionsSample = cfgP.sections(cfgSample) missingSection = [] dicoMissingSection = {} @@ -31,10 +32,10 @@ def main(): for sec in sectionsSample: if sec not in sections: missingSection += [sec] - dicoMissingSection[sec] = RawConfigParser.items(cfgSample, sec) + dicoMissingSection[sec] = cfgP.items(cfgSample, sec) else: - setSample = set(RawConfigParser.options(cfgSample, sec)) - setNormal = set(RawConfigParser.options(cfg, sec)) + setSample = set(cfgP.options(cfgSample, sec)) + setNormal = set(cfgP.options(cfg, sec)) if setSample != setNormal: missing_items = list(setSample.difference(setNormal)) missingItem += [sec] @@ -57,23 +58,43 @@ def main(): for item in dicoMissingItem[section]: print(" - "+item[0]) print("+--------------------------------------------------------------------+") + resp = raw_input("Do you want to auto fix it? [y/n] ") if resp != 'y': return else: - for section in missingSection: - cfg.add_section(section) - for item, value in dicoMissingSection[section]: - cfg.set(section, item, value) + #Do not keep item order in section. New items appened for section in missingItem: for item, value in dicoMissingItem[section]: cfg.set(section, item, value) + #Keep sections order while updating the config file + new_dico = add_items_to_correct_position(cfgSample._sections, cfg._sections, missingSection, dicoMissingSection) + cfg._sections = new_dico + with open(configfile, 'w') as f: cfg.write(f) +''' Return a new dico with the section ordered as the old configuration with the updated one added ''' +def add_items_to_correct_position(sample_dico, old_dico, missingSection, dicoMissingSection): + new_dico = OrderedDict() + + positions = {} + for pos_i, sec in enumerate(sample_dico): + if sec in missingSection: + positions[pos_i] = sec + + for pos_i, sec in enumerate(old_dico): + if pos_i in positions: + missSection = positions[pos_i] + new_dico[missSection] = sample_dico[missSection] + + new_dico[sec] = old_dico[sec] + return new_dico + + if __name__ == "__main__": main() From 6e80eb09540a5886d1851e45c47109f5573e2d5f Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Tue, 28 Mar 2017 09:55:59 +0200 Subject: [PATCH 3/5] Typo --- bin/Update-conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/Update-conf.py b/bin/Update-conf.py index b945dabe..7ceada75 100755 --- a/bin/Update-conf.py +++ b/bin/Update-conf.py @@ -64,12 +64,12 @@ def main(): if resp != 'y': return else: - #Do not keep item order in section. New items appened + #Do not keep item ordering in section. New items appened for section in missingItem: for item, value in dicoMissingItem[section]: cfg.set(section, item, value) - #Keep sections order while updating the config file + #Keep sections ordering while updating the config file new_dico = add_items_to_correct_position(cfgSample._sections, cfg._sections, missingSection, dicoMissingSection) cfg._sections = new_dico From e75d6afdd9f2fe998ee06975e8a03300aab5ea49 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Tue, 25 Apr 2017 08:34:21 +0200 Subject: [PATCH 4/5] Added all_modules.txt rule in gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 46d87d71..a99c47fd 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,4 @@ bin/packages/config.cfg # installed files nltk_data/ +doc/all_modules.txt From 318a557c560705ad5f9cf77b66fa885a6417ef14 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Tue, 25 Apr 2017 09:32:05 +0200 Subject: [PATCH 5/5] Added possibility to keep a backup of the old config + check if config is valid before starting scripts. --- bin/LAUNCH.sh | 18 +++++++++++++++++- bin/Update-conf.py | 20 ++++++++++++++++---- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 3e8814c0..af44e794 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -112,9 +112,17 @@ function launching_queues { } function launching_scripts { + echo -e "\t* Checking configuration" + bash -c "./Update-conf.py" + exitStatus=$? + if [ $exitStatus -ge 1 ]; then + echo -e $RED"\t* Configuration not up-to-date"$DEFAULT + exit + fi + echo -e $GREEN"\t* Configuration up-to-date"$DEFAULT + screen -dmS "Script" sleep 0.1 - echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModulesInformationV2.py -k 0 -c 1; read x' @@ -253,7 +261,15 @@ for i in ${!options[@]}; do bash -c "./Shutdown.py" ;; Update-config) + echo -e "\t* Checking configuration" bash -c "./Update-conf.py" + exitStatus=$? + if [ $exitStatus -ge 1 ]; then + echo -e $RED"\t* Configuration not up-to-date"$DEFAULT + exit + else + echo -e $GREEN"\t* Configuration up-to-date"$DEFAULT + fi ;; esac fi diff --git a/bin/Update-conf.py b/bin/Update-conf.py index 7ceada75..863ff436 100755 --- a/bin/Update-conf.py +++ b/bin/Update-conf.py @@ -5,11 +5,15 @@ import ConfigParser from ConfigParser import ConfigParser as cfgP import os from collections import OrderedDict +import sys +import shutil +#return true if the configuration is up-to-date def main(): configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') + configfileBackup = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') + '.backup' if not os.path.exists(configfile): raise Exception('Unable to find the configuration file. \ Did you set environment variables? \ @@ -45,8 +49,8 @@ def main(): dicoMissingItem[sec] = list_items if len(missingSection) == 0 and len(missingItem) == 0: - print("Configuration up-to-date") - return + #print("Configuration up-to-date") + return True print("/!\\ Configuration not complete. Missing following configuration: /!\\") print("+--------------------------------------------------------------------+") for section in missingSection: @@ -62,8 +66,12 @@ def main(): resp = raw_input("Do you want to auto fix it? [y/n] ") if resp != 'y': - return + return False else: + resp2 = raw_input("Do you want to keep a backup of the old configuration file? [y/n] ") + if resp2 == 'y': + shutil.move(configfile, configfileBackup) + #Do not keep item ordering in section. New items appened for section in missingItem: for item, value in dicoMissingItem[section]: @@ -75,6 +83,7 @@ def main(): with open(configfile, 'w') as f: cfg.write(f) + return True ''' Return a new dico with the section ordered as the old configuration with the updated one added ''' @@ -96,5 +105,8 @@ def add_items_to_correct_position(sample_dico, old_dico, missingSection, dicoMis if __name__ == "__main__": - main() + if main(): + sys.exit() + else: + sys.exit(1)