diff --git a/.gitignore b/.gitignore index 46d87d71..a99c47fd 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,4 @@ bin/packages/config.cfg # installed files nltk_data/ +doc/all_modules.txt diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index e082b7f0..af44e794 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -112,9 +112,17 @@ function launching_queues { } function launching_scripts { + echo -e "\t* Checking configuration" + bash -c "./Update-conf.py" + exitStatus=$? + if [ $exitStatus -ge 1 ]; then + echo -e $RED"\t* Configuration not up-to-date"$DEFAULT + exit + fi + echo -e $GREEN"\t* Configuration up-to-date"$DEFAULT + screen -dmS "Script" sleep 0.1 - echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModulesInformationV2.py -k 0 -c 1; read x' @@ -183,7 +191,7 @@ islogged=`screen -ls | awk '/\.Logging\t/ {print strtonum($1)}'` isqueued=`screen -ls | awk '/\.Queue\t/ {print strtonum($1)}'` isscripted=`screen -ls | awk '/\.Script\t/ {print strtonum($1)}'` -options=("Redis" "LevelDB" "Logs" "Queues" "Scripts" "Killall" "Shutdown") +options=("Redis" "LevelDB" "Logs" "Queues" "Scripts" "Killall" "Shutdown" "Update-config") menu() { echo "What do you want to Launch?:" @@ -252,6 +260,17 @@ for i in ${!options[@]}; do Shutdown) bash -c "./Shutdown.py" ;; + Update-config) + echo -e "\t* Checking configuration" + bash -c "./Update-conf.py" + exitStatus=$? + if [ $exitStatus -ge 1 ]; then + echo -e $RED"\t* Configuration not up-to-date"$DEFAULT + exit + else + echo -e $GREEN"\t* Configuration up-to-date"$DEFAULT + fi + ;; esac fi done diff --git a/bin/Update-conf.py b/bin/Update-conf.py new file mode 100755 index 00000000..863ff436 --- /dev/null +++ b/bin/Update-conf.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python2 +# -*-coding:UTF-8 -* + +import ConfigParser +from ConfigParser import ConfigParser as cfgP +import os +from collections import OrderedDict +import sys +import shutil + + +#return true if the configuration is up-to-date +def main(): + + configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') + configfileBackup = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') + '.backup' + if not os.path.exists(configfile): + raise Exception('Unable to find the configuration file. \ + Did you set environment variables? \ + Or activate the virtualenv.') + configfileSample = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg.sample') + + cfg = ConfigParser.ConfigParser() + cfg.read(configfile) + cfgSample = ConfigParser.ConfigParser() + cfgSample.read(configfileSample) + + sections = cfgP.sections(cfg) + sectionsSample = cfgP.sections(cfgSample) + + missingSection = [] + dicoMissingSection = {} + missingItem = [] + dicoMissingItem = {} + + for sec in sectionsSample: + if sec not in sections: + missingSection += [sec] + dicoMissingSection[sec] = cfgP.items(cfgSample, sec) + else: + setSample = set(cfgP.options(cfgSample, sec)) + setNormal = set(cfgP.options(cfg, sec)) + if setSample != setNormal: + missing_items = list(setSample.difference(setNormal)) + missingItem += [sec] + list_items = [] + for i in missing_items: + list_items.append( (i, cfgSample.get(sec, i)) ) + dicoMissingItem[sec] = list_items + + if len(missingSection) == 0 and len(missingItem) == 0: + #print("Configuration up-to-date") + return True + print("/!\\ Configuration not complete. Missing following configuration: /!\\") + print("+--------------------------------------------------------------------+") + for section in missingSection: + print("["+section+"]") + for item in dicoMissingSection[section]: + print(" - "+item[0]) + for section in missingItem: + print("["+section+"]") + for item in dicoMissingItem[section]: + print(" - "+item[0]) + print("+--------------------------------------------------------------------+") + + resp = raw_input("Do you want to auto fix it? [y/n] ") + + if resp != 'y': + return False + else: + resp2 = raw_input("Do you want to keep a backup of the old configuration file? [y/n] ") + if resp2 == 'y': + shutil.move(configfile, configfileBackup) + + #Do not keep item ordering in section. New items appened + for section in missingItem: + for item, value in dicoMissingItem[section]: + cfg.set(section, item, value) + + #Keep sections ordering while updating the config file + new_dico = add_items_to_correct_position(cfgSample._sections, cfg._sections, missingSection, dicoMissingSection) + cfg._sections = new_dico + + with open(configfile, 'w') as f: + cfg.write(f) + return True + + +''' Return a new dico with the section ordered as the old configuration with the updated one added ''' +def add_items_to_correct_position(sample_dico, old_dico, missingSection, dicoMissingSection): + new_dico = OrderedDict() + + positions = {} + for pos_i, sec in enumerate(sample_dico): + if sec in missingSection: + positions[pos_i] = sec + + for pos_i, sec in enumerate(old_dico): + if pos_i in positions: + missSection = positions[pos_i] + new_dico[missSection] = sample_dico[missSection] + + new_dico[sec] = old_dico[sec] + return new_dico + + +if __name__ == "__main__": + if main(): + sys.exit() + else: + sys.exit(1) + diff --git a/bin/packages/config.cfg.sample b/bin/packages/config.cfg.sample index a634e4f1..0e91a993 100644 --- a/bin/packages/config.cfg.sample +++ b/bin/packages/config.cfg.sample @@ -123,6 +123,7 @@ cc_tld = r'\.de$' [Indexer] type = whoosh path = indexdir +register = indexdir/all_index.txt #size in Mb index_max_size = 2000