Merge pull request #116 from mokaddem/auto-conf

Auto-configuration
This commit is contained in:
Alexandre Dulaunoy 2017-04-27 15:22:39 +02:00 committed by GitHub
commit 55be38987f
4 changed files with 135 additions and 2 deletions

1
.gitignore vendored
View file

@ -26,3 +26,4 @@ bin/packages/config.cfg
# installed files
nltk_data/
doc/all_modules.txt

View file

@ -112,9 +112,17 @@ function launching_queues {
}
function launching_scripts {
echo -e "\t* Checking configuration"
bash -c "./Update-conf.py"
exitStatus=$?
if [ $exitStatus -ge 1 ]; then
echo -e $RED"\t* Configuration not up-to-date"$DEFAULT
exit
fi
echo -e $GREEN"\t* Configuration up-to-date"$DEFAULT
screen -dmS "Script"
sleep 0.1
echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModulesInformationV2.py -k 0 -c 1; read x'
@ -183,7 +191,7 @@ islogged=`screen -ls | awk '/\.Logging\t/ {print strtonum($1)}'`
isqueued=`screen -ls | awk '/\.Queue\t/ {print strtonum($1)}'`
isscripted=`screen -ls | awk '/\.Script\t/ {print strtonum($1)}'`
options=("Redis" "LevelDB" "Logs" "Queues" "Scripts" "Killall" "Shutdown")
options=("Redis" "LevelDB" "Logs" "Queues" "Scripts" "Killall" "Shutdown" "Update-config")
menu() {
echo "What do you want to Launch?:"
@ -252,6 +260,17 @@ for i in ${!options[@]}; do
Shutdown)
bash -c "./Shutdown.py"
;;
Update-config)
echo -e "\t* Checking configuration"
bash -c "./Update-conf.py"
exitStatus=$?
if [ $exitStatus -ge 1 ]; then
echo -e $RED"\t* Configuration not up-to-date"$DEFAULT
exit
else
echo -e $GREEN"\t* Configuration up-to-date"$DEFAULT
fi
;;
esac
fi
done

112
bin/Update-conf.py Executable file
View file

@ -0,0 +1,112 @@
#!/usr/bin/env python2
# -*-coding:UTF-8 -*
import ConfigParser
from ConfigParser import ConfigParser as cfgP
import os
from collections import OrderedDict
import sys
import shutil
#return true if the configuration is up-to-date
def main():
configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
configfileBackup = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') + '.backup'
if not os.path.exists(configfile):
raise Exception('Unable to find the configuration file. \
Did you set environment variables? \
Or activate the virtualenv.')
configfileSample = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg.sample')
cfg = ConfigParser.ConfigParser()
cfg.read(configfile)
cfgSample = ConfigParser.ConfigParser()
cfgSample.read(configfileSample)
sections = cfgP.sections(cfg)
sectionsSample = cfgP.sections(cfgSample)
missingSection = []
dicoMissingSection = {}
missingItem = []
dicoMissingItem = {}
for sec in sectionsSample:
if sec not in sections:
missingSection += [sec]
dicoMissingSection[sec] = cfgP.items(cfgSample, sec)
else:
setSample = set(cfgP.options(cfgSample, sec))
setNormal = set(cfgP.options(cfg, sec))
if setSample != setNormal:
missing_items = list(setSample.difference(setNormal))
missingItem += [sec]
list_items = []
for i in missing_items:
list_items.append( (i, cfgSample.get(sec, i)) )
dicoMissingItem[sec] = list_items
if len(missingSection) == 0 and len(missingItem) == 0:
#print("Configuration up-to-date")
return True
print("/!\\ Configuration not complete. Missing following configuration: /!\\")
print("+--------------------------------------------------------------------+")
for section in missingSection:
print("["+section+"]")
for item in dicoMissingSection[section]:
print(" - "+item[0])
for section in missingItem:
print("["+section+"]")
for item in dicoMissingItem[section]:
print(" - "+item[0])
print("+--------------------------------------------------------------------+")
resp = raw_input("Do you want to auto fix it? [y/n] ")
if resp != 'y':
return False
else:
resp2 = raw_input("Do you want to keep a backup of the old configuration file? [y/n] ")
if resp2 == 'y':
shutil.move(configfile, configfileBackup)
#Do not keep item ordering in section. New items appened
for section in missingItem:
for item, value in dicoMissingItem[section]:
cfg.set(section, item, value)
#Keep sections ordering while updating the config file
new_dico = add_items_to_correct_position(cfgSample._sections, cfg._sections, missingSection, dicoMissingSection)
cfg._sections = new_dico
with open(configfile, 'w') as f:
cfg.write(f)
return True
''' Return a new dico with the section ordered as the old configuration with the updated one added '''
def add_items_to_correct_position(sample_dico, old_dico, missingSection, dicoMissingSection):
new_dico = OrderedDict()
positions = {}
for pos_i, sec in enumerate(sample_dico):
if sec in missingSection:
positions[pos_i] = sec
for pos_i, sec in enumerate(old_dico):
if pos_i in positions:
missSection = positions[pos_i]
new_dico[missSection] = sample_dico[missSection]
new_dico[sec] = old_dico[sec]
return new_dico
if __name__ == "__main__":
if main():
sys.exit()
else:
sys.exit(1)

View file

@ -123,6 +123,7 @@ cc_tld = r'\.de$'
[Indexer]
type = whoosh
path = indexdir
register = indexdir/all_index.txt
#size in Mb
index_max_size = 2000