Merge pull request #116 from mokaddem/auto-conf

Auto-configuration

.gitignore

@@ -26,3 +26,4 @@ bin/packages/config.cfg
 
 # installed files
 nltk_data/
+doc/all_modules.txt

bin/LAUNCH.sh

@@ -112,9 +112,17 @@ function launching_queues {
 }
 
 function launching_scripts {
+    echo -e "\t* Checking configuration"
+    bash -c "./Update-conf.py"
+    exitStatus=$?
+    if [ $exitStatus -ge 1 ]; then
+        echo -e $RED"\t* Configuration not up-to-date"$DEFAULT
+        exit
+    fi
+    echo -e $GREEN"\t* Configuration up-to-date"$DEFAULT
+
     screen -dmS "Script"
     sleep 0.1
-
     echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
 
     screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModulesInformationV2.py -k 0 -c 1; read x'
@@ -183,7 +191,7 @@ islogged=`screen -ls | awk '/\.Logging\t/ {print strtonum($1)}'`
 isqueued=`screen -ls | awk '/\.Queue\t/ {print strtonum($1)}'`
 isscripted=`screen -ls | awk '/\.Script\t/ {print strtonum($1)}'`
 
-options=("Redis" "LevelDB" "Logs" "Queues" "Scripts" "Killall" "Shutdown")
+options=("Redis" "LevelDB" "Logs" "Queues" "Scripts" "Killall" "Shutdown" "Update-config")
 
 menu() {
     echo "What do you want to Launch?:"
@@ -252,6 +260,17 @@ for i in ${!options[@]}; do
             Shutdown)
                 bash -c "./Shutdown.py"
                 ;;
+            Update-config)
+                echo -e "\t* Checking configuration"
+                bash -c "./Update-conf.py"
+                exitStatus=$?
+                if [ $exitStatus -ge 1 ]; then
+                    echo -e $RED"\t* Configuration not up-to-date"$DEFAULT
+                    exit
+                else
+                    echo -e $GREEN"\t* Configuration up-to-date"$DEFAULT
+                fi
+                ;;
         esac
     fi
 done

bin/Update-conf.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python2
+# -*-coding:UTF-8 -*
+
+import ConfigParser
+from ConfigParser import ConfigParser as cfgP
+import os
+from collections import OrderedDict
+import sys
+import shutil
+
+
+#return true if the configuration is up-to-date
+def main():
+
+    configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
+    configfileBackup = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') + '.backup'
+    if not os.path.exists(configfile):
+        raise Exception('Unable to find the configuration file. \
+                        Did you set environment variables? \
+                        Or activate the virtualenv.')
+    configfileSample  = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg.sample')
+
+    cfg = ConfigParser.ConfigParser()
+    cfg.read(configfile)
+    cfgSample = ConfigParser.ConfigParser()
+    cfgSample.read(configfileSample)
+
+    sections = cfgP.sections(cfg)
+    sectionsSample = cfgP.sections(cfgSample)
+    
+    missingSection = []
+    dicoMissingSection = {}
+    missingItem = []
+    dicoMissingItem = {}
+
+    for sec in sectionsSample:
+        if sec not in sections:
+            missingSection += [sec]
+            dicoMissingSection[sec] = cfgP.items(cfgSample, sec)
+        else:
+            setSample = set(cfgP.options(cfgSample, sec))
+            setNormal = set(cfgP.options(cfg, sec))
+            if setSample != setNormal:
+                missing_items = list(setSample.difference(setNormal))
+                missingItem += [sec]
+                list_items = []
+                for i in missing_items:
+                    list_items.append( (i, cfgSample.get(sec, i)) )
+                dicoMissingItem[sec] = list_items
+
+    if len(missingSection) == 0 and len(missingItem) == 0:
+        #print("Configuration up-to-date")
+        return True
+    print("/!\\ Configuration not complete. Missing following configuration: /!\\")
+    print("+--------------------------------------------------------------------+")
+    for section in missingSection:
+        print("["+section+"]")
+        for item in dicoMissingSection[section]:
+            print("  - "+item[0])
+    for section in missingItem:
+        print("["+section+"]")
+        for item in dicoMissingItem[section]:
+            print("  - "+item[0])
+    print("+--------------------------------------------------------------------+")
+
+    resp = raw_input("Do you want to auto fix it? [y/n] ")
+
+    if resp != 'y':
+        return False
+    else:
+        resp2 = raw_input("Do you want to keep a backup of the old configuration file? [y/n] ")
+        if resp2 == 'y':
+            shutil.move(configfile, configfileBackup)
+
+        #Do not keep item ordering in section. New items appened
+        for section in missingItem:
+            for item, value in dicoMissingItem[section]:
+                cfg.set(section, item, value)
+
+        #Keep sections ordering while updating the config file
+        new_dico = add_items_to_correct_position(cfgSample._sections, cfg._sections, missingSection, dicoMissingSection)
+        cfg._sections = new_dico
+
+        with open(configfile, 'w') as f:
+            cfg.write(f)
+        return True
+
+
+''' Return a new dico with the section ordered as the old configuration with the updated one added '''
+def add_items_to_correct_position(sample_dico, old_dico, missingSection, dicoMissingSection):
+    new_dico = OrderedDict()
+    
+    positions = {}
+    for pos_i, sec in enumerate(sample_dico):
+        if sec in missingSection:
+            positions[pos_i] = sec
+
+    for pos_i, sec in enumerate(old_dico):
+        if pos_i in positions:
+            missSection = positions[pos_i]
+            new_dico[missSection] = sample_dico[missSection]
+
+        new_dico[sec] = old_dico[sec]
+    return new_dico
+
+
+if __name__ == "__main__":
+    if main():
+        sys.exit()
+    else:
+        sys.exit(1)
+

bin/packages/config.cfg.sample

@@ -123,6 +123,7 @@ cc_tld = r'\.de$'
 [Indexer]
 type = whoosh
 path = indexdir
+register = indexdir/all_index.txt
 #size in Mb
 index_max_size = 2000