fix: [LAUNCH] use envar, add options (launch all, flask, killall)

bin/LAUNCH.sh

@@ -12,10 +12,14 @@ CYAN="\\033[1;36m"
 [ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
 [ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
 [ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
 
 export PATH=$AIL_HOME:$PATH
 export PATH=$AIL_REDIS:$PATH
 export PATH=$AIL_ARDB:$PATH
+export PATH=$AIL_BIN:$PATH
+export PATH=$AIL_FLASK:$PATH
 
 function helptext {
     echo -e $YELLOW"
@@ -35,15 +39,18 @@ function helptext {
     - All the ZMQ queuing modules.
     - All the ZMQ processing modules.
     - All Redis in memory servers.
-    - All Level-DB on disk servers.
+    - All ARDB on disk servers.
     "$DEFAULT"
     (Inside screen Daemons)
-    "$RED"
-    But first of all you'll need to edit few path where you installed
-    your redis & ardb servers.
     "$DEFAULT"
     Usage:
     -----
+    LAUNCH.sh
+      [-l | --launchAuto]
+      [-k | --killAll]
+      [-c | --configUpdate]
+      [-t | --thirdpartyUpdate]
+      [-h | --help]
     "
 }
 
@@ -68,16 +75,16 @@ function launching_ardb {
     echo -e $GREEN"\t* Launching ARDB servers"$DEFAULT
 
     sleep 0.1
-    screen -S "ARDB_AIL" -X screen -t "6382" bash -c 'ardb-server '$conf_dir'6382.conf ; read x'
+    screen -S "ARDB_AIL" -X screen -t "6382" bash -c 'cd '${AIL_HOME}'; ardb-server '$conf_dir'6382.conf ; read x'
 }
 
 function launching_logs {
     screen -dmS "Logging_AIL"
     sleep 0.1
     echo -e $GREEN"\t* Launching logging process"$DEFAULT
-    screen -S "Logging_AIL" -X screen -t "LogQueue" bash -c 'log_subscriber -p 6380 -c Queuing -l ../logs/; read x'
+    screen -S "Logging_AIL" -X screen -t "LogQueue" bash -c 'cd '${AIL_BIN}'; log_subscriber -p 6380 -c Queuing -l ../logs/; read x'
     sleep 0.1
-    screen -S "Logging_AIL" -X screen -t "LogScript" bash -c 'log_subscriber -p 6380 -c Script -l ../logs/; read x'
+    screen -S "Logging_AIL" -X screen -t "LogScript" bash -c 'cd '${AIL_BIN}'; log_subscriber -p 6380 -c Script -l ../logs/; read x'
 }
 
 function launching_queues {
@@ -85,90 +92,100 @@ function launching_queues {
     sleep 0.1
 
     echo -e $GREEN"\t* Launching all the queues"$DEFAULT
-    screen -S "Queue_AIL" -X screen -t "Queues" bash -c 'python3 launch_queues.py; read x'
+    screen -S "Queue_AIL" -X screen -t "Queues" bash -c 'cd '${AIL_BIN}'; python3 launch_queues.py; read x'
 }
 
-function launching_scripts {
+function checking_configuration {
+    bin_dir=${AIL_HOME}/bin
     echo -e "\t* Checking configuration"
-    bash -c "python3 Update-conf.py"
+    if [ "$1" == "automatic" ]; then
+        bash -c "python3 $bin_dir/Update-conf.py True"
+    else
+        bash -c "python3 $bin_dir/Update-conf.py False"
+    fi
+
     exitStatus=$?
     if [ $exitStatus -ge 1 ]; then
         echo -e $RED"\t* Configuration not up-to-date"$DEFAULT
         exit
     fi
     echo -e $GREEN"\t* Configuration up-to-date"$DEFAULT
+}
+
+function launching_scripts {
+    checking_configuration $1;
 
     screen -dmS "Script_AIL"
     sleep 0.1
     echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
 
-    screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c './ModulesInformationV2.py -k 0 -c 1; read x'
+    screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c 'cd '${AIL_BIN}'; ./ModulesInformationV2.py -k 0 -c 1; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Mixer" bash -c './Mixer.py; read x'
+    screen -S "Script_AIL" -X screen -t "Mixer" bash -c 'cd '${AIL_BIN}'; ./Mixer.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Global" bash -c './Global.py; read x'
+    screen -S "Script_AIL" -X screen -t "Global" bash -c 'cd '${AIL_BIN}'; ./Global.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Duplicates" bash -c './Duplicates.py; read x'
+    screen -S "Script_AIL" -X screen -t "Duplicates" bash -c 'cd '${AIL_BIN}'; ./Duplicates.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Lines" bash -c './Lines.py; read x'
+    screen -S "Script_AIL" -X screen -t "Lines" bash -c 'cd '${AIL_BIN}'; ./Lines.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c './DomClassifier.py; read x'
+    screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c 'cd '${AIL_BIN}'; ./DomClassifier.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Categ" bash -c './Categ.py; read x'
+    screen -S "Script_AIL" -X screen -t "Categ" bash -c 'cd '${AIL_BIN}'; ./Categ.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Tokenize" bash -c './Tokenize.py; read x'
+    screen -S "Script_AIL" -X screen -t "Tokenize" bash -c 'cd '${AIL_BIN}'; ./Tokenize.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "CreditCards" bash -c './CreditCards.py; read x'
+    screen -S "Script_AIL" -X screen -t "CreditCards" bash -c 'cd '${AIL_BIN}'; ./CreditCards.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Onion" bash -c './Onion.py; read x'
+    screen -S "Script_AIL" -X screen -t "Onion" bash -c 'cd '${AIL_BIN}'; ./Onion.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Mail" bash -c './Mail.py; read x'
+    screen -S "Script_AIL" -X screen -t "Mail" bash -c 'cd '${AIL_BIN}'; ./Mail.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "ApiKey" bash -c './ApiKey.py; read x'
+    screen -S "Script_AIL" -X screen -t "ApiKey" bash -c 'cd '${AIL_BIN}'; ./ApiKey.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Web" bash -c './Web.py; read x'
+    screen -S "Script_AIL" -X screen -t "Web" bash -c 'cd '${AIL_BIN}'; ./Web.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Credential" bash -c './Credential.py; read x'
+    screen -S "Script_AIL" -X screen -t "Credential" bash -c 'cd '${AIL_BIN}'; ./Credential.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Curve" bash -c './Curve.py; read x'
+    screen -S "Script_AIL" -X screen -t "Curve" bash -c 'cd '${AIL_BIN}'; ./Curve.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "CurveManageTopSets" bash -c './CurveManageTopSets.py; read x'
+    screen -S "Script_AIL" -X screen -t "CurveManageTopSets" bash -c 'cd '${AIL_BIN}'; ./CurveManageTopSets.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "RegexForTermsFrequency" bash -c './RegexForTermsFrequency.py; read x'
+    screen -S "Script_AIL" -X screen -t "RegexForTermsFrequency" bash -c 'cd '${AIL_BIN}'; ./RegexForTermsFrequency.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "SetForTermsFrequency" bash -c './SetForTermsFrequency.py; read x'
+    screen -S "Script_AIL" -X screen -t "SetForTermsFrequency" bash -c 'cd '${AIL_BIN}'; ./SetForTermsFrequency.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Indexer" bash -c './Indexer.py; read x'
+    screen -S "Script_AIL" -X screen -t "Indexer" bash -c 'cd '${AIL_BIN}'; ./Indexer.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Keys" bash -c './Keys.py; read x'
+    screen -S "Script_AIL" -X screen -t "Keys" bash -c 'cd '${AIL_BIN}'; ./Keys.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Base64" bash -c './Base64.py; read x'
+    screen -S "Script_AIL" -X screen -t "Base64" bash -c 'cd '${AIL_BIN}'; ./Base64.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c './Bitcoin.py; read x'
+    screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c 'cd '${AIL_BIN}'; ./Bitcoin.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Phone" bash -c './Phone.py; read x'
+    screen -S "Script_AIL" -X screen -t "Phone" bash -c 'cd '${AIL_BIN}'; ./Phone.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Release" bash -c './Release.py; read x'
+    screen -S "Script_AIL" -X screen -t "Release" bash -c 'cd '${AIL_BIN}'; ./Release.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Cve" bash -c './Cve.py; read x'
+    screen -S "Script_AIL" -X screen -t "Cve" bash -c 'cd '${AIL_BIN}'; ./Cve.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "WebStats" bash -c './WebStats.py; read x'
+    screen -S "Script_AIL" -X screen -t "WebStats" bash -c 'cd '${AIL_BIN}'; ./WebStats.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c './ModuleStats.py; read x'
+    screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c 'cd '${AIL_BIN}'; ./ModuleStats.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c './SQLInjectionDetection.py; read x'
+    screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c 'cd '${AIL_BIN}'; ./SQLInjectionDetection.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "LibInjection" bash -c './LibInjection.py; read x'
+    screen -S "Script_AIL" -X screen -t "LibInjection" bash -c 'cd '${AIL_BIN}'; ./LibInjection.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "alertHandler" bash -c './alertHandler.py; read x'
+    screen -S "Script_AIL" -X screen -t "alertHandler" bash -c 'cd '${AIL_BIN}'; ./alertHandler.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c './MISP_The_Hive_feeder.py; read x'
+    screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c 'cd '${AIL_BIN}'; ./MISP_The_Hive_feeder.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Tags" bash -c './Tags.py; read x'
+    screen -S "Script_AIL" -X screen -t "Tags" bash -c 'cd '${AIL_BIN}'; ./Tags.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x'
+    screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c 'cd '${AIL_BIN}'; ./SentimentAnalysis.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c './submit_paste.py; read x'
+    screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c 'cd '${AIL_BIN}'; ./submit_paste.py; read x'
 
 }
 
@@ -217,17 +234,120 @@ function checking_ardb {
     sleep 0.2
     bash -c $redis_dir'redis-cli -p 6382 PING | grep "PONG" &> /dev/null'
     if [ ! $? == 0 ]; then
-       echo -e $RED"\t6382 not ready"$DEFAULT
+       echo -e $RED"\t6382 ARDB not ready"$DEFAULT
        flag_ardb=1
     fi
 
     return $flag_ardb;
 }
 
-#If no params, display the help
+function launch_redis {
-#[[ $@ ]] || { helptext; exit 1;}
+    if [[ ! $isredis ]]; then
+        launching_redis;
+    else
+        echo -e $RED"\t* A screen is already launched"$DEFAULT
+    fi
+}
+
+function launch_ardb {
+    if [[ ! $isardb ]]; then
+        launching_ardb;
+    else
+        echo -e $RED"\t* A screen is already launched"$DEFAULT
+    fi
+}
+
+function launch_logs {
+    if [[ ! $islogged ]]; then
+        launching_logs;
+    else
+        echo -e $RED"\t* A screen is already launched"$DEFAULT
+    fi
+}
+
+function launch_queues {
+    if [[ ! $isqueued ]]; then
+        launching_queues;
+    else
+        echo -e $RED"\t* A screen is already launched"$DEFAULT
+    fi
+}
+
+function launch_scripts {
+    if [[ ! $isscripted ]]; then
+      sleep 1
+        if checking_redis && checking_ardb; then
+            launching_scripts $1;
+        else
+            while [[ (checking_ardb) && (checking_redis) ]]; do
+                echo -e $YELLOW"\tScript not started, waiting 5 secondes"$DEFAULT
+                sleep 5
+                if checking_redis && checking_ardb; then
+                    launching_scripts $1;
+                else
+                    echo -e $RED"\tScript not started"$DEFAULT
+                fi;
+            done
+        fi;
+    else
+        echo -e $RED"\t* A screen is already launched"$DEFAULT
+    fi
+}
+
+function launch_flask {
+    if [[ ! $isflasked ]]; then
+        flask_dir=${AIL_FLASK}
+        screen -dmS "Flask_AIL"
+        sleep 0.1
+        echo -e $GREEN"\t* Launching Flask server"$DEFAULT
+        screen -S "Flask_AIL" -X screen -t "Flask_server" bash -c "cd $flask_dir; ls; ./Flask_server.py; read x"
+    else
+        echo -e $RED"\t* A Flask screen is already launched"$DEFAULT
+    fi
+}
+
+function killall {
+    if [[ $isredis || $isardb || $islogged || $isqueued || $isscripted || $isflasked ]]; then
+        echo -e $GREEN"Gracefully closing redis servers"$DEFAULT
+        shutting_down_redis;
+        sleep 0.2
+        echo -e $GREEN"Gracefully closing ardb servers"$DEFAULT
+        shutting_down_ardb;
+        echo -e $GREEN"Killing all"$DEFAULT
+        kill $isredis $isardb $islogged $isqueued $isscripted $isflasked
+        sleep 0.2
+        echo -e $ROSE`screen -ls`$DEFAULT
+        echo -e $GREEN"\t* $isredis $isardb $islogged $isqueued $isscripted killed."$DEFAULT
+    else
+        echo -e $RED"\t* No screen to kill"$DEFAULT
+    fi
+}
+
+function shutdown {
+    bash -c "./Shutdown.py"
+}
+
+function update_thirdparty {
+    echo -e "\t* Updating thirdparty..."
+    bash -c "(cd ${AIL_FLASK}; ./update_thirdparty.sh)"
+    exitStatus=$?
+    if [ $exitStatus -ge 1 ]; then
+        echo -e $RED"\t* Thirdparty not up-to-date"$DEFAULT
+        exit
+    else
+        echo -e $GREEN"\t* Thirdparty updated"$DEFAULT
+    fi
+}
+
+function launch_all {
+    launch_redis;
+    launch_ardb;
+    launch_logs;
+    launch_queues;
+    launch_scripts $1;
+    launch_flask;
+}
 
-helptext;
 
 ############### TESTS ###################
 isredis=`screen -ls | egrep '[0-9]+.Redis_AIL' | cut -d. -f1`
@@ -235,8 +355,14 @@ isardb=`screen -ls | egrep '[0-9]+.ARDB_AIL' | cut -d. -f1`
 islogged=`screen -ls | egrep '[0-9]+.Logging_AIL' | cut -d. -f1`
 isqueued=`screen -ls | egrep '[0-9]+.Queue_AIL' | cut -d. -f1`
 isscripted=`screen -ls | egrep '[0-9]+.Script_AIL' | cut -d. -f1`
+isflasked=`screen -ls | egrep '[0-9]+.Flask_AIL' | cut -d. -f1`
 
-options=("Redis" "Ardb" "Logs" "Queues" "Scripts" "Killall" "Shutdown" "Update-config" "Update-thirdparty")
+#If no params, display the menu
+[[ $@ ]] || {
+
+    helptext;
+
+    options=("Redis" "Ardb" "Logs" "Queues" "Scripts" "Flask" "Killall" "Shutdown" "Update-config" "Update-thirdparty")
 
     menu() {
         echo "What do you want to Launch?:"
@@ -261,92 +387,57 @@ for i in ${!options[@]}; do
         if [[ "${choices[i]}" ]]; then
             case ${options[i]} in
                 Redis)
-                if [[ ! $isredis ]]; then
+                    launch_redis
-                    launching_redis;
-                else
-                    echo -e $RED"\t* A screen is already launched"$DEFAULT
-                fi
                     ;;
                 Ardb)
-                if [[ ! $isardb ]]; then
+                    launch_ardb;
-                    launching_ardb;
-                else
-                    echo -e $RED"\t* A screen is already launched"$DEFAULT
-                fi
                     ;;
                 Logs)
-                if [[ ! $islogged ]]; then
+                    launch_logs;
-                    launching_logs;
-                else
-                    echo -e $RED"\t* A screen is already launched"$DEFAULT
-                fi
                     ;;
                 Queues)
-                if [[ ! $isqueued ]]; then
+                    launch_queues;
-                    launching_queues;
-                else
-                    echo -e $RED"\t* A screen is already launched"$DEFAULT
-                fi
                     ;;
                 Scripts)
-                if [[ ! $isscripted ]]; then
+                    launch_scripts;
-                  sleep 1
+                    ;;
-                    if checking_redis && checking_ardb; then
+                Flask)
-                        launching_scripts;
+                    launch_flask;
-                    else
-                        echo -e $YELLOW"\tScript not started, waiting 5 secondes"$DEFAULT
-                        sleep 5
-                        if checking_redis && checking_ardb; then
-                            launching_scripts;
-                        else
-                            echo -e $RED"\tScript not started"$DEFAULT
-                        fi;
-                    fi;
-                else
-                    echo -e $RED"\t* A screen is already launched"$DEFAULT
-                fi
                     ;;
                 Killall)
-                if [[ $isredis || $isardb || $islogged || $isqueued || $isscripted ]]; then
+                    killall;
-                    echo -e $GREEN"Gracefully closing redis servers"$DEFAULT
-                    shutting_down_redis;
-                    sleep 0.2
-                    echo -e $GREEN"Gracefully closing ardb servers"$DEFAULT
-                    shutting_down_ardb;
-                    echo -e $GREEN"Killing all"$DEFAULT
-                    kill $isredis $isardb $islogged $isqueued $isscripted
-                    sleep 0.2
-                    echo -e $ROSE`screen -ls`$DEFAULT
-                    echo -e $GREEN"\t* $isredis $isardb $islogged $isqueued $isscripted killed."$DEFAULT
-                else
-                    echo -e $RED"\t* No screen to kill"$DEFAULT
-                fi
                     ;;
                 Shutdown)
-                bash -c "./Shutdown.py"
+                    shutdown;
                     ;;
                 Update-config)
-                echo -e "\t* Checking configuration"
+                    checking_configuration "manual";
-                bash -c "./Update-conf.py"
-                exitStatus=$?
-                if [ $exitStatus -ge 1 ]; then
-                    echo -e $RED"\t* Configuration not up-to-date"$DEFAULT
-                    exit
-                else
-                    echo -e $GREEN"\t* Configuration up-to-date"$DEFAULT
-                fi
                     ;;
                 Update-thirdparty)
-                echo -e "\t* Updating thirdparty..."
+                    update_thirdparty;
-                bash -c "(cd ../var/www && ./update_thirdparty.sh)"
-                exitStatus=$?
-                if [ $exitStatus -ge 1 ]; then
-                    echo -e $RED"\t* Configuration not up-to-date"$DEFAULT
-                    exit
-                else
-                    echo -e $GREEN"\t* Configuration up-to-date"$DEFAULT
-                fi
                     ;;
             esac
         fi
     done
+
+    exit
+}
+
+while [ "$1" != "" ]; do
+    case $1 in
+        -l | --launchAuto )         launch_all "automatic";
+                                    ;;
+        -k | --killAll )            killall;
+                                    ;;
+        -c | --configUpdate )       checking_configuration "manual";
+                                    ;;
+        -t | --thirdpartyUpdate )    update_thirdparty;
+                                    ;;
+        -h | --help )               helptext;
+                                    exit
+                                    ;;
+        * )                         helptext
+                                    exit 1
+    esac
+    shift
+done

bin/Update-conf.py

@@ -11,6 +11,15 @@ import shutil
 
 #return true if the configuration is up-to-date
 def main():
+    if len(sys.argv) != 2:
+        print('usage:', 'Update-conf.py', 'Automatic (boolean)')
+        exit(1)
+    else:
+        automatic = sys.argv[1]
+        if automatic == 'True':
+            automatic = True
+        else:
+            automatic = False
 
     configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
     configfileBackup = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') + '.backup'
@@ -63,12 +72,19 @@ def main():
             print("  - "+item[0])
     print("+--------------------------------------------------------------------+")
 
+    if automatic:
+        resp = 'y'
+    else:
         resp = input("Do you want to auto fix it? [y/n] ")
 
     if resp != 'y':
         return False
+    else:
+        if automatic:
+            resp2 = 'y'
         else:
             resp2 = input("Do you want to keep a backup of the old configuration file? [y/n] ")
+
         if resp2 == 'y':
             shutil.move(configfile, configfileBackup)
 

configs/6382.conf

@@ -1,7 +1,7 @@
 # Ardb configuration file example, modified from redis's conf file.
 
 # Home dir for ardb instance, it can be referenced by ${ARDB_HOME} in this config file
-home  ../DATA_ARDB/
+home  DATA_ARDB/
 
 # Note on units: when memory size is needed, it is possible to specify
 # it in the usual form of 1k 5GB 4M and so forth: