chg: [Update + Launcher] use background update + add new options in launcher

bin/LAUNCH.sh

@@ -297,6 +297,30 @@ function checking_ardb {
     return $flag_ardb;
 }
 
+function wait_until_redis_is_ready {
+    redis_not_ready=true
+    while $redis_not_ready; do
+        if checking_redis; then
+            redis_not_ready=false;
+        else
+            sleep 1
+        fi
+    done
+    echo -e $YELLOW"\t* Redis Launched"$DEFAULT
+}
+
+function wait_until_ardb_is_ready {
+    ardb_not_ready=true;
+    while $ardb_not_ready; do
+        if checking_ardb; then
+            ardb_not_ready=false
+        else
+            sleep 3
+        fi
+    done
+    echo -e $YELLOW"\t* ARDB Launched"$DEFAULT
+}
+
 function launch_redis {
     if [[ ! $isredis ]]; then
         launching_redis;
@@ -378,7 +402,7 @@ function launch_feeder {
 }
 
 function killall {
-    if [[ $isredis || $isardb || $islogged || $isqueued || $isscripted || $isflasked || $isfeeded ]]; then
+    if [[ $isredis || $isardb || $islogged || $isqueued || $isscripted || $isflasked || $isfeeded || $iscrawler ]]; then
         if [[ $isredis ]]; then
             echo -e $GREEN"Gracefully closing redis servers"$DEFAULT
             shutting_down_redis;
@@ -389,10 +413,10 @@ function killall {
             shutting_down_ardb;
         fi
         echo -e $GREEN"Killing all"$DEFAULT
-        kill $isredis $isardb $islogged $isqueued $isscripted $isflasked $isfeeded
+        kill $isredis $isardb $islogged $isqueued $isscripted $isflasked $isfeeded $iscrawler
         sleep 0.2
         echo -e $ROSE`screen -ls`$DEFAULT
-        echo -e $GREEN"\t* $isredis $isardb $islogged $isqueued $isscripted killed."$DEFAULT
+        echo -e $GREEN"\t* $isredis $isardb $islogged $isqueued $isscripted $isflasked $isfeeded $iscrawler killed."$DEFAULT
     else
         echo -e $RED"\t* No screen to kill"$DEFAULT
     fi
@@ -511,30 +535,36 @@ function launch_all {
 
 while [ "$1" != "" ]; do
     case $1 in
-        -l | --launchAuto )         launch_all "automatic";
+        -l | --launchAuto )           launch_all "automatic";
-                                    ;;
+                                      ;;
-        -lr | --launchRedis )       launch_redis;
+        -lr | --launchRedis )         launch_redis;
-                                    ;;
+                                      ;;
-        -la | --launchARDB )        launch_ardb;
+        -la | --launchARDB )          launch_ardb;
-                                    ;;
+                                      ;;
-        -k | --killAll )            killall;
+        -lrv | --launchRedisVerify )  launch_redis;
-                                    ;;
+                                      wait_until_redis_is_ready;
-        -u | --update )             update;
+                                      ;;
-                                    ;;
+        -lav | --launchARDBVerify )   launch_ardb;
-        -t | --thirdpartyUpdate )   update_thirdparty;
+                                      wait_until_ardb_is_ready;
-                                    ;;
+                                      ;;
-        -c | --crawler )            launching_crawler;
+        -k | --killAll )              killall;
-                                    ;;
+                                      ;;
-        -f | --launchFeeder )       launch_feeder;
+        -u | --update )               update;
-                                    ;;
+                                      ;;
-        -h | --help )               helptext;
+        -t | --thirdpartyUpdate )     update_thirdparty;
-                                    exit
+                                      ;;
-                                    ;;
+        -c | --crawler )              launching_crawler;
-        -kh | --khelp )             helptext;
+                                      ;;
+        -f | --launchFeeder )         launch_feeder;
+                                      ;;
+        -h | --help )                 helptext;
+                                      exit
+                                      ;;
+        -kh | --khelp )               helptext;
 
-                                    ;;
+                                      ;;
-        * )                         helptext
+        * )                           helptext
-                                    exit 1
+                                      exit 1
     esac
     shift
 done

bin/update-background.py

@@ -38,21 +38,20 @@ if __name__ == "__main__":
         if ail_version == 'v1.5':
             onions_update_status = r_serv.get('v1.5:onions')
             if onions_update_status is None:
-                if int(onions_update_status) != 1:
+                update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Onions.py')
-                    update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Onions.py')
+                process = subprocess.run(['python' ,update_file])
-                    process = subprocess.run(['unbuffer', 'python' ,'update_file'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
             metadata_update_status = r_serv.get('v1.5:metadata')
             if metadata_update_status is None:
-                if int(metadata_update_status) != 1:
+                update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Metadata.py')
-                    pass
+                process = subprocess.run(['python' ,update_file])
-                    #launch update
+
             tags_update_status = r_serv.get('v1.5:tags')
             if tags_update_status is None:
-                if int(tags_update_status) != 1:
+                update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Tags.py')
-                    pass
+                process = subprocess.run(['python' ,update_file])
-                    #launch update
+
             tags_background_update_status = r_serv.get('v1.5:tags_background')
             if tags_background_update_status is None:
-                if int(tags_background_update_status) != 1:
+                update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v1.4', 'Update-ARDB_Tags_background.py')
-                    pass
+                process = subprocess.run(['python' ,update_file])
-                    #launch update

update/v1.4/Update.sh

@@ -12,17 +12,48 @@ export PATH=$AIL_ARDB:$PATH
 export PATH=$AIL_BIN:$PATH
 export PATH=$AIL_FLASK:$PATH
 
+GREEN="\\033[1;32m"
+DEFAULT="\\033[0;39m"
+
+echo -e $GREEN"Shutting down AIL ..."$DEFAULT
+bash ${AIL_BIN}/LAUNCH.sh -k &
+wait
+
 echo ""
-bash -c "bash ${AIL_HOME}/update/bin/Update_Redis.sh"
+#bash -c "bash ${AIL_HOME}/update/bin/Update_Redis.sh"
 #bash -c "bash ${AIL_HOME}/update/bin/Update_ARDB.sh"
 
 echo ""
-echo "Fixing ARDB ..."
+echo -e $GREEN"Update DomainClassifier"$DEFAULT
+echo ""
+pip3 install --upgrade --force-reinstall git+https://github.com/D4-project/BGP-Ranking.git/@28013297efb039d2ebbce96ee2d89493f6ae56b0#subdirectory=client&egg=pybgpranking
+pip3 install --upgrade --force-reinstall git+https://github.com/adulau/DomainClassifier.git
+wait
 echo ""
-bash -c "unbuffer python ${AIL_HOME}/update/v1.5/Update.py"
 
-echo "Shutting down ARDB ..."
+echo ""
-bash -c "bash ${AIL_BIN}/LAUNCH.sh -k"
+echo -e $GREEN"Update Web thirdparty"$DEFAULT
+echo ""
+bash ${AIL_FLASK}update_thirdparty.sh &
+wait
+echo ""
+
+bash ${AIL_BIN}LAUNCH.sh -lav &
+wait
+echo ""
+
+echo ""
+echo -e $GREEN"Fixing ARDB ..."$DEFAULT
+echo ""
+python ${AIL_HOME}/update/v1.4/Update.py &
+wait
+echo ""
+echo ""
+
+echo ""
+echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
+bash ${AIL_BIN}/LAUNCH.sh -k &
+wait
 
 echo ""