fix: [LAUNCH] use envar, add options (launch all, flask, killall)

This commit is contained in:
Terrtia 2018-07-24 10:41:10 +02:00
parent 72d7cd1681
commit d4f0c14620
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
3 changed files with 267 additions and 160 deletions

View file

@ -12,10 +12,14 @@ CYAN="\\033[1;36m"
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
export PATH=$AIL_HOME:$PATH
export PATH=$AIL_REDIS:$PATH
export PATH=$AIL_ARDB:$PATH
export PATH=$AIL_BIN:$PATH
export PATH=$AIL_FLASK:$PATH
function helptext {
echo -e $YELLOW"
@ -35,15 +39,18 @@ function helptext {
- All the ZMQ queuing modules.
- All the ZMQ processing modules.
- All Redis in memory servers.
- All Level-DB on disk servers.
- All ARDB on disk servers.
"$DEFAULT"
(Inside screen Daemons)
"$RED"
But first of all you'll need to edit few path where you installed
your redis & ardb servers.
"$DEFAULT"
Usage:
-----
LAUNCH.sh
[-l | --launchAuto]
[-k | --killAll]
[-c | --configUpdate]
[-t | --thirdpartyUpdate]
[-h | --help]
"
}
@ -68,16 +75,16 @@ function launching_ardb {
echo -e $GREEN"\t* Launching ARDB servers"$DEFAULT
sleep 0.1
screen -S "ARDB_AIL" -X screen -t "6382" bash -c 'ardb-server '$conf_dir'6382.conf ; read x'
screen -S "ARDB_AIL" -X screen -t "6382" bash -c 'cd '${AIL_HOME}'; ardb-server '$conf_dir'6382.conf ; read x'
}
function launching_logs {
screen -dmS "Logging_AIL"
sleep 0.1
echo -e $GREEN"\t* Launching logging process"$DEFAULT
screen -S "Logging_AIL" -X screen -t "LogQueue" bash -c 'log_subscriber -p 6380 -c Queuing -l ../logs/; read x'
screen -S "Logging_AIL" -X screen -t "LogQueue" bash -c 'cd '${AIL_BIN}'; log_subscriber -p 6380 -c Queuing -l ../logs/; read x'
sleep 0.1
screen -S "Logging_AIL" -X screen -t "LogScript" bash -c 'log_subscriber -p 6380 -c Script -l ../logs/; read x'
screen -S "Logging_AIL" -X screen -t "LogScript" bash -c 'cd '${AIL_BIN}'; log_subscriber -p 6380 -c Script -l ../logs/; read x'
}
function launching_queues {
@ -85,90 +92,100 @@ function launching_queues {
sleep 0.1
echo -e $GREEN"\t* Launching all the queues"$DEFAULT
screen -S "Queue_AIL" -X screen -t "Queues" bash -c 'python3 launch_queues.py; read x'
screen -S "Queue_AIL" -X screen -t "Queues" bash -c 'cd '${AIL_BIN}'; python3 launch_queues.py; read x'
}
function launching_scripts {
function checking_configuration {
bin_dir=${AIL_HOME}/bin
echo -e "\t* Checking configuration"
bash -c "python3 Update-conf.py"
if [ "$1" == "automatic" ]; then
bash -c "python3 $bin_dir/Update-conf.py True"
else
bash -c "python3 $bin_dir/Update-conf.py False"
fi
exitStatus=$?
if [ $exitStatus -ge 1 ]; then
echo -e $RED"\t* Configuration not up-to-date"$DEFAULT
exit
fi
echo -e $GREEN"\t* Configuration up-to-date"$DEFAULT
}
function launching_scripts {
checking_configuration $1;
screen -dmS "Script_AIL"
sleep 0.1
echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c './ModulesInformationV2.py -k 0 -c 1; read x'
screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c 'cd '${AIL_BIN}'; ./ModulesInformationV2.py -k 0 -c 1; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "Mixer" bash -c './Mixer.py; read x'
screen -S "Script_AIL" -X screen -t "Mixer" bash -c 'cd '${AIL_BIN}'; ./Mixer.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "Global" bash -c './Global.py; read x'
screen -S "Script_AIL" -X screen -t "Global" bash -c 'cd '${AIL_BIN}'; ./Global.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "Duplicates" bash -c './Duplicates.py; read x'
screen -S "Script_AIL" -X screen -t "Duplicates" bash -c 'cd '${AIL_BIN}'; ./Duplicates.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "Lines" bash -c './Lines.py; read x'
screen -S "Script_AIL" -X screen -t "Lines" bash -c 'cd '${AIL_BIN}'; ./Lines.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c './DomClassifier.py; read x'
screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c 'cd '${AIL_BIN}'; ./DomClassifier.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "Categ" bash -c './Categ.py; read x'
screen -S "Script_AIL" -X screen -t "Categ" bash -c 'cd '${AIL_BIN}'; ./Categ.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "Tokenize" bash -c './Tokenize.py; read x'
screen -S "Script_AIL" -X screen -t "Tokenize" bash -c 'cd '${AIL_BIN}'; ./Tokenize.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "CreditCards" bash -c './CreditCards.py; read x'
screen -S "Script_AIL" -X screen -t "CreditCards" bash -c 'cd '${AIL_BIN}'; ./CreditCards.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "Onion" bash -c './Onion.py; read x'
screen -S "Script_AIL" -X screen -t "Onion" bash -c 'cd '${AIL_BIN}'; ./Onion.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "Mail" bash -c './Mail.py; read x'
screen -S "Script_AIL" -X screen -t "Mail" bash -c 'cd '${AIL_BIN}'; ./Mail.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "ApiKey" bash -c './ApiKey.py; read x'
screen -S "Script_AIL" -X screen -t "ApiKey" bash -c 'cd '${AIL_BIN}'; ./ApiKey.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "Web" bash -c './Web.py; read x'
screen -S "Script_AIL" -X screen -t "Web" bash -c 'cd '${AIL_BIN}'; ./Web.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "Credential" bash -c './Credential.py; read x'
screen -S "Script_AIL" -X screen -t "Credential" bash -c 'cd '${AIL_BIN}'; ./Credential.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "Curve" bash -c './Curve.py; read x'
screen -S "Script_AIL" -X screen -t "Curve" bash -c 'cd '${AIL_BIN}'; ./Curve.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "CurveManageTopSets" bash -c './CurveManageTopSets.py; read x'
screen -S "Script_AIL" -X screen -t "CurveManageTopSets" bash -c 'cd '${AIL_BIN}'; ./CurveManageTopSets.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "RegexForTermsFrequency" bash -c './RegexForTermsFrequency.py; read x'
screen -S "Script_AIL" -X screen -t "RegexForTermsFrequency" bash -c 'cd '${AIL_BIN}'; ./RegexForTermsFrequency.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "SetForTermsFrequency" bash -c './SetForTermsFrequency.py; read x'
screen -S "Script_AIL" -X screen -t "SetForTermsFrequency" bash -c 'cd '${AIL_BIN}'; ./SetForTermsFrequency.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "Indexer" bash -c './Indexer.py; read x'
screen -S "Script_AIL" -X screen -t "Indexer" bash -c 'cd '${AIL_BIN}'; ./Indexer.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "Keys" bash -c './Keys.py; read x'
screen -S "Script_AIL" -X screen -t "Keys" bash -c 'cd '${AIL_BIN}'; ./Keys.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "Base64" bash -c './Base64.py; read x'
screen -S "Script_AIL" -X screen -t "Base64" bash -c 'cd '${AIL_BIN}'; ./Base64.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c './Bitcoin.py; read x'
screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c 'cd '${AIL_BIN}'; ./Bitcoin.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "Phone" bash -c './Phone.py; read x'
screen -S "Script_AIL" -X screen -t "Phone" bash -c 'cd '${AIL_BIN}'; ./Phone.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "Release" bash -c './Release.py; read x'
screen -S "Script_AIL" -X screen -t "Release" bash -c 'cd '${AIL_BIN}'; ./Release.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "Cve" bash -c './Cve.py; read x'
screen -S "Script_AIL" -X screen -t "Cve" bash -c 'cd '${AIL_BIN}'; ./Cve.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "WebStats" bash -c './WebStats.py; read x'
screen -S "Script_AIL" -X screen -t "WebStats" bash -c 'cd '${AIL_BIN}'; ./WebStats.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c './ModuleStats.py; read x'
screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c 'cd '${AIL_BIN}'; ./ModuleStats.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c './SQLInjectionDetection.py; read x'
screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c 'cd '${AIL_BIN}'; ./SQLInjectionDetection.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "LibInjection" bash -c './LibInjection.py; read x'
screen -S "Script_AIL" -X screen -t "LibInjection" bash -c 'cd '${AIL_BIN}'; ./LibInjection.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "alertHandler" bash -c './alertHandler.py; read x'
screen -S "Script_AIL" -X screen -t "alertHandler" bash -c 'cd '${AIL_BIN}'; ./alertHandler.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c './MISP_The_Hive_feeder.py; read x'
screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c 'cd '${AIL_BIN}'; ./MISP_The_Hive_feeder.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "Tags" bash -c './Tags.py; read x'
screen -S "Script_AIL" -X screen -t "Tags" bash -c 'cd '${AIL_BIN}'; ./Tags.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x'
screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c 'cd '${AIL_BIN}'; ./SentimentAnalysis.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c './submit_paste.py; read x'
screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c 'cd '${AIL_BIN}'; ./submit_paste.py; read x'
}
@ -217,17 +234,120 @@ function checking_ardb {
sleep 0.2
bash -c $redis_dir'redis-cli -p 6382 PING | grep "PONG" &> /dev/null'
if [ ! $? == 0 ]; then
echo -e $RED"\t6382 not ready"$DEFAULT
echo -e $RED"\t6382 ARDB not ready"$DEFAULT
flag_ardb=1
fi
return $flag_ardb;
}
#If no params, display the help
#[[ $@ ]] || { helptext; exit 1;}
function launch_redis {
if [[ ! $isredis ]]; then
launching_redis;
else
echo -e $RED"\t* A screen is already launched"$DEFAULT
fi
}
function launch_ardb {
if [[ ! $isardb ]]; then
launching_ardb;
else
echo -e $RED"\t* A screen is already launched"$DEFAULT
fi
}
function launch_logs {
if [[ ! $islogged ]]; then
launching_logs;
else
echo -e $RED"\t* A screen is already launched"$DEFAULT
fi
}
function launch_queues {
if [[ ! $isqueued ]]; then
launching_queues;
else
echo -e $RED"\t* A screen is already launched"$DEFAULT
fi
}
function launch_scripts {
if [[ ! $isscripted ]]; then
sleep 1
if checking_redis && checking_ardb; then
launching_scripts $1;
else
while [[ (checking_ardb) && (checking_redis) ]]; do
echo -e $YELLOW"\tScript not started, waiting 5 secondes"$DEFAULT
sleep 5
if checking_redis && checking_ardb; then
launching_scripts $1;
else
echo -e $RED"\tScript not started"$DEFAULT
fi;
done
fi;
else
echo -e $RED"\t* A screen is already launched"$DEFAULT
fi
}
function launch_flask {
if [[ ! $isflasked ]]; then
flask_dir=${AIL_FLASK}
screen -dmS "Flask_AIL"
sleep 0.1
echo -e $GREEN"\t* Launching Flask server"$DEFAULT
screen -S "Flask_AIL" -X screen -t "Flask_server" bash -c "cd $flask_dir; ls; ./Flask_server.py; read x"
else
echo -e $RED"\t* A Flask screen is already launched"$DEFAULT
fi
}
function killall {
if [[ $isredis || $isardb || $islogged || $isqueued || $isscripted || $isflasked ]]; then
echo -e $GREEN"Gracefully closing redis servers"$DEFAULT
shutting_down_redis;
sleep 0.2
echo -e $GREEN"Gracefully closing ardb servers"$DEFAULT
shutting_down_ardb;
echo -e $GREEN"Killing all"$DEFAULT
kill $isredis $isardb $islogged $isqueued $isscripted $isflasked
sleep 0.2
echo -e $ROSE`screen -ls`$DEFAULT
echo -e $GREEN"\t* $isredis $isardb $islogged $isqueued $isscripted killed."$DEFAULT
else
echo -e $RED"\t* No screen to kill"$DEFAULT
fi
}
function shutdown {
bash -c "./Shutdown.py"
}
function update_thirdparty {
echo -e "\t* Updating thirdparty..."
bash -c "(cd ${AIL_FLASK}; ./update_thirdparty.sh)"
exitStatus=$?
if [ $exitStatus -ge 1 ]; then
echo -e $RED"\t* Thirdparty not up-to-date"$DEFAULT
exit
else
echo -e $GREEN"\t* Thirdparty updated"$DEFAULT
fi
}
function launch_all {
launch_redis;
launch_ardb;
launch_logs;
launch_queues;
launch_scripts $1;
launch_flask;
}
helptext;
############### TESTS ###################
isredis=`screen -ls | egrep '[0-9]+.Redis_AIL' | cut -d. -f1`
@ -235,118 +355,89 @@ isardb=`screen -ls | egrep '[0-9]+.ARDB_AIL' | cut -d. -f1`
islogged=`screen -ls | egrep '[0-9]+.Logging_AIL' | cut -d. -f1`
isqueued=`screen -ls | egrep '[0-9]+.Queue_AIL' | cut -d. -f1`
isscripted=`screen -ls | egrep '[0-9]+.Script_AIL' | cut -d. -f1`
isflasked=`screen -ls | egrep '[0-9]+.Flask_AIL' | cut -d. -f1`
options=("Redis" "Ardb" "Logs" "Queues" "Scripts" "Killall" "Shutdown" "Update-config" "Update-thirdparty")
#If no params, display the menu
[[ $@ ]] || {
menu() {
echo "What do you want to Launch?:"
for i in ${!options[@]}; do
printf "%3d%s) %s\n" $((i+1)) "${choices[i]:- }" "${options[i]}"
helptext;
options=("Redis" "Ardb" "Logs" "Queues" "Scripts" "Flask" "Killall" "Shutdown" "Update-config" "Update-thirdparty")
menu() {
echo "What do you want to Launch?:"
for i in ${!options[@]}; do
printf "%3d%s) %s\n" $((i+1)) "${choices[i]:- }" "${options[i]}"
done
[[ "$msg" ]] && echo "$msg"; :
}
prompt="Check an option (again to uncheck, ENTER when done): "
while menu && read -rp "$prompt" numinput && [[ "$numinput" ]]; do
for num in $numinput; do
[[ "$num" != *[![:digit:]]* ]] && (( num > 0 && num <= ${#options[@]} )) || {
msg="Invalid option: $num"; break
}
((num--)); msg="${options[num]} was ${choices[num]:+un}checked"
[[ "${choices[num]}" ]] && choices[num]="" || choices[num]="+"
done
done
[[ "$msg" ]] && echo "$msg"; :
for i in ${!options[@]}; do
if [[ "${choices[i]}" ]]; then
case ${options[i]} in
Redis)
launch_redis
;;
Ardb)
launch_ardb;
;;
Logs)
launch_logs;
;;
Queues)
launch_queues;
;;
Scripts)
launch_scripts;
;;
Flask)
launch_flask;
;;
Killall)
killall;
;;
Shutdown)
shutdown;
;;
Update-config)
checking_configuration "manual";
;;
Update-thirdparty)
update_thirdparty;
;;
esac
fi
done
exit
}
prompt="Check an option (again to uncheck, ENTER when done): "
while menu && read -rp "$prompt" numinput && [[ "$numinput" ]]; do
for num in $numinput; do
[[ "$num" != *[![:digit:]]* ]] && (( num > 0 && num <= ${#options[@]} )) || {
msg="Invalid option: $num"; break
}
((num--)); msg="${options[num]} was ${choices[num]:+un}checked"
[[ "${choices[num]}" ]] && choices[num]="" || choices[num]="+"
done
done
for i in ${!options[@]}; do
if [[ "${choices[i]}" ]]; then
case ${options[i]} in
Redis)
if [[ ! $isredis ]]; then
launching_redis;
else
echo -e $RED"\t* A screen is already launched"$DEFAULT
fi
;;
Ardb)
if [[ ! $isardb ]]; then
launching_ardb;
else
echo -e $RED"\t* A screen is already launched"$DEFAULT
fi
;;
Logs)
if [[ ! $islogged ]]; then
launching_logs;
else
echo -e $RED"\t* A screen is already launched"$DEFAULT
fi
;;
Queues)
if [[ ! $isqueued ]]; then
launching_queues;
else
echo -e $RED"\t* A screen is already launched"$DEFAULT
fi
;;
Scripts)
if [[ ! $isscripted ]]; then
sleep 1
if checking_redis && checking_ardb; then
launching_scripts;
else
echo -e $YELLOW"\tScript not started, waiting 5 secondes"$DEFAULT
sleep 5
if checking_redis && checking_ardb; then
launching_scripts;
else
echo -e $RED"\tScript not started"$DEFAULT
fi;
fi;
else
echo -e $RED"\t* A screen is already launched"$DEFAULT
fi
;;
Killall)
if [[ $isredis || $isardb || $islogged || $isqueued || $isscripted ]]; then
echo -e $GREEN"Gracefully closing redis servers"$DEFAULT
shutting_down_redis;
sleep 0.2
echo -e $GREEN"Gracefully closing ardb servers"$DEFAULT
shutting_down_ardb;
echo -e $GREEN"Killing all"$DEFAULT
kill $isredis $isardb $islogged $isqueued $isscripted
sleep 0.2
echo -e $ROSE`screen -ls`$DEFAULT
echo -e $GREEN"\t* $isredis $isardb $islogged $isqueued $isscripted killed."$DEFAULT
else
echo -e $RED"\t* No screen to kill"$DEFAULT
fi
;;
Shutdown)
bash -c "./Shutdown.py"
;;
Update-config)
echo -e "\t* Checking configuration"
bash -c "./Update-conf.py"
exitStatus=$?
if [ $exitStatus -ge 1 ]; then
echo -e $RED"\t* Configuration not up-to-date"$DEFAULT
exit
else
echo -e $GREEN"\t* Configuration up-to-date"$DEFAULT
fi
;;
Update-thirdparty)
echo -e "\t* Updating thirdparty..."
bash -c "(cd ../var/www && ./update_thirdparty.sh)"
exitStatus=$?
if [ $exitStatus -ge 1 ]; then
echo -e $RED"\t* Configuration not up-to-date"$DEFAULT
exit
else
echo -e $GREEN"\t* Configuration up-to-date"$DEFAULT
fi
;;
esac
fi
while [ "$1" != "" ]; do
case $1 in
-l | --launchAuto ) launch_all "automatic";
;;
-k | --killAll ) killall;
;;
-c | --configUpdate ) checking_configuration "manual";
;;
-t | --thirdpartyUpdate ) update_thirdparty;
;;
-h | --help ) helptext;
exit
;;
* ) helptext
exit 1
esac
shift
done

View file

@ -11,6 +11,15 @@ import shutil
#return true if the configuration is up-to-date
def main():
if len(sys.argv) != 2:
print('usage:', 'Update-conf.py', 'Automatic (boolean)')
exit(1)
else:
automatic = sys.argv[1]
if automatic == 'True':
automatic = True
else:
automatic = False
configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
configfileBackup = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') + '.backup'
@ -63,12 +72,19 @@ def main():
print(" - "+item[0])
print("+--------------------------------------------------------------------+")
resp = input("Do you want to auto fix it? [y/n] ")
if automatic:
resp = 'y'
else:
resp = input("Do you want to auto fix it? [y/n] ")
if resp != 'y':
return False
else:
resp2 = input("Do you want to keep a backup of the old configuration file? [y/n] ")
if automatic:
resp2 = 'y'
else:
resp2 = input("Do you want to keep a backup of the old configuration file? [y/n] ")
if resp2 == 'y':
shutil.move(configfile, configfileBackup)

View file

@ -1,7 +1,7 @@
# Ardb configuration file example, modified from redis's conf file.
# Home dir for ardb instance, it can be referenced by ${ARDB_HOME} in this config file
home ../DATA_ARDB/
home DATA_ARDB/
# Note on units: when memory size is needed, it is possible to specify
# it in the usual form of 1k 5GB 4M and so forth: