From 5cf81f82913b4546f323e7d27a432bf88fb61ff4 Mon Sep 17 00:00:00 2001 From: niclas Date: Tue, 16 Jan 2024 08:44:32 +0100 Subject: [PATCH] add [install] container setup ail + lacus --- other_installers/LXD/INSTALL.sh | 303 +++++++++++++++++++++++++++++++- other_installers/LXD/README.md | 24 ++- 2 files changed, 320 insertions(+), 7 deletions(-) diff --git a/other_installers/LXD/INSTALL.sh b/other_installers/LXD/INSTALL.sh index d795336a..1c265c4f 100644 --- a/other_installers/LXD/INSTALL.sh +++ b/other_installers/LXD/INSTALL.sh @@ -7,14 +7,19 @@ setVars() { RED='\033[0;31m' NC='\033[0m' # No Color - PROJECT_NAME=$(generateName "AIL") STORAGE_POOL_NAME=$(generateName "AIL") NETWORK_NAME=$(generateName "AIL") NETWORK_NAME=${NETWORK_NAME:0:14} + PROFILE=$(generateName "AIL") UBUNTU="ubuntu:22.04" +} - AIL_CONTAINER=$(generateName "AIL") +setDefaults(){ + default_ail_project=$(generateName "AIL") + default_ail_name=$(generateName "AIL") + default_lacus="Yes" + default_lacus_name=$(generateName "LACUS") } error() { @@ -59,8 +64,24 @@ generateName(){ setupLXD(){ lxc project create "$PROJECT_NAME" lxc project switch "$PROJECT_NAME" - lxc storage create "$STORAGE_POOL_NAME" "dir" - lxc network create "$NETWORK_NAME" + + if checkRessourceExist "storage" "$STORAGE_POOL_NAME"; then + error "Storage '$STORAGE_POOL_NAME' already exists." + exit 1 + fi + lxc storage create "$STORAGE_POOL_NAME" zfs source="$PARTITION_NAME" + + if checkRessourceExist "network" "$NETWORK_NAME"; then + error "Network '$NETWORK_NAME' already exists." + fi + lxc network create "$NETWORK_NAME" --type=bridge + + if checkRessourceExist "profile" "$PROFILE"; then + error "Profile '$PROFILE' already exists." + fi + lxc profile create "$PROFILE" + lxc profile device add "$PROFILE" root disk path="/" pool="$STORAGE_POOL_NAME" + lxc profile device add "$PROFILE" eth0 nic name=eth0 network="$NETWORK_NAME" } waitForContainer() { @@ -81,9 +102,279 @@ waitForContainer() { interrupt() { warn "Script interrupted by user. Delete project and exit ..." deleteLXDProject "$PROJECT_NAME" - lxc storage delete "$APP_STORAGE" - lxc storage delete "$DB_STORAGE" lxc network delete "$NETWORK_NAME" exit 130 } +deleteLXDProject(){ + local project="$1" + + echo "Starting cleanup ..." + echo "Deleting container in project" + for container in $(lxc query "/1.0/containers?recursion=1&project=${project}" | jq .[].name -r); do + lxc delete --project "${project}" -f "${container}" + done + + echo "Deleting images in project" + for image in $(lxc query "/1.0/images?recursion=1&project=${project}" | jq .[].fingerprint -r); do + lxc image delete --project "${project}" "${image}" + done + + echo "Deleting profiles in project" + for profile in $(lxc query "/1.0/profiles?recursion=1&project=${project}" | jq .[].name -r); do + if [ "${profile}" = "default" ]; then + printf 'config: {}\ndevices: {}' | lxc profile edit --project "${project}" default + continue + fi + lxc profile delete --project "${project}" "${profile}" + done + + echo "Deleting project" + lxc project delete "${project}" +} + +createAILContainer(){ + lxc launch $UBUNTU "$AIL_CONTAINER" --profile "$PROFILE" + waitForContainer "$AIL_CONTAINER" + lxc exec "$AIL_CONTAINER" -- sed -i "/#\$nrconf{restart} = 'i';/s/.*/\$nrconf{restart} = 'a';/" /etc/needrestart/needrestart.conf + lxc exec "$AIL_CONTAINER" -- apt update + lxc exec "$AIL_CONTAINER" -- apt upgrade -y + lxc exec "$AIL_CONTAINER" -- useradd -m -s /bin/bash ail + if lxc exec "$AIL_CONTAINER" -- id ail; then + lxc exec "$AIL_CONTAINER" -- usermod -aG sudo ail + success "User ail created." + else + error "User ail not created." + exit 1 + fi + lxc exec "$AIL_CONTAINER" -- bash -c "echo 'ail ALL=(ALL) NOPASSWD: ALL' | sudo tee /etc/sudoers.d/ail" + lxc exec "$AIL_CONTAINER" --cwd=/home/ail -- sudo -u ail bash -c "git clone https://github.com/ail-project/ail-framework.git" + lxc exec "$AIL_CONTAINER" --cwd=/home/ail/ail-framework -- sudo -u ail bash -c "./installing_deps.sh" + lxc exec "$AIL_CONTAINER" -- sed -i '/^\[Flask\]/,/^\[/ s/host = 127\.0\.0\.1/host = 0.0.0.0/' /home/ail/ail-framework/configs/core.cfg + lxc exec "$AIL_CONTAINER" --cwd=/home/ail/ail-framework/bin -- sudo -u ail bash -c "./LAUNCH.sh -l" + lxc exec "$AIL_CONTAINER" -- sed -i "/^\$nrconf{restart} = 'a';/s/.*/#\$nrconf{restart} = 'i';/" /etc/needrestart/needrestart.conf +} + +createLacusContainer(){ + lxc launch $UBUNTU "$LACUS_CONTAINER" --profile "$PROFILE" + waitForContainer "$LACUS_CONTAINER" + lxc exec "$LACUS_CONTAINER" -- sed -i "/#\$nrconf{restart} = 'i';/s/.*/\$nrconf{restart} = 'a';/" /etc/needrestart/needrestart.conf + lxc exec "$LACUS_CONTAINER" -- apt update + lxc exec "$LACUS_CONTAINER" -- apt upgrade -y + lxc exec "$LACUS_CONTAINER" -- apt install pipx -y + lxc exec "$LACUS_CONTAINER" -- pipx install poetry + lxc exec "$LACUS_CONTAINER" -- pipx ensurepath + lxc exec "$LACUS_CONTAINER" -- apt install build-essential tcl -y + lxc exec "$LACUS_CONTAINER" -- git clone https://github.com/redis/redis.git + lxc exec "$LACUS_CONTAINER" --cwd=/root/redis -- git checkout 7.2 + lxc exec "$LACUS_CONTAINER" --cwd=/root/redis -- make + lxc exec "$LACUS_CONTAINER" --cwd=/root/redis -- make test + lxc exec "$LACUS_CONTAINER" -- git clone https://github.com/ail-project/lacus.git + lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- /root/.local/bin/poetry install + AIL_VENV_PATH=$(lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "/root/.local/bin/poetry env info -p") + # lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "source ${AIL_VENV_PATH}/bin/activate" + # lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- /root/.local/bin/poetry shell + lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "source ${AIL_VENV_PATH}/bin/activate && playwright install-deps" + lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "echo LACUS_HOME=/root/lacus >> .env" + lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "export PATH='/root/.local/bin:$PATH' && yes | /root/.local/bin/poetry run update --init" + lxc exec "$LACUS_CONTAINER" -- sed -i "/^\$nrconf{restart} = 'a';/s/.*/#\$nrconf{restart} = 'i';/" /etc/needrestart/needrestart.conf +} + +nonInteractiveConfig(){ + VALID_ARGS=$(getopt -o h --long help,production,project:ail-name:,no-lacus,lacus-name: -- "$@") + if [[ $? -ne 0 ]]; then + exit 1; + fi + + eval set -- "$VALID_ARGS" + while [ $# -gt 0 ]; do + case "$1" in + -h | --help) + usage + exit 0 + ;; + --project) + ail_project=$2 + shift 2 + ;; + --ail-name) + ail_name=$2 + shift 2 + ;; + --no-lacus) + lacus="N" + shift + ;; + --lacus-name) + lacus_name=$2 + shift 2 + ;; + *) + break + ;; + esac + done + + # Set global values + PROJECT_NAME=${ail_project:-$default_ail_project} + AIL_CONTAINER=${ail_name:-$default_ail_name} + lacus=${lacus:-$default_lacus} + LACUS=$(echo "$lacus" | grep -iE '^y(es)?$' > /dev/null && echo true || echo false) + LACUS_CONTAINER=${lacus_name:-$default_lacus_name} +} + +validateArgs(){ + # Check Names + local names=("$PROJECT_NAME" "$AIL_CONTAINER") + for i in "${names[@]}"; do + if ! checkNamingConvention "$i"; then + exit 1 + fi + done + + if $LACUS && ! checkNamingConvention "$LACUS_CONTAINER"; then + exit 1 + fi + + # Check for Project + if checkRessourceExist "project" "$PROJECT_NAME"; then + error "Project '$PROJECT_NAME' already exists." + exit 1 + fi + + # Check Container Names + local containers=("$AIL_CONTAINER") + + declare -A name_counts + for name in "${containers[@]}"; do + ((name_counts["$name"]++)) + done + + if $LACUS;then + ((name_counts["$LACUS_CONTAINER"]++)) + fi + + for name in "${!name_counts[@]}"; do + if ((name_counts["$name"] >= 2)); then + error "At least two container have the same name: $name" + exit 1 + fi + done +} + +checkRessourceExist() { + local resource_type="$1" + local resource_name="$2" + + case "$resource_type" in + "container") + lxc info "$resource_name" &>/dev/null + ;; + "image") + lxc image list --format=json | jq -e --arg alias "$resource_name" '.[] | select(.aliases[].name == $alias) | .fingerprint' &>/dev/null + ;; + "project") + lxc project list --format=json | jq -e --arg name "$resource_name" '.[] | select(.name == $name) | .name' &>/dev/null + ;; + "storage") + lxc storage list --format=json | jq -e --arg name "$resource_name" '.[] | select(.name == $name) | .name' &>/dev/null + ;; + "network") + lxc network list --format=json | jq -e --arg name "$resource_name" '.[] | select(.name == $name) | .name' &>/dev/null + ;; + "profile") + lxc profile list --format=json | jq -e --arg name "$resource_name" '.[] | select(.name == $name) | .name' &>/dev/null + ;; + esac + + return $? +} + +checkNamingConvention(){ + local input="$1" + local pattern="^[a-zA-Z0-9-]+$" + + if ! [[ "$input" =~ $pattern ]]; then + error "Invalid Name $input. Please use only alphanumeric characters and hyphens." + return 1 + fi + return 0 +} + +usage() { + echo "Usage: $0 [OPTIONS]" + echo + echo "Options:" + echo " -h, --help Display this help message and exit." + echo " --project Specify the project name." + echo " --ail-name Specify the AIL container name." + echo " --no-lacus Do not create Lacus container." + echo " --lacus-name Specify the Lacus container name." + echo " -i, --interactive Run the script in interactive mode." + echo + echo "This script sets up LXD containers for AIL and optionally Lacus." + echo "It creates a new LXD project, and configures the network and storage." + echo "Then it launches and configures the specified containers." + echo + echo "Examples:" + echo " $0 --project myProject --ail-name ailContainer" + echo " $0 --interactive" +} + +# ------------------ MAIN ------------------ + +setDefaults + +# Check if interactive mode +INTERACTIVE=false +for arg in "$@"; do + if [[ $arg == "-i" ]] || [[ $arg == "--interactive" ]]; then + INTERACTIVE=true + break + fi +done + +if [ "$INTERACTIVE" = true ]; then + interactiveConfig +else + nonInteractiveConfig "$@" +fi + +validateArgs +setVars + +trap 'interrupt' INT +trap 'err ${LINENO}' ERR + +info "Setup LXD Project" +setupLXD + +info "Create AIL Container" +createAILContainer + +if $LACUS; then + info "Create Lacus Container" + createLacusContainer +fi + +# Print info +ail_ip=$(lxc list $AIL_CONTAINER --format=json | jq -r '.[0].state.network.eth0.addresses[] | select(.family=="inet").address') +if $LACUS; then + lacus_ip=$(lxc list $LACUS_CONTAINER --format=json | jq -r '.[0].state.network.eth0.addresses[] | select(.family=="inet").address') +fi +ail_email=$(lxc exec $AIL_CONTAINER -- bash -c "grep '^email=' /home/ail/ail-framework/DEFAULT_PASSWORD | cut -d'=' -f2") +ail_password=$(lxc exec $AIL_CONTAINER -- bash -c "grep '^password=' /home/ail/ail-framework/DEFAULT_PASSWORD | cut -d'=' -f2") +ail_API_Key=$(lxc exec $AIL_CONTAINER -- bash -c "grep '^API_Key=' /home/ail/ail-framework/DEFAULT_PASSWORD | cut -d'=' -f2") + +echo "--------------------------------------------------------------------------------------------" +echo -e "${BLUE}AIL ${NC}is up and running on $ail_ip" +echo "--------------------------------------------------------------------------------------------" +echo -e "${BLUE}AIL ${NC}credentials:" +echo -e "Email: ${GREEN}$ail_email${NC}" +echo -e "Password: ${GREEN}$ail_password${NC}" +echo -e "API Key: ${GREEN}$ail_API_Key${NC}" +echo "--------------------------------------------------------------------------------------------" +if $LACUS; then + echo -e "${BLUE}Lacus ${NC}is up and running on $lacus_ip" +fi +echo "--------------------------------------------------------------------------------------------" diff --git a/other_installers/LXD/README.md b/other_installers/LXD/README.md index fb0f9110..28af09ab 100644 --- a/other_installers/LXD/README.md +++ b/other_installers/LXD/README.md @@ -1,2 +1,24 @@ # AIL-framework-LXD -This installer is based on the [LXD](https://linuxcontainers.org/lxd/introduction/) container manager and can be used to install AIL on Linux. +This installer is based on the [LXD](https://canonical.com/lxd) container manager and can be used to install AIL on Linux. It also supports the installation of [Lacus](https://github.com/ail-project/lacus) a crawler for the AIL framework. + +## Requirements +- [LXD](https://canonical.com/lxd) 5.19 +- jq 1.6 + +## Usage +Make sure you have all the requirements installed on your system. + +### Interactive mode +Run the INSTALL.sh script with the --interactive flag to enter the interactive mode, which guides you through the configuration process: +```bash +bash INSTALL.sh --interactive +``` + +### Non-interactive mode +If you want to install AIL without the interactive mode, you can use the following command: +```bash +bash INSTALL.sh [OPTIONS] +``` + +## Configuration +If you installed Lacus, you can configure AIL to use it as a crawler. For further information, please refer to the [HOWTO](https://github.com/ail-project/ail-framework/blob/master/HOWTO.md)