From 6c77f4219cc607b0759429a723e1162da9d6c838 Mon Sep 17 00:00:00 2001 From: fukusuket <41001169+fukusuket@users.noreply.github.com> Date: Sat, 6 Jan 2024 23:38:35 +0900 Subject: [PATCH 1/9] fix: [README] api.md link typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7b5818fc..617e0d48 100644 --- a/README.md +++ b/README.md @@ -138,7 +138,7 @@ CIRCL organises training on how to use or extend the AIL framework. AIL training ## API -The API documentation is available in [doc/README.md](doc/README.md) +The API documentation is available in [doc/api.md](doc/api.md) ## HOWTO From 5418f47e9a2aa12f9e35760769eef3a5af3cc698 Mon Sep 17 00:00:00 2001 From: niclas Date: Mon, 15 Jan 2024 08:26:34 +0100 Subject: [PATCH 2/9] Initial commit --- other_installers/LXD/INSTALL.sh | 89 +++++++++++++++++++++++++++++++++ other_installers/LXD/README.md | 2 + 2 files changed, 91 insertions(+) create mode 100644 other_installers/LXD/INSTALL.sh create mode 100644 other_installers/LXD/README.md diff --git a/other_installers/LXD/INSTALL.sh b/other_installers/LXD/INSTALL.sh new file mode 100644 index 00000000..d795336a --- /dev/null +++ b/other_installers/LXD/INSTALL.sh @@ -0,0 +1,89 @@ +#!/bin/bash + +setVars() { + GREEN='\033[0;32m' + YELLOW='\033[1;33m' + BLUE='\033[0;34m' + RED='\033[0;31m' + NC='\033[0m' # No Color + + PROJECT_NAME=$(generateName "AIL") + STORAGE_POOL_NAME=$(generateName "AIL") + NETWORK_NAME=$(generateName "AIL") + NETWORK_NAME=${NETWORK_NAME:0:14} + + UBUNTU="ubuntu:22.04" + + AIL_CONTAINER=$(generateName "AIL") +} + +error() { + echo -e "${RED}ERROR: $1${NC}" +} + +warn() { + echo -e "${YELLOW}WARNING: $1${NC}" +} + +info() { + echo -e "${BLUE}INFO: $1${NC}" +} + +success() { + echo -e "${GREEN}SUCCESS: $1${NC}" +} + +err() { + local parent_lineno="$1" + local message="$2" + local code="${3:-1}" + + if [[ -n "$message" ]] ; then + error "Line ${parent_lineno}: ${message}: exiting with status ${code}" + else + error "Line ${parent_lineno}: exiting with status ${code}" + fi + + deleteLXDProject "$PROJECT_NAME" + lxc storage delete "$APP_STORAGE" + lxc storage delete "$DB_STORAGE" + lxc network delete "$NETWORK_NAME" + exit "${code}" +} + +generateName(){ + local name="$1" + echo "${name}-$(date +%Y%m%d%H%M%S)" +} + +setupLXD(){ + lxc project create "$PROJECT_NAME" + lxc project switch "$PROJECT_NAME" + lxc storage create "$STORAGE_POOL_NAME" "dir" + lxc network create "$NETWORK_NAME" +} + +waitForContainer() { + local container_name="$1" + + sleep 3 + while true; do + status=$(lxc list --format=json | jq -e --arg name "$container_name" '.[] | select(.name == $name) | .status') + if [ "$status" = "\"Running\"" ]; then + echo -e "${BLUE}$container_name ${GREEN}is running.${NC}" + break + fi + echo "Waiting for $container_name container to start." + sleep 5 + done +} + +interrupt() { + warn "Script interrupted by user. Delete project and exit ..." + deleteLXDProject "$PROJECT_NAME" + lxc storage delete "$APP_STORAGE" + lxc storage delete "$DB_STORAGE" + lxc network delete "$NETWORK_NAME" + exit 130 +} + diff --git a/other_installers/LXD/README.md b/other_installers/LXD/README.md new file mode 100644 index 00000000..fb0f9110 --- /dev/null +++ b/other_installers/LXD/README.md @@ -0,0 +1,2 @@ +# AIL-framework-LXD +This installer is based on the [LXD](https://linuxcontainers.org/lxd/introduction/) container manager and can be used to install AIL on Linux. From 5cf81f82913b4546f323e7d27a432bf88fb61ff4 Mon Sep 17 00:00:00 2001 From: niclas Date: Tue, 16 Jan 2024 08:44:32 +0100 Subject: [PATCH 3/9] add [install] container setup ail + lacus --- other_installers/LXD/INSTALL.sh | 303 +++++++++++++++++++++++++++++++- other_installers/LXD/README.md | 24 ++- 2 files changed, 320 insertions(+), 7 deletions(-) diff --git a/other_installers/LXD/INSTALL.sh b/other_installers/LXD/INSTALL.sh index d795336a..1c265c4f 100644 --- a/other_installers/LXD/INSTALL.sh +++ b/other_installers/LXD/INSTALL.sh @@ -7,14 +7,19 @@ setVars() { RED='\033[0;31m' NC='\033[0m' # No Color - PROJECT_NAME=$(generateName "AIL") STORAGE_POOL_NAME=$(generateName "AIL") NETWORK_NAME=$(generateName "AIL") NETWORK_NAME=${NETWORK_NAME:0:14} + PROFILE=$(generateName "AIL") UBUNTU="ubuntu:22.04" +} - AIL_CONTAINER=$(generateName "AIL") +setDefaults(){ + default_ail_project=$(generateName "AIL") + default_ail_name=$(generateName "AIL") + default_lacus="Yes" + default_lacus_name=$(generateName "LACUS") } error() { @@ -59,8 +64,24 @@ generateName(){ setupLXD(){ lxc project create "$PROJECT_NAME" lxc project switch "$PROJECT_NAME" - lxc storage create "$STORAGE_POOL_NAME" "dir" - lxc network create "$NETWORK_NAME" + + if checkRessourceExist "storage" "$STORAGE_POOL_NAME"; then + error "Storage '$STORAGE_POOL_NAME' already exists." + exit 1 + fi + lxc storage create "$STORAGE_POOL_NAME" zfs source="$PARTITION_NAME" + + if checkRessourceExist "network" "$NETWORK_NAME"; then + error "Network '$NETWORK_NAME' already exists." + fi + lxc network create "$NETWORK_NAME" --type=bridge + + if checkRessourceExist "profile" "$PROFILE"; then + error "Profile '$PROFILE' already exists." + fi + lxc profile create "$PROFILE" + lxc profile device add "$PROFILE" root disk path="/" pool="$STORAGE_POOL_NAME" + lxc profile device add "$PROFILE" eth0 nic name=eth0 network="$NETWORK_NAME" } waitForContainer() { @@ -81,9 +102,279 @@ waitForContainer() { interrupt() { warn "Script interrupted by user. Delete project and exit ..." deleteLXDProject "$PROJECT_NAME" - lxc storage delete "$APP_STORAGE" - lxc storage delete "$DB_STORAGE" lxc network delete "$NETWORK_NAME" exit 130 } +deleteLXDProject(){ + local project="$1" + + echo "Starting cleanup ..." + echo "Deleting container in project" + for container in $(lxc query "/1.0/containers?recursion=1&project=${project}" | jq .[].name -r); do + lxc delete --project "${project}" -f "${container}" + done + + echo "Deleting images in project" + for image in $(lxc query "/1.0/images?recursion=1&project=${project}" | jq .[].fingerprint -r); do + lxc image delete --project "${project}" "${image}" + done + + echo "Deleting profiles in project" + for profile in $(lxc query "/1.0/profiles?recursion=1&project=${project}" | jq .[].name -r); do + if [ "${profile}" = "default" ]; then + printf 'config: {}\ndevices: {}' | lxc profile edit --project "${project}" default + continue + fi + lxc profile delete --project "${project}" "${profile}" + done + + echo "Deleting project" + lxc project delete "${project}" +} + +createAILContainer(){ + lxc launch $UBUNTU "$AIL_CONTAINER" --profile "$PROFILE" + waitForContainer "$AIL_CONTAINER" + lxc exec "$AIL_CONTAINER" -- sed -i "/#\$nrconf{restart} = 'i';/s/.*/\$nrconf{restart} = 'a';/" /etc/needrestart/needrestart.conf + lxc exec "$AIL_CONTAINER" -- apt update + lxc exec "$AIL_CONTAINER" -- apt upgrade -y + lxc exec "$AIL_CONTAINER" -- useradd -m -s /bin/bash ail + if lxc exec "$AIL_CONTAINER" -- id ail; then + lxc exec "$AIL_CONTAINER" -- usermod -aG sudo ail + success "User ail created." + else + error "User ail not created." + exit 1 + fi + lxc exec "$AIL_CONTAINER" -- bash -c "echo 'ail ALL=(ALL) NOPASSWD: ALL' | sudo tee /etc/sudoers.d/ail" + lxc exec "$AIL_CONTAINER" --cwd=/home/ail -- sudo -u ail bash -c "git clone https://github.com/ail-project/ail-framework.git" + lxc exec "$AIL_CONTAINER" --cwd=/home/ail/ail-framework -- sudo -u ail bash -c "./installing_deps.sh" + lxc exec "$AIL_CONTAINER" -- sed -i '/^\[Flask\]/,/^\[/ s/host = 127\.0\.0\.1/host = 0.0.0.0/' /home/ail/ail-framework/configs/core.cfg + lxc exec "$AIL_CONTAINER" --cwd=/home/ail/ail-framework/bin -- sudo -u ail bash -c "./LAUNCH.sh -l" + lxc exec "$AIL_CONTAINER" -- sed -i "/^\$nrconf{restart} = 'a';/s/.*/#\$nrconf{restart} = 'i';/" /etc/needrestart/needrestart.conf +} + +createLacusContainer(){ + lxc launch $UBUNTU "$LACUS_CONTAINER" --profile "$PROFILE" + waitForContainer "$LACUS_CONTAINER" + lxc exec "$LACUS_CONTAINER" -- sed -i "/#\$nrconf{restart} = 'i';/s/.*/\$nrconf{restart} = 'a';/" /etc/needrestart/needrestart.conf + lxc exec "$LACUS_CONTAINER" -- apt update + lxc exec "$LACUS_CONTAINER" -- apt upgrade -y + lxc exec "$LACUS_CONTAINER" -- apt install pipx -y + lxc exec "$LACUS_CONTAINER" -- pipx install poetry + lxc exec "$LACUS_CONTAINER" -- pipx ensurepath + lxc exec "$LACUS_CONTAINER" -- apt install build-essential tcl -y + lxc exec "$LACUS_CONTAINER" -- git clone https://github.com/redis/redis.git + lxc exec "$LACUS_CONTAINER" --cwd=/root/redis -- git checkout 7.2 + lxc exec "$LACUS_CONTAINER" --cwd=/root/redis -- make + lxc exec "$LACUS_CONTAINER" --cwd=/root/redis -- make test + lxc exec "$LACUS_CONTAINER" -- git clone https://github.com/ail-project/lacus.git + lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- /root/.local/bin/poetry install + AIL_VENV_PATH=$(lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "/root/.local/bin/poetry env info -p") + # lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "source ${AIL_VENV_PATH}/bin/activate" + # lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- /root/.local/bin/poetry shell + lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "source ${AIL_VENV_PATH}/bin/activate && playwright install-deps" + lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "echo LACUS_HOME=/root/lacus >> .env" + lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "export PATH='/root/.local/bin:$PATH' && yes | /root/.local/bin/poetry run update --init" + lxc exec "$LACUS_CONTAINER" -- sed -i "/^\$nrconf{restart} = 'a';/s/.*/#\$nrconf{restart} = 'i';/" /etc/needrestart/needrestart.conf +} + +nonInteractiveConfig(){ + VALID_ARGS=$(getopt -o h --long help,production,project:ail-name:,no-lacus,lacus-name: -- "$@") + if [[ $? -ne 0 ]]; then + exit 1; + fi + + eval set -- "$VALID_ARGS" + while [ $# -gt 0 ]; do + case "$1" in + -h | --help) + usage + exit 0 + ;; + --project) + ail_project=$2 + shift 2 + ;; + --ail-name) + ail_name=$2 + shift 2 + ;; + --no-lacus) + lacus="N" + shift + ;; + --lacus-name) + lacus_name=$2 + shift 2 + ;; + *) + break + ;; + esac + done + + # Set global values + PROJECT_NAME=${ail_project:-$default_ail_project} + AIL_CONTAINER=${ail_name:-$default_ail_name} + lacus=${lacus:-$default_lacus} + LACUS=$(echo "$lacus" | grep -iE '^y(es)?$' > /dev/null && echo true || echo false) + LACUS_CONTAINER=${lacus_name:-$default_lacus_name} +} + +validateArgs(){ + # Check Names + local names=("$PROJECT_NAME" "$AIL_CONTAINER") + for i in "${names[@]}"; do + if ! checkNamingConvention "$i"; then + exit 1 + fi + done + + if $LACUS && ! checkNamingConvention "$LACUS_CONTAINER"; then + exit 1 + fi + + # Check for Project + if checkRessourceExist "project" "$PROJECT_NAME"; then + error "Project '$PROJECT_NAME' already exists." + exit 1 + fi + + # Check Container Names + local containers=("$AIL_CONTAINER") + + declare -A name_counts + for name in "${containers[@]}"; do + ((name_counts["$name"]++)) + done + + if $LACUS;then + ((name_counts["$LACUS_CONTAINER"]++)) + fi + + for name in "${!name_counts[@]}"; do + if ((name_counts["$name"] >= 2)); then + error "At least two container have the same name: $name" + exit 1 + fi + done +} + +checkRessourceExist() { + local resource_type="$1" + local resource_name="$2" + + case "$resource_type" in + "container") + lxc info "$resource_name" &>/dev/null + ;; + "image") + lxc image list --format=json | jq -e --arg alias "$resource_name" '.[] | select(.aliases[].name == $alias) | .fingerprint' &>/dev/null + ;; + "project") + lxc project list --format=json | jq -e --arg name "$resource_name" '.[] | select(.name == $name) | .name' &>/dev/null + ;; + "storage") + lxc storage list --format=json | jq -e --arg name "$resource_name" '.[] | select(.name == $name) | .name' &>/dev/null + ;; + "network") + lxc network list --format=json | jq -e --arg name "$resource_name" '.[] | select(.name == $name) | .name' &>/dev/null + ;; + "profile") + lxc profile list --format=json | jq -e --arg name "$resource_name" '.[] | select(.name == $name) | .name' &>/dev/null + ;; + esac + + return $? +} + +checkNamingConvention(){ + local input="$1" + local pattern="^[a-zA-Z0-9-]+$" + + if ! [[ "$input" =~ $pattern ]]; then + error "Invalid Name $input. Please use only alphanumeric characters and hyphens." + return 1 + fi + return 0 +} + +usage() { + echo "Usage: $0 [OPTIONS]" + echo + echo "Options:" + echo " -h, --help Display this help message and exit." + echo " --project Specify the project name." + echo " --ail-name Specify the AIL container name." + echo " --no-lacus Do not create Lacus container." + echo " --lacus-name Specify the Lacus container name." + echo " -i, --interactive Run the script in interactive mode." + echo + echo "This script sets up LXD containers for AIL and optionally Lacus." + echo "It creates a new LXD project, and configures the network and storage." + echo "Then it launches and configures the specified containers." + echo + echo "Examples:" + echo " $0 --project myProject --ail-name ailContainer" + echo " $0 --interactive" +} + +# ------------------ MAIN ------------------ + +setDefaults + +# Check if interactive mode +INTERACTIVE=false +for arg in "$@"; do + if [[ $arg == "-i" ]] || [[ $arg == "--interactive" ]]; then + INTERACTIVE=true + break + fi +done + +if [ "$INTERACTIVE" = true ]; then + interactiveConfig +else + nonInteractiveConfig "$@" +fi + +validateArgs +setVars + +trap 'interrupt' INT +trap 'err ${LINENO}' ERR + +info "Setup LXD Project" +setupLXD + +info "Create AIL Container" +createAILContainer + +if $LACUS; then + info "Create Lacus Container" + createLacusContainer +fi + +# Print info +ail_ip=$(lxc list $AIL_CONTAINER --format=json | jq -r '.[0].state.network.eth0.addresses[] | select(.family=="inet").address') +if $LACUS; then + lacus_ip=$(lxc list $LACUS_CONTAINER --format=json | jq -r '.[0].state.network.eth0.addresses[] | select(.family=="inet").address') +fi +ail_email=$(lxc exec $AIL_CONTAINER -- bash -c "grep '^email=' /home/ail/ail-framework/DEFAULT_PASSWORD | cut -d'=' -f2") +ail_password=$(lxc exec $AIL_CONTAINER -- bash -c "grep '^password=' /home/ail/ail-framework/DEFAULT_PASSWORD | cut -d'=' -f2") +ail_API_Key=$(lxc exec $AIL_CONTAINER -- bash -c "grep '^API_Key=' /home/ail/ail-framework/DEFAULT_PASSWORD | cut -d'=' -f2") + +echo "--------------------------------------------------------------------------------------------" +echo -e "${BLUE}AIL ${NC}is up and running on $ail_ip" +echo "--------------------------------------------------------------------------------------------" +echo -e "${BLUE}AIL ${NC}credentials:" +echo -e "Email: ${GREEN}$ail_email${NC}" +echo -e "Password: ${GREEN}$ail_password${NC}" +echo -e "API Key: ${GREEN}$ail_API_Key${NC}" +echo "--------------------------------------------------------------------------------------------" +if $LACUS; then + echo -e "${BLUE}Lacus ${NC}is up and running on $lacus_ip" +fi +echo "--------------------------------------------------------------------------------------------" diff --git a/other_installers/LXD/README.md b/other_installers/LXD/README.md index fb0f9110..28af09ab 100644 --- a/other_installers/LXD/README.md +++ b/other_installers/LXD/README.md @@ -1,2 +1,24 @@ # AIL-framework-LXD -This installer is based on the [LXD](https://linuxcontainers.org/lxd/introduction/) container manager and can be used to install AIL on Linux. +This installer is based on the [LXD](https://canonical.com/lxd) container manager and can be used to install AIL on Linux. It also supports the installation of [Lacus](https://github.com/ail-project/lacus) a crawler for the AIL framework. + +## Requirements +- [LXD](https://canonical.com/lxd) 5.19 +- jq 1.6 + +## Usage +Make sure you have all the requirements installed on your system. + +### Interactive mode +Run the INSTALL.sh script with the --interactive flag to enter the interactive mode, which guides you through the configuration process: +```bash +bash INSTALL.sh --interactive +``` + +### Non-interactive mode +If you want to install AIL without the interactive mode, you can use the following command: +```bash +bash INSTALL.sh [OPTIONS] +``` + +## Configuration +If you installed Lacus, you can configure AIL to use it as a crawler. For further information, please refer to the [HOWTO](https://github.com/ail-project/ail-framework/blob/master/HOWTO.md) From 39aa74aa5242e1796759a84cb3d6647ec299664e Mon Sep 17 00:00:00 2001 From: niclas Date: Tue, 16 Jan 2024 11:41:40 +0100 Subject: [PATCH 4/9] Add [install] install tor on lacus --- other_installers/LXD/INSTALL.sh | 28 +++++++++++++++------- other_installers/LXD/systemd/lacus.service | 19 +++++++++++++++ 2 files changed, 38 insertions(+), 9 deletions(-) create mode 100644 other_installers/LXD/systemd/lacus.service diff --git a/other_installers/LXD/INSTALL.sh b/other_installers/LXD/INSTALL.sh index 1c265c4f..80204def 100644 --- a/other_installers/LXD/INSTALL.sh +++ b/other_installers/LXD/INSTALL.sh @@ -172,12 +172,23 @@ createLacusContainer(){ lxc exec "$LACUS_CONTAINER" -- git clone https://github.com/ail-project/lacus.git lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- /root/.local/bin/poetry install AIL_VENV_PATH=$(lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "/root/.local/bin/poetry env info -p") - # lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "source ${AIL_VENV_PATH}/bin/activate" - # lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- /root/.local/bin/poetry shell lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "source ${AIL_VENV_PATH}/bin/activate && playwright install-deps" lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "echo LACUS_HOME=/root/lacus >> .env" - lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "export PATH='/root/.local/bin:$PATH' && yes | /root/.local/bin/poetry run update --init" + lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "export PATH='/root/.local/bin:$PATH' && echo 'no' | /root/.local/bin/poetry run update --init" + # Install Tor + lxc exec "$LACUS_CONTAINER" -- apt install apt-transport-https -y + lxc exec "$LACUS_CONTAINER" -- bash -c "echo 'deb https://deb.torproject.org/torproject.org $(lsb_release -cs) main' >> /etc/apt/sources.list.d/tor.list" + lxc exec "$LACUS_CONTAINER" -- bash -c "echo 'deb-src https://deb.torproject.org/torproject.org $(lsb_release -cs) main' >> /etc/apt/sources.list.d/tor.list" + lxc exec "$LACUS_CONTAINER" -- bash -c "wget -qO- https://deb.torproject.org/torproject.org/A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89.asc | gpg --dearmor | tee /usr/share/keyrings/tor-archive-keyring.gpg > /dev/null" + lxc exec "$LACUS_CONTAINER" -- apt update + lxc exec "$LACUS_CONTAINER" -- apt install tor deb.torproject.org-keyring -y lxc exec "$LACUS_CONTAINER" -- sed -i "/^\$nrconf{restart} = 'a';/s/.*/#\$nrconf{restart} = 'i';/" /etc/needrestart/needrestart.conf + # Start Lacus + lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- cp ./confg/logging.json.sample ./confg/logging.json + lxc file push ./systemd/lacus.service "$LACUS_CONTAINER"/etc/systemd/system/lacus.service + lxc exec "$LACUS_CONTAINER" -- systemctl daemon-reload + lxc exec "$LACUS_CONTAINER" -- systemctl enable lacus.service + lxc exec "$LACUS_CONTAINER" -- systemctl start lacus.service } nonInteractiveConfig(){ @@ -358,14 +369,13 @@ if $LACUS; then fi # Print info -ail_ip=$(lxc list $AIL_CONTAINER --format=json | jq -r '.[0].state.network.eth0.addresses[] | select(.family=="inet").address') +ail_ip=$(lxc list "$AIL_CONTAINER" --format=json | jq -r '.[0].state.network.eth0.addresses[] | select(.family=="inet").address') +ail_email=$(lxc exec "$AIL_CONTAINER" -- bash -c "grep '^email=' /home/ail/ail-framework/DEFAULT_PASSWORD | cut -d'=' -f2") +ail_password=$(lxc exec "$AIL_CONTAINER" -- bash -c "grep '^password=' /home/ail/ail-framework/DEFAULT_PASSWORD | cut -d'=' -f2") +ail_API_Key=$(lxc exec "$AIL_CONTAINER" -- bash -c "grep '^API_Key=' /home/ail/ail-framework/DEFAULT_PASSWORD | cut -d'=' -f2") if $LACUS; then - lacus_ip=$(lxc list $LACUS_CONTAINER --format=json | jq -r '.[0].state.network.eth0.addresses[] | select(.family=="inet").address') + lacus_ip=$(lxc list "$LACUS_CONTAINER" --format=json | jq -r '.[0].state.network.eth0.addresses[] | select(.family=="inet").address') fi -ail_email=$(lxc exec $AIL_CONTAINER -- bash -c "grep '^email=' /home/ail/ail-framework/DEFAULT_PASSWORD | cut -d'=' -f2") -ail_password=$(lxc exec $AIL_CONTAINER -- bash -c "grep '^password=' /home/ail/ail-framework/DEFAULT_PASSWORD | cut -d'=' -f2") -ail_API_Key=$(lxc exec $AIL_CONTAINER -- bash -c "grep '^API_Key=' /home/ail/ail-framework/DEFAULT_PASSWORD | cut -d'=' -f2") - echo "--------------------------------------------------------------------------------------------" echo -e "${BLUE}AIL ${NC}is up and running on $ail_ip" echo "--------------------------------------------------------------------------------------------" diff --git a/other_installers/LXD/systemd/lacus.service b/other_installers/LXD/systemd/lacus.service new file mode 100644 index 00000000..d6f67142 --- /dev/null +++ b/other_installers/LXD/systemd/lacus.service @@ -0,0 +1,19 @@ +[Unit] +Description=lacus service +After=network.target + +[Service] +User=root +Group=root +Type=forking +WorkingDirectory=/root/lacus +Environment="PATH=/root/.local/bin/poetry:/usr/bin" +Environment="LACUS_HOME=/root/lacus" +ExecStart=/bin/bash -c "exec /root/.local/bin/poetry run start" +ExecStop=/bin/bash -c "exec /root/.local/bin/poetry run stop" +StandardOutput=append:/var/log/lacus_message.log +StandardError=append:/var/log/lacus_error.log + + +[Install] +WantedBy=multi-user.target \ No newline at end of file From 82aec1bf354bf9e5ee760374d57d2568c7a45968 Mon Sep 17 00:00:00 2001 From: niclas Date: Tue, 16 Jan 2024 12:59:06 +0100 Subject: [PATCH 5/9] Fix [tor install] tor install on lacus --- other_installers/LXD/INSTALL.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/other_installers/LXD/INSTALL.sh b/other_installers/LXD/INSTALL.sh index 80204def..264b18fb 100644 --- a/other_installers/LXD/INSTALL.sh +++ b/other_installers/LXD/INSTALL.sh @@ -177,14 +177,14 @@ createLacusContainer(){ lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "export PATH='/root/.local/bin:$PATH' && echo 'no' | /root/.local/bin/poetry run update --init" # Install Tor lxc exec "$LACUS_CONTAINER" -- apt install apt-transport-https -y - lxc exec "$LACUS_CONTAINER" -- bash -c "echo 'deb https://deb.torproject.org/torproject.org $(lsb_release -cs) main' >> /etc/apt/sources.list.d/tor.list" - lxc exec "$LACUS_CONTAINER" -- bash -c "echo 'deb-src https://deb.torproject.org/torproject.org $(lsb_release -cs) main' >> /etc/apt/sources.list.d/tor.list" + lxc exec "$LACUS_CONTAINER" -- bash -c "echo 'deb [signed-by=/usr/share/keyrings/tor-archive-keyring.gpg] https://deb.torproject.org/torproject.org $(lsb_release -cs) main' >> /etc/apt/sources.list.d/tor.list" + lxc exec "$LACUS_CONTAINER" -- bash -c "echo 'deb-src [signed-by=/usr/share/keyrings/tor-archive-keyring.gpg] https://deb.torproject.org/torproject.org $(lsb_release -cs) main' >> /etc/apt/sources.list.d/tor.list" lxc exec "$LACUS_CONTAINER" -- bash -c "wget -qO- https://deb.torproject.org/torproject.org/A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89.asc | gpg --dearmor | tee /usr/share/keyrings/tor-archive-keyring.gpg > /dev/null" lxc exec "$LACUS_CONTAINER" -- apt update lxc exec "$LACUS_CONTAINER" -- apt install tor deb.torproject.org-keyring -y lxc exec "$LACUS_CONTAINER" -- sed -i "/^\$nrconf{restart} = 'a';/s/.*/#\$nrconf{restart} = 'i';/" /etc/needrestart/needrestart.conf # Start Lacus - lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- cp ./confg/logging.json.sample ./confg/logging.json + lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- cp ./config/logging.json.sample ./config/logging.json lxc file push ./systemd/lacus.service "$LACUS_CONTAINER"/etc/systemd/system/lacus.service lxc exec "$LACUS_CONTAINER" -- systemctl daemon-reload lxc exec "$LACUS_CONTAINER" -- systemctl enable lacus.service From 8159b10e5d26f384af1b49e96f985f734840951d Mon Sep 17 00:00:00 2001 From: niclas Date: Tue, 16 Jan 2024 14:21:37 +0100 Subject: [PATCH 6/9] Add [interactive install] interactive install --- other_installers/LXD/INSTALL.sh | 114 +++++++++++++++++++++++++++++--- 1 file changed, 106 insertions(+), 8 deletions(-) diff --git a/other_installers/LXD/INSTALL.sh b/other_installers/LXD/INSTALL.sh index 264b18fb..2c278a89 100644 --- a/other_installers/LXD/INSTALL.sh +++ b/other_installers/LXD/INSTALL.sh @@ -1,12 +1,12 @@ #!/bin/bash -setVars() { - GREEN='\033[0;32m' - YELLOW='\033[1;33m' - BLUE='\033[0;34m' - RED='\033[0;31m' - NC='\033[0m' # No Color +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +RED='\033[0;31m' +NC='\033[0m' # No Color +setVars() { STORAGE_POOL_NAME=$(generateName "AIL") NETWORK_NAME=$(generateName "AIL") NETWORK_NAME=${NETWORK_NAME:0:14} @@ -20,6 +20,7 @@ setDefaults(){ default_ail_name=$(generateName "AIL") default_lacus="Yes" default_lacus_name=$(generateName "LACUS") + default_partition="" } error() { @@ -69,7 +70,7 @@ setupLXD(){ error "Storage '$STORAGE_POOL_NAME' already exists." exit 1 fi - lxc storage create "$STORAGE_POOL_NAME" zfs source="$PARTITION_NAME" + lxc storage create "$STORAGE_POOL_NAME" zfs source="$PARTITION" if checkRessourceExist "network" "$NETWORK_NAME"; then error "Network '$NETWORK_NAME' already exists." @@ -191,8 +192,100 @@ createLacusContainer(){ lxc exec "$LACUS_CONTAINER" -- systemctl start lacus.service } +interactiveConfig(){ + echo + echo "################################################################################" + echo -e "# Welcome to the ${BLUE}AIL-framework-LXD${NC} Installer Script #" + echo "#------------------------------------------------------------------------------#" + echo -e "# This installer script will guide you through the installation process of #" + echo -e "# ${BLUE}AIL${NC} using LXD. #" + echo -e "# #" + echo "################################################################################" + echo + + declare -A nameCheckArray + + # Ask for LXD project name + while true; do + read -r -p "Name of the AIL LXD-project (default: $default_ail_project): " ail_project + PROJECT_NAME=${ail_project:-$default_ail_project} + if ! checkNamingConvention "$PROJECT_NAME"; then + continue + fi + if checkRessourceExist "project" "$PROJECT_NAME"; then + error "Project '$PROJECT_NAME' already exists." + continue + fi + break + done + + # Ask for AIL container name + while true; do + read -r -p "Name of the AIL container (default: $default_ail_name): " ail_name + AIL_CONTAINER=${ail_name:-$default_ail_name} + if [[ ${nameCheckArray[$AIL_CONTAINER]+_} ]]; then + error "Name '$AIL_CONTAINER' has already been used. Please choose a different name." + continue + fi + if ! checkNamingConvention "$AIL_CONTAINER"; then + continue + fi + nameCheckArray[$AIL_CONTAINER]=1 + break + done + + # Ask for Lacus installation + read -r -p "Do you want to install Lacus (y/n, default: $default_lacus): " lacus + lacus=${lacus:-$default_lacus} + LACUS=$(echo "$lacus" | grep -iE '^y(es)?$' > /dev/null && echo true || echo false) + if $LACUS; then + # Ask for LACUS container name + while true; do + read -r -p "Name of the Lacus container (default: $default_lacus_name): " lacus_name + LACUS_CONTAINER=${lacus_name:-$default_lacus_name} + if [[ ${nameCheckArray[$LACUS_CONTAINER]+_} ]]; then + error "Name '$LACUS_CONTAINER' has already been used. Please choose a different name." + continue + fi + if ! checkNamingConvention "$LACUS_CONTAINER"; then + continue + fi + nameCheckArray[$LACUS_CONTAINER]=1 + break + done + + fi + + # Ask for dedicated partitions + read -r -p "Dedicated partition for AIL LXD-project (leave blank if none): " partition + PARTITION=${partition:-$default_partition} + + # Output values set by the user + echo -e "\nValues set:" + echo "--------------------------------------------------------------------------------------------------------------------" + echo -e "PROJECT_NAME: ${GREEN}$PROJECT_NAME${NC}" + echo "--------------------------------------------------------------------------------------------------------------------" + echo -e "AIL_CONTAINER: ${GREEN}$AIL_CONTAINER${NC}" + echo "--------------------------------------------------------------------------------------------------------------------" + echo -e "LACUS: ${GREEN}$LACUS${NC}" + if $LACUS; then + echo -e "LACUS_CONTAINER: ${GREEN}$LACUS_CONTAINER${NC}" + echo "--------------------------------------------------------------------------------------------------------------------" + fi + echo -e "PARTITION: ${GREEN}$PARTITION${NC}" + echo "--------------------------------------------------------------------------------------------------------------------" + + # Ask for confirmation + read -r -p "Do you want to proceed with the installation? (y/n): " confirm + confirm=${confirm:-$default_confirm} + if [[ $confirm != "y" ]]; then + warn "Installation aborted." + exit 1 + fi +} + nonInteractiveConfig(){ - VALID_ARGS=$(getopt -o h --long help,production,project:ail-name:,no-lacus,lacus-name: -- "$@") + VALID_ARGS=$(getopt -o h --long help,production,project:ail-name:,no-lacus,lacus-name:,partition: -- "$@") if [[ $? -ne 0 ]]; then exit 1; fi @@ -204,6 +297,10 @@ nonInteractiveConfig(){ usage exit 0 ;; + --partition) + partition=$2 + shift 2 + ;; --project) ail_project=$2 shift 2 @@ -232,6 +329,7 @@ nonInteractiveConfig(){ lacus=${lacus:-$default_lacus} LACUS=$(echo "$lacus" | grep -iE '^y(es)?$' > /dev/null && echo true || echo false) LACUS_CONTAINER=${lacus_name:-$default_lacus_name} + PARTITION=${partition:-$default_partition} } validateArgs(){ From 45aae5569126e88685ec9676a335b8654d8bb4f8 Mon Sep 17 00:00:00 2001 From: niclas Date: Tue, 16 Jan 2024 14:35:18 +0100 Subject: [PATCH 7/9] Update [README] add installer options --- other_installers/LXD/README.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/other_installers/LXD/README.md b/other_installers/LXD/README.md index 28af09ab..59c779cf 100644 --- a/other_installers/LXD/README.md +++ b/other_installers/LXD/README.md @@ -20,5 +20,16 @@ If you want to install AIL without the interactive mode, you can use the followi bash INSTALL.sh [OPTIONS] ``` +The following options are available: +| Flag | Default Value | Description | +| ------------------------------- | ----------------------- | ------------------------------------------------------------------------ | +| `-i`, `--interactive` | N/A | Activates an interactive installation process. | +| `--project ` | `AIL-` | Name of the LXD project for organizing and running the containers. | +| `--ail-name ` | `AIL-` | The name of the container responsible for running the AIL application. | +| `--no-lacus` | `false` | Determines whether to install the Lacus container. | +| `--lacus-name ` | `LACUS-` | The name of the container responsible for running the Lacus application. | +| `--partition ` | `` | Dedicated partition for LXD-project storage. | + + ## Configuration -If you installed Lacus, you can configure AIL to use it as a crawler. For further information, please refer to the [HOWTO](https://github.com/ail-project/ail-framework/blob/master/HOWTO.md) +If you installed Lacus, you can configure AIL to use it as a crawler. For further information, please refer to the [HOWTO.md](https://github.com/ail-project/ail-framework/blob/master/HOWTO.md) From 89365844ab310c28f83ec739d419d90e5dce9fb2 Mon Sep 17 00:00:00 2001 From: niclas Date: Fri, 19 Jan 2024 15:56:44 +0100 Subject: [PATCH 8/9] Add [autobuild] build script + systemd config --- other_installers/LXD/.gitignore | 4 + other_installers/LXD/build/ailbuilder.py | 134 ++++++ other_installers/LXD/build/build.sh | 389 ++++++++++++++++++ other_installers/LXD/build/conf/lacus.service | 19 + .../LXD/build/conf/sign.json.template | 7 + .../LXD/build/conf/tracker.json.template | 28 ++ .../LXD/build/systemd/ailbuilder.service | 13 + other_installers/LXD/build/systemd/setup.sh | 88 ++++ other_installers/LXD/build/systemd/update.sh | 36 ++ 9 files changed, 718 insertions(+) create mode 100644 other_installers/LXD/.gitignore create mode 100644 other_installers/LXD/build/ailbuilder.py create mode 100644 other_installers/LXD/build/build.sh create mode 100644 other_installers/LXD/build/conf/lacus.service create mode 100644 other_installers/LXD/build/conf/sign.json.template create mode 100644 other_installers/LXD/build/conf/tracker.json.template create mode 100644 other_installers/LXD/build/systemd/ailbuilder.service create mode 100644 other_installers/LXD/build/systemd/setup.sh create mode 100644 other_installers/LXD/build/systemd/update.sh diff --git a/other_installers/LXD/.gitignore b/other_installers/LXD/.gitignore new file mode 100644 index 00000000..8c8b5b17 --- /dev/null +++ b/other_installers/LXD/.gitignore @@ -0,0 +1,4 @@ +build/conf/sign.json +build/conf/tracker.json +images/ + diff --git a/other_installers/LXD/build/ailbuilder.py b/other_installers/LXD/build/ailbuilder.py new file mode 100644 index 00000000..3044c806 --- /dev/null +++ b/other_installers/LXD/build/ailbuilder.py @@ -0,0 +1,134 @@ +import json +import requests +import subprocess +import re +import os +from time import sleep +from typing import List, Optional +from pathlib import Path + +BUILD_PATH = "/opt/ailbuilder/build" + +class Repo: + """Base class for repository tracking and update checking.""" + + def __init__(self, id: str, args: List[str], name: str, outputdir: str) -> None: + self.id = id + self.args = args + self.name = name + self.outputdir = outputdir + self.last_seen_update = None + + def _check_for_new_update(self) -> bool: + latest_update = self._get_latest_update() + if latest_update and (latest_update != self.last_seen_update): + print(f"New update found for {self.id}") + self.last_seen_update = latest_update + return True + return False + + def _get_latest_update(self): + raise NotImplementedError + + def _save_state(self): + try: + with open(f'{BUILD_PATH}/systemd/state.json', 'r') as file: + states = json.load(file) + except FileNotFoundError: + states = {} + + states[self.id] = self.last_seen_update + + with open(f'{BUILD_PATH}/systemd/state.json', 'w') as file: + json.dump(states, file) + + def load_state(self): + try: + with open(f'{BUILD_PATH}/systemd/state.json', 'r') as file: + states = json.load(file) + except FileNotFoundError: + states = {} + + self.last_seen_update = states.get(self.id, None) + + def build(self) -> None: + if self._check_for_new_update(): + try: + cmd = [f'{BUILD_PATH}/build.sh'] + self.args + ["-o", self.outputdir] + print(f"Running {cmd}") + result = subprocess.run(cmd, check=False) + if result.returncode != 0: + print(f"Failed to run {cmd} for {self.id}") + return + most_recent_dir = max((d for d in Path(self.outputdir).iterdir() if d.is_dir()), key=os.path.getctime, default=None) + relative_path = most_recent_dir.relative_to(Path(self.outputdir)) + if os.path.exists(f"{self.outputdir}/latest_{self.name}"): + os.remove(f"{self.outputdir}/latest_{self.name}") + os.symlink(relative_path, f"{self.outputdir}/latest_{self.name}") + print(f"Created symlink {self.outputdir}/latest_{self.name} to {relative_path}") + self._save_state() + except Exception as e: + print(f"Failed to run {cmd} for {self.id}: {e}") + +class GitHub(Repo): + """Class for tracking GitHub repositories.""" + + def __init__(self, id: str, mode: str, args: List[str], name: str, outputdir: str) -> None: + super().__init__(id, args, name, outputdir) + self.mode = mode + + def _get_latest_update(self) -> Optional[str]: + print(f"Fetching {self.mode} for {self.id}") + url=f'https://api.github.com/repos/{self.id}/{self.mode}' + response = requests.get(url) + if response.status_code == 200: + return response.json()[0]['sha'] + else: + print(f"Failed to fetch {self.mode} for {self.id}") + return None + +class APT(Repo): + """Class for tracking APT packages.""" + + def __init__(self, id: str, args: List[str], name: str, outputdir: str) -> None: + super().__init__(id, args, name, outputdir) + + def _get_latest_update(self) -> Optional[str]: + try: + cmd = ["apt-cache", "policy", self.id] + print (f"Running {cmd}") + output = subprocess.check_output(cmd).decode('utf-8') + match = re.search(r'Candidate: (\S+)', output) + if match: + return match.group(1) + else: + return None + except: + return None + +def main(): + with open(f'{BUILD_PATH}/conf/tracker.json') as f: + config = json.load(f) + + repos = [] + for repo in config["github"]: + repos.append(GitHub(repo["id"], repo["mode"], repo["args"], repo["name"], config["outputdir"])) + + aptpkg = [] + for package in config["apt"]: + aptpkg.append(APT(package["id"], package["args"], package["name"], config["outputdir"])) + + for repo in repos + aptpkg: + if config["sign"]: + repo.args.append("-s") + repo.load_state() + + while True: + for repo in repos: + repo.build() + for package in aptpkg: + package.build() + sleep(config["check_interval"]) + +if __name__ == "__main__": + main() diff --git a/other_installers/LXD/build/build.sh b/other_installers/LXD/build/build.sh new file mode 100644 index 00000000..b785364e --- /dev/null +++ b/other_installers/LXD/build/build.sh @@ -0,0 +1,389 @@ +#!/bin/bash + +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +RED='\033[0;31m' +NC='\033[0m' # No Color + +setVars() { + DEPEDENCIES=("lxc" "jq") + PATH_TO_BUILD="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + + PROJECT_NAME=$(generateName "AIL") + STORAGE_POOL_NAME=$(generateName "AIL") + NETWORK_NAME=$(generateName "AIL") + NETWORK_NAME=${NETWORK_NAME:0:14} + + UBUNTU="ubuntu:22.04" + + AIL_CONTAINER=$(generateName "AIL") + LACUS_CONTAINER=$(generateName "LACUS") + LACUS_SERVICE_FILE="$PATH_TO_BUILD/conf/lacus.service" +} + +setDefaults(){ + default_ail=false + default_ail_image="AIL" + default_lacus=false + default_lacus_image="Lacus" + default_outputdir="" + default_sign=false +} + +error() { + echo -e "${RED}ERROR: $1${NC}" +} + +warn() { + echo -e "${YELLOW}WARNING: $1${NC}" +} + +info() { + echo -e "${BLUE}INFO: $1${NC}" +} + +success() { + echo -e "${GREEN}SUCCESS: $1${NC}" +} + +err() { + local parent_lineno="$1" + local message="$2" + local code="${3:-1}" + + if [[ -n "$message" ]] ; then + error "Line ${parent_lineno}: ${message}: exiting with status ${code}" + else + error "Line ${parent_lineno}: exiting with status ${code}" + fi + + deleteLXDProject "$PROJECT_NAME" + lxc storage delete "$APP_STORAGE" + lxc storage delete "$DB_STORAGE" + lxc network delete "$NETWORK_NAME" + exit "${code}" +} + +generateName(){ + local name="$1" + echo "${name}-$(date +%Y%m%d%H%M%S)" +} + + +waitForContainer() { + local container_name="$1" + + sleep 3 + while true; do + status=$(lxc list --format=json | jq -e --arg name "$container_name" '.[] | select(.name == $name) | .status') + if [ "$status" = "\"Running\"" ]; then + echo -e "${BLUE}$container_name ${GREEN}is running.${NC}" + break + fi + echo "Waiting for $container_name container to start." + sleep 5 + done +} + +interrupt() { + warn "Script interrupted by user. Delete project and exit ..." + deleteLXDProject "$PROJECT_NAME" + lxc network delete "$NETWORK_NAME" + exit 130 +} + +cleanupProject(){ + local project="$1" + + info "Starting cleanup ..." + echo "Deleting container in project" + for container in $(lxc query "/1.0/containers?recursion=1&project=${project}" | jq .[].name -r); do + lxc delete --project "${project}" -f "${container}" + done + + echo "Deleting images in project" + for image in $(lxc query "/1.0/images?recursion=1&project=${project}" | jq .[].fingerprint -r); do + lxc image delete --project "${project}" "${image}" + done + + echo "Deleting project" + lxc project delete "${project}" +} + +cleanup(){ + cleanupProject "$PROJECT_NAME" + lxc storage delete "$STORAGE_POOL_NAME" + lxc network delete "$NETWORK_NAME" +} + +createAILContainer(){ + lxc launch $UBUNTU "$AIL_CONTAINER" -p default --storage "$STORAGE_POOL_NAME" --network "$NETWORK_NAME" + waitForContainer "$AIL_CONTAINER" + lxc exec "$AIL_CONTAINER" -- sed -i "/#\$nrconf{restart} = 'i';/s/.*/\$nrconf{restart} = 'a';/" /etc/needrestart/needrestart.conf + lxc exec "$AIL_CONTAINER" -- apt update + lxc exec "$AIL_CONTAINER" -- apt upgrade -y + lxc exec "$AIL_CONTAINER" -- useradd -m -s /bin/bash ail + if lxc exec "$AIL_CONTAINER" -- id ail; then + lxc exec "$AIL_CONTAINER" -- usermod -aG sudo ail + success "User ail created." + else + error "User ail not created." + exit 1 + fi + lxc exec "$AIL_CONTAINER" -- bash -c "echo 'ail ALL=(ALL) NOPASSWD: ALL' | sudo tee /etc/sudoers.d/ail" + lxc exec "$AIL_CONTAINER" --cwd=/home/ail -- sudo -u ail bash -c "git clone https://github.com/ail-project/ail-framework.git" + lxc exec "$AIL_CONTAINER" --cwd=/home/ail/ail-framework -- sudo -u ail bash -c "./installing_deps.sh" + lxc exec "$AIL_CONTAINER" -- sed -i '/^\[Flask\]/,/^\[/ s/host = 127\.0\.0\.1/host = 0.0.0.0/' /home/ail/ail-framework/configs/core.cfg + lxc exec "$AIL_CONTAINER" --cwd=/home/ail/ail-framework/bin -- sudo -u ail bash -c "./LAUNCH.sh -l" + lxc exec "$AIL_CONTAINER" -- sed -i "/^\$nrconf{restart} = 'a';/s/.*/#\$nrconf{restart} = 'i';/" /etc/needrestart/needrestart.conf +} + +createLacusContainer(){ + lxc launch $UBUNTU "$LACUS_CONTAINER" -p default --storage "$STORAGE_POOL_NAME" --network "$NETWORK_NAME" + waitForContainer "$LACUS_CONTAINER" + lxc exec "$LACUS_CONTAINER" -- sed -i "/#\$nrconf{restart} = 'i';/s/.*/\$nrconf{restart} = 'a';/" /etc/needrestart/needrestart.conf + lxc exec "$LACUS_CONTAINER" -- apt update + lxc exec "$LACUS_CONTAINER" -- apt upgrade -y + lxc exec "$LACUS_CONTAINER" -- apt install pipx -y + lxc exec "$LACUS_CONTAINER" -- pipx install poetry + lxc exec "$LACUS_CONTAINER" -- pipx ensurepath + lxc exec "$LACUS_CONTAINER" -- apt install build-essential tcl -y + lxc exec "$LACUS_CONTAINER" -- git clone https://github.com/redis/redis.git + lxc exec "$LACUS_CONTAINER" --cwd=/root/redis -- git checkout 7.2 + lxc exec "$LACUS_CONTAINER" --cwd=/root/redis -- make + lxc exec "$LACUS_CONTAINER" --cwd=/root/redis -- make test + lxc exec "$LACUS_CONTAINER" -- git clone https://github.com/ail-project/lacus.git + lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- /root/.local/bin/poetry install + AIL_VENV_PATH=$(lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "/root/.local/bin/poetry env info -p") + lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "source ${AIL_VENV_PATH}/bin/activate && playwright install-deps" + lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "echo LACUS_HOME=/root/lacus >> .env" + lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- bash -c "export PATH='/root/.local/bin:$PATH' && echo 'no' | /root/.local/bin/poetry run update --init" + # Install Tor + lxc exec "$LACUS_CONTAINER" -- apt install apt-transport-https -y + lxc exec "$LACUS_CONTAINER" -- bash -c "echo 'deb [signed-by=/usr/share/keyrings/tor-archive-keyring.gpg] https://deb.torproject.org/torproject.org $(lsb_release -cs) main' >> /etc/apt/sources.list.d/tor.list" + lxc exec "$LACUS_CONTAINER" -- bash -c "echo 'deb-src [signed-by=/usr/share/keyrings/tor-archive-keyring.gpg] https://deb.torproject.org/torproject.org $(lsb_release -cs) main' >> /etc/apt/sources.list.d/tor.list" + lxc exec "$LACUS_CONTAINER" -- bash -c "wget -qO- https://deb.torproject.org/torproject.org/A3C4F0F979CAA22CDBA8F512EE8CBC9E886DDD89.asc | gpg --dearmor | tee /usr/share/keyrings/tor-archive-keyring.gpg > /dev/null" + lxc exec "$LACUS_CONTAINER" -- apt update + lxc exec "$LACUS_CONTAINER" -- apt install tor deb.torproject.org-keyring -y + lxc exec "$LACUS_CONTAINER" -- sed -i "/^\$nrconf{restart} = 'a';/s/.*/#\$nrconf{restart} = 'i';/" /etc/needrestart/needrestart.conf + # Start Lacus + lxc exec "$LACUS_CONTAINER" --cwd=/root/lacus -- cp ./config/logging.json.sample ./config/logging.json + lxc file push "$LACUS_SERVICE_FILE" "$LACUS_CONTAINER"/etc/systemd/system/lacus.service + lxc exec "$LACUS_CONTAINER" -- systemctl daemon-reload + lxc exec "$LACUS_CONTAINER" -- systemctl enable lacus.service + lxc exec "$LACUS_CONTAINER" -- systemctl start lacus.service +} + +createLXDImage() { + local container="$1" + local image_name="$2" + local path_to_repo="$3" + local user="$4" + + local commit_id + local version + commit_id=$(getCommitID "$container" "$path_to_repo") + version=$(getVersion "$container" "$path_to_repo" "$user") + + lxc stop "$container" + lxc publish "$container" --alias "$image_name" + lxc image export "$image_name" "$OUTPUTDIR" + local file_name + file_name="${image_name}_${version}_${commit_id}.tar.gz" + pushd "$OUTPUTDIR" && mv -i "$(ls -t | head -n1)" "$file_name" + popd || { error "Failed to rename image file"; exit 1; } + sleep 2 + if $SIGN; then + sign "$file_name" + fi +} + +getCommitID() { + local container="$1" + local path_to_repo="$2" + local current_branch + current_branch=$(lxc exec "$container" -- cat "$path_to_repo"/.git/HEAD | awk '{print $2}') + local commit_id + commit_id=$(lxc exec "$container" -- cat "$path_to_repo"/.git/"$current_branch") + echo "$commit_id" +} + +getVersion() { + local container="$1" + local path_to_repo="$2" + local user="$3" + local version + version=$(lxc exec "$container" --cwd="$path_to_repo" -- sudo -u "$user" bash -c "git tag | sort -V | tail -n 1") + echo "$version" +} + +sign() { + if ! command -v gpg &> /dev/null; then + error "GPG is not installed. Please install it before running this script with signing." + exit 1 + fi + local file=$1 + SIGN_CONFIG_FILE="$PATH_TO_BUILD/conf/sign.json" + + if [[ ! -f "$SIGN_CONFIG_FILE" ]]; then + error "Config file not found: $SIGN_CONFIG_FILE" + exit 1 + fi + + GPG_KEY_ID=$(jq -r '.EMAIL' "$SIGN_CONFIG_FILE") + GPG_KEY_PASSPHRASE=$(jq -r '.PASSPHRASE' "$SIGN_CONFIG_FILE") + + # Check if the GPG key is available + if ! gpg --list-keys | grep -q "$GPG_KEY_ID"; then + warn "GPG key not found: $GPG_KEY_ID. Create new key." + # Setup GPG key + KEY_NAME=$(jq -r '.NAME' "$SIGN_CONFIG_FILE") + KEY_EMAIL=$(jq -r '.EMAIL' "$SIGN_CONFIG_FILE") + KEY_COMMENT=$(jq -r '.COMMENT' "$SIGN_CONFIG_FILE") + KEY_EXPIRE=$(jq -r '.EXPIRE_DATE' "$SIGN_CONFIG_FILE") + KEY_PASSPHRASE=$(jq -r '.PASSPHRASE' "$SIGN_CONFIG_FILE") + BATCH_FILE=$(mktemp -d)/batch + + cat > "$BATCH_FILE" < /dev/null; then + echo -e "${RED}Error: $dep is not installed.${NC}" + exit 1 + fi + done +} + +usage() { + echo "Usage: $0 [OPTIONS]" +} + +# ------------------ MAIN ------------------ +checkSoftwareDependencies "${DEPEDENCIES[@]}" +setVars +setDefaults + +VALID_ARGS=$(getopt -o ho:s --long help,outputdir:,sign,ail,lacus,ail-name:,lacus-name: -- "$@") +if [[ $? -ne 0 ]]; then + exit 1; +fi + +eval set -- "$VALID_ARGS" +while [ $# -gt 0 ]; do + case "$1" in + -h | --help) + usage + exit 0 + ;; + --ail) + ail=true + shift + ;; + --lacus) + lacus=true + shift + ;; + --ail-name) + ail_image=$2 + shift 2 + ;; + --lacus-name) + lacus_image=$2 + shift 2 + ;; + -o | --outputdir) + outputdir=$2 + shift 2 + ;; + -s | --sign) + sign=true + shift + ;; + *) + break + ;; + esac +done + +AIL=${ail:-$default_ail} +LACUS=${lacus:-$default_lacus} +AIL_IMAGE=${ail_image:-$default_ail_image} +LACUS_IMAGE=${lacus_image:-$default_lacus_image} +OUTPUTDIR=${outputdir:-$default_outputdir} +SIGN=${sign:-$default_sign} + +if [ ! -e "$OUTPUTDIR" ]; then + error "The specified directory does not exist." + exit 1 +fi + +if ! $AIL && ! $LACUS; then + error "No image specified!" + exit 1 +fi + +echo "----------------------------------------" +echo "Startting creating LXD images ..." +echo "----------------------------------------" + +trap cleanup EXIT + +lxc project create "$PROJECT_NAME" +lxc project switch "$PROJECT_NAME" +lxc storage create "$STORAGE_POOL_NAME" "dir" +lxc network create "$NETWORK_NAME" + +if $AIL; then + createAILContainer + createLXDImage "$AIL_CONTAINER" "$AIL_IMAGE" "/home/ail/ail-framework" "ail" +fi + +if $LACUS; then + createLacusContainer + createLXDImage "$LACUS_CONTAINER" "$LACUS_IMAGE" "/root/lacus" "root" +fi + +echo "----------------------------------------" +echo "Build script finished." +echo "----------------------------------------" \ No newline at end of file diff --git a/other_installers/LXD/build/conf/lacus.service b/other_installers/LXD/build/conf/lacus.service new file mode 100644 index 00000000..d6f67142 --- /dev/null +++ b/other_installers/LXD/build/conf/lacus.service @@ -0,0 +1,19 @@ +[Unit] +Description=lacus service +After=network.target + +[Service] +User=root +Group=root +Type=forking +WorkingDirectory=/root/lacus +Environment="PATH=/root/.local/bin/poetry:/usr/bin" +Environment="LACUS_HOME=/root/lacus" +ExecStart=/bin/bash -c "exec /root/.local/bin/poetry run start" +ExecStop=/bin/bash -c "exec /root/.local/bin/poetry run stop" +StandardOutput=append:/var/log/lacus_message.log +StandardError=append:/var/log/lacus_error.log + + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/other_installers/LXD/build/conf/sign.json.template b/other_installers/LXD/build/conf/sign.json.template new file mode 100644 index 00000000..e3554a77 --- /dev/null +++ b/other_installers/LXD/build/conf/sign.json.template @@ -0,0 +1,7 @@ +{ + "NAME": "admin", + "EMAIL": "admin@admin.test", + "COMMENT": "Key for signing images", + "EXPIRE_DATE": 0, + "PASSPHRASE": "admin" +} \ No newline at end of file diff --git a/other_installers/LXD/build/conf/tracker.json.template b/other_installers/LXD/build/conf/tracker.json.template new file mode 100644 index 00000000..ea97ceb0 --- /dev/null +++ b/other_installers/LXD/build/conf/tracker.json.template @@ -0,0 +1,28 @@ +{ + "check_interval": 600, + "outputdir": "/opt/ailbuilder/images", + "sign": true, + "github": [ + { + "name": "AIL", + "id": "ail-project/ail-framework", + "mode": "commits", + "args": [ + "--ail", + "--ail-name", + "AIL" + ] + }, + { + "name": "Lacus", + "id": "ail-project/lacus", + "mode": "commits", + "args": [ + "--lacus", + "--lacus-name", + "Lacus" + ] + } + ], + "apt": [] +} \ No newline at end of file diff --git a/other_installers/LXD/build/systemd/ailbuilder.service b/other_installers/LXD/build/systemd/ailbuilder.service new file mode 100644 index 00000000..8935a2a7 --- /dev/null +++ b/other_installers/LXD/build/systemd/ailbuilder.service @@ -0,0 +1,13 @@ +[Unit] +Description=Service for building AIL and Lacus LXD images +After=network.target + +[Service] +Type=simple +User=ailbuilder +ExecStart=/usr/bin/python3 /opt/ailbuilder/build/ailbuilder.py +Restart=on-failure +Environment=PYTHONUNBUFFERED=1 + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/other_installers/LXD/build/systemd/setup.sh b/other_installers/LXD/build/systemd/setup.sh new file mode 100644 index 00000000..42230a66 --- /dev/null +++ b/other_installers/LXD/build/systemd/setup.sh @@ -0,0 +1,88 @@ +#!/bin/bash + +DIR="$(dirname "$0")" +SERVICE_FILE="ailbuilder.service" +SERVICE_PATH="/etc/systemd/system/" +AILBUILDER_PATH="/opt/ailbuilder" +BUILD_DIR="${DIR}/../../build" +BATCH_FILE="/tmp/key_batch" +SIGN_CONFIG_FILE="../conf/sign.json" + +log() { + echo "[$(date +'%Y-%m-%dT%H:%M:%S%z')]: $*" +} + +echo "Start setting up ailbuilder service ..." + +if [[ $EUID -ne 0 ]]; then + log "This script must be run as root or with sudo privileges" + exit 1 +fi + +if [[ ! -d "$AILBUILDER_PATH" ]]; then + mkdir -p "$AILBUILDER_PATH"/images || { log "Failed to create directory $AILBUILDER_PATH"; exit 1; } +fi + +if [[ -d "$BUILD_DIR" ]]; then + cp -r "$BUILD_DIR" "$AILBUILDER_PATH/" || { log "Failed to copy build directory"; exit 1; } +else + log "Build directory $BUILD_DIR does not exist" + exit 1 +fi + +# Create user if it doesn't exist +if ! id "ailbuilder" &>/dev/null; then + useradd -r -s /bin/false ailbuilder || { log "Failed to create user ailbuilder"; exit 1; } +fi + +# Set ownership and permissions +chown -R ailbuilder: "$AILBUILDER_PATH" || { log "Failed to change ownership"; exit 1; } +chmod -R u+x "$AILBUILDER_PATH/build/"*.py || { log "Failed to set execute permission on scripts"; exit 1; } +chmod -R u+x "$AILBUILDER_PATH/build/"*.sh || { log "Failed to set execute permission on scripts"; exit 1; } +chmod -R u+w "$AILBUILDER_PATH/images/" || { log "Failed to set execute permission on images dir"; exit 1; } + +# Add user to lxd group +sudo usermod -aG lxd ailbuilder || { log "Failed to add user ailbuilder to lxd group"; exit 1; } +mkdir -p /home/ailbuilder || { log "Failed to create directory /home/ailbuilder"; exit 1; } +chown -R ailbuilder: "/home/ailbuilder" || { log "Failed to change ownership"; exit 1; } +chmod -R u+w "/home/ailbuilder" || { log "Failed to set execute permission on home dir"; exit 1; } + +# Setup GPG key +KEY_NAME=$(jq -r '.NAME' "$SIGN_CONFIG_FILE") +KEY_EMAIL=$(jq -r '.EMAIL' "$SIGN_CONFIG_FILE") +KEY_COMMENT=$(jq -r '.COMMENT' "$SIGN_CONFIG_FILE") +KEY_EXPIRE=$(jq -r '.EXPIRE_DATE' "$SIGN_CONFIG_FILE") +KEY_PASSPHRASE=$(jq -r '.PASSPHRASE' "$SIGN_CONFIG_FILE") + +if ! sudo -u ailbuilder bash -c "gpg --list-keys | grep -q $KEY_EMAIL"; then + cat > "$BATCH_FILE" < Date: Tue, 30 Jan 2024 09:15:50 +0100 Subject: [PATCH 9/9] Update README.md --- other_installers/LXD/README.md | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/other_installers/LXD/README.md b/other_installers/LXD/README.md index 59c779cf..2c504f9f 100644 --- a/other_installers/LXD/README.md +++ b/other_installers/LXD/README.md @@ -33,3 +33,36 @@ The following options are available: ## Configuration If you installed Lacus, you can configure AIL to use it as a crawler. For further information, please refer to the [HOWTO.md](https://github.com/ail-project/ail-framework/blob/master/HOWTO.md) + +## Using Images to run AIL +If you want to use images to install AIL, you can download them from the ail-project [image website](https://images.ail-project.org/) + +After downloading the images, you can import them into LXD using the following command: +```bash +lxc image import --alias +``` +Now you can use the image to create a container: +```bash +lxc launch +``` + +To log into the container you need to know the automatically generated password. You can get it with the following command: +```bash +lxc exec -- bash -c "grep '^password=' /home/ail/ail-framework/DEFAULT_PASSWORD | cut -d'=' -f2" +``` + +If you also want to use Lacus, you can do the same with the Lacus image. After that, you can configure AIL to use Lacus as a crawler. For further information, please refer to the [HOWTO.md](https://github.com/ail-project/ail-framework/blob/master/HOWTO.md). + +## Building the images locally +If you want to build the images locally, you can use the `build.sh` script: +```bash +bash build.sh [OPTIONS] +``` +| Flag | Default Value | Description | +| ------------------------------- | ------------- | --------------------------------------------------------------------------------- | +| `--ail` | `false` | Activates the creation of the AIL container. | +| `--lacus` | `false` | Activates the creation of the Lacus container. | +| `--ail-name ` | `AIL` | Specifies the name of the AIL container. The default is a generic name "AIL". | +| `--lacus-name ` | `Lacus` | Specifies the name of the Lacus container. The default is a generic name "Lacus". | +| `-o`, `--outputdir ` | `` | Sets the output directory for the LXD image files. | +| `-s`, `--sign` | `false` | Enables the signing of the generated LXD image files. |