diff --git a/.github/workflows/build-lxd.yml b/.github/workflows/build-lxd.yml index e46f26542..2e57ea25d 100644 --- a/.github/workflows/build-lxd.yml +++ b/.github/workflows/build-lxd.yml @@ -64,32 +64,27 @@ jobs: LXC: "${{ needs.determine-runner.outputs.lxc_cmd }}" steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: "${{ env.VERSION }}" - - name: Cleanup lxd - run: test -z "$("$LXC" profile device show default | grep eth0)" || "$LXC" profile device remove default eth0 - - uses: bdx0/action-incus@v1 - continue-on-error: true - with: - lxd_version: latest/stable - - name: Fix LXD + - name: Setup incus + if: ${{ needs.determine-runner.outputs.runner_label != 'ubuntu-20.04-arm64' }} run: | - sudo iptables -I DOCKER-USER -i lxdbr0 -j ACCEPT - sudo iptables -I DOCKER-USER -o lxdbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + curl https://pkgs.zabbly.com/get/incus-stable | sudo sh -x sudo iptables -I DOCKER-USER -i incusbr0 -j ACCEPT sudo iptables -I DOCKER-USER -o incusbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + sudo incus admin init --auto - name: Build LXD image env: USE_INCUS: "${{ needs.determine-runner.outputs.lxc_cmd == 'incus' && 'yes' || 'no' }}" run: | - BRANCH="$VERSION" ./build/build-LXD.sh + BRANCH="${VERSION##refs/heads/}" ./build/build-LXD.sh - name: Pack LXD image id: pack-lxd run: | . ./build/buildlib.sh ARTIFACT_FILE="NextcloudPi_LXD_${LXD_ARCH:-x86}_${VERSION//\//_}" - "$LXC" image export -q ncp/"${version}" "output/${ARTIFACT_FILE}" + sudo "$LXC" image export -q ncp/"${version}" "output/${ARTIFACT_FILE}" echo "artifact_file=${ARTIFACT_FILE}.tar.gz" >> $GITHUB_OUTPUT - name: upload LXD image to artifact store uses: actions/upload-artifact@v3 @@ -329,16 +324,15 @@ jobs: VERSION: "${{ inputs.git_ref || github.ref }}" LXC: "${{ needs.determine-runner.outputs.lxc_cmd }}" PREVIOUS_IMAGE_URL: "https://github.com/nextcloud/nextcloudpi/releases/download/v1.53.2/NextcloudPi_LXD_x86_v1.53.2.tar.gz" + USE_INCUS: "${{ needs.determine-runner.outputs.lxc_cmd == 'incus' && 'yes' || 'no' }}" steps: - - uses: whywaita/setup-lxd@v1 - if: ${{ needs.determine-runner.outputs.lxc_cmd == 'lxc' }} - continue-on-error: true - with: - lxd_version: latest/stable - - name: Fix LXD + - name: Setup incus + if: ${{ needs.determine-runner.outputs.runner_label != 'ubuntu-20.04-arm64' }} run: | - sudo iptables -I DOCKER-USER -i lxdbr0 -j ACCEPT - sudo iptables -I DOCKER-USER -o lxdbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + curl https://pkgs.zabbly.com/get/incus-stable | sudo sh -x + sudo iptables -I DOCKER-USER -i incusbr0 -j ACCEPT + sudo iptables -I DOCKER-USER -o incusbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + sudo incus admin init --auto - name: Checkout code uses: actions/checkout@v3 with: @@ -363,26 +357,26 @@ jobs: - name: Launch ncp container run: | set -x - "$LXC" delete -q -f ncp || true - "$LXC" image import -q "./ncp.tar.gz" --alias "ncp/update" - systemd-run --user --scope -p "Delegate=yes" "$LXC" launch -q "ncp/update" ncp - "$LXC" exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done' - "$LXC" exec ncp -- rm -f /opt/ncdata/data/nextcloud.log + sudo "$LXC" delete -q -f ncp || true + sudo "$LXC" image import -q "./ncp.tar.gz" --alias "ncp/update" + systemd-run --user --scope -p "Delegate=yes" sudo "$LXC" launch -q "ncp/update" ncp + sudo "$LXC" exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done' + sudo "$LXC" exec ncp -- rm -f /opt/ncdata/data/nextcloud.log sleep 30 - ip="$("$LXC" list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)" + ip="$(sudo "$LXC" list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)" ip="${ip/% *}" echo "${ip} nextcloudpi.local" | sudo tee /etc/hosts - name: Activate and Test LXD Image working-directory: ./tests run: | - "$LXC" exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' & + sudo "$LXC" exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' & python activation_tests.py --no-gui "nextcloudpi.local" 443 4443 || { echo "Activation test failed!" echo "Geckodriver logs:" tail -n 20 geckodriver.log >&2 || true echo "================" echo "ncp.log: " - "$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true + sudo "$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true exit 1 } python nextcloud_tests.py --no-gui --skip-release-check "nextcloudpi.local" 443 4443 || { @@ -391,11 +385,11 @@ jobs: tail -n 20 geckodriver.log >&2 || true echo "================" echo "ncp.log: " - "$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true + sudo "$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true echo "================" echo "nextcloud log: " - datadir="$("$LXC" exec ncp -- ncc config:system:get datadirectory)" - "$LXC" exec ncp -- cat "$datadir/nextcloud.log" || true + datadir="$(sudo "$LXC" exec ncp -- ncc config:system:get datadirectory)" + sudo "$LXC" exec ncp -- cat "$datadir/nextcloud.log" || true exit 1 } python system_tests.py --non-interactive --skip-update-test || { @@ -417,29 +411,29 @@ jobs: echo "VERSION=${VERSION}" >> "$GITHUB_ENV" echo "Running update to ${VERSION}" - current_nc_version="$("$LXC" exec ncp -- ncc status | grep "version:" | awk '{ print $3 }')" + current_nc_version="$(sudo "$LXC" exec ncp -- ncc status | grep "version:" | awk '{ print $3 }')" latest_nc_version="$(cat etc/ncp.cfg | jq -r '.nextcloud_version')" - "$LXC" exec ncp -- bash -c "DBG=x ncp-update ${UPDATE_ARGS[*]}" - "$LXC" exec ncp -- /usr/local/bin/ncc status + sudo "$LXC" exec ncp -- bash -c "DBG=x ncp-update ${UPDATE_ARGS[*]}" + sudo "$LXC" exec ncp -- /usr/local/bin/ncc status if [[ "$current_nc_version" =~ "$latest_nc_version".* ]] then echo "Nextcloud is up to date - skipping NC update test." else - "$LXC" exec ncp -- bash -c "DBG=x ncp-update-nc ${latest_nc_version?}" + sudo "$LXC" exec ncp -- bash -c "DBG=x ncp-update-nc ${latest_nc_version?}" fi - "$LXC" exec ncp -- rm -f /opt/ncdata/data/nextcloud.log + sudo "$LXC" exec ncp -- rm -f /opt/ncdata/data/nextcloud.log - "$LXC" stop ncp + sudo "$LXC" stop ncp - name: Relaunch container run: | set -x - systemd-run --user --scope -p "Delegate=yes" "$LXC" start ncp - "$LXC" exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done' + systemd-run --user --scope -p "Delegate=yes" sudo "$LXC" start ncp + sudo "$LXC" exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done' sleep 30 - ip="$("$LXC" list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)" + ip="$(sudo "$LXC" list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)" ip="${ip/% *}" echo "${ip} nextcloudpi.local" | sudo tee /etc/hosts - name: Test LXD Image @@ -451,14 +445,14 @@ jobs: tail -n 20 geckodriver.log >&2 || true echo "================" echo "ncp.log: " - "$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true + sudo "$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true echo "================" echo "nextcloud log: " - datadir="$("$LXC" exec ncp -- ncc config:system:get datadirectory)" - "$LXC" exec ncp -- cat "$datadir/nextcloud.log" || true + datadir="$(sudo "$LXC" exec ncp -- ncc config:system:get datadirectory)" + sudo "$LXC" exec ncp -- cat "$datadir/nextcloud.log" || true exit 1 } - "$LXC" exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' & + sudo "$LXC" exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' & python system_tests.py --non-interactive --skip-update-test || { echo "System test failed!" exit 1 @@ -467,46 +461,46 @@ jobs: id: distupgrade run: | set -x - "$LXC" exec ncp -- cat /etc/os-release | grep 'VERSION_ID="11"' || { - echo "can't upgrade from Debian $("$LXC" exec ncp -- cat /etc/os-release | grep VERSION_ID=)" + sudo "$LXC" exec ncp -- cat /etc/os-release | grep 'VERSION_ID="11"' || { + echo "can't upgrade from Debian $(sudo "$LXC" exec ncp -- cat /etc/os-release | grep VERSION_ID=)" exit 1 } - "$LXC" exec ncp -- bash -c "DEBIAN_FRONTEND=noninteractive ncp-dist-upgrade" + sudo "$LXC" exec ncp -- bash -c "DEBIAN_FRONTEND=noninteractive ncp-dist-upgrade" - "$LXC" exec ncp -- rm -f /opt/ncdata/data/nextcloud.log + sudo "$LXC" exec ncp -- rm -f /opt/ncdata/data/nextcloud.log - "$LXC" stop ncp + sudo "$LXC" stop ncp - name: Relaunch container run: | set -x - systemd-run --user --scope -p "Delegate=yes" "$LXC" start ncp - "$LXC" exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done' + systemd-run --user --scope -p "Delegate=yes" sudo "$LXC" start ncp + sudo "$LXC" exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done' sleep 30 - ip="$("$LXC" list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)" + ip="$(sudo "$LXC" list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)" ip="${ip/% *}" echo "${ip} nextcloudpi.local" | sudo tee /etc/hosts - name: Test LXD Image working-directory: ./tests run: | - "$LXC" exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' & + sudo "$LXC" exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' & python nextcloud_tests.py --no-gui "nextcloudpi.local" 443 4443 || { echo "Nextcloud test failed!" echo "Geckodriver logs:" tail -n 20 geckodriver.log >&2 || true echo "================" echo "ncp.log: " - "$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true + sudo "$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true echo "================" echo "nextcloud log: " - datadir="$("$LXC" exec ncp -- ncc config:system:get datadirectory)" - "$LXC" exec ncp -- cat "$datadir/nextcloud.log" || true + datadir="$(sudo "$LXC" exec ncp -- ncc config:system:get datadirectory)" + sudo "$LXC" exec ncp -- cat "$datadir/nextcloud.log" || true exit 1 } python system_tests.py --non-interactive || { echo "System test failed!" exit 1 } - "$LXC" stop ncp + sudo "$LXC" stop ncp test-fresh-install: needs: @@ -518,23 +512,19 @@ jobs: ARTIFACT_NAME: ${{ needs.build-current.outputs.artifact_name }} ARTIFACT_FILE: ${{ needs.build-current.outputs.artifact_file }} LXC: "${{ needs.determine-runner.outputs.lxc_cmd }}" + USE_INCUS: "${{ needs.determine-runner.outputs.lxc_cmd == 'incus' && 'yes' || 'no' }}" steps: - name: Checkout code uses: actions/checkout@v3 with: ref: "${{ env.VERSION }}" - - name: Cleanup lxd - run: test -z "$("$LXC" profile device show default | grep eth0)" || "$LXC" profile device remove default eth0 - - uses: whywaita/setup-lxd@v1 - if: ${{ needs.determine-runner.outputs.lxc_cmd == 'lxc' }} - continue-on-error: true - with: - lxd_version: latest/stable - - name: Fix LXD - continue-on-error: true + - name: Setup incus + if: ${{ needs.determine-runner.outputs.runner_label != 'ubuntu-20.04-arm64' }} run: | - sudo iptables -I DOCKER-USER -i lxdbr0 -j ACCEPT - sudo iptables -I DOCKER-USER -o lxdbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + curl https://pkgs.zabbly.com/get/incus-stable | sudo sh -x + sudo iptables -I DOCKER-USER -i incusbr0 -j ACCEPT + sudo iptables -I DOCKER-USER -o incusbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + sudo incus admin init --auto - name: Setup Firefox uses: browser-actions/setup-firefox@latest continue-on-error: true diff --git a/build/build-LXD.sh b/build/build-LXD.sh index 8fef861af..ee5732eb3 100755 --- a/build/build-LXD.sh +++ b/build/build-LXD.sh @@ -33,11 +33,13 @@ prepare_dirs # tmp cache output debian_version="$(. etc/library.sh > /dev/null 2>&1; echo "${RELEASE%%-security}")" -LXC_CMD=lxc -[[ "$USE_INCUS" == "yes" ]] && LXC_CMD=incus +LXC_CMD=(lxc) +[[ "$USE_INCUS" == "yes" ]] && LXC_CMD=(incus) -$LXC_CMD delete -f ncp 2>/dev/null || true -LXC_CREATE=($LXC_CMD init -p default) +"${LXC_CMD[@]}" info || LXC_CMD=(sudo "${LXC_CMD[0]}") + +"${LXC_CMD[@]}" delete -f ncp 2>/dev/null || true +LXC_CREATE=("${LXC_CMD[@]}" init -p default) [[ -n "$LXD_EXTRA_PROFILE" ]] && LXC_CREATE+=(-p "$LXD_EXTRA_PROFILE") if [[ -n "$LXD_ARCH" ]] && [[ "$LXD_ARCH" != "x86" ]] then @@ -64,19 +66,23 @@ LXC_CREATE+=(ncp) set -x EXEC_ARGS=() [[ -z "$BRANCH" ]] || EXEC_ARGS+=(--env "BRANCH=${BRANCH}") -systemd-run --user --scope -p "Delegate=yes" $LXC_CMD start ncp -q || \ -sudo systemd-run --scope -p "Delegate=yes" $LXC_CMD start ncp -q -$LXC_CMD config device add ncp buildcode disk source="$(pwd)" path=/build -$LXC_CMD exec ncp "${EXEC_ARGS[@]}" -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done' -$LXC_CMD exec ncp "${EXEC_ARGS[@]}" -- bash -c 'CODE_DIR=/build DBG=x bash /build/install.sh' -$LXC_CMD exec ncp "${EXEC_ARGS[@]}" -- bash -c 'source /build/etc/library.sh; run_app_unsafe /build/post-inst.sh' -$LXC_CMD exec ncp "${EXEC_ARGS[@]}" -- bash -c "echo '$(basename "$IMG")' > /usr/local/etc/ncp-baseimage" -$LXC_CMD stop ncp -$LXC_CMD config device remove ncp buildcode -$LXC_CMD publish -q ncp -f --alias ncp/"${version}" +systemd-run --user --scope -p "Delegate=yes" "${LXC_CMD[@]}" start ncp -q || \ +sudo systemd-run --scope -p "Delegate=yes" "${LXC_CMD[@]}" start ncp -q || { + rc=$? + "${LXC_CMD[@]}" info --show-log ncp + exit $rc +} +"${LXC_CMD[@]}" config device add ncp buildcode disk source="$(pwd)" path=/build +"${LXC_CMD[@]}" exec ncp "${EXEC_ARGS[@]}" -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done' +"${LXC_CMD[@]}" exec ncp "${EXEC_ARGS[@]}" -- bash -c 'CODE_DIR=/build DBG=x bash /build/install.sh' +"${LXC_CMD[@]}" exec ncp "${EXEC_ARGS[@]}" -- bash -c 'source /build/etc/library.sh; run_app_unsafe /build/post-inst.sh' +"${LXC_CMD[@]}" exec ncp "${EXEC_ARGS[@]}" -- bash -c "echo '$(basename "$IMG")' > /usr/local/etc/ncp-baseimage" +"${LXC_CMD[@]}" stop ncp +"${LXC_CMD[@]}" config device remove ncp buildcode +"${LXC_CMD[@]}" publish -q ncp -f --alias ncp/"${version}" ## pack -[[ " $* " =~ .*" --pack ".* ]] && $LXC_CMD image export -q ncp/"${version}" "$TAR" +[[ " $* " =~ .*" --pack ".* ]] && "${LXC_CMD[@]}" image export -q ncp/"${version}" "$TAR" exit 0 diff --git a/tests/system_tests.py b/tests/system_tests.py index b3124604d..3ef7d6750 100755 --- a/tests/system_tests.py +++ b/tests/system_tests.py @@ -59,10 +59,6 @@ '/.ncp-image', ] -lxc_command = 'lxc' -if 'USE_INCUS' in os.environ and os.environ['USE_INCUS'] == 'yes': - lxc_command = 'incus' - class tc: "terminal colors" @@ -298,9 +294,19 @@ def set_cohorte_id(cohorte_id: int) -> CompletedProcess: except: dockers_running = '' + lxc_command = ['lxc'] if 'USE_INCUS' not in os.environ or os.environ['USE_INCUS'] != 'yes' else ['incus'] + + try: + lxc_test = run(lxc_command + ['info'], stdout=PIPE, check=True) + if lxc_test.returncode != 0: + raise Exception(f"failed to execute {lxc_command} info") + except: + lxc_test = run(['sudo'] + lxc_command + ['info'], stdout=PIPE, check='True') + lxc_command = ['sudo', 'lxc'] + # detect if we are running this in a LXC instance try: - lxc_running = run([lxc_command, 'info', 'ncp'], stdout=PIPE, check = True) + lxc_running = run(lxc_command + ['info', 'ncp'], stdout=PIPE, check = True) except: lxc_running = False @@ -328,7 +334,7 @@ def set_cohorte_id(cohorte_id: int) -> CompletedProcess: # LXC method elif lxc_running: print( tc.brown + "* local LXC instance detected" + tc.normal) - pre_cmd = [lxc_command, 'exec', 'ncp', '--'] + pre_cmd = lxc_command + ['exec', 'ncp', '--'] elif systemd_container_running: pre_cmd = ['systemd-run', '--wait', '-P', '--machine=ncp']