diff --git a/.github/workflows/e2e-build-images.yaml b/.github/workflows/e2e-build-images.yaml index bd034833e..557a5a605 100644 --- a/.github/workflows/e2e-build-images.yaml +++ b/.github/workflows/e2e-build-images.yaml @@ -20,6 +20,7 @@ jobs: runs-on: ubuntu-latest outputs: matrix: ${{ steps.set-supported-releases.outputs.matrix }} + versions: ${{ steps.set-supported-releases.outputs.versions }} steps: - name: Install needed binaries run: | @@ -30,23 +31,32 @@ jobs: run: | set -eu - all="$(distro-info --supported-esm) $(distro-info --supported)" - all="$(echo $all | tr ' ' '\n' | sort -u)" + codenames="$(distro-info --supported-esm)\n$(distro-info --supported)\n" + versions="$(distro-info --supported-esm --release)\n$(distro-info --supported --release)\n" - releases="" + # Paste the codenames and versions together, sort them, and remove duplicates + codenames_with_versions="$(paste <(printf "$versions" | cut -d' ' -f1) <(printf "$codenames") | sort -u)" - for r in ${all}; do + releases="" + versions="" + while IFS=$'\t' read -r version codename; do # Filter out unsupported LTS releases - if [ "${r}" = "trusty" -o "${r}" = "xenial" -o "${r}" = "bionic" ]; then + if [[ "${codename}" =~ trusty|xenial|bionic ]]; then continue fi if [ -n "${releases}" ]; then releases="${releases}, " fi - releases="${releases}'${r}'" - done + releases="${releases}'${codename}'" + + if [ -n "${versions}" ]; then + versions="${versions}, " + fi + versions="${versions}\"${codename}\": \"${version}\"" + done <<< "$codenames_with_versions" + echo versions="{${versions}}" >> $GITHUB_OUTPUT echo matrix="${releases}" >> $GITHUB_OUTPUT build-template: @@ -75,6 +85,8 @@ jobs: chmod 600 ~/.ssh/adsys-e2e.pem - name: Check if template needs to be created id: check-vm-template + env: + versions: ${{ needs.supported-releases.outputs.versions }} run: | set -eu @@ -83,7 +95,9 @@ jobs: force="--force" fi - IMAGE_VERSION=$(go run ./e2e/cmd/build_base_image/00_check_vm_image --codename ${{ matrix.codename }} ${force}) + version="$(echo $versions | jq -r .${{ matrix.codename }})" + codename="${{ matrix.codename }}" + IMAGE_VERSION=$(go run ./e2e/cmd/build_base_image/00_check_vm_image --codename ${codename} --version ${version} ${force}) if [ ! -z "${IMAGE_VERSION}" ]; then echo image-version=$IMAGE_VERSION >> $GITHUB_OUTPUT fi diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml index d46e9414a..8d0593e2a 100644 --- a/.github/workflows/e2e-tests.yaml +++ b/.github/workflows/e2e-tests.yaml @@ -36,33 +36,42 @@ jobs: - uses: azure/login@v2 with: creds: ${{ secrets.AZURE_CREDENTIALS }} + - uses: actions/checkout@v4 + with: + repository: ${{ inputs.repository || github.repository }} + ref: ${{ inputs.branch || github.ref }} + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod - name: Build matrix id: set-supported-releases run: | set -eu - all="$(distro-info --supported-esm) $(distro-info --supported)" - all="$(echo $all | tr ' ' '\n' | sort -u)" + codenames="$(distro-info --supported-esm)\n$(distro-info --supported)\n" + versions="$(distro-info --supported-esm --release)\n$(distro-info --supported --release)\n" - releases="" + # Paste the codenames and versions together, sort them, and remove duplicates + codenames_with_versions="$(paste <(printf "$versions" | cut -d' ' -f1) <(printf "$codenames") | sort -u)" - for r in ${all}; do + releases="" + while IFS=$'\t' read -r version codename; do # Filter out unsupported LTS releases - if [ "${r}" = "trusty" -o "${r}" = "xenial" -o "${r}" = "bionic" ]; then + if [[ "${codename}" =~ trusty|xenial|bionic ]]; then continue fi # Filter out releases with no corresponding Azure images - images="$(az vm image list --publisher Canonical --offer 0001-com-ubuntu-minimal-${r} --all)" - if [ "$(jq '(. | length) == 0' <<<${images})" = "true" ]; then + latestImageVersion="$(go run ./e2e/cmd/build_base_image/00_check_vm_image --codename ${codename} --version ${version} --force)" + if [ -z "${latestImageVersion}" ]; then continue fi if [ -n "${releases}" ]; then releases="${releases}, " fi - releases="${releases}'${r}'" - done + releases="${releases}'${codename}'" + done <<< "$codenames_with_versions" echo matrix="${releases}" >> $GITHUB_OUTPUT diff --git a/e2e/cmd/build_base_image/00_check_vm_image/main.go b/e2e/cmd/build_base_image/00_check_vm_image/main.go index f44b983e8..85d220ee3 100644 --- a/e2e/cmd/build_base_image/00_check_vm_image/main.go +++ b/e2e/cmd/build_base_image/00_check_vm_image/main.go @@ -16,7 +16,7 @@ import ( "github.com/ubuntu/adsys/e2e/internal/command" ) -var codename string +var codename, version string var force bool func main() { @@ -40,11 +40,13 @@ If the --force flag is set, the script will return the latest image URN regardless of custom image availability. Options: - --codename Required: codename of the Ubuntu release (e.g. focal) + --codename Required: codename of the Ubuntu release (e.g. noble) + --version Required: version of the Ubuntu release (e.g. 24.04) -f, --force Force the script to return the latest image URN regardless of whether we have a custom image or not `, filepath.Base(os.Args[0])) cmd.AddStringFlag(&codename, "codename", "", "") + cmd.AddStringFlag(&version, "version", "", "") cmd.AddBoolFlag(&force, "force", false, "") cmd.AddBoolFlag(&force, "f", false, "") @@ -52,8 +54,8 @@ Options: } func validate(_ context.Context, _ *command.Command) error { - if codename == "" { - return errors.New("codename must be specified") + if codename == "" || version == "" { + return errors.New("codename and version must be specified") } return nil @@ -62,7 +64,7 @@ func validate(_ context.Context, _ *command.Command) error { func action(ctx context.Context, _ *command.Command) error { var noStable, noDaily bool - availableImages, err := az.ImageList(ctx, codename) + availableImages, err := az.ImageList(ctx, codename, version) if err != nil { return err } diff --git a/e2e/internal/az/image.go b/e2e/internal/az/image.go index 8b5325dc6..d11e40a86 100644 --- a/e2e/internal/az/image.go +++ b/e2e/internal/az/image.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "slices" + "strconv" "strings" "github.com/maruel/natural" @@ -15,6 +16,10 @@ import ( // NullImageVersion is the version returned when no image version is found. const NullImageVersion = "0.0.0" +// oldImageVersionCodenames is a list of Ubuntu codenames for which Azure images +// are built using the previous image versioning format. +var oldImageVersionCodenames = []string{"focal", "jammy", "mantic"} + // Image contains information about an Azure image. type Image struct { Architecture string `json:"architecture"` @@ -39,12 +44,34 @@ func ImageDefinitionName(codename string) string { } // ImageList returns a list of Azure images for the given codename. -func ImageList(ctx context.Context, codename string) (Images, error) { - out, _, err := RunCommand(ctx, "vm", "image", "list", - "--publisher", "Canonical", - "--offer", fmt.Sprintf("0001-com-ubuntu-minimal-%s", codename), - "--all", - ) +func ImageList(ctx context.Context, codename, version string) (Images, error) { + versionParts := strings.Split(version, ".") + if len(versionParts) != 2 { + return nil, fmt.Errorf("invalid version format: %s", version) + } + + // Determine if this is an LTS release + year, err := strconv.Atoi(versionParts[0]) + if err != nil { + return nil, fmt.Errorf("failed to parse version year as a number: %w", err) + } + isLTS := year%2 == 0 && versionParts[1] == "04" + + version = fmt.Sprintf("ubuntu-%s", strings.Join(versionParts, "_")) + if isLTS { + version += "-lts" + } + + var filterArgs []string + filterArgs = []string{"--offer", version, "--sku", "minimal"} + if slices.Contains(oldImageVersionCodenames, codename) { + filterArgs = []string{"--offer", fmt.Sprintf("0001-com-ubuntu-minimal-%s", codename)} + } + + args := []string{"vm", "image", "list", "--publisher", "Canonical", "--all"} + args = append(args, filterArgs...) + + out, _, err := RunCommand(ctx, args...) if err != nil { return nil, err } @@ -108,7 +135,12 @@ func (i Image) isDailyImage() bool { } func (i Image) isGen2Image() bool { - return strings.Contains(i.SKU, "gen2") + // gen2 images are explicitly defined in the old versioning scheme. + if strings.Contains(i.Offer, "0001-com-ubuntu") { + return strings.Contains(i.SKU, "gen2") + } + + return !strings.Contains(i.SKU, "gen1") } // LatestImageVersion returns the latest image version for the given image definition. diff --git a/e2e/scripts/Dockerfile.build b/e2e/scripts/Dockerfile.build index 64aa24d72..38947a3fa 100644 --- a/e2e/scripts/Dockerfile.build +++ b/e2e/scripts/Dockerfile.build @@ -11,7 +11,10 @@ RUN set -ex \ && apt-get upgrade -y --no-install-recommends ARG CODENAME=latest -RUN if [ "$CODENAME" != "noble" ]; then \ + +# Keep the codename list easy to maintain, keeping in mind that we might need to +# add more as the need to backport various packages arises. +RUN if echo "focal" | grep -q "$CODENAME"; then \ set -ex \ && apt-get install -y software-properties-common \ && add-apt-repository -y ppa:ubuntu-enterprise-desktop/golang; \ diff --git a/e2e/scripts/first-run.sh b/e2e/scripts/first-run.sh index 419d3b50f..676d736e1 100644 --- a/e2e/scripts/first-run.sh +++ b/e2e/scripts/first-run.sh @@ -4,7 +4,7 @@ set -eu # This script runs on the first boot of the VM. echo "Setting hostname..." -hostname="$(lsb_release -cs)-$(openssl rand -hex 4)" +hostname="$(lsb_release -cs)-$(openssl rand -hex 2)" hostnamectl set-hostname "$hostname" echo "Adding hostname to hosts file..." diff --git a/e2e/scripts/provision.sh b/e2e/scripts/provision.sh index dd7f3d19b..baedfa5e6 100644 --- a/e2e/scripts/provision.sh +++ b/e2e/scripts/provision.sh @@ -32,10 +32,10 @@ echo "Updating DNS resolver to use AD DNS..." echo "DNS=10.1.0.4" >> /etc/systemd/resolved.conf systemctl restart systemd-resolved -# Work around an issue on Jammy and Noble where systemd-networkd times out due -# to eth0 losing connectivity shortly after boot, even though network works fine -# as reported by Azure. -if [[ "$(lsb_release -cs)" =~ ^(jammy|noble)$ ]]; then +# Work around an issue on newer Ubuntu versions (starting with Jammy) where +# systemd-networkd times out due to eth0 losing connectivity shortly after boot, +# even though network works fine as reported by Azure. +if [ ! "$(lsb_release -cs)" = "focal" ]; then echo "Disabling misbehaving systemd-networkd-wait-online.service..." systemctl mask systemd-networkd-wait-online.service fi