tests(e2e): support new Azure image versioning scheme (#1039)

The Canonical CPC team recently changed the versioning scheme for Ubuntu
Azure VM images from:

```json
{
    "architecture": "x64",
    "offer": "0001-com-ubuntu-minimal-mantic",
    "publisher": "Canonical",
    "sku": "minimal-23_10-gen2",
    "urn": "Canonical:0001-com-ubuntu-minimal-mantic:minimal-23_10-gen2:23.10.202406260",
    "version": "23.10.202406260"
}
```

To something like:

```json
{
    "architecture": "x64",
    "offer": "ubuntu-24_04-lts",
    "publisher": "Canonical",
    "sku": "minimal",
    "urn": "Canonical:ubuntu-24_04-lts:minimal:24.04.202404230",
    "version": "24.04.202404230"
}
```
To make matters worse, previous supported Ubuntu versions (Focal, Jammy,
Mantic) still use the old version and there’s no plan to migrate them as
well, thus we have to maintain both versioning schemes.

Update the relevant script to differentiate between old and new
codenames according to the specification above. We now require an
additional `version` CLI parameter since the new versioning scheme does
not use the codename at all. This is a hard requirement just for the
sake of consistency, even if the previous versioning scheme does not
need a version.

The other main difference is that with the new versioning scheme images
are Gen2 by default, while Gen1 images are explicitly noted in the SKU.

Fixes UDENG-3379

------------------

The additional commits reverse some codename-specific logic in order to
add support for Oracular.
Here's a run on my fork with all cells passing (including Oracular):
https://github.com/GabrielNagy/adsys/actions/runs/9766528441

.github/workflows/e2e-build-images.yaml

@@ -20,6 +20,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       matrix: ${{ steps.set-supported-releases.outputs.matrix }}
+      versions: ${{ steps.set-supported-releases.outputs.versions }}
     steps:
       - name: Install needed binaries
         run: |
@@ -30,23 +31,32 @@ jobs:
         run: |
           set -eu
 
-          all="$(distro-info --supported-esm) $(distro-info --supported)"
-          all="$(echo $all | tr ' ' '\n' | sort -u)"
+          codenames="$(distro-info --supported-esm)\n$(distro-info --supported)\n"
+          versions="$(distro-info --supported-esm --release)\n$(distro-info --supported --release)\n"
 
-          releases=""
+          # Paste the codenames and versions together, sort them, and remove duplicates
+          codenames_with_versions="$(paste <(printf "$versions" | cut -d' ' -f1) <(printf "$codenames") | sort -u)"
 
-          for r in ${all}; do
+          releases=""
+          versions=""
+          while IFS=$'\t' read -r version codename; do
               # Filter out unsupported LTS releases
-              if [ "${r}" = "trusty" -o "${r}" = "xenial" -o "${r}" = "bionic" ]; then
+              if [[ "${codename}" =~ trusty|xenial|bionic ]]; then
                   continue
               fi
 
               if [ -n "${releases}" ]; then
                   releases="${releases}, "
               fi
-              releases="${releases}'${r}'"
-          done
+              releases="${releases}'${codename}'"
+
+              if [ -n "${versions}" ]; then
+                  versions="${versions}, "
+              fi
+              versions="${versions}\"${codename}\": \"${version}\""
+          done <<< "$codenames_with_versions"
 
+          echo versions="{${versions}}" >> $GITHUB_OUTPUT
           echo matrix="${releases}" >> $GITHUB_OUTPUT
 
   build-template:
@@ -75,6 +85,8 @@ jobs:
           chmod 600 ~/.ssh/adsys-e2e.pem
       - name: Check if template needs to be created
         id: check-vm-template
+        env:
+          versions: ${{ needs.supported-releases.outputs.versions }}
         run: |
           set -eu
 
@@ -83,7 +95,9 @@ jobs:
               force="--force"
           fi
 
-          IMAGE_VERSION=$(go run ./e2e/cmd/build_base_image/00_check_vm_image --codename ${{ matrix.codename }} ${force})
+          version="$(echo $versions | jq -r .${{ matrix.codename }})"
+          codename="${{ matrix.codename }}"
+          IMAGE_VERSION=$(go run ./e2e/cmd/build_base_image/00_check_vm_image --codename ${codename} --version ${version} ${force})
           if [ ! -z "${IMAGE_VERSION}" ]; then
               echo image-version=$IMAGE_VERSION >> $GITHUB_OUTPUT
           fi

.github/workflows/e2e-tests.yaml

@@ -36,33 +36,42 @@ jobs:
       - uses: azure/login@v2
         with:
           creds: ${{ secrets.AZURE_CREDENTIALS }}
+      - uses: actions/checkout@v4
+        with:
+          repository: ${{ inputs.repository || github.repository }}
+          ref: ${{ inputs.branch || github.ref }}
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
       - name: Build matrix
         id: set-supported-releases
         run: |
           set -eu
 
-          all="$(distro-info --supported-esm) $(distro-info --supported)"
-          all="$(echo $all | tr ' ' '\n' | sort -u)"
+          codenames="$(distro-info --supported-esm)\n$(distro-info --supported)\n"
+          versions="$(distro-info --supported-esm --release)\n$(distro-info --supported --release)\n"
 
-          releases=""
+          # Paste the codenames and versions together, sort them, and remove duplicates
+          codenames_with_versions="$(paste <(printf "$versions" | cut -d' ' -f1) <(printf "$codenames") | sort -u)"
 
-          for r in ${all}; do
+          releases=""
+          while IFS=$'\t' read -r version codename; do
               # Filter out unsupported LTS releases
-              if [ "${r}" = "trusty" -o "${r}" = "xenial" -o "${r}" = "bionic" ]; then
+              if [[ "${codename}" =~ trusty|xenial|bionic ]]; then
                   continue
               fi
 
               # Filter out releases with no corresponding Azure images
-              images="$(az vm image list --publisher Canonical --offer 0001-com-ubuntu-minimal-${r} --all)"
-              if [ "$(jq '(. | length) == 0' <<<${images})" = "true"  ]; then
+              latestImageVersion="$(go run ./e2e/cmd/build_base_image/00_check_vm_image --codename ${codename} --version ${version} --force)"
+              if [ -z "${latestImageVersion}" ]; then
                   continue
               fi
 
               if [ -n "${releases}" ]; then
                   releases="${releases}, "
               fi
-              releases="${releases}'${r}'"
-          done
+              releases="${releases}'${codename}'"
+          done <<< "$codenames_with_versions"
 
           echo matrix="${releases}" >> $GITHUB_OUTPUT
 

e2e/cmd/build_base_image/00_check_vm_image/main.go

@@ -16,7 +16,7 @@ import (
 	"github.com/ubuntu/adsys/e2e/internal/command"
 )
 
-var codename string
+var codename, version string
 var force bool
 
 func main() {
@@ -40,20 +40,22 @@ If the --force flag is set, the script will return the latest image URN
 regardless of custom image availability.
 
 Options:
- --codename              Required: codename of the Ubuntu release (e.g. focal)
+ --codename              Required: codename of the Ubuntu release (e.g. noble)
+ --version               Required: version of the Ubuntu release (e.g. 24.04)
  -f, --force             Force the script to return the latest image URN
                          regardless of whether we have a custom image or not
 `, filepath.Base(os.Args[0]))
 	cmd.AddStringFlag(&codename, "codename", "", "")
+	cmd.AddStringFlag(&version, "version", "", "")
 	cmd.AddBoolFlag(&force, "force", false, "")
 	cmd.AddBoolFlag(&force, "f", false, "")
 
 	return cmd.Execute(context.Background())
 }
 
 func validate(_ context.Context, _ *command.Command) error {
-	if codename == "" {
-		return errors.New("codename must be specified")
+	if codename == "" || version == "" {
+		return errors.New("codename and version must be specified")
 	}
 
 	return nil
@@ -62,7 +64,7 @@ func validate(_ context.Context, _ *command.Command) error {
 func action(ctx context.Context, _ *command.Command) error {
 	var noStable, noDaily bool
 
-	availableImages, err := az.ImageList(ctx, codename)
+	availableImages, err := az.ImageList(ctx, codename, version)
 	if err != nil {
 		return err
 	}

e2e/internal/az/image.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"slices"
+	"strconv"
 	"strings"
 
 	"github.com/maruel/natural"
@@ -15,6 +16,10 @@ import (
 // NullImageVersion is the version returned when no image version is found.
 const NullImageVersion = "0.0.0"
 
+// oldImageVersionCodenames is a list of Ubuntu codenames for which Azure images
+// are built using the previous image versioning format.
+var oldImageVersionCodenames = []string{"focal", "jammy", "mantic"}
+
 // Image contains information about an Azure image.
 type Image struct {
 	Architecture string `json:"architecture"`
@@ -39,12 +44,34 @@ func ImageDefinitionName(codename string) string {
 }
 
 // ImageList returns a list of Azure images for the given codename.
-func ImageList(ctx context.Context, codename string) (Images, error) {
-	out, _, err := RunCommand(ctx, "vm", "image", "list",
-		"--publisher", "Canonical",
-		"--offer", fmt.Sprintf("0001-com-ubuntu-minimal-%s", codename),
-		"--all",
-	)
+func ImageList(ctx context.Context, codename, version string) (Images, error) {
+	versionParts := strings.Split(version, ".")
+	if len(versionParts) != 2 {
+		return nil, fmt.Errorf("invalid version format: %s", version)
+	}
+
+	// Determine if this is an LTS release
+	year, err := strconv.Atoi(versionParts[0])
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse version year as a number: %w", err)
+	}
+	isLTS := year%2 == 0 && versionParts[1] == "04"
+
+	version = fmt.Sprintf("ubuntu-%s", strings.Join(versionParts, "_"))
+	if isLTS {
+		version += "-lts"
+	}
+
+	var filterArgs []string
+	filterArgs = []string{"--offer", version, "--sku", "minimal"}
+	if slices.Contains(oldImageVersionCodenames, codename) {
+		filterArgs = []string{"--offer", fmt.Sprintf("0001-com-ubuntu-minimal-%s", codename)}
+	}
+
+	args := []string{"vm", "image", "list", "--publisher", "Canonical", "--all"}
+	args = append(args, filterArgs...)
+
+	out, _, err := RunCommand(ctx, args...)
 	if err != nil {
 		return nil, err
 	}
@@ -108,7 +135,12 @@ func (i Image) isDailyImage() bool {
 }
 
 func (i Image) isGen2Image() bool {
-	return strings.Contains(i.SKU, "gen2")
+	// gen2 images are explicitly defined in the old versioning scheme.
+	if strings.Contains(i.Offer, "0001-com-ubuntu") {
+		return strings.Contains(i.SKU, "gen2")
+	}
+
+	return !strings.Contains(i.SKU, "gen1")
 }
 
 // LatestImageVersion returns the latest image version for the given image definition.

e2e/scripts/Dockerfile.build

@@ -11,7 +11,10 @@ RUN set -ex \
     && apt-get upgrade -y --no-install-recommends
 
 ARG CODENAME=latest
-RUN if [ "$CODENAME" != "noble" ]; then \
+
+# Keep the codename list easy to maintain, keeping in mind that we might need to
+# add more as the need to backport various packages arises.
+RUN if echo "focal" | grep -q "$CODENAME"; then \
         set -ex \
         && apt-get install -y software-properties-common \
         && add-apt-repository -y ppa:ubuntu-enterprise-desktop/golang; \

e2e/scripts/first-run.sh

@@ -4,7 +4,7 @@ set -eu
 
 # This script runs on the first boot of the VM.
 echo "Setting hostname..."
-hostname="$(lsb_release -cs)-$(openssl rand -hex 4)"
+hostname="$(lsb_release -cs)-$(openssl rand -hex 2)"
 hostnamectl set-hostname "$hostname"
 
 echo "Adding hostname to hosts file..."

e2e/scripts/provision.sh

@@ -32,10 +32,10 @@ echo "Updating DNS resolver to use AD DNS..."
 echo "DNS=10.1.0.4" >> /etc/systemd/resolved.conf
 systemctl restart systemd-resolved
 
-# Work around an issue on Jammy and Noble where systemd-networkd times out due
-# to eth0 losing connectivity shortly after boot, even though network works fine
-# as reported by Azure.
-if [[ "$(lsb_release -cs)" =~ ^(jammy|noble)$ ]]; then
+# Work around an issue on newer Ubuntu versions (starting with Jammy) where
+# systemd-networkd times out due to eth0 losing connectivity shortly after boot,
+# even though network works fine as reported by Azure.
+if [ ! "$(lsb_release -cs)" = "focal" ]; then
     echo "Disabling misbehaving systemd-networkd-wait-online.service..."
     systemctl mask systemd-networkd-wait-online.service
 fi