From e8758dcbad6d3188dfccd235dbab04c19dd1a6ed Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Fri, 22 Dec 2023 16:10:11 +0400 Subject: [PATCH] chore: support http downloads for assets in talosctl cluster create This allows to pass direct URLs to Image Factory assets for disk image/ISO/vmlinuz/initramfs, so that we can test Image Factory with Talos. Also add an integration test for Image Factory. Signed-off-by: Andrey Smirnov --- .drone.jsonnet | 84 ++++++++++++++++++++++++ cmd/talosctl/cmd/mgmt/cluster/create.go | 78 +++++++++++++++++++++- hack/test/e2e-image-factory.sh | 87 +++++++++++++++++++++++++ hack/test/e2e.sh | 2 +- 4 files changed, 249 insertions(+), 2 deletions(-) create mode 100755 hack/test/e2e-image-factory.sh diff --git a/.drone.jsonnet b/.drone.jsonnet index e0edf078ab..f5474a8e8c 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -465,6 +465,71 @@ local integration_qemu_trusted_boot = Step('e2e-qemu-trusted-boot', target='e2e- EXTRA_TEST_ARGS: '-talos.trustedboot', }); +local integration_factory_16_iso = Step('factory-1.6-iso', target='e2e-image-factory', privileged=true, depends_on=[load_artifacts], environment={ + FACTORY_BOOT_METHOD: 'iso', + FACTORY_VERSION: 'v1.6.0', + FACTORY_SCHEMATIC: '376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba', + KUBERNETES_VERSION: '1.29.0', + FACTORY_UPGRADE: 'true', + FACTORY_UPGRADE_SCHEMATIC: 'cf9b7aab9ed7c365d5384509b4d31c02fdaa06d2b3ac6cc0bc806f28130eff1f', + FACTORY_UPGRADE_VERSION: 'v1.6.1', +}); + +local integration_factory_16_image = Step('factory-1.6-image', depends_on=[integration_factory_16_iso], target='e2e-image-factory', privileged=true, environment={ + FACTORY_BOOT_METHOD: 'disk-image', + FACTORY_VERSION: 'v1.6.0', + FACTORY_SCHEMATIC: '376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba', + KUBERNETES_VERSION: '1.29.0', + FACTORY_UPGRADE: 'true', + FACTORY_UPGRADE_SCHEMATIC: 'cf9b7aab9ed7c365d5384509b4d31c02fdaa06d2b3ac6cc0bc806f28130eff1f', + FACTORY_UPGRADE_VERSION: 'v1.6.1', +}); + +local integration_factory_16_pxe = Step('factory-1.6-pxe', depends_on=[integration_factory_16_image], target='e2e-image-factory', privileged=true, environment={ + FACTORY_BOOT_METHOD: 'pxe', + FACTORY_VERSION: 'v1.6.1', + FACTORY_SCHEMATIC: '376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba', + KUBERNETES_VERSION: '1.29.0', +}); + +local integration_factory_16_secureboot = Step('factory-1.6-secureboot', depends_on=[integration_factory_16_pxe], target='e2e-image-factory', privileged=true, environment={ + FACTORY_BOOT_METHOD: 'secureboot-iso', + FACTORY_VERSION: 'v1.6.0', + FACTORY_SCHEMATIC: 'cf9b7aab9ed7c365d5384509b4d31c02fdaa06d2b3ac6cc0bc806f28130eff1f', + KUBERNETES_VERSION: '1.29.0', + FACTORY_UPGRADE: 'true', + FACTORY_UPGRADE_SCHEMATIC: '376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba', + FACTORY_UPGRADE_VERSION: 'v1.6.1', +}); + +local integration_factory_15_iso = Step('factory-1.5-iso', depends_on=[integration_factory_16_secureboot], target='e2e-image-factory', privileged=true, environment={ + FACTORY_BOOT_METHOD: 'iso', + FACTORY_VERSION: 'v1.5.5', + FACTORY_SCHEMATIC: '376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba', + KUBERNETES_VERSION: '1.28.5', + FACTORY_UPGRADE: 'true', + FACTORY_UPGRADE_SCHEMATIC: 'cf9b7aab9ed7c365d5384509b4d31c02fdaa06d2b3ac6cc0bc806f28130eff1f', + FACTORY_UPGRADE_VERSION: 'v1.5.5', +}); + +local integration_factory_13_iso = Step('factory-1.3-iso', depends_on=[integration_factory_15_iso], target='e2e-image-factory', privileged=true, environment={ + FACTORY_BOOT_METHOD: 'iso', + FACTORY_VERSION: 'v1.3.7', + FACTORY_SCHEMATIC: '376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba', + KUBERNETES_VERSION: '1.26.5', + FACTORY_UPGRADE: 'true', + FACTORY_UPGRADE_SCHEMATIC: 'cf9b7aab9ed7c365d5384509b4d31c02fdaa06d2b3ac6cc0bc806f28130eff1f', + FACTORY_UPGRADE_VERSION: 'v1.3.7', +}); + +local integration_factory_13_image = Step('factory-1.3-image', depends_on=[integration_factory_13_iso], target='e2e-image-factory', privileged=true, environment={ + FACTORY_BOOT_METHOD: 'disk-image', + FACTORY_VERSION: 'v1.3.7', + FACTORY_SCHEMATIC: '376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba', + KUBERNETES_VERSION: '1.26.5', +}); + + local build_race = Step('build-race', target='initramfs installer', depends_on=[load_artifacts], environment={ IMAGE_REGISTRY: local_registry, PUSH: true, TAG_SUFFIX: '-race', WITH_RACE: '1', PLATFORM: 'linux/amd64' }); local integration_qemu_race = Step('e2e-qemu-race', target='e2e-qemu', privileged=true, depends_on=[build_race], environment={ IMAGE_REGISTRY: local_registry, TAG_SUFFIX: '-race' }); @@ -642,6 +707,15 @@ local integration_pipelines = [ Pipeline('integration-images', default_pipeline_steps + [integration_images, integration_sbcs]) + integration_trigger(['integration-images']), Pipeline('integration-reproducibility-test', default_pipeline_steps + [integration_reproducibility_test]) + integration_trigger(['integration-reproducibility']), Pipeline('integration-cloud-images', default_pipeline_steps + [integration_images, integration_cloud_images]) + literal_trigger(['integration-cloud-images']), + Pipeline('image-factory', default_pipeline_steps + [ + integration_factory_16_iso, + integration_factory_16_image, + integration_factory_16_pxe, + integration_factory_16_secureboot, + integration_factory_15_iso, + integration_factory_13_iso, + integration_factory_13_image, + ]) + literal_trigger(['image-factory']), // cron pipelines, triggered on schedule events Pipeline('cron-integration-qemu', default_pipeline_steps + [integration_qemu, push_edge], [default_cron_pipeline]) + cron_trigger(['thrice-daily', 'nightly']), @@ -666,6 +740,16 @@ local integration_pipelines = [ Pipeline('cron-integration-qemu-csi', default_pipeline_steps + [integration_qemu_csi], [default_cron_pipeline]) + cron_trigger(['nightly']), Pipeline('cron-integration-images', default_pipeline_steps + [integration_images, integration_sbcs], [default_cron_pipeline]) + cron_trigger(['nightly']), Pipeline('cron-integration-reproducibility-test', default_pipeline_steps + [integration_reproducibility_test], [default_cron_pipeline]) + cron_trigger(['nightly']), + Pipeline('cron-image-factory', default_pipeline_steps + [ + integration_factory_16_iso, + integration_factory_16_image, + integration_factory_16_pxe, + integration_factory_16_secureboot, + integration_factory_15_iso, + integration_factory_13_iso, + integration_factory_13_image, + ], + [default_cron_pipeline]) + cron_trigger(['nightly']), ]; diff --git a/cmd/talosctl/cmd/mgmt/cluster/create.go b/cmd/talosctl/cmd/mgmt/cluster/create.go index 97ebc15d11..d3ca2b873f 100644 --- a/cmd/talosctl/cmd/mgmt/cluster/create.go +++ b/cmd/talosctl/cmd/mgmt/cluster/create.go @@ -10,6 +10,7 @@ import ( "fmt" "math/big" "net/netip" + "net/url" "os" "path/filepath" stdruntime "runtime" @@ -18,6 +19,7 @@ import ( "time" "github.com/dustin/go-humanize" + "github.com/hashicorp/go-getter/v2" "github.com/siderolabs/go-blockdevice/blockdevice/encryption" "github.com/siderolabs/go-kubeconfig" "github.com/siderolabs/go-procfs/procfs" @@ -180,8 +182,82 @@ var createCmd = &cobra.Command{ }, } +func downloadBootAssets(ctx context.Context) error { + // download & cache images if provides as URLs + for _, downloadableImage := range []*string{ + &nodeVmlinuzPath, + &nodeInitramfsPath, + &nodeISOPath, + &nodeDiskImagePath, + } { + if *downloadableImage == "" { + continue + } + + u, err := url.Parse(*downloadableImage) + if err != nil || !(u.Scheme == "http" || u.Scheme == "https") { + // not a URL + continue + } + + defaultStateDir, err := clientconfig.GetTalosDirectory() + if err != nil { + return err + } + + cacheDir := filepath.Join(defaultStateDir, "cache") + + if os.MkdirAll(cacheDir, 0o755) != nil { + return err + } + + destPath := strings.ReplaceAll( + strings.ReplaceAll(u.String(), "/", "-"), + ":", "-") + + _, err = os.Stat(filepath.Join(cacheDir, destPath)) + if err == nil { + *downloadableImage = filepath.Join(cacheDir, destPath) + + // already cached + continue + } + + fmt.Fprintf(os.Stderr, "downloading asset from %q to %q\n", u.String(), filepath.Join(cacheDir, destPath)) + + client := getter.Client{ + Getters: []getter.Getter{ + &getter.HttpGetter{ + HeadFirstTimeout: 30 * time.Minute, + ReadTimeout: 30 * time.Minute, + }, + }, + } + + _, err = client.Get(ctx, &getter.Request{ + Src: u.String(), + Dst: filepath.Join(cacheDir, destPath), + GetMode: getter.ModeFile, + }) + if err != nil { + // clean up the destination on failure + os.Remove(filepath.Join(cacheDir, destPath)) //nolint:errcheck + + return err + } + + *downloadableImage = filepath.Join(cacheDir, destPath) + } + + return nil +} + //nolint:gocyclo,cyclop -func create(ctx context.Context, flags *pflag.FlagSet) (err error) { +func create(ctx context.Context, flags *pflag.FlagSet) error { + if err := downloadBootAssets(ctx); err != nil { + return err + } + if controlplanes < 1 { return fmt.Errorf("number of controlplanes can't be less than 1") } diff --git a/hack/test/e2e-image-factory.sh b/hack/test/e2e-image-factory.sh new file mode 100755 index 0000000000..ee6b70c8d6 --- /dev/null +++ b/hack/test/e2e-image-factory.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash + +set -eou pipefail + +# shellcheck source=/dev/null +source ./hack/test/e2e.sh + +PROVISIONER=qemu +CLUSTER_NAME="e2e-${PROVISIONER}" + +FACTORY_HOSTNAME=${FACTORY_HOSTNAME:-factory.talos.dev} +PXE_FACTORY_HOSTNAME=${PXE_FACTORY_HOSTNAME:-pxe.factory.talos.dev} +FACTORY_SCHEME=${FACTORY_SCHEME:-https} +INSTALLER_IMAGE_NAME=${INSTALLER_IMAGE_NAME:-installer} + +case "${FACTORY_BOOT_METHOD:-iso}" in + iso) + QEMU_FLAGS+=("--iso-path=${FACTORY_SCHEME}://${FACTORY_HOSTNAME}/image/${FACTORY_SCHEMATIC}/${FACTORY_VERSION}/metal-amd64.iso") + ;; + disk-image) + QEMU_FLAGS+=("--disk-image-path=${FACTORY_SCHEME}://${FACTORY_HOSTNAME}/image/${FACTORY_SCHEMATIC}/${FACTORY_VERSION}/metal-amd64.raw.xz") + ;; + ipxe) + QEMU_FLAGS+=("--ipxe-boot-script=${FACTORY_SCHEME}://${PXE_FACTORY_HOSTNAME}/pxe/${FACTORY_SCHEMATIC}/${FACTORY_VERSION}/metal-amd64") + ;; + secureboot-iso) + QEMU_FLAGS+=("--iso-path=${FACTORY_SCHEME}://${FACTORY_HOSTNAME}/image/${FACTORY_SCHEMATIC}/${FACTORY_VERSION}/metal-amd64-secureboot.iso" "--with-tpm2" "--encrypt-ephemeral" "--encrypt-state" "--disk-encryption-key-types=tpm") + INSTALLER_IMAGE_NAME=installer-secureboot + ;; +esac + +function assert_secureboot { + if [[ "${FACTORY_BOOT_METHOD:-iso}" != "secureboot-iso" ]]; then + return + fi + + ${TALOSCTL} get securitystate -o json + ${TALOSCTL} get securitystate -o json | jq -e '.spec.secureBoot == true' +} + +function create_cluster { + build_registry_mirrors + + "${TALOSCTL}" cluster create \ + --provisioner="${PROVISIONER}" \ + --name="${CLUSTER_NAME}" \ + --kubernetes-version="${KUBERNETES_VERSION}" \ + --controlplanes=3 \ + --workers="${QEMU_WORKERS:-1}" \ + --disk=15360 \ + --mtu=1450 \ + --memory=2048 \ + --memory-workers="${QEMU_MEMORY_WORKERS:-2048}" \ + --cpus="${QEMU_CPUS:-2}" \ + --cpus-workers="${QEMU_CPUS_WORKERS:-2}" \ + --cidr=172.20.1.0/24 \ + --cni-bundle-url="${ARTIFACTS}/talosctl-cni-bundle-\${ARCH}.tar.gz" \ + --skip-injecting-config \ + --with-apply-config \ + --talos-version="${FACTORY_VERSION}" \ + --install-image="${FACTORY_HOSTNAME}/${INSTALLER_IMAGE_NAME}/${FACTORY_SCHEMATIC}:${FACTORY_VERSION}" \ + --crashdump \ + "${REGISTRY_MIRROR_FLAGS[@]}" \ + "${QEMU_FLAGS[@]}" + + ${TALOSCTL} config node 172.20.1.2 +} + +function destroy_cluster() { + "${TALOSCTL}" cluster destroy --name "${CLUSTER_NAME}" --provisioner "${PROVISIONER}" +} + +create_cluster + +${TALOSCTL} health --run-e2e +${TALOSCTL} version | grep "${FACTORY_VERSION}" +${TALOSCTL} get extensions | grep "${FACTORY_SCHEMATIC}" +assert_secureboot + +if [[ "${FACTORY_UPGRADE:-false}" == "true" ]]; then + ${TALOSCTL} upgrade -i "${FACTORY_HOSTNAME}/${INSTALLER_IMAGE_NAME}/${FACTORY_UPGRADE_SCHEMATIC:-$FACTORY_SCHEMATIC}:${FACTORY_UPGRADE_VERSION:-$FACTORY_VERSION}" + ${TALOSCTL} version | grep "${FACTORY_UPGRADE_VERSION:-$FACTORY_VERSION}" + ${TALOSCTL} get extensions | grep "${FACTORY_UPGRADE_SCHEMATIC:-$FACTORY_SCHEMATIC}" + assert_secureboot +fi + +destroy_cluster diff --git a/hack/test/e2e.sh b/hack/test/e2e.sh index 519a06a24f..933169e2ed 100755 --- a/hack/test/e2e.sh +++ b/hack/test/e2e.sh @@ -217,7 +217,7 @@ function build_registry_mirrors { done else # use the value from the environment, if present - REGISTRY_MIRROR_FLAGS=("${REGISTRY_MIRROR_FLAGS:-}") + REGISTRY_MIRROR_FLAGS=(${REGISTRY_MIRROR_FLAGS:-}) fi }