diff --git a/cmd/kops/integration_test.go b/cmd/kops/integration_test.go index 4d144e6521875..a29d67a375455 100644 --- a/cmd/kops/integration_test.go +++ b/cmd/kops/integration_test.go @@ -326,9 +326,19 @@ func TestMixedInstancesSpotASG(t *testing.T) { newIntegrationTest("mixedinstances.example.com", "mixed_instances_spot").withZones(3).runTestCloudformation(t) } -// TestContainerdCloudformation runs the test on a containerd configuration -func TestContainerdCloudformation(t *testing.T) { - newIntegrationTest("containerd.example.com", "containerd-cloudformation").runTestCloudformation(t) +// TestContainerd runs the test on a containerd configuration +func TestContainerd(t *testing.T) { + newIntegrationTest("containerd.example.com", "containerd").runTestCloudformation(t) +} + +// TestContainerdCustom runs the test on a custom containerd URL configuration +func TestContainerdCustom(t *testing.T) { + newIntegrationTest("containerd.example.com", "containerd-custom").runTestCloudformation(t) +} + +// TestDockerCustom runs the test on a custom Docker URL configuration +func TestDockerCustom(t *testing.T) { + newIntegrationTest("docker.example.com", "docker-custom").runTestCloudformation(t) } // TestLaunchConfigurationASG tests ASGs using launch configurations instead of launch templates diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md index fdd7a315910bc..52487304edaad 100644 --- a/docs/cluster_spec.md +++ b/docs/cluster_spec.md @@ -907,21 +907,49 @@ spec: ## containerd +### Configuration + It is possible to override the [containerd](https://github.com/containerd/containerd/blob/master/README.md) daemon options for all the nodes in the cluster. See the [API docs](https://pkg.go.dev/k8s.io/kops/pkg/apis/kops#ContainerdConfig) for the full list of options. ```yaml spec: containerd: - version: 1.3.3 + version: 1.4.3 logLevel: info configOverride: "" ``` -## docker +### Custom Packages + +kOps uses the `.tar.gz` packages for installing containerd on any supported OS. This makes it easy to use a custom build or pre-release packages, by specifying its URL and sha256: + +```yaml +spec: + containerd: + packages: + urlAmd64: https://github.com/containerd/containerd/releases/download/v1.4.3/cri-containerd-cni-1.4.3-linux-amd64.tar.gz + hashAmd64: 2697a342e3477c211ab48313e259fd7e32ad1f5ded19320e6a559f50a82bff3d +``` + +The format of the custom package must be identical to the official packages: + +```bash +tar tf cri-containerd-cni-1.4.3-linux-amd64.tar.gz + usr/local/bin/containerd + usr/local/bin/containerd-shim + usr/local/bin/containerd-shim-runc-v1 + usr/local/bin/containerd-shim-runc-v2 + usr/local/bin/crictl + usr/local/bin/critest + usr/local/bin/ctr + usr/local/sbin/runc +``` + +## Docker It is possible to override Docker daemon options for all masters and nodes in the cluster. See the [API docs](https://pkg.go.dev/k8s.io/kops/pkg/apis/kops#DockerConfig) for the full list of options. -### registryMirrors +### Registry Mirrors If you have a bunch of Docker instances (physical or vm) running, each time one of them pulls an image that is not present on the host, it will fetch it from the internet (DockerHub). By caching these images, you can keep the traffic within your local network and avoid egress bandwidth usage. This setting benefits not only cluster provisioning but also image pulling. @@ -948,7 +976,7 @@ spec: **NOTE:** When this field is set to `true`, it is entirely up to the user to install and configure Docker. -### storage +### Storage The Docker [Storage Driver](https://docs.docker.com/engine/reference/commandline/dockerd/#daemon-storage-driver) can be specified in order to override the default. Be sure the driver you choose is supported by your operating system and docker version. @@ -961,7 +989,7 @@ docker: - "dm.use_deferred_removal=true" ``` -### networking +### Networking In order for containers started with `docker run` instead of Kubernetes to have network and internet access you need to enable the necessary [iptables](https://docs.docker.com/network/iptables/) rules: @@ -971,6 +999,33 @@ docker: ipTables: true ``` +### Custom Packages + +kOps uses the `.tgz` (static) packages for installing Docker on any supported OS. This makes it easy to use a custom build or pre-release packages, by specifying its URL and sha256: + +```yaml +spec: + containerd: + packages: + urlAmd64: https://download.docker.com/linux/static/stable/x86_64/docker-20.10.1.tgz + hashAmd64: 8790f3b94ee07ca69a9fdbd1310cbffc729af0a07e5bf9f34a79df1e13d2e50e +``` + +The format of the custom package must be identical to the official packages: + +```bash +tar tf docker-20.10.1.tgz + docker/containerd + docker/containerd-shim + docker/containerd-shim-runc-v2 + docker/ctr + docker/docker + docker/docker-init + docker/docker-proxy + docker/dockerd + docker/runc +``` + ## sshKeyName In some cases, it may be desirable to use an existing AWS SSH key instead of allowing kOps to create a new one. diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml index 291acd122371e..9c39bbdd48788 100644 --- a/k8s/crds/kops.k8s.io_clusters.yaml +++ b/k8s/crds/kops.k8s.io_clusters.yaml @@ -483,28 +483,45 @@ spec: description: Component configurations properties: address: - description: Address of containerd's GRPC server (default "/run/containerd/containerd.sock") + description: Address of containerd's GRPC server (default "/run/containerd/containerd.sock"). type: string configOverride: - description: Complete containerd config file provided by the user + description: ConfigOverride is the complete containerd config + file provided by the user. type: string logLevel: - description: Logging level [trace, debug, info, warn, error, fatal, - panic] (default "info") + description: LogLevel controls the logging details [trace, debug, + info, warn, error, fatal, panic] (default "info"). type: string + packages: + description: Packages overrides the URL and hash for the packages. + properties: + hashAmd64: + description: HashAmd64 overrides the hash for the AMD64 package. + type: string + hashArm64: + description: HashArm64 overrides the hash for the ARM64 package. + type: string + urlAmd64: + description: UrlAmd64 overrides the URL for the AMD64 package. + type: string + urlArm64: + description: UrlArm64 overrides the URL for the ARM64 package. + type: string + type: object root: - description: Directory for persistent data (default "/var/lib/containerd") + description: Root directory for persistent data (default "/var/lib/containerd"). type: string skipInstall: - description: Prevents kops from installing and modifying containerd - in any way (default "false") + description: SkipInstall prevents kOps from installing and modifying + containerd in any way (default "false"). type: boolean state: - description: Directory for execution state files (default "/run/containerd") + description: State directory for execution state files (default + "/run/containerd"). type: string version: - description: Consumed by nodeup and used to pick the containerd - version + description: Version used to pick the containerd package. type: string type: object dnsControllerGossipConfig: @@ -634,6 +651,22 @@ spec: description: MTU is the containers network MTU format: int32 type: integer + packages: + description: Packages overrides the URL and hash for the packages. + properties: + hashAmd64: + description: HashAmd64 overrides the hash for the AMD64 package. + type: string + hashArm64: + description: HashArm64 overrides the hash for the ARM64 package. + type: string + urlAmd64: + description: UrlAmd64 overrides the URL for the AMD64 package. + type: string + urlArm64: + description: UrlArm64 overrides the URL for the ARM64 package. + type: string + type: object registryMirrors: description: RegistryMirrors is a referred list of docker registry mirror diff --git a/pkg/apis/kops/cluster.go b/pkg/apis/kops/cluster.go index 59e77a2541630..ea0ffbe43359f 100644 --- a/pkg/apis/kops/cluster.go +++ b/pkg/apis/kops/cluster.go @@ -789,3 +789,14 @@ type RollingUpdate struct { // +optional MaxSurge *intstr.IntOrString `json:"maxSurge,omitempty"` } + +type PackagesConfig struct { + // HashAmd64 overrides the hash for the AMD64 package. + HashAmd64 *string `json:"hashAmd64,omitempty"` + // HashArm64 overrides the hash for the ARM64 package. + HashArm64 *string `json:"hashArm64,omitempty"` + // UrlAmd64 overrides the URL for the AMD64 package. + UrlAmd64 *string `json:"urlAmd64,omitempty"` + // UrlArm64 overrides the URL for the ARM64 package. + UrlArm64 *string `json:"urlArm64,omitempty"` +} diff --git a/pkg/apis/kops/containerdconfig.go b/pkg/apis/kops/containerdconfig.go index 138da1b35bfda..4217cd34529b8 100644 --- a/pkg/apis/kops/containerdconfig.go +++ b/pkg/apis/kops/containerdconfig.go @@ -18,18 +18,20 @@ package kops // ContainerdConfig is the configuration for containerd type ContainerdConfig struct { - // Address of containerd's GRPC server (default "/run/containerd/containerd.sock") + // Address of containerd's GRPC server (default "/run/containerd/containerd.sock"). Address *string `json:"address,omitempty" flag:"address"` - // Complete containerd config file provided by the user + // ConfigOverride is the complete containerd config file provided by the user. ConfigOverride *string `json:"configOverride,omitempty"` - // Logging level [trace, debug, info, warn, error, fatal, panic] (default "info") + // LogLevel controls the logging details [trace, debug, info, warn, error, fatal, panic] (default "info"). LogLevel *string `json:"logLevel,omitempty" flag:"log-level"` - // Directory for persistent data (default "/var/lib/containerd") + // Packages overrides the URL and hash for the packages. + Packages *PackagesConfig `json:"packages,omitempty"` + // Root directory for persistent data (default "/var/lib/containerd"). Root *string `json:"root,omitempty" flag:"root"` - // Prevents kops from installing and modifying containerd in any way (default "false") + // SkipInstall prevents kOps from installing and modifying containerd in any way (default "false"). SkipInstall bool `json:"skipInstall,omitempty"` - // Directory for execution state files (default "/run/containerd") + // State directory for execution state files (default "/run/containerd"). State *string `json:"state,omitempty" flag:"state"` - // Consumed by nodeup and used to pick the containerd version + // Version used to pick the containerd package. Version *string `json:"version,omitempty"` } diff --git a/pkg/apis/kops/dockerconfig.go b/pkg/apis/kops/dockerconfig.go index d79b5d308fc28..4ad5f573e6593 100644 --- a/pkg/apis/kops/dockerconfig.go +++ b/pkg/apis/kops/dockerconfig.go @@ -60,6 +60,8 @@ type DockerConfig struct { MetricsAddress *string `json:"metricsAddress,omitempty" flag:"metrics-addr"` // MTU is the containers network MTU MTU *int32 `json:"mtu,omitempty" flag:"mtu"` + // Packages overrides the URL and hash for the packages. + Packages *PackagesConfig `json:"packages,omitempty"` // RegistryMirrors is a referred list of docker registry mirror RegistryMirrors []string `json:"registryMirrors,omitempty" flag:"registry-mirror,repeat"` // Runtimes registers an additional OCI compatible runtime (default []) diff --git a/pkg/apis/kops/v1alpha2/cluster.go b/pkg/apis/kops/v1alpha2/cluster.go index 2066ebf3b6eee..ae827b7c4f073 100644 --- a/pkg/apis/kops/v1alpha2/cluster.go +++ b/pkg/apis/kops/v1alpha2/cluster.go @@ -674,3 +674,14 @@ type RollingUpdate struct { // +optional MaxSurge *intstr.IntOrString `json:"maxSurge,omitempty"` } + +type PackagesConfig struct { + // HashAmd64 overrides the hash for the AMD64 package. + HashAmd64 *string `json:"hashAmd64,omitempty"` + // HashArm64 overrides the hash for the ARM64 package. + HashArm64 *string `json:"hashArm64,omitempty"` + // UrlAmd64 overrides the URL for the AMD64 package. + UrlAmd64 *string `json:"urlAmd64,omitempty"` + // UrlArm64 overrides the URL for the ARM64 package. + UrlArm64 *string `json:"urlArm64,omitempty"` +} diff --git a/pkg/apis/kops/v1alpha2/containerdconfig.go b/pkg/apis/kops/v1alpha2/containerdconfig.go index d1b9d8bca8870..e5aa79219b310 100644 --- a/pkg/apis/kops/v1alpha2/containerdconfig.go +++ b/pkg/apis/kops/v1alpha2/containerdconfig.go @@ -18,18 +18,20 @@ package v1alpha2 // ContainerdConfig is the configuration for containerd type ContainerdConfig struct { - // Address of containerd's GRPC server (default "/run/containerd/containerd.sock") + // Address of containerd's GRPC server (default "/run/containerd/containerd.sock"). Address *string `json:"address,omitempty" flag:"address"` - // Complete containerd config file provided by the user + // ConfigOverride is the complete containerd config file provided by the user. ConfigOverride *string `json:"configOverride,omitempty"` - // Logging level [trace, debug, info, warn, error, fatal, panic] (default "info") + // LogLevel controls the logging details [trace, debug, info, warn, error, fatal, panic] (default "info"). LogLevel *string `json:"logLevel,omitempty" flag:"log-level"` - // Directory for persistent data (default "/var/lib/containerd") + // Packages overrides the URL and hash for the packages. + Packages *PackagesConfig `json:"packages,omitempty"` + // Root directory for persistent data (default "/var/lib/containerd"). Root *string `json:"root,omitempty" flag:"root"` - // Prevents kops from installing and modifying containerd in any way (default "false") + // SkipInstall prevents kOps from installing and modifying containerd in any way (default "false"). SkipInstall bool `json:"skipInstall,omitempty"` - // Directory for execution state files (default "/run/containerd") + // State directory for execution state files (default "/run/containerd"). State *string `json:"state,omitempty" flag:"state"` - // Consumed by nodeup and used to pick the containerd version + // Version used to pick the containerd package. Version *string `json:"version,omitempty"` } diff --git a/pkg/apis/kops/v1alpha2/dockerconfig.go b/pkg/apis/kops/v1alpha2/dockerconfig.go index 1d014e75d9102..e38f3cfd7676c 100644 --- a/pkg/apis/kops/v1alpha2/dockerconfig.go +++ b/pkg/apis/kops/v1alpha2/dockerconfig.go @@ -60,6 +60,8 @@ type DockerConfig struct { MetricsAddress *string `json:"metricsAddress,omitempty" flag:"metrics-addr"` // MTU is the containers network MTU MTU *int32 `json:"mtu,omitempty" flag:"mtu"` + // Packages overrides the URL and hash for the packages. + Packages *PackagesConfig `json:"packages,omitempty"` // RegistryMirrors is a referred list of docker registry mirror RegistryMirrors []string `json:"registryMirrors,omitempty" flag:"registry-mirror,repeat"` // Runtimes registers an additional OCI compatible runtime (default []) diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 9abb204713354..570128f6c716e 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -853,6 +853,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*PackagesConfig)(nil), (*kops.PackagesConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha2_PackagesConfig_To_kops_PackagesConfig(a.(*PackagesConfig), b.(*kops.PackagesConfig), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*kops.PackagesConfig)(nil), (*PackagesConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_kops_PackagesConfig_To_v1alpha2_PackagesConfig(a.(*kops.PackagesConfig), b.(*PackagesConfig), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*RBACAuthorizationSpec)(nil), (*kops.RBACAuthorizationSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha2_RBACAuthorizationSpec_To_kops_RBACAuthorizationSpec(a.(*RBACAuthorizationSpec), b.(*kops.RBACAuthorizationSpec), scope) }); err != nil { @@ -2682,6 +2692,15 @@ func autoConvert_v1alpha2_ContainerdConfig_To_kops_ContainerdConfig(in *Containe out.Address = in.Address out.ConfigOverride = in.ConfigOverride out.LogLevel = in.LogLevel + if in.Packages != nil { + in, out := &in.Packages, &out.Packages + *out = new(kops.PackagesConfig) + if err := Convert_v1alpha2_PackagesConfig_To_kops_PackagesConfig(*in, *out, s); err != nil { + return err + } + } else { + out.Packages = nil + } out.Root = in.Root out.SkipInstall = in.SkipInstall out.State = in.State @@ -2698,6 +2717,15 @@ func autoConvert_kops_ContainerdConfig_To_v1alpha2_ContainerdConfig(in *kops.Con out.Address = in.Address out.ConfigOverride = in.ConfigOverride out.LogLevel = in.LogLevel + if in.Packages != nil { + in, out := &in.Packages, &out.Packages + *out = new(PackagesConfig) + if err := Convert_kops_PackagesConfig_To_v1alpha2_PackagesConfig(*in, *out, s); err != nil { + return err + } + } else { + out.Packages = nil + } out.Root = in.Root out.SkipInstall = in.SkipInstall out.State = in.State @@ -2840,6 +2868,15 @@ func autoConvert_v1alpha2_DockerConfig_To_kops_DockerConfig(in *DockerConfig, ou out.LogOpt = in.LogOpt out.MetricsAddress = in.MetricsAddress out.MTU = in.MTU + if in.Packages != nil { + in, out := &in.Packages, &out.Packages + *out = new(kops.PackagesConfig) + if err := Convert_v1alpha2_PackagesConfig_To_kops_PackagesConfig(*in, *out, s); err != nil { + return err + } + } else { + out.Packages = nil + } out.RegistryMirrors = in.RegistryMirrors out.Runtimes = in.Runtimes out.SelinuxEnabled = in.SelinuxEnabled @@ -2878,6 +2915,15 @@ func autoConvert_kops_DockerConfig_To_v1alpha2_DockerConfig(in *kops.DockerConfi out.LogOpt = in.LogOpt out.MetricsAddress = in.MetricsAddress out.MTU = in.MTU + if in.Packages != nil { + in, out := &in.Packages, &out.Packages + *out = new(PackagesConfig) + if err := Convert_kops_PackagesConfig_To_v1alpha2_PackagesConfig(*in, *out, s); err != nil { + return err + } + } else { + out.Packages = nil + } out.RegistryMirrors = in.RegistryMirrors out.Runtimes = in.Runtimes out.SelinuxEnabled = in.SelinuxEnabled @@ -5600,6 +5646,32 @@ func Convert_kops_OpenstackRouter_To_v1alpha2_OpenstackRouter(in *kops.Openstack return autoConvert_kops_OpenstackRouter_To_v1alpha2_OpenstackRouter(in, out, s) } +func autoConvert_v1alpha2_PackagesConfig_To_kops_PackagesConfig(in *PackagesConfig, out *kops.PackagesConfig, s conversion.Scope) error { + out.HashAmd64 = in.HashAmd64 + out.HashArm64 = in.HashArm64 + out.UrlAmd64 = in.UrlAmd64 + out.UrlArm64 = in.UrlArm64 + return nil +} + +// Convert_v1alpha2_PackagesConfig_To_kops_PackagesConfig is an autogenerated conversion function. +func Convert_v1alpha2_PackagesConfig_To_kops_PackagesConfig(in *PackagesConfig, out *kops.PackagesConfig, s conversion.Scope) error { + return autoConvert_v1alpha2_PackagesConfig_To_kops_PackagesConfig(in, out, s) +} + +func autoConvert_kops_PackagesConfig_To_v1alpha2_PackagesConfig(in *kops.PackagesConfig, out *PackagesConfig, s conversion.Scope) error { + out.HashAmd64 = in.HashAmd64 + out.HashArm64 = in.HashArm64 + out.UrlAmd64 = in.UrlAmd64 + out.UrlArm64 = in.UrlArm64 + return nil +} + +// Convert_kops_PackagesConfig_To_v1alpha2_PackagesConfig is an autogenerated conversion function. +func Convert_kops_PackagesConfig_To_v1alpha2_PackagesConfig(in *kops.PackagesConfig, out *PackagesConfig, s conversion.Scope) error { + return autoConvert_kops_PackagesConfig_To_v1alpha2_PackagesConfig(in, out, s) +} + func autoConvert_v1alpha2_RBACAuthorizationSpec_To_kops_RBACAuthorizationSpec(in *RBACAuthorizationSpec, out *kops.RBACAuthorizationSpec, s conversion.Scope) error { return nil } diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go index b99f6d2bd129a..6251a782227d4 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go @@ -1007,6 +1007,11 @@ func (in *ContainerdConfig) DeepCopyInto(out *ContainerdConfig) { *out = new(string) **out = **in } + if in.Packages != nil { + in, out := &in.Packages, &out.Packages + *out = new(PackagesConfig) + (*in).DeepCopyInto(*out) + } if in.Root != nil { in, out := &in.Root, &out.Root *out = new(string) @@ -1247,6 +1252,11 @@ func (in *DockerConfig) DeepCopyInto(out *DockerConfig) { *out = new(int32) **out = **in } + if in.Packages != nil { + in, out := &in.Packages, &out.Packages + *out = new(PackagesConfig) + (*in).DeepCopyInto(*out) + } if in.RegistryMirrors != nil { in, out := &in.RegistryMirrors, &out.RegistryMirrors *out = make([]string, len(*in)) @@ -3850,6 +3860,42 @@ func (in *OpenstackRouter) DeepCopy() *OpenstackRouter { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PackagesConfig) DeepCopyInto(out *PackagesConfig) { + *out = *in + if in.HashAmd64 != nil { + in, out := &in.HashAmd64, &out.HashAmd64 + *out = new(string) + **out = **in + } + if in.HashArm64 != nil { + in, out := &in.HashArm64, &out.HashArm64 + *out = new(string) + **out = **in + } + if in.UrlAmd64 != nil { + in, out := &in.UrlAmd64, &out.UrlAmd64 + *out = new(string) + **out = **in + } + if in.UrlArm64 != nil { + in, out := &in.UrlArm64, &out.UrlArm64 + *out = new(string) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PackagesConfig. +func (in *PackagesConfig) DeepCopy() *PackagesConfig { + if in == nil { + return nil + } + out := new(PackagesConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RBACAuthorizationSpec) DeepCopyInto(out *RBACAuthorizationSpec) { *out = *in diff --git a/pkg/apis/kops/validation/validation.go b/pkg/apis/kops/validation/validation.go index 6e038a289c7bc..f3dfe1e15b0ac 100644 --- a/pkg/apis/kops/validation/validation.go +++ b/pkg/apis/kops/validation/validation.go @@ -20,6 +20,7 @@ import ( "errors" "fmt" "net" + "net/url" "regexp" "strings" @@ -1116,7 +1117,50 @@ func validateContainerdConfig(config *kops.ContainerdConfig, fldPath *field.Path fmt.Sprintf("unable to parse version string: %s", err.Error()))) } if sv.LT(semver.MustParse("1.3.4")) { - allErrs = append(allErrs, field.Invalid(fldPath.Child("version"), config.Version, "unsupported legacy version")) + allErrs = append(allErrs, field.Invalid(fldPath.Child("version"), config.Version, + "unsupported legacy version")) + } + } + + if config.Packages != nil { + if config.Packages.UrlAmd64 != nil && config.Packages.HashAmd64 != nil { + u := fi.StringValue(config.Packages.UrlAmd64) + _, err := url.Parse(u) + if err != nil { + allErrs = append(allErrs, field.Invalid(fldPath.Child("packageUrl"), config.Packages.UrlAmd64, + fmt.Sprintf("cannot parse package URL: %v", err))) + } + h := fi.StringValue(config.Packages.HashAmd64) + if len(h) > 64 { + allErrs = append(allErrs, field.Invalid(fldPath.Child("packageHash"), config.Packages.HashAmd64, + "Package hash must be 64 characters long")) + } + } else if config.Packages.UrlAmd64 != nil { + allErrs = append(allErrs, field.Invalid(fldPath.Child("packageUrl"), config.Packages.HashAmd64, + "Package hash must also be set")) + } else if config.Packages.HashAmd64 != nil { + allErrs = append(allErrs, field.Invalid(fldPath.Child("packageHash"), config.Packages.HashAmd64, + "Package URL must also be set")) + } + + if config.Packages.UrlArm64 != nil && config.Packages.HashArm64 != nil { + u := fi.StringValue(config.Packages.UrlArm64) + _, err := url.Parse(u) + if err != nil { + allErrs = append(allErrs, field.Invalid(fldPath.Child("packageUrlArm64"), config.Packages.UrlArm64, + fmt.Sprintf("cannot parse package URL: %v", err))) + } + h := fi.StringValue(config.Packages.HashArm64) + if len(h) > 64 { + allErrs = append(allErrs, field.Invalid(fldPath.Child("packageHashArm64"), config.Packages.HashArm64, + "Package hash must be 64 characters long")) + } + } else if config.Packages.UrlArm64 != nil { + allErrs = append(allErrs, field.Invalid(fldPath.Child("packageUrlArm64"), config.Packages.HashArm64, + "Package hash must also be set")) + } else if config.Packages.HashArm64 != nil { + allErrs = append(allErrs, field.Invalid(fldPath.Child("packageHashArm64"), config.Packages.HashArm64, + "Package URL must also be set")) } } @@ -1136,7 +1180,50 @@ func validateDockerConfig(config *kops.DockerConfig, fldPath *field.Path) field. allErrs = append(allErrs, field.Invalid(fldPath.Child("version"), config.Version, "version is no longer available: https://www.docker.com/blog/changes-dockerproject-org-apt-yum-repositories")) } else if sv.LT(semver.MustParse("17.3.0")) { - allErrs = append(allErrs, field.Invalid(fldPath.Child("version"), config.Version, "unsupported legacy version")) + allErrs = append(allErrs, field.Invalid(fldPath.Child("version"), config.Version, + "unsupported legacy version")) + } + } + + if config.Packages != nil { + if config.Packages.UrlAmd64 != nil && config.Packages.HashAmd64 != nil { + u := fi.StringValue(config.Packages.UrlAmd64) + _, err := url.Parse(u) + if err != nil { + allErrs = append(allErrs, field.Invalid(fldPath.Child("packageUrl"), config.Packages.UrlAmd64, + fmt.Sprintf("unable parse package URL string: %v", err))) + } + h := fi.StringValue(config.Packages.HashAmd64) + if len(h) > 64 { + allErrs = append(allErrs, field.Invalid(fldPath.Child("packageHash"), config.Packages.HashAmd64, + "Package hash must be 64 characters long")) + } + } else if config.Packages.UrlAmd64 != nil { + allErrs = append(allErrs, field.Invalid(fldPath.Child("packageUrl"), config.Packages.HashAmd64, + "Package hash must also be set")) + } else if config.Packages.HashAmd64 != nil { + allErrs = append(allErrs, field.Invalid(fldPath.Child("packageHash"), config.Packages.HashAmd64, + "Package URL must also be set")) + } + + if config.Packages.UrlArm64 != nil && config.Packages.HashArm64 != nil { + u := fi.StringValue(config.Packages.UrlArm64) + _, err := url.Parse(u) + if err != nil { + allErrs = append(allErrs, field.Invalid(fldPath.Child("packageUrlArm64"), config.Packages.UrlArm64, + fmt.Sprintf("unable parse package URL string: %v", err))) + } + h := fi.StringValue(config.Packages.HashArm64) + if len(h) > 64 { + allErrs = append(allErrs, field.Invalid(fldPath.Child("packageHashArm64"), config.Packages.HashArm64, + "Package hash must be 64 characters long")) + } + } else if config.Packages.UrlArm64 != nil { + allErrs = append(allErrs, field.Invalid(fldPath.Child("packageUrlArm64"), config.Packages.HashArm64, + "Package hash must also be set")) + } else if config.Packages.HashArm64 != nil { + allErrs = append(allErrs, field.Invalid(fldPath.Child("packageHashArm64"), config.Packages.HashArm64, + "Package URL must also be set")) } } diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go index de06d0f1e3946..2e0a49319b99a 100644 --- a/pkg/apis/kops/zz_generated.deepcopy.go +++ b/pkg/apis/kops/zz_generated.deepcopy.go @@ -1130,6 +1130,11 @@ func (in *ContainerdConfig) DeepCopyInto(out *ContainerdConfig) { *out = new(string) **out = **in } + if in.Packages != nil { + in, out := &in.Packages, &out.Packages + *out = new(PackagesConfig) + (*in).DeepCopyInto(*out) + } if in.Root != nil { in, out := &in.Root, &out.Root *out = new(string) @@ -1370,6 +1375,11 @@ func (in *DockerConfig) DeepCopyInto(out *DockerConfig) { *out = new(int32) **out = **in } + if in.Packages != nil { + in, out := &in.Packages, &out.Packages + *out = new(PackagesConfig) + (*in).DeepCopyInto(*out) + } if in.RegistryMirrors != nil { in, out := &in.RegistryMirrors, &out.RegistryMirrors *out = make([]string, len(*in)) @@ -4064,6 +4074,42 @@ func (in *OpenstackRouter) DeepCopy() *OpenstackRouter { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PackagesConfig) DeepCopyInto(out *PackagesConfig) { + *out = *in + if in.HashAmd64 != nil { + in, out := &in.HashAmd64, &out.HashAmd64 + *out = new(string) + **out = **in + } + if in.HashArm64 != nil { + in, out := &in.HashArm64, &out.HashArm64 + *out = new(string) + **out = **in + } + if in.UrlAmd64 != nil { + in, out := &in.UrlAmd64, &out.UrlAmd64 + *out = new(string) + **out = **in + } + if in.UrlArm64 != nil { + in, out := &in.UrlArm64, &out.UrlArm64 + *out = new(string) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PackagesConfig. +func (in *PackagesConfig) DeepCopy() *PackagesConfig { + if in == nil { + return nil + } + out := new(PackagesConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RBACAuthorizationSpec) DeepCopyInto(out *RBACAuthorizationSpec) { *out = *in diff --git a/tests/integration/update_cluster/containerd-cloudformation/cloudformation.json b/tests/integration/update_cluster/containerd-custom/cloudformation.json similarity index 100% rename from tests/integration/update_cluster/containerd-cloudformation/cloudformation.json rename to tests/integration/update_cluster/containerd-custom/cloudformation.json diff --git a/tests/integration/update_cluster/containerd-custom/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/containerd-custom/cloudformation.json.extracted.yaml new file mode 100644 index 0000000000000..fe5c8b6aa203d --- /dev/null +++ b/tests/integration/update_cluster/containerd-custom/cloudformation.json.extracted.yaml @@ -0,0 +1,534 @@ +Resources.AWSEC2LaunchTemplatemasterustest1amasterscontainerdexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.19.0-alpha.3/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.19.0-alpha.3/nodeup-linux-amd64,https://kubeupv2.s3.amazonaws.com/kops/1.19.0-alpha.3/linux/amd64/nodeup + NODEUP_HASH_AMD64=6980fda4fa37bbdc043738cf4ddac6388eb57f561895c69299c1b0ee263d465d + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.19.0-alpha.3/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.19.0-alpha.3/nodeup-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.19.0-alpha.3/linux/arm64/nodeup + NODEUP_HASH_ARM64=dcc7f9f3c180ee76a511627e46da0ac69cdcb518cdf3be348e5ed046d491eb87 + + export AWS_REGION=us-test-1 + + + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, url1, url2... + download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: null + containerRuntime: containerd + containerd: + configOverride: |- + version = 2 + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".cni] + conf_template = "/etc/containerd/cni-config.template" + logLevel: info + packages: + hashAmd64: "0000000000000000000000000000000000000000000000000000000000000000" + urlAmd64: https://github.com/containerd/containerd/releases/download/v1.3.9/cri-containerd-cni-1.3.9-linux-amd64.tar.gz + version: 1.4.3 + docker: + skipInstall: true + encryptionConfig: null + etcdClusters: + events: + version: 3.4.13 + main: + version: 3.4.13 + kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.19.0 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 + kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: containerd.example.com + configureCloudRoutes: true + image: k8s.gcr.io/kube-controller-manager:v1.19.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.19.0 + logLevel: 2 + kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.19.0 + leaderElection: + leaderElect: true + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + masterKubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 3f03e5c160a8b658d30b34824a1c00abadbac96e62c4d01bf5c9271a2debc3ab@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/amd64/kubelet + - 79bb0d2f05487ff533999a639c075043c70a0a1ba25c1629eb1eef6ebe3ba70f@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 0000000000000000000000000000000000000000000000000000000000000000@https://github.com/containerd/containerd/releases/download/v1.3.9/cri-containerd-cni-1.3.9-linux-amd64.tar.gz + arm64: + - d8fa5a9739ecc387dfcc55afa91ac6f4b0ccd01f1423c423dbd312d787bbb6bf@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/arm64/kubelet + - d4adf1b6b97252025cb2f7febf55daa3f42dc305822e3da133f77fd33071ec2f@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 6e3f80e8451ecbe7b3559247721c3e226be6b228acaadee7e13683f80c20e81c@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.0.tgz + ClusterName: containerd.example.com + ConfigBase: memfs://clusters.example.com/containerd.example.com + InstanceGroupName: master-us-test-1a + InstanceGroupRole: Master + KubeletConfig: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + nodeLabels: + kubernetes.io/role: master + node-role.kubernetes.io/master: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + channels: + - memfs://clusters.example.com/containerd.example.com/addons/bootstrap-channel.yaml + etcdManifests: + - memfs://clusters.example.com/containerd.example.com/manifests/etcd/main.yaml + - memfs://clusters.example.com/containerd.example.com/manifests/etcd/events.yaml + protokubeImage: + amd64: + hash: 7b3c7f6adbda11b1ec740bd6b969c84f249b7eee818af95f2d321963088245a8 + name: protokube:1.19.0-alpha.3 + sources: + - https://artifacts.k8s.io/binaries/kops/1.19.0-alpha.3/images/protokube-amd64.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.19.0-alpha.3/images-protokube-amd64.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.19.0-alpha.3/images/protokube-amd64.tar.gz + arm64: + hash: 69270ca9c1c950be65af40337adfccec0a728930fa3224bb0d2e88f181f39ead + name: protokube:1.19.0-alpha.3 + sources: + - https://artifacts.k8s.io/binaries/kops/1.19.0-alpha.3/images/protokube-arm64.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.19.0-alpha.3/images-protokube-arm64.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.19.0-alpha.3/images/protokube-arm64.tar.gz + staticManifests: + - key: kube-apiserver-healthcheck + path: manifests/static/kube-apiserver-healthcheck.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" +Resources.AWSEC2LaunchTemplatenodescontainerdexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.19.0-alpha.3/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.19.0-alpha.3/nodeup-linux-amd64,https://kubeupv2.s3.amazonaws.com/kops/1.19.0-alpha.3/linux/amd64/nodeup + NODEUP_HASH_AMD64=6980fda4fa37bbdc043738cf4ddac6388eb57f561895c69299c1b0ee263d465d + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.19.0-alpha.3/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.19.0-alpha.3/nodeup-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.19.0-alpha.3/linux/arm64/nodeup + NODEUP_HASH_ARM64=dcc7f9f3c180ee76a511627e46da0ac69cdcb518cdf3be348e5ed046d491eb87 + + export AWS_REGION=us-test-1 + + + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, url1, url2... + download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: null + containerRuntime: containerd + containerd: + configOverride: |- + version = 2 + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".cni] + conf_template = "/etc/containerd/cni-config.template" + logLevel: info + packages: + hashAmd64: "0000000000000000000000000000000000000000000000000000000000000000" + urlAmd64: https://github.com/containerd/containerd/releases/download/v1.3.9/cri-containerd-cni-1.3.9-linux-amd64.tar.gz + version: 1.4.3 + docker: + skipInstall: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.19.0 + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 3f03e5c160a8b658d30b34824a1c00abadbac96e62c4d01bf5c9271a2debc3ab@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/amd64/kubelet + - 79bb0d2f05487ff533999a639c075043c70a0a1ba25c1629eb1eef6ebe3ba70f@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 0000000000000000000000000000000000000000000000000000000000000000@https://github.com/containerd/containerd/releases/download/v1.3.9/cri-containerd-cni-1.3.9-linux-amd64.tar.gz + arm64: + - d8fa5a9739ecc387dfcc55afa91ac6f4b0ccd01f1423c423dbd312d787bbb6bf@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/arm64/kubelet + - d4adf1b6b97252025cb2f7febf55daa3f42dc305822e3da133f77fd33071ec2f@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 6e3f80e8451ecbe7b3559247721c3e226be6b228acaadee7e13683f80c20e81c@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.0.tgz + ClusterName: containerd.example.com + ConfigBase: memfs://clusters.example.com/containerd.example.com + InstanceGroupName: nodes + InstanceGroupRole: Node + KubeletConfig: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + nodeLabels: + kubernetes.io/role: node + node-role.kubernetes.io/node: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podManifestPath: /etc/kubernetes/manifests + channels: + - memfs://clusters.example.com/containerd.example.com/addons/bootstrap-channel.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/containerd-cloudformation/id_rsa.pub b/tests/integration/update_cluster/containerd-custom/id_rsa.pub similarity index 100% rename from tests/integration/update_cluster/containerd-cloudformation/id_rsa.pub rename to tests/integration/update_cluster/containerd-custom/id_rsa.pub diff --git a/tests/integration/update_cluster/containerd-custom/in-v1alpha2.yaml b/tests/integration/update_cluster/containerd-custom/in-v1alpha2.yaml new file mode 100644 index 0000000000000..87e108d2a613b --- /dev/null +++ b/tests/integration/update_cluster/containerd-custom/in-v1alpha2.yaml @@ -0,0 +1,85 @@ +apiVersion: kops.k8s.io/v1alpha2 +kind: Cluster +metadata: + creationTimestamp: "2016-12-10T22:42:27Z" + name: containerd.example.com +spec: + kubernetesApiAccess: + - 0.0.0.0/0 + channel: stable + cloudProvider: aws + configBase: memfs://clusters.example.com/containerd.example.com + containerRuntime: containerd + containerd: + packages: + urlAmd64: https://github.com/containerd/containerd/releases/download/v1.3.9/cri-containerd-cni-1.3.9-linux-amd64.tar.gz + hashAmd64: "0000000000000000000000000000000000000000000000000000000000000000" + etcdClusters: + - etcdMembers: + - instanceGroup: master-us-test-1a + name: us-test-1a + name: main + - etcdMembers: + - instanceGroup: master-us-test-1a + name: us-test-1a + name: events + iam: {} + kubelet: + anonymousAuth: false + kubernetesVersion: v1.19.0 + masterInternalName: api.internal.containerd.example.com + masterPublicName: api.containerd.example.com + networkCIDR: 172.20.0.0/16 + networking: + kubenet: {} + nonMasqueradeCIDR: 100.64.0.0/10 + sshAccess: + - 0.0.0.0/0 + topology: + masters: public + nodes: public + subnets: + - cidr: 172.20.32.0/19 + name: us-test-1a + type: Public + zone: us-test-1a + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2016-12-10T22:42:28Z" + name: nodes + labels: + kops.k8s.io/cluster: containerd.example.com +spec: + associatePublicIp: true + image: kope.io/k8s-1.14-debian-stretch-amd64-hvm-ebs-2019-08-16 + machineType: t2.medium + maxSize: 2 + minSize: 2 + role: Node + subnets: + - us-test-1a + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2016-12-10T22:42:28Z" + name: master-us-test-1a + labels: + kops.k8s.io/cluster: containerd.example.com +spec: + associatePublicIp: true + image: kope.io/k8s-1.14-debian-stretch-amd64-hvm-ebs-2019-08-16 + machineType: m3.medium + maxSize: 1 + minSize: 1 + role: Master + subnets: + - us-test-1a + + diff --git a/tests/integration/update_cluster/containerd/cloudformation.json b/tests/integration/update_cluster/containerd/cloudformation.json new file mode 100644 index 0000000000000..e9844062c1390 --- /dev/null +++ b/tests/integration/update_cluster/containerd/cloudformation.json @@ -0,0 +1,1084 @@ +{ + "Resources": { + "AWSAutoScalingAutoScalingGroupmasterustest1amasterscontainerdexamplecom": { + "Type": "AWS::AutoScaling::AutoScalingGroup", + "Properties": { + "AutoScalingGroupName": "master-us-test-1a.masters.containerd.example.com", + "LaunchTemplate": { + "LaunchTemplateId": { + "Ref": "AWSEC2LaunchTemplatemasterustest1amasterscontainerdexamplecom" + }, + "Version": { + "Fn::GetAtt": [ + "AWSEC2LaunchTemplatemasterustest1amasterscontainerdexamplecom", + "LatestVersionNumber" + ] + } + }, + "MaxSize": "1", + "MinSize": "1", + "VPCZoneIdentifier": [ + { + "Ref": "AWSEC2Subnetustest1acontainerdexamplecom" + } + ], + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "containerd.example.com", + "PropagateAtLaunch": true + }, + { + "Key": "Name", + "Value": "master-us-test-1a.masters.containerd.example.com", + "PropagateAtLaunch": true + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role", + "Value": "master", + "PropagateAtLaunch": true + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master", + "Value": "", + "PropagateAtLaunch": true + }, + { + "Key": "k8s.io/role/master", + "Value": "1", + "PropagateAtLaunch": true + }, + { + "Key": "kops.k8s.io/instancegroup", + "Value": "master-us-test-1a", + "PropagateAtLaunch": true + }, + { + "Key": "kubernetes.io/cluster/containerd.example.com", + "Value": "owned", + "PropagateAtLaunch": true + } + ], + "MetricsCollection": [ + { + "Granularity": "1Minute", + "Metrics": [ + "GroupDesiredCapacity", + "GroupInServiceInstances", + "GroupMaxSize", + "GroupMinSize", + "GroupPendingInstances", + "GroupStandbyInstances", + "GroupTerminatingInstances", + "GroupTotalInstances" + ] + } + ] + } + }, + "AWSAutoScalingAutoScalingGroupnodescontainerdexamplecom": { + "Type": "AWS::AutoScaling::AutoScalingGroup", + "Properties": { + "AutoScalingGroupName": "nodes.containerd.example.com", + "LaunchTemplate": { + "LaunchTemplateId": { + "Ref": "AWSEC2LaunchTemplatenodescontainerdexamplecom" + }, + "Version": { + "Fn::GetAtt": [ + "AWSEC2LaunchTemplatenodescontainerdexamplecom", + "LatestVersionNumber" + ] + } + }, + "MaxSize": "2", + "MinSize": "2", + "VPCZoneIdentifier": [ + { + "Ref": "AWSEC2Subnetustest1acontainerdexamplecom" + } + ], + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "containerd.example.com", + "PropagateAtLaunch": true + }, + { + "Key": "Name", + "Value": "nodes.containerd.example.com", + "PropagateAtLaunch": true + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role", + "Value": "node", + "PropagateAtLaunch": true + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node", + "Value": "", + "PropagateAtLaunch": true + }, + { + "Key": "k8s.io/role/node", + "Value": "1", + "PropagateAtLaunch": true + }, + { + "Key": "kops.k8s.io/instancegroup", + "Value": "nodes", + "PropagateAtLaunch": true + }, + { + "Key": "kubernetes.io/cluster/containerd.example.com", + "Value": "owned", + "PropagateAtLaunch": true + } + ], + "MetricsCollection": [ + { + "Granularity": "1Minute", + "Metrics": [ + "GroupDesiredCapacity", + "GroupInServiceInstances", + "GroupMaxSize", + "GroupMinSize", + "GroupPendingInstances", + "GroupStandbyInstances", + "GroupTerminatingInstances", + "GroupTotalInstances" + ] + } + ] + } + }, + "AWSEC2DHCPOptionscontainerdexamplecom": { + "Type": "AWS::EC2::DHCPOptions", + "Properties": { + "DomainName": "us-test-1.compute.internal", + "DomainNameServers": [ + "AmazonProvidedDNS" + ], + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "containerd.example.com" + }, + { + "Key": "Name", + "Value": "containerd.example.com" + }, + { + "Key": "kubernetes.io/cluster/containerd.example.com", + "Value": "owned" + } + ] + } + }, + "AWSEC2InternetGatewaycontainerdexamplecom": { + "Type": "AWS::EC2::InternetGateway", + "Properties": { + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "containerd.example.com" + }, + { + "Key": "Name", + "Value": "containerd.example.com" + }, + { + "Key": "kubernetes.io/cluster/containerd.example.com", + "Value": "owned" + } + ] + } + }, + "AWSEC2LaunchTemplatemasterustest1amasterscontainerdexamplecom": { + "Type": "AWS::EC2::LaunchTemplate", + "Properties": { + "LaunchTemplateName": "master-us-test-1a.masters.containerd.example.com", + "LaunchTemplateData": { + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/xvda", + "Ebs": { + "VolumeType": "gp2", + "VolumeSize": 64, + "DeleteOnTermination": true, + "Encrypted": false + } + }, + { + "DeviceName": "/dev/sdc", + "VirtualName": "ephemeral0" + } + ], + "IamInstanceProfile": { + "Name": { + "Ref": "AWSIAMInstanceProfilemasterscontainerdexamplecom" + } + }, + "ImageId": "ami-11400000", + "InstanceType": "m3.medium", + "KeyName": "kubernetes.containerd.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57", + "MetadataOptions": { + "HttpPutResponseHopLimit": 1, + "HttpTokens": "optional" + }, + "NetworkInterfaces": [ + { + "AssociatePublicIpAddress": true, + "DeleteOnTermination": true, + "DeviceIndex": 0, + "Groups": [ + { + "Ref": "AWSEC2SecurityGroupmasterscontainerdexamplecom" + } + ] + } + ], + "TagSpecifications": [ + { + "ResourceType": "instance", + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "containerd.example.com" + }, + { + "Key": "Name", + "Value": "master-us-test-1a.masters.containerd.example.com" + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role", + "Value": "master" + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master", + "Value": "" + }, + { + "Key": "k8s.io/role/master", + "Value": "1" + }, + { + "Key": "kops.k8s.io/instancegroup", + "Value": "master-us-test-1a" + }, + { + "Key": "kubernetes.io/cluster/containerd.example.com", + "Value": "owned" + } + ] + }, + { + "ResourceType": "volume", + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "containerd.example.com" + }, + { + "Key": "Name", + "Value": "master-us-test-1a.masters.containerd.example.com" + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role", + "Value": "master" + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master", + "Value": "" + }, + { + "Key": "k8s.io/role/master", + "Value": "1" + }, + { + "Key": "kops.k8s.io/instancegroup", + "Value": "master-us-test-1a" + }, + { + "Key": "kubernetes.io/cluster/containerd.example.com", + "Value": "owned" + } + ] + } + ], + "UserData": "extracted" + } + } + }, + "AWSEC2LaunchTemplatenodescontainerdexamplecom": { + "Type": "AWS::EC2::LaunchTemplate", + "Properties": { + "LaunchTemplateName": "nodes.containerd.example.com", + "LaunchTemplateData": { + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/xvda", + "Ebs": { + "VolumeType": "gp2", + "VolumeSize": 128, + "DeleteOnTermination": true, + "Encrypted": false + } + } + ], + "IamInstanceProfile": { + "Name": { + "Ref": "AWSIAMInstanceProfilenodescontainerdexamplecom" + } + }, + "ImageId": "ami-11400000", + "InstanceType": "t2.medium", + "KeyName": "kubernetes.containerd.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57", + "MetadataOptions": { + "HttpPutResponseHopLimit": 1, + "HttpTokens": "optional" + }, + "NetworkInterfaces": [ + { + "AssociatePublicIpAddress": true, + "DeleteOnTermination": true, + "DeviceIndex": 0, + "Groups": [ + { + "Ref": "AWSEC2SecurityGroupnodescontainerdexamplecom" + } + ] + } + ], + "TagSpecifications": [ + { + "ResourceType": "instance", + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "containerd.example.com" + }, + { + "Key": "Name", + "Value": "nodes.containerd.example.com" + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role", + "Value": "node" + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node", + "Value": "" + }, + { + "Key": "k8s.io/role/node", + "Value": "1" + }, + { + "Key": "kops.k8s.io/instancegroup", + "Value": "nodes" + }, + { + "Key": "kubernetes.io/cluster/containerd.example.com", + "Value": "owned" + } + ] + }, + { + "ResourceType": "volume", + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "containerd.example.com" + }, + { + "Key": "Name", + "Value": "nodes.containerd.example.com" + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role", + "Value": "node" + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node", + "Value": "" + }, + { + "Key": "k8s.io/role/node", + "Value": "1" + }, + { + "Key": "kops.k8s.io/instancegroup", + "Value": "nodes" + }, + { + "Key": "kubernetes.io/cluster/containerd.example.com", + "Value": "owned" + } + ] + } + ], + "UserData": "extracted" + } + } + }, + "AWSEC2Route00000": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "AWSEC2RouteTablecontainerdexamplecom" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "GatewayId": { + "Ref": "AWSEC2InternetGatewaycontainerdexamplecom" + } + } + }, + "AWSEC2RouteTablecontainerdexamplecom": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "AWSEC2VPCcontainerdexamplecom" + }, + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "containerd.example.com" + }, + { + "Key": "Name", + "Value": "containerd.example.com" + }, + { + "Key": "kubernetes.io/cluster/containerd.example.com", + "Value": "owned" + }, + { + "Key": "kubernetes.io/kops/role", + "Value": "public" + } + ] + } + }, + "AWSEC2SecurityGroupEgressmasterscontainerdexamplecomegressall0to000000": { + "Type": "AWS::EC2::SecurityGroupEgress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupmasterscontainerdexamplecom" + }, + "FromPort": 0, + "ToPort": 0, + "IpProtocol": "-1", + "CidrIp": "0.0.0.0/0" + } + }, + "AWSEC2SecurityGroupEgressnodescontainerdexamplecomegressall0to000000": { + "Type": "AWS::EC2::SecurityGroupEgress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupnodescontainerdexamplecom" + }, + "FromPort": 0, + "ToPort": 0, + "IpProtocol": "-1", + "CidrIp": "0.0.0.0/0" + } + }, + "AWSEC2SecurityGroupIngresshttpsexternaltomaster00000": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupmasterscontainerdexamplecom" + }, + "FromPort": 443, + "ToPort": 443, + "IpProtocol": "tcp", + "CidrIp": "0.0.0.0/0" + } + }, + "AWSEC2SecurityGroupIngressmasterscontainerdexamplecomingressall0to0masterscontainerdexamplecom": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupmasterscontainerdexamplecom" + }, + "SourceSecurityGroupId": { + "Ref": "AWSEC2SecurityGroupmasterscontainerdexamplecom" + }, + "FromPort": 0, + "ToPort": 0, + "IpProtocol": "-1" + } + }, + "AWSEC2SecurityGroupIngressmasterscontainerdexamplecomingressall0to0nodescontainerdexamplecom": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupnodescontainerdexamplecom" + }, + "SourceSecurityGroupId": { + "Ref": "AWSEC2SecurityGroupmasterscontainerdexamplecom" + }, + "FromPort": 0, + "ToPort": 0, + "IpProtocol": "-1" + } + }, + "AWSEC2SecurityGroupIngressnodescontainerdexamplecomingressall0to0nodescontainerdexamplecom": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupnodescontainerdexamplecom" + }, + "SourceSecurityGroupId": { + "Ref": "AWSEC2SecurityGroupnodescontainerdexamplecom" + }, + "FromPort": 0, + "ToPort": 0, + "IpProtocol": "-1" + } + }, + "AWSEC2SecurityGroupIngressnodescontainerdexamplecomingresstcp1to2379masterscontainerdexamplecom": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupmasterscontainerdexamplecom" + }, + "SourceSecurityGroupId": { + "Ref": "AWSEC2SecurityGroupnodescontainerdexamplecom" + }, + "FromPort": 1, + "ToPort": 2379, + "IpProtocol": "tcp" + } + }, + "AWSEC2SecurityGroupIngressnodescontainerdexamplecomingresstcp2382to4000masterscontainerdexamplecom": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupmasterscontainerdexamplecom" + }, + "SourceSecurityGroupId": { + "Ref": "AWSEC2SecurityGroupnodescontainerdexamplecom" + }, + "FromPort": 2382, + "ToPort": 4000, + "IpProtocol": "tcp" + } + }, + "AWSEC2SecurityGroupIngressnodescontainerdexamplecomingresstcp4003to65535masterscontainerdexamplecom": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupmasterscontainerdexamplecom" + }, + "SourceSecurityGroupId": { + "Ref": "AWSEC2SecurityGroupnodescontainerdexamplecom" + }, + "FromPort": 4003, + "ToPort": 65535, + "IpProtocol": "tcp" + } + }, + "AWSEC2SecurityGroupIngressnodescontainerdexamplecomingressudp1to65535masterscontainerdexamplecom": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupmasterscontainerdexamplecom" + }, + "SourceSecurityGroupId": { + "Ref": "AWSEC2SecurityGroupnodescontainerdexamplecom" + }, + "FromPort": 1, + "ToPort": 65535, + "IpProtocol": "udp" + } + }, + "AWSEC2SecurityGroupIngresssshexternaltomaster00000": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupmasterscontainerdexamplecom" + }, + "FromPort": 22, + "ToPort": 22, + "IpProtocol": "tcp", + "CidrIp": "0.0.0.0/0" + } + }, + "AWSEC2SecurityGroupIngresssshexternaltonode00000": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupnodescontainerdexamplecom" + }, + "FromPort": 22, + "ToPort": 22, + "IpProtocol": "tcp", + "CidrIp": "0.0.0.0/0" + } + }, + "AWSEC2SecurityGroupmasterscontainerdexamplecom": { + "Type": "AWS::EC2::SecurityGroup", + "Properties": { + "GroupName": "masters.containerd.example.com", + "VpcId": { + "Ref": "AWSEC2VPCcontainerdexamplecom" + }, + "GroupDescription": "Security group for masters", + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "containerd.example.com" + }, + { + "Key": "Name", + "Value": "masters.containerd.example.com" + }, + { + "Key": "kubernetes.io/cluster/containerd.example.com", + "Value": "owned" + } + ] + } + }, + "AWSEC2SecurityGroupnodescontainerdexamplecom": { + "Type": "AWS::EC2::SecurityGroup", + "Properties": { + "GroupName": "nodes.containerd.example.com", + "VpcId": { + "Ref": "AWSEC2VPCcontainerdexamplecom" + }, + "GroupDescription": "Security group for nodes", + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "containerd.example.com" + }, + { + "Key": "Name", + "Value": "nodes.containerd.example.com" + }, + { + "Key": "kubernetes.io/cluster/containerd.example.com", + "Value": "owned" + } + ] + } + }, + "AWSEC2SubnetRouteTableAssociationustest1acontainerdexamplecom": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "SubnetId": { + "Ref": "AWSEC2Subnetustest1acontainerdexamplecom" + }, + "RouteTableId": { + "Ref": "AWSEC2RouteTablecontainerdexamplecom" + } + } + }, + "AWSEC2Subnetustest1acontainerdexamplecom": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "VpcId": { + "Ref": "AWSEC2VPCcontainerdexamplecom" + }, + "CidrBlock": "172.20.32.0/19", + "AvailabilityZone": "us-test-1a", + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "containerd.example.com" + }, + { + "Key": "Name", + "Value": "us-test-1a.containerd.example.com" + }, + { + "Key": "SubnetType", + "Value": "Public" + }, + { + "Key": "kubernetes.io/cluster/containerd.example.com", + "Value": "owned" + }, + { + "Key": "kubernetes.io/role/elb", + "Value": "1" + } + ] + } + }, + "AWSEC2VPCDHCPOptionsAssociationcontainerdexamplecom": { + "Type": "AWS::EC2::VPCDHCPOptionsAssociation", + "Properties": { + "VpcId": { + "Ref": "AWSEC2VPCcontainerdexamplecom" + }, + "DhcpOptionsId": { + "Ref": "AWSEC2DHCPOptionscontainerdexamplecom" + } + } + }, + "AWSEC2VPCGatewayAttachmentcontainerdexamplecom": { + "Type": "AWS::EC2::VPCGatewayAttachment", + "Properties": { + "VpcId": { + "Ref": "AWSEC2VPCcontainerdexamplecom" + }, + "InternetGatewayId": { + "Ref": "AWSEC2InternetGatewaycontainerdexamplecom" + } + } + }, + "AWSEC2VPCcontainerdexamplecom": { + "Type": "AWS::EC2::VPC", + "Properties": { + "CidrBlock": "172.20.0.0/16", + "EnableDnsHostnames": true, + "EnableDnsSupport": true, + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "containerd.example.com" + }, + { + "Key": "Name", + "Value": "containerd.example.com" + }, + { + "Key": "kubernetes.io/cluster/containerd.example.com", + "Value": "owned" + } + ] + } + }, + "AWSEC2Volumeustest1aetcdeventscontainerdexamplecom": { + "Type": "AWS::EC2::Volume", + "Properties": { + "AvailabilityZone": "us-test-1a", + "Size": 20, + "VolumeType": "gp2", + "Encrypted": false, + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "containerd.example.com" + }, + { + "Key": "Name", + "Value": "us-test-1a.etcd-events.containerd.example.com" + }, + { + "Key": "k8s.io/etcd/events", + "Value": "us-test-1a/us-test-1a" + }, + { + "Key": "k8s.io/role/master", + "Value": "1" + }, + { + "Key": "kubernetes.io/cluster/containerd.example.com", + "Value": "owned" + } + ] + } + }, + "AWSEC2Volumeustest1aetcdmaincontainerdexamplecom": { + "Type": "AWS::EC2::Volume", + "Properties": { + "AvailabilityZone": "us-test-1a", + "Size": 20, + "VolumeType": "gp2", + "Encrypted": false, + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "containerd.example.com" + }, + { + "Key": "Name", + "Value": "us-test-1a.etcd-main.containerd.example.com" + }, + { + "Key": "k8s.io/etcd/main", + "Value": "us-test-1a/us-test-1a" + }, + { + "Key": "k8s.io/role/master", + "Value": "1" + }, + { + "Key": "kubernetes.io/cluster/containerd.example.com", + "Value": "owned" + } + ] + } + }, + "AWSIAMInstanceProfilemasterscontainerdexamplecom": { + "Type": "AWS::IAM::InstanceProfile", + "Properties": { + "InstanceProfileName": "masters.containerd.example.com", + "Roles": [ + { + "Ref": "AWSIAMRolemasterscontainerdexamplecom" + } + ] + } + }, + "AWSIAMInstanceProfilenodescontainerdexamplecom": { + "Type": "AWS::IAM::InstanceProfile", + "Properties": { + "InstanceProfileName": "nodes.containerd.example.com", + "Roles": [ + { + "Ref": "AWSIAMRolenodescontainerdexamplecom" + } + ] + } + }, + "AWSIAMPolicymasterscontainerdexamplecom": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyName": "masters.containerd.example.com", + "Roles": [ + { + "Ref": "AWSIAMRolemasterscontainerdexamplecom" + } + ], + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "ec2:DescribeAccountAttributes", + "ec2:DescribeInstances", + "ec2:DescribeInternetGateways", + "ec2:DescribeRegions", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeVolumes" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "ec2:CreateSecurityGroup", + "ec2:CreateTags", + "ec2:CreateVolume", + "ec2:DescribeVolumesModifications", + "ec2:ModifyInstanceAttribute", + "ec2:ModifyVolume" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "ec2:AttachVolume", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:CreateRoute", + "ec2:DeleteRoute", + "ec2:DeleteSecurityGroup", + "ec2:DeleteVolume", + "ec2:DetachVolume", + "ec2:RevokeSecurityGroupIngress" + ], + "Condition": { + "StringEquals": { + "ec2:ResourceTag/KubernetesCluster": "containerd.example.com" + } + }, + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeTags", + "ec2:DescribeLaunchTemplateVersions" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "autoscaling:SetDesiredCapacity", + "autoscaling:TerminateInstanceInAutoScalingGroup", + "autoscaling:UpdateAutoScalingGroup" + ], + "Condition": { + "StringEquals": { + "autoscaling:ResourceTag/KubernetesCluster": "containerd.example.com" + } + }, + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "elasticloadbalancing:AddTags", + "elasticloadbalancing:AttachLoadBalancerToSubnets", + "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer", + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateLoadBalancerPolicy", + "elasticloadbalancing:CreateLoadBalancerListeners", + "elasticloadbalancing:ConfigureHealthCheck", + "elasticloadbalancing:DeleteLoadBalancer", + "elasticloadbalancing:DeleteLoadBalancerListeners", + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DetachLoadBalancerFromSubnets", + "elasticloadbalancing:DeregisterInstancesFromLoadBalancer", + "elasticloadbalancing:ModifyLoadBalancerAttributes", + "elasticloadbalancing:RegisterInstancesWithLoadBalancer", + "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "ec2:DescribeVpcs", + "elasticloadbalancing:AddTags", + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:CreateTargetGroup", + "elasticloadbalancing:DeleteListener", + "elasticloadbalancing:DeleteTargetGroup", + "elasticloadbalancing:DeregisterTargets", + "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DescribeLoadBalancerPolicies", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeTargetHealth", + "elasticloadbalancing:ModifyListener", + "elasticloadbalancing:ModifyTargetGroup", + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:SetLoadBalancerPoliciesOfListener" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "iam:ListServerCertificates", + "iam:GetServerCertificate" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "route53:ChangeResourceRecordSets", + "route53:ListResourceRecordSets", + "route53:GetHostedZone" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:route53:::hostedzone/Z1AFAKE1ZON3YO" + ] + }, + { + "Action": [ + "route53:GetChange" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:route53:::change/*" + ] + }, + { + "Action": [ + "route53:ListHostedZones" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + } + ], + "Version": "2012-10-17" + } + } + }, + "AWSIAMPolicynodescontainerdexamplecom": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyName": "nodes.containerd.example.com", + "Roles": [ + { + "Ref": "AWSIAMRolenodescontainerdexamplecom" + } + ], + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "ec2:DescribeInstances", + "ec2:DescribeRegions" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + } + ], + "Version": "2012-10-17" + } + } + }, + "AWSIAMRolemasterscontainerdexamplecom": { + "Type": "AWS::IAM::Role", + "Properties": { + "RoleName": "masters.containerd.example.com", + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + } + }, + "AWSIAMRolenodescontainerdexamplecom": { + "Type": "AWS::IAM::Role", + "Properties": { + "RoleName": "nodes.containerd.example.com", + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + } + } + } +} diff --git a/tests/integration/update_cluster/containerd-cloudformation/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/containerd/cloudformation.json.extracted.yaml similarity index 100% rename from tests/integration/update_cluster/containerd-cloudformation/cloudformation.json.extracted.yaml rename to tests/integration/update_cluster/containerd/cloudformation.json.extracted.yaml diff --git a/tests/integration/update_cluster/containerd/id_rsa.pub b/tests/integration/update_cluster/containerd/id_rsa.pub new file mode 100755 index 0000000000000..81cb0127830e7 --- /dev/null +++ b/tests/integration/update_cluster/containerd/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ== diff --git a/tests/integration/update_cluster/containerd-cloudformation/in-v1alpha2.yaml b/tests/integration/update_cluster/containerd/in-v1alpha2.yaml similarity index 100% rename from tests/integration/update_cluster/containerd-cloudformation/in-v1alpha2.yaml rename to tests/integration/update_cluster/containerd/in-v1alpha2.yaml diff --git a/tests/integration/update_cluster/docker-custom/cloudformation.json b/tests/integration/update_cluster/docker-custom/cloudformation.json new file mode 100644 index 0000000000000..262b0a1d6662d --- /dev/null +++ b/tests/integration/update_cluster/docker-custom/cloudformation.json @@ -0,0 +1,1084 @@ +{ + "Resources": { + "AWSAutoScalingAutoScalingGroupmasterustest1amastersdockerexamplecom": { + "Type": "AWS::AutoScaling::AutoScalingGroup", + "Properties": { + "AutoScalingGroupName": "master-us-test-1a.masters.docker.example.com", + "LaunchTemplate": { + "LaunchTemplateId": { + "Ref": "AWSEC2LaunchTemplatemasterustest1amastersdockerexamplecom" + }, + "Version": { + "Fn::GetAtt": [ + "AWSEC2LaunchTemplatemasterustest1amastersdockerexamplecom", + "LatestVersionNumber" + ] + } + }, + "MaxSize": "1", + "MinSize": "1", + "VPCZoneIdentifier": [ + { + "Ref": "AWSEC2Subnetustest1adockerexamplecom" + } + ], + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "docker.example.com", + "PropagateAtLaunch": true + }, + { + "Key": "Name", + "Value": "master-us-test-1a.masters.docker.example.com", + "PropagateAtLaunch": true + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role", + "Value": "master", + "PropagateAtLaunch": true + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master", + "Value": "", + "PropagateAtLaunch": true + }, + { + "Key": "k8s.io/role/master", + "Value": "1", + "PropagateAtLaunch": true + }, + { + "Key": "kops.k8s.io/instancegroup", + "Value": "master-us-test-1a", + "PropagateAtLaunch": true + }, + { + "Key": "kubernetes.io/cluster/docker.example.com", + "Value": "owned", + "PropagateAtLaunch": true + } + ], + "MetricsCollection": [ + { + "Granularity": "1Minute", + "Metrics": [ + "GroupDesiredCapacity", + "GroupInServiceInstances", + "GroupMaxSize", + "GroupMinSize", + "GroupPendingInstances", + "GroupStandbyInstances", + "GroupTerminatingInstances", + "GroupTotalInstances" + ] + } + ] + } + }, + "AWSAutoScalingAutoScalingGroupnodesdockerexamplecom": { + "Type": "AWS::AutoScaling::AutoScalingGroup", + "Properties": { + "AutoScalingGroupName": "nodes.docker.example.com", + "LaunchTemplate": { + "LaunchTemplateId": { + "Ref": "AWSEC2LaunchTemplatenodesdockerexamplecom" + }, + "Version": { + "Fn::GetAtt": [ + "AWSEC2LaunchTemplatenodesdockerexamplecom", + "LatestVersionNumber" + ] + } + }, + "MaxSize": "2", + "MinSize": "2", + "VPCZoneIdentifier": [ + { + "Ref": "AWSEC2Subnetustest1adockerexamplecom" + } + ], + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "docker.example.com", + "PropagateAtLaunch": true + }, + { + "Key": "Name", + "Value": "nodes.docker.example.com", + "PropagateAtLaunch": true + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role", + "Value": "node", + "PropagateAtLaunch": true + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node", + "Value": "", + "PropagateAtLaunch": true + }, + { + "Key": "k8s.io/role/node", + "Value": "1", + "PropagateAtLaunch": true + }, + { + "Key": "kops.k8s.io/instancegroup", + "Value": "nodes", + "PropagateAtLaunch": true + }, + { + "Key": "kubernetes.io/cluster/docker.example.com", + "Value": "owned", + "PropagateAtLaunch": true + } + ], + "MetricsCollection": [ + { + "Granularity": "1Minute", + "Metrics": [ + "GroupDesiredCapacity", + "GroupInServiceInstances", + "GroupMaxSize", + "GroupMinSize", + "GroupPendingInstances", + "GroupStandbyInstances", + "GroupTerminatingInstances", + "GroupTotalInstances" + ] + } + ] + } + }, + "AWSEC2DHCPOptionsdockerexamplecom": { + "Type": "AWS::EC2::DHCPOptions", + "Properties": { + "DomainName": "us-test-1.compute.internal", + "DomainNameServers": [ + "AmazonProvidedDNS" + ], + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "docker.example.com" + }, + { + "Key": "Name", + "Value": "docker.example.com" + }, + { + "Key": "kubernetes.io/cluster/docker.example.com", + "Value": "owned" + } + ] + } + }, + "AWSEC2InternetGatewaydockerexamplecom": { + "Type": "AWS::EC2::InternetGateway", + "Properties": { + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "docker.example.com" + }, + { + "Key": "Name", + "Value": "docker.example.com" + }, + { + "Key": "kubernetes.io/cluster/docker.example.com", + "Value": "owned" + } + ] + } + }, + "AWSEC2LaunchTemplatemasterustest1amastersdockerexamplecom": { + "Type": "AWS::EC2::LaunchTemplate", + "Properties": { + "LaunchTemplateName": "master-us-test-1a.masters.docker.example.com", + "LaunchTemplateData": { + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/xvda", + "Ebs": { + "VolumeType": "gp2", + "VolumeSize": 64, + "DeleteOnTermination": true, + "Encrypted": false + } + }, + { + "DeviceName": "/dev/sdc", + "VirtualName": "ephemeral0" + } + ], + "IamInstanceProfile": { + "Name": { + "Ref": "AWSIAMInstanceProfilemastersdockerexamplecom" + } + }, + "ImageId": "ami-11400000", + "InstanceType": "m3.medium", + "KeyName": "kubernetes.docker.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57", + "MetadataOptions": { + "HttpPutResponseHopLimit": 1, + "HttpTokens": "optional" + }, + "NetworkInterfaces": [ + { + "AssociatePublicIpAddress": true, + "DeleteOnTermination": true, + "DeviceIndex": 0, + "Groups": [ + { + "Ref": "AWSEC2SecurityGroupmastersdockerexamplecom" + } + ] + } + ], + "TagSpecifications": [ + { + "ResourceType": "instance", + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "docker.example.com" + }, + { + "Key": "Name", + "Value": "master-us-test-1a.masters.docker.example.com" + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role", + "Value": "master" + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master", + "Value": "" + }, + { + "Key": "k8s.io/role/master", + "Value": "1" + }, + { + "Key": "kops.k8s.io/instancegroup", + "Value": "master-us-test-1a" + }, + { + "Key": "kubernetes.io/cluster/docker.example.com", + "Value": "owned" + } + ] + }, + { + "ResourceType": "volume", + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "docker.example.com" + }, + { + "Key": "Name", + "Value": "master-us-test-1a.masters.docker.example.com" + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role", + "Value": "master" + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/master", + "Value": "" + }, + { + "Key": "k8s.io/role/master", + "Value": "1" + }, + { + "Key": "kops.k8s.io/instancegroup", + "Value": "master-us-test-1a" + }, + { + "Key": "kubernetes.io/cluster/docker.example.com", + "Value": "owned" + } + ] + } + ], + "UserData": "extracted" + } + } + }, + "AWSEC2LaunchTemplatenodesdockerexamplecom": { + "Type": "AWS::EC2::LaunchTemplate", + "Properties": { + "LaunchTemplateName": "nodes.docker.example.com", + "LaunchTemplateData": { + "BlockDeviceMappings": [ + { + "DeviceName": "/dev/xvda", + "Ebs": { + "VolumeType": "gp2", + "VolumeSize": 128, + "DeleteOnTermination": true, + "Encrypted": false + } + } + ], + "IamInstanceProfile": { + "Name": { + "Ref": "AWSIAMInstanceProfilenodesdockerexamplecom" + } + }, + "ImageId": "ami-11400000", + "InstanceType": "t2.medium", + "KeyName": "kubernetes.docker.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57", + "MetadataOptions": { + "HttpPutResponseHopLimit": 1, + "HttpTokens": "optional" + }, + "NetworkInterfaces": [ + { + "AssociatePublicIpAddress": true, + "DeleteOnTermination": true, + "DeviceIndex": 0, + "Groups": [ + { + "Ref": "AWSEC2SecurityGroupnodesdockerexamplecom" + } + ] + } + ], + "TagSpecifications": [ + { + "ResourceType": "instance", + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "docker.example.com" + }, + { + "Key": "Name", + "Value": "nodes.docker.example.com" + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role", + "Value": "node" + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node", + "Value": "" + }, + { + "Key": "k8s.io/role/node", + "Value": "1" + }, + { + "Key": "kops.k8s.io/instancegroup", + "Value": "nodes" + }, + { + "Key": "kubernetes.io/cluster/docker.example.com", + "Value": "owned" + } + ] + }, + { + "ResourceType": "volume", + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "docker.example.com" + }, + { + "Key": "Name", + "Value": "nodes.docker.example.com" + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/role", + "Value": "node" + }, + { + "Key": "k8s.io/cluster-autoscaler/node-template/label/node-role.kubernetes.io/node", + "Value": "" + }, + { + "Key": "k8s.io/role/node", + "Value": "1" + }, + { + "Key": "kops.k8s.io/instancegroup", + "Value": "nodes" + }, + { + "Key": "kubernetes.io/cluster/docker.example.com", + "Value": "owned" + } + ] + } + ], + "UserData": "extracted" + } + } + }, + "AWSEC2Route00000": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "AWSEC2RouteTabledockerexamplecom" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "GatewayId": { + "Ref": "AWSEC2InternetGatewaydockerexamplecom" + } + } + }, + "AWSEC2RouteTabledockerexamplecom": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "AWSEC2VPCdockerexamplecom" + }, + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "docker.example.com" + }, + { + "Key": "Name", + "Value": "docker.example.com" + }, + { + "Key": "kubernetes.io/cluster/docker.example.com", + "Value": "owned" + }, + { + "Key": "kubernetes.io/kops/role", + "Value": "public" + } + ] + } + }, + "AWSEC2SecurityGroupEgressmastersdockerexamplecomegressall0to000000": { + "Type": "AWS::EC2::SecurityGroupEgress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupmastersdockerexamplecom" + }, + "FromPort": 0, + "ToPort": 0, + "IpProtocol": "-1", + "CidrIp": "0.0.0.0/0" + } + }, + "AWSEC2SecurityGroupEgressnodesdockerexamplecomegressall0to000000": { + "Type": "AWS::EC2::SecurityGroupEgress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupnodesdockerexamplecom" + }, + "FromPort": 0, + "ToPort": 0, + "IpProtocol": "-1", + "CidrIp": "0.0.0.0/0" + } + }, + "AWSEC2SecurityGroupIngresshttpsexternaltomaster00000": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupmastersdockerexamplecom" + }, + "FromPort": 443, + "ToPort": 443, + "IpProtocol": "tcp", + "CidrIp": "0.0.0.0/0" + } + }, + "AWSEC2SecurityGroupIngressmastersdockerexamplecomingressall0to0mastersdockerexamplecom": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupmastersdockerexamplecom" + }, + "SourceSecurityGroupId": { + "Ref": "AWSEC2SecurityGroupmastersdockerexamplecom" + }, + "FromPort": 0, + "ToPort": 0, + "IpProtocol": "-1" + } + }, + "AWSEC2SecurityGroupIngressmastersdockerexamplecomingressall0to0nodesdockerexamplecom": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupnodesdockerexamplecom" + }, + "SourceSecurityGroupId": { + "Ref": "AWSEC2SecurityGroupmastersdockerexamplecom" + }, + "FromPort": 0, + "ToPort": 0, + "IpProtocol": "-1" + } + }, + "AWSEC2SecurityGroupIngressnodesdockerexamplecomingressall0to0nodesdockerexamplecom": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupnodesdockerexamplecom" + }, + "SourceSecurityGroupId": { + "Ref": "AWSEC2SecurityGroupnodesdockerexamplecom" + }, + "FromPort": 0, + "ToPort": 0, + "IpProtocol": "-1" + } + }, + "AWSEC2SecurityGroupIngressnodesdockerexamplecomingresstcp1to2379mastersdockerexamplecom": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupmastersdockerexamplecom" + }, + "SourceSecurityGroupId": { + "Ref": "AWSEC2SecurityGroupnodesdockerexamplecom" + }, + "FromPort": 1, + "ToPort": 2379, + "IpProtocol": "tcp" + } + }, + "AWSEC2SecurityGroupIngressnodesdockerexamplecomingresstcp2382to4000mastersdockerexamplecom": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupmastersdockerexamplecom" + }, + "SourceSecurityGroupId": { + "Ref": "AWSEC2SecurityGroupnodesdockerexamplecom" + }, + "FromPort": 2382, + "ToPort": 4000, + "IpProtocol": "tcp" + } + }, + "AWSEC2SecurityGroupIngressnodesdockerexamplecomingresstcp4003to65535mastersdockerexamplecom": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupmastersdockerexamplecom" + }, + "SourceSecurityGroupId": { + "Ref": "AWSEC2SecurityGroupnodesdockerexamplecom" + }, + "FromPort": 4003, + "ToPort": 65535, + "IpProtocol": "tcp" + } + }, + "AWSEC2SecurityGroupIngressnodesdockerexamplecomingressudp1to65535mastersdockerexamplecom": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupmastersdockerexamplecom" + }, + "SourceSecurityGroupId": { + "Ref": "AWSEC2SecurityGroupnodesdockerexamplecom" + }, + "FromPort": 1, + "ToPort": 65535, + "IpProtocol": "udp" + } + }, + "AWSEC2SecurityGroupIngresssshexternaltomaster00000": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupmastersdockerexamplecom" + }, + "FromPort": 22, + "ToPort": 22, + "IpProtocol": "tcp", + "CidrIp": "0.0.0.0/0" + } + }, + "AWSEC2SecurityGroupIngresssshexternaltonode00000": { + "Type": "AWS::EC2::SecurityGroupIngress", + "Properties": { + "GroupId": { + "Ref": "AWSEC2SecurityGroupnodesdockerexamplecom" + }, + "FromPort": 22, + "ToPort": 22, + "IpProtocol": "tcp", + "CidrIp": "0.0.0.0/0" + } + }, + "AWSEC2SecurityGroupmastersdockerexamplecom": { + "Type": "AWS::EC2::SecurityGroup", + "Properties": { + "GroupName": "masters.docker.example.com", + "VpcId": { + "Ref": "AWSEC2VPCdockerexamplecom" + }, + "GroupDescription": "Security group for masters", + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "docker.example.com" + }, + { + "Key": "Name", + "Value": "masters.docker.example.com" + }, + { + "Key": "kubernetes.io/cluster/docker.example.com", + "Value": "owned" + } + ] + } + }, + "AWSEC2SecurityGroupnodesdockerexamplecom": { + "Type": "AWS::EC2::SecurityGroup", + "Properties": { + "GroupName": "nodes.docker.example.com", + "VpcId": { + "Ref": "AWSEC2VPCdockerexamplecom" + }, + "GroupDescription": "Security group for nodes", + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "docker.example.com" + }, + { + "Key": "Name", + "Value": "nodes.docker.example.com" + }, + { + "Key": "kubernetes.io/cluster/docker.example.com", + "Value": "owned" + } + ] + } + }, + "AWSEC2SubnetRouteTableAssociationustest1adockerexamplecom": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "SubnetId": { + "Ref": "AWSEC2Subnetustest1adockerexamplecom" + }, + "RouteTableId": { + "Ref": "AWSEC2RouteTabledockerexamplecom" + } + } + }, + "AWSEC2Subnetustest1adockerexamplecom": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "VpcId": { + "Ref": "AWSEC2VPCdockerexamplecom" + }, + "CidrBlock": "172.20.32.0/19", + "AvailabilityZone": "us-test-1a", + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "docker.example.com" + }, + { + "Key": "Name", + "Value": "us-test-1a.docker.example.com" + }, + { + "Key": "SubnetType", + "Value": "Public" + }, + { + "Key": "kubernetes.io/cluster/docker.example.com", + "Value": "owned" + }, + { + "Key": "kubernetes.io/role/elb", + "Value": "1" + } + ] + } + }, + "AWSEC2VPCDHCPOptionsAssociationdockerexamplecom": { + "Type": "AWS::EC2::VPCDHCPOptionsAssociation", + "Properties": { + "VpcId": { + "Ref": "AWSEC2VPCdockerexamplecom" + }, + "DhcpOptionsId": { + "Ref": "AWSEC2DHCPOptionsdockerexamplecom" + } + } + }, + "AWSEC2VPCGatewayAttachmentdockerexamplecom": { + "Type": "AWS::EC2::VPCGatewayAttachment", + "Properties": { + "VpcId": { + "Ref": "AWSEC2VPCdockerexamplecom" + }, + "InternetGatewayId": { + "Ref": "AWSEC2InternetGatewaydockerexamplecom" + } + } + }, + "AWSEC2VPCdockerexamplecom": { + "Type": "AWS::EC2::VPC", + "Properties": { + "CidrBlock": "172.20.0.0/16", + "EnableDnsHostnames": true, + "EnableDnsSupport": true, + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "docker.example.com" + }, + { + "Key": "Name", + "Value": "docker.example.com" + }, + { + "Key": "kubernetes.io/cluster/docker.example.com", + "Value": "owned" + } + ] + } + }, + "AWSEC2Volumeustest1aetcdeventsdockerexamplecom": { + "Type": "AWS::EC2::Volume", + "Properties": { + "AvailabilityZone": "us-test-1a", + "Size": 20, + "VolumeType": "gp2", + "Encrypted": false, + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "docker.example.com" + }, + { + "Key": "Name", + "Value": "us-test-1a.etcd-events.docker.example.com" + }, + { + "Key": "k8s.io/etcd/events", + "Value": "us-test-1a/us-test-1a" + }, + { + "Key": "k8s.io/role/master", + "Value": "1" + }, + { + "Key": "kubernetes.io/cluster/docker.example.com", + "Value": "owned" + } + ] + } + }, + "AWSEC2Volumeustest1aetcdmaindockerexamplecom": { + "Type": "AWS::EC2::Volume", + "Properties": { + "AvailabilityZone": "us-test-1a", + "Size": 20, + "VolumeType": "gp2", + "Encrypted": false, + "Tags": [ + { + "Key": "KubernetesCluster", + "Value": "docker.example.com" + }, + { + "Key": "Name", + "Value": "us-test-1a.etcd-main.docker.example.com" + }, + { + "Key": "k8s.io/etcd/main", + "Value": "us-test-1a/us-test-1a" + }, + { + "Key": "k8s.io/role/master", + "Value": "1" + }, + { + "Key": "kubernetes.io/cluster/docker.example.com", + "Value": "owned" + } + ] + } + }, + "AWSIAMInstanceProfilemastersdockerexamplecom": { + "Type": "AWS::IAM::InstanceProfile", + "Properties": { + "InstanceProfileName": "masters.docker.example.com", + "Roles": [ + { + "Ref": "AWSIAMRolemastersdockerexamplecom" + } + ] + } + }, + "AWSIAMInstanceProfilenodesdockerexamplecom": { + "Type": "AWS::IAM::InstanceProfile", + "Properties": { + "InstanceProfileName": "nodes.docker.example.com", + "Roles": [ + { + "Ref": "AWSIAMRolenodesdockerexamplecom" + } + ] + } + }, + "AWSIAMPolicymastersdockerexamplecom": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyName": "masters.docker.example.com", + "Roles": [ + { + "Ref": "AWSIAMRolemastersdockerexamplecom" + } + ], + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "ec2:DescribeAccountAttributes", + "ec2:DescribeInstances", + "ec2:DescribeInternetGateways", + "ec2:DescribeRegions", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeVolumes" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "ec2:CreateSecurityGroup", + "ec2:CreateTags", + "ec2:CreateVolume", + "ec2:DescribeVolumesModifications", + "ec2:ModifyInstanceAttribute", + "ec2:ModifyVolume" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "ec2:AttachVolume", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:CreateRoute", + "ec2:DeleteRoute", + "ec2:DeleteSecurityGroup", + "ec2:DeleteVolume", + "ec2:DetachVolume", + "ec2:RevokeSecurityGroupIngress" + ], + "Condition": { + "StringEquals": { + "ec2:ResourceTag/KubernetesCluster": "docker.example.com" + } + }, + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeTags", + "ec2:DescribeLaunchTemplateVersions" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "autoscaling:SetDesiredCapacity", + "autoscaling:TerminateInstanceInAutoScalingGroup", + "autoscaling:UpdateAutoScalingGroup" + ], + "Condition": { + "StringEquals": { + "autoscaling:ResourceTag/KubernetesCluster": "docker.example.com" + } + }, + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "elasticloadbalancing:AddTags", + "elasticloadbalancing:AttachLoadBalancerToSubnets", + "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer", + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateLoadBalancerPolicy", + "elasticloadbalancing:CreateLoadBalancerListeners", + "elasticloadbalancing:ConfigureHealthCheck", + "elasticloadbalancing:DeleteLoadBalancer", + "elasticloadbalancing:DeleteLoadBalancerListeners", + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DetachLoadBalancerFromSubnets", + "elasticloadbalancing:DeregisterInstancesFromLoadBalancer", + "elasticloadbalancing:ModifyLoadBalancerAttributes", + "elasticloadbalancing:RegisterInstancesWithLoadBalancer", + "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "ec2:DescribeVpcs", + "elasticloadbalancing:AddTags", + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:CreateTargetGroup", + "elasticloadbalancing:DeleteListener", + "elasticloadbalancing:DeleteTargetGroup", + "elasticloadbalancing:DeregisterTargets", + "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DescribeLoadBalancerPolicies", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeTargetHealth", + "elasticloadbalancing:ModifyListener", + "elasticloadbalancing:ModifyTargetGroup", + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:SetLoadBalancerPoliciesOfListener" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "iam:ListServerCertificates", + "iam:GetServerCertificate" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "route53:ChangeResourceRecordSets", + "route53:ListResourceRecordSets", + "route53:GetHostedZone" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:route53:::hostedzone/Z1AFAKE1ZON3YO" + ] + }, + { + "Action": [ + "route53:GetChange" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:route53:::change/*" + ] + }, + { + "Action": [ + "route53:ListHostedZones" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + } + ], + "Version": "2012-10-17" + } + } + }, + "AWSIAMPolicynodesdockerexamplecom": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyName": "nodes.docker.example.com", + "Roles": [ + { + "Ref": "AWSIAMRolenodesdockerexamplecom" + } + ], + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "ec2:DescribeInstances", + "ec2:DescribeRegions" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + } + ], + "Version": "2012-10-17" + } + } + }, + "AWSIAMRolemastersdockerexamplecom": { + "Type": "AWS::IAM::Role", + "Properties": { + "RoleName": "masters.docker.example.com", + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + } + }, + "AWSIAMRolenodesdockerexamplecom": { + "Type": "AWS::IAM::Role", + "Properties": { + "RoleName": "nodes.docker.example.com", + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + } + } + } +} diff --git a/tests/integration/update_cluster/docker-custom/cloudformation.json.extracted.yaml b/tests/integration/update_cluster/docker-custom/cloudformation.json.extracted.yaml new file mode 100644 index 0000000000000..a682df1a3d400 --- /dev/null +++ b/tests/integration/update_cluster/docker-custom/cloudformation.json.extracted.yaml @@ -0,0 +1,559 @@ +Resources.AWSEC2LaunchTemplatemasterustest1amastersdockerexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.19.0-alpha.3/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.19.0-alpha.3/nodeup-linux-amd64,https://kubeupv2.s3.amazonaws.com/kops/1.19.0-alpha.3/linux/amd64/nodeup + NODEUP_HASH_AMD64=6980fda4fa37bbdc043738cf4ddac6388eb57f561895c69299c1b0ee263d465d + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.19.0-alpha.3/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.19.0-alpha.3/nodeup-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.19.0-alpha.3/linux/arm64/nodeup + NODEUP_HASH_ARM64=dcc7f9f3c180ee76a511627e46da0ac69cdcb518cdf3be348e5ed046d491eb87 + + export AWS_REGION=us-test-1 + + + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, url1, url2... + download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: null + containerRuntime: docker + containerd: + configOverride: | + disabled_plugins = ["cri"] + logLevel: info + docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + packages: + hashAmd64: 000000000000000000000000000000000000000000000000000000000000000a + hashArm64: 000000000000000000000000000000000000000000000000000000000000000b + urlAmd64: https://download.docker.com/linux/static/stable/x86_64/docker-20.10.1.tgz + urlArm64: https://download.docker.com/linux/static/stable/aarch64/docker-20.10.1.tgz + storage: overlay2,overlay,aufs + version: 19.03.14 + encryptionConfig: null + etcdClusters: + events: + version: 3.4.13 + main: + version: 3.4.13 + kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.19.0 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 + kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: docker.example.com + configureCloudRoutes: true + image: k8s.gcr.io/kube-controller-manager:v1.19.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.19.0 + logLevel: 2 + kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.19.0 + leaderElection: + leaderElect: true + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + masterKubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 3f03e5c160a8b658d30b34824a1c00abadbac96e62c4d01bf5c9271a2debc3ab@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/amd64/kubelet + - 79bb0d2f05487ff533999a639c075043c70a0a1ba25c1629eb1eef6ebe3ba70f@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 000000000000000000000000000000000000000000000000000000000000000a@https://download.docker.com/linux/static/stable/x86_64/docker-20.10.1.tgz + arm64: + - d8fa5a9739ecc387dfcc55afa91ac6f4b0ccd01f1423c423dbd312d787bbb6bf@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/arm64/kubelet + - d4adf1b6b97252025cb2f7febf55daa3f42dc305822e3da133f77fd33071ec2f@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 000000000000000000000000000000000000000000000000000000000000000b@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.1.tgz + ClusterName: docker.example.com + ConfigBase: memfs://clusters.example.com/docker.example.com + InstanceGroupName: master-us-test-1a + InstanceGroupRole: Master + KubeletConfig: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nodeLabels: + kubernetes.io/role: master + node-role.kubernetes.io/master: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + channels: + - memfs://clusters.example.com/docker.example.com/addons/bootstrap-channel.yaml + etcdManifests: + - memfs://clusters.example.com/docker.example.com/manifests/etcd/main.yaml + - memfs://clusters.example.com/docker.example.com/manifests/etcd/events.yaml + protokubeImage: + amd64: + hash: 7b3c7f6adbda11b1ec740bd6b969c84f249b7eee818af95f2d321963088245a8 + name: protokube:1.19.0-alpha.3 + sources: + - https://artifacts.k8s.io/binaries/kops/1.19.0-alpha.3/images/protokube-amd64.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.19.0-alpha.3/images-protokube-amd64.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.19.0-alpha.3/images/protokube-amd64.tar.gz + arm64: + hash: 69270ca9c1c950be65af40337adfccec0a728930fa3224bb0d2e88f181f39ead + name: protokube:1.19.0-alpha.3 + sources: + - https://artifacts.k8s.io/binaries/kops/1.19.0-alpha.3/images/protokube-arm64.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.19.0-alpha.3/images-protokube-arm64.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.19.0-alpha.3/images/protokube-arm64.tar.gz + staticManifests: + - key: kube-apiserver-healthcheck + path: manifests/static/kube-apiserver-healthcheck.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" +Resources.AWSEC2LaunchTemplatenodesdockerexamplecom.Properties.LaunchTemplateData.UserData: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.19.0-alpha.3/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.19.0-alpha.3/nodeup-linux-amd64,https://kubeupv2.s3.amazonaws.com/kops/1.19.0-alpha.3/linux/amd64/nodeup + NODEUP_HASH_AMD64=6980fda4fa37bbdc043738cf4ddac6388eb57f561895c69299c1b0ee263d465d + NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.19.0-alpha.3/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.19.0-alpha.3/nodeup-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.19.0-alpha.3/linux/arm64/nodeup + NODEUP_HASH_ARM64=dcc7f9f3c180ee76a511627e46da0ac69cdcb518cdf3be348e5ed046d491eb87 + + export AWS_REGION=us-test-1 + + + + + function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} + } + + # Retry a download until we get it. args: name, sha, url1, url2... + download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done + } + + validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi + } + + function split-commas() { + echo $1 | tr "," "\n" + } + + function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup + } + + function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) + } + + #################################################################################### + + /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + + echo "== nodeup node config starting ==" + ensure-install-dir + + cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' + cloudConfig: null + containerRuntime: docker + containerd: + configOverride: | + disabled_plugins = ["cri"] + logLevel: info + docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + packages: + hashAmd64: 000000000000000000000000000000000000000000000000000000000000000a + hashArm64: 000000000000000000000000000000000000000000000000000000000000000b + urlAmd64: https://download.docker.com/linux/static/stable/x86_64/docker-20.10.1.tgz + urlArm64: https://download.docker.com/linux/static/stable/aarch64/docker-20.10.1.tgz + storage: overlay2,overlay,aufs + version: 19.03.14 + kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.19.0 + logLevel: 2 + kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + + __EOF_CLUSTER_SPEC + + cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' + {} + + __EOF_IG_SPEC + + cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' + Assets: + amd64: + - 3f03e5c160a8b658d30b34824a1c00abadbac96e62c4d01bf5c9271a2debc3ab@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/amd64/kubelet + - 79bb0d2f05487ff533999a639c075043c70a0a1ba25c1629eb1eef6ebe3ba70f@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/amd64/kubectl + - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz + - 000000000000000000000000000000000000000000000000000000000000000a@https://download.docker.com/linux/static/stable/x86_64/docker-20.10.1.tgz + arm64: + - d8fa5a9739ecc387dfcc55afa91ac6f4b0ccd01f1423c423dbd312d787bbb6bf@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/arm64/kubelet + - d4adf1b6b97252025cb2f7febf55daa3f42dc305822e3da133f77fd33071ec2f@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/arm64/kubectl + - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz + - 000000000000000000000000000000000000000000000000000000000000000b@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.1.tgz + ClusterName: docker.example.com + ConfigBase: memfs://clusters.example.com/docker.example.com + InstanceGroupName: nodes + InstanceGroupRole: Node + KubeletConfig: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nodeLabels: + kubernetes.io/role: node + node-role.kubernetes.io/node: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + channels: + - memfs://clusters.example.com/docker.example.com/addons/bootstrap-channel.yaml + + __EOF_KUBE_ENV + + download-release + echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/docker-custom/id_rsa.pub b/tests/integration/update_cluster/docker-custom/id_rsa.pub new file mode 100755 index 0000000000000..81cb0127830e7 --- /dev/null +++ b/tests/integration/update_cluster/docker-custom/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ== diff --git a/tests/integration/update_cluster/docker-custom/in-v1alpha2.yaml b/tests/integration/update_cluster/docker-custom/in-v1alpha2.yaml new file mode 100644 index 0000000000000..d48b24a7324ff --- /dev/null +++ b/tests/integration/update_cluster/docker-custom/in-v1alpha2.yaml @@ -0,0 +1,87 @@ +apiVersion: kops.k8s.io/v1alpha2 +kind: Cluster +metadata: + creationTimestamp: "2016-12-10T22:42:27Z" + name: docker.example.com +spec: + kubernetesApiAccess: + - 0.0.0.0/0 + channel: stable + cloudProvider: aws + configBase: memfs://clusters.example.com/docker.example.com + containerRuntime: docker + docker: + packages: + urlAmd64: https://download.docker.com/linux/static/stable/x86_64/docker-20.10.1.tgz + hashAmd64: "000000000000000000000000000000000000000000000000000000000000000a" + urlArm64: https://download.docker.com/linux/static/stable/aarch64/docker-20.10.1.tgz + hashArm64: "000000000000000000000000000000000000000000000000000000000000000b" + etcdClusters: + - etcdMembers: + - instanceGroup: master-us-test-1a + name: us-test-1a + name: main + - etcdMembers: + - instanceGroup: master-us-test-1a + name: us-test-1a + name: events + iam: {} + kubelet: + anonymousAuth: false + kubernetesVersion: v1.19.0 + masterInternalName: api.internal.docker.example.com + masterPublicName: api.docker.example.com + networkCIDR: 172.20.0.0/16 + networking: + kubenet: {} + nonMasqueradeCIDR: 100.64.0.0/10 + sshAccess: + - 0.0.0.0/0 + topology: + masters: public + nodes: public + subnets: + - cidr: 172.20.32.0/19 + name: us-test-1a + type: Public + zone: us-test-1a + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2016-12-10T22:42:28Z" + name: nodes + labels: + kops.k8s.io/cluster: docker.example.com +spec: + associatePublicIp: true + image: kope.io/k8s-1.14-debian-stretch-amd64-hvm-ebs-2019-08-16 + machineType: t2.medium + maxSize: 2 + minSize: 2 + role: Node + subnets: + - us-test-1a + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2016-12-10T22:42:28Z" + name: master-us-test-1a + labels: + kops.k8s.io/cluster: docker.example.com +spec: + associatePublicIp: true + image: kope.io/k8s-1.14-debian-stretch-amd64-hvm-ebs-2019-08-16 + machineType: m3.medium + maxSize: 1 + minSize: 1 + role: Master + subnets: + - us-test-1a + + diff --git a/upup/pkg/fi/cloudup/containerd.go b/upup/pkg/fi/cloudup/containerd.go index 53ab704f5e863..0af1718e06bf9 100644 --- a/upup/pkg/fi/cloudup/containerd.go +++ b/upup/pkg/fi/cloudup/containerd.go @@ -39,12 +39,28 @@ const ( ) func findContainerdAsset(c *kops.Cluster, assetBuilder *assets.AssetBuilder, arch architectures.Architecture) (*url.URL, *hashing.Hash, error) { - if c.Spec.Containerd == nil || fi.StringValue(c.Spec.Containerd.Version) == "" { - return nil, nil, fmt.Errorf("unable to find containerd version") + if c.Spec.Containerd == nil { + return nil, nil, fmt.Errorf("unable to find containerd config") } + containerd := c.Spec.Containerd - version := fi.StringValue(c.Spec.Containerd.Version) + if containerd.Packages != nil { + if arch == architectures.ArchitectureAmd64 && containerd.Packages.UrlAmd64 != nil && containerd.Packages.HashAmd64 != nil { + assetUrl := fi.StringValue(containerd.Packages.UrlAmd64) + assetHash := fi.StringValue(containerd.Packages.HashAmd64) + return findAssetsUrlHash(assetBuilder, assetUrl, assetHash) + } + if arch == architectures.ArchitectureArm64 && containerd.Packages.UrlArm64 != nil && containerd.Packages.HashArm64 != nil { + assetUrl := fi.StringValue(containerd.Packages.UrlArm64) + assetHash := fi.StringValue(containerd.Packages.HashArm64) + return findAssetsUrlHash(assetBuilder, assetUrl, assetHash) + } + } + version := fi.StringValue(containerd.Version) + if version == "" { + return nil, nil, fmt.Errorf("unable to find containerd version") + } assetUrl, assetHash, err := findContainerdVersionUrlHash(arch, version) if err != nil { return nil, nil, err diff --git a/upup/pkg/fi/cloudup/docker.go b/upup/pkg/fi/cloudup/docker.go index 8f1a01ea71f4e..e86e874cd1d8d 100644 --- a/upup/pkg/fi/cloudup/docker.go +++ b/upup/pkg/fi/cloudup/docker.go @@ -42,12 +42,28 @@ const ( ) func findDockerAsset(c *kops.Cluster, assetBuilder *assets.AssetBuilder, arch architectures.Architecture) (*url.URL, *hashing.Hash, error) { - if c.Spec.Docker == nil || fi.StringValue(c.Spec.Docker.Version) == "" { - return nil, nil, fmt.Errorf("unable to find Docker version") + if c.Spec.Docker == nil { + return nil, nil, fmt.Errorf("unable to find Docker config") } + docker := c.Spec.Docker - version := fi.StringValue(c.Spec.Docker.Version) + if docker.Packages != nil { + if arch == architectures.ArchitectureAmd64 && docker.Packages.UrlAmd64 != nil && docker.Packages.HashAmd64 != nil { + assetUrl := fi.StringValue(docker.Packages.UrlAmd64) + assetHash := fi.StringValue(docker.Packages.HashAmd64) + return findAssetsUrlHash(assetBuilder, assetUrl, assetHash) + } + if arch == architectures.ArchitectureArm64 && docker.Packages.UrlArm64 != nil && docker.Packages.HashArm64 != nil { + assetUrl := fi.StringValue(docker.Packages.UrlArm64) + assetHash := fi.StringValue(docker.Packages.HashArm64) + return findAssetsUrlHash(assetBuilder, assetUrl, assetHash) + } + } + version := fi.StringValue(docker.Version) + if version == "" { + return nil, nil, fmt.Errorf("unable to find Docker version") + } assetUrl, assetHash, err := findDockerVersionUrlHash(arch, version) if err != nil { return nil, nil, err