Merge pull request #15342 from olemarkus/ecr-credentials-provider

Switch to using external ECR credential provider for k8s 1.27

docs/releases/1.27-NOTES.md

@@ -14,6 +14,8 @@ This behaviour can be overridden by setting `spec.etcdClusters[*].manager.backup
 * As of Kubernetes version 1.27, all nodes will default to running with instance-metadata-service tokens required, with a max hop limit of 1.
 Newly created clusters will be configured as necessary to have these settings.
 
+* As of Kubernetes version 1.27, credentials for private ECR repositories will be handled by the out-of-tree credential provider. This is an additional binary that each instance downloads from the assets repository.
+
 ## GCP
 
 ## Openstack

nodeup/pkg/model/kubelet.go

@@ -49,7 +49,8 @@ const (
 	// kubeletService is the name of the kubelet service
 	kubeletService = "kubelet.service"
 
-	kubeletConfigFilePath = "/var/lib/kubelet/kubelet.conf"
+	kubeletConfigFilePath            = "/var/lib/kubelet/kubelet.conf"
+	credentialProviderConfigFilePath = "/var/lib/kubelet/credential-provider.conf"
 )
 
 // KubeletBuilder installs kubelet
@@ -157,6 +158,12 @@ func (b *KubeletBuilder) Build(c *fi.NodeupModelBuilderContext) error {
 		return err
 	}
 
+	if b.Cluster.UsesExternalECRCredentialsProvider() {
+		if err := b.addECRCP(c); err != nil {
+			return fmt.Errorf("failed to add ECR credential provider: %w", err)
+		}
+	}
+
 	if kubeletConfig.CgroupDriver == "systemd" && b.NodeupConfig.ContainerRuntime == "containerd" {
 
 		{
@@ -241,16 +248,25 @@ func buildKubeletComponentConfig(kubeletConfig *kops.KubeletConfigSpec) (*nodeta
 	return t, nil
 }
 
-// kubeletPath returns the path of the kubelet based on distro
-func (b *KubeletBuilder) kubeletPath() string {
-	kubeletCommand := "/usr/local/bin/kubelet"
+func (b *KubeletBuilder) binaryPath() string {
+	path := "/usr/local/bin"
 	if b.Distribution == distributions.DistributionFlatcar {
-		kubeletCommand = "/opt/kubernetes/bin/kubelet"
+		path = "/opt/kubernetes/bin"
 	}
 	if b.Distribution == distributions.DistributionContainerOS {
-		kubeletCommand = "/home/kubernetes/bin/kubelet"
+		path = "/home/kubernetes/bin"
 	}
-	return kubeletCommand
+	return path
+}
+
+// kubeletPath returns the path of the kubelet based on distro
+func (b *KubeletBuilder) kubeletPath() string {
+	return b.binaryPath() + "/kubelet"
+}
+
+// ecrcpPath returns the path of the ECR credentials provider based on distro and archiecture
+func (b *KubeletBuilder) ecrcpPath() string {
+	return b.binaryPath() + "/ecr-credential-provider"
 }
 
 // buildManifestDirectory creates the directory where kubelet expects static manifests to reside
@@ -330,6 +346,11 @@ func (b *KubeletBuilder) buildSystemdEnvironmentFile(kubeletConfig *kops.Kubelet
 
 	flags += " --config=" + kubeletConfigFilePath
 
+	if b.Cluster.UsesExternalECRCredentialsProvider() {
+		flags += " --image-credential-provider-config=" + credentialProviderConfigFilePath
+		flags += " --image-credential-provider-bin-dir=" + b.binaryPath()
+	}
+
 	sysconfig := "DAEMON_ARGS=\"" + flags + "\"\n"
 	// Makes kubelet read /root/.docker/config.json properly
 	sysconfig = sysconfig + "HOME=\"/root" + "\"\n"
@@ -413,6 +434,56 @@ func (b *KubeletBuilder) usesContainerizedMounter() bool {
 	}
 }
 
+// addECRCP installs the ECR credential provider
+func (b *KubeletBuilder) addECRCP(c *fi.NodeupModelBuilderContext) error {
+	{
+		assetName := "ecr-credential-provider-linux-" + string(b.Architecture)
+		assetPath := ""
+		asset, err := b.Assets.Find(assetName, assetPath)
+		if err != nil {
+			return fmt.Errorf("error trying to locate asset %q: %v", assetName, err)
+		}
+		if asset == nil {
+			return fmt.Errorf("unable to locate asset %q", assetName)
+		}
+
+		t := &nodetasks.File{
+			Path:     b.ecrcpPath(),
+			Contents: asset,
+			Type:     nodetasks.FileType_File,
+			Mode:     s("0755"),
+		}
+		c.AddTask(t)
+	}
+
+	{
+		configContent := `apiVersion: kubelet.config.k8s.io/v1
+kind: CredentialProviderConfig
+providers:
+  - name: ecr-credential-provider
+    matchImages:
+      - "*.dkr.ecr.*.amazonaws.com"
+      - "*.dkr.ecr.*.amazonaws.cn"
+      - "*.dkr.ecr-fips.*.amazonaws.com"
+      - "*.dkr.ecr.us-iso-east-1.c2s.ic.gov"
+      - "*.dkr.ecr.us-isob-east-1.sc2s.sgov.gov"
+    defaultCacheDuration: "12h"
+    apiVersion: credentialprovider.kubelet.k8s.io/v1
+    args:
+      - get-credentials
+`
+
+		t := &nodetasks.File{
+			Path:     credentialProviderConfigFilePath,
+			Contents: fi.NewStringResource(configContent),
+			Type:     nodetasks.FileType_File,
+			Mode:     s("0644"),
+		}
+		c.AddTask(t)
+	}
+	return nil
+}
+
 // addContainerizedMounter downloads and installs the containerized mounter, that we need on ContainerOS
 func (b *KubeletBuilder) addContainerizedMounter(c *fi.NodeupModelBuilderContext) error {
 	if !b.usesContainerizedMounter() {

pkg/apis/kops/cluster.go

@@ -220,6 +220,9 @@ type AWSSpec struct {
 	// Spotinst cloud-config specs
 	SpotinstProduct     *string `json:"spotinstProduct,omitempty"`
 	SpotinstOrientation *string `json:"spotinstOrientation,omitempty"`
+
+	// BinariesLocation is the location of the AWS cloud provider binaries.
+	BinariesLocation *string `json:"binaryLocation,omitempty"`
 }
 
 // DOSpec configures the Digital Ocean cloud provider.
@@ -895,6 +898,10 @@ func (c *Cluster) UsesNoneDNS() bool {
 	return false
 }
 
+func (c *Cluster) UsesExternalECRCredentialsProvider() bool {
+	return c.IsKubernetesGTE("1.27") && c.Spec.GetCloudProvider() == CloudProviderAWS
+}
+
 func (c *Cluster) APIInternalName() string {
 	return "api.internal." + c.ObjectMeta.Name
 }

pkg/apis/kops/v1alpha3/cluster.go

@@ -216,6 +216,9 @@ type AWSSpec struct {
 	// Spotinst cloud-config specs
 	SpotinstProduct     *string `json:"spotinstProduct,omitempty"`
 	SpotinstOrientation *string `json:"spotinstOrientation,omitempty"`
+
+	// BinariesLocation is the location of the AWS cloud provider binaries.
+	BinariesLocation *string `json:"binaryLocation,omitempty"`
 }
 
 // DOSpec configures the Digital Ocean cloud provider.

tests/integration/update_cluster/minimal/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data

@@ -250,7 +250,7 @@ CloudProvider: aws
 ConfigBase: memfs://clusters.example.com/minimal.example.com
 InstanceGroupName: master-us-test-1a
 InstanceGroupRole: ControlPlane
-NodeupConfigHash: N/3CvlvqplvxWXN3+r1r09mA/Y55Wt/klapVL1nCd2w=
+NodeupConfigHash: rJVuGlmAuBwCe2RWXKgT1dI+bdFv3FhvakVQhTVudXU=
 
 __EOF_KUBE_ENV
 

tests/integration/update_cluster/minimal/data/aws_launch_template_nodes.minimal.example.com_user_data

@@ -174,7 +174,7 @@ ConfigServer:
   - https://kops-controller.internal.minimal.example.com:3988/
 InstanceGroupName: nodes
 InstanceGroupRole: Node
-NodeupConfigHash: g+og1yXE09z0oZE2ggS/s5xmrp7CHsM9PbOhPFs8cNs=
+NodeupConfigHash: jg6YaOqnx4/XWWzsT5/JFgNHr3tC6qgvfQ6M1CfY6Jk=
 
 __EOF_KUBE_ENV
 

tests/integration/update_cluster/minimal/data/aws_s3_object_nodeupconfig-master-us-test-1a_content

@@ -56,6 +56,7 @@ Assets:
   amd64:
   - be5e79c70e926019e588c8c5c44d93efb1042f3be6f1db0de14e707141564d29@https://storage.googleapis.com/kubernetes-release/release/v1.27.0-alpha.3/bin/linux/amd64/kubelet
   - 4d416a4bed7c1bb576e284fc6b0bb843f879c08fa8e4beb7e2376305d2dc8299@https://storage.googleapis.com/kubernetes-release/release/v1.27.0-alpha.3/bin/linux/amd64/kubectl
+  - 5035d7814c95cd3cedbc5efb447ef25a4942ef05caab2159746d55ce1698c74a@https://artifacts.k8s.io/binaries/cloud-provider-aws/v1.27.1/linux/amd64/ecr-credential-provider-linux-amd64
   - 962100bbc4baeaaa5748cdbfce941f756b1531c2eadb290129401498bfac21e7@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-amd64-v0.9.1.tgz
   - bb9a9ccd6517e2a54da748a9f60dc9aa9d79d19d4724663f2386812f083968e2@https://github.com/containerd/containerd/releases/download/v1.6.20/containerd-1.6.20-linux-amd64.tar.gz
   - db772be63147a4e747b4fe286c7c16a2edc4a8458bd3092ea46aaee77750e8ce@https://github.com/opencontainers/runc/releases/download/v1.1.4/runc.amd64
@@ -64,6 +65,7 @@ Assets:
   arm64:
   - 2a15e9c291dce3e07db5d9948d5c37a4e52c9884343f63287189e4f0d4c3df76@https://storage.googleapis.com/kubernetes-release/release/v1.27.0-alpha.3/bin/linux/arm64/kubelet
   - c5fad9f96ab5fe04b8927ce83ceefc4db65b032303a23b35e353276479450d2a@https://storage.googleapis.com/kubernetes-release/release/v1.27.0-alpha.3/bin/linux/arm64/kubectl
+  - b3d567bda9e2996fc1fbd9d13506bd16763d3865b5c7b0b3c4b48c6088c04481@https://artifacts.k8s.io/binaries/cloud-provider-aws/v1.27.1/linux/arm64/ecr-credential-provider-linux-arm64
   - ef17764ffd6cdcb16d76401bac1db6acc050c9b088f1be5efa0e094ea3b01df0@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-arm64-v0.9.1.tgz
   - c3e6a054b18b20fce06c7c3ed53f0989bb4b255c849bede446ebca955f07a9ce@https://github.com/containerd/containerd/releases/download/v1.6.20/containerd-1.6.20-linux-arm64.tar.gz
   - dbb71e737eaef454a406ce21fd021bd8f1b35afb7635016745992bbd7c17a223@https://github.com/opencontainers/runc/releases/download/v1.1.4/runc.arm64

tests/integration/update_cluster/minimal/data/aws_s3_object_nodeupconfig-nodes_content

@@ -2,12 +2,14 @@ Assets:
   amd64:
   - be5e79c70e926019e588c8c5c44d93efb1042f3be6f1db0de14e707141564d29@https://storage.googleapis.com/kubernetes-release/release/v1.27.0-alpha.3/bin/linux/amd64/kubelet
   - 4d416a4bed7c1bb576e284fc6b0bb843f879c08fa8e4beb7e2376305d2dc8299@https://storage.googleapis.com/kubernetes-release/release/v1.27.0-alpha.3/bin/linux/amd64/kubectl
+  - 5035d7814c95cd3cedbc5efb447ef25a4942ef05caab2159746d55ce1698c74a@https://artifacts.k8s.io/binaries/cloud-provider-aws/v1.27.1/linux/amd64/ecr-credential-provider-linux-amd64
   - 962100bbc4baeaaa5748cdbfce941f756b1531c2eadb290129401498bfac21e7@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-amd64-v0.9.1.tgz
   - bb9a9ccd6517e2a54da748a9f60dc9aa9d79d19d4724663f2386812f083968e2@https://github.com/containerd/containerd/releases/download/v1.6.20/containerd-1.6.20-linux-amd64.tar.gz
   - db772be63147a4e747b4fe286c7c16a2edc4a8458bd3092ea46aaee77750e8ce@https://github.com/opencontainers/runc/releases/download/v1.1.4/runc.amd64
   arm64:
   - 2a15e9c291dce3e07db5d9948d5c37a4e52c9884343f63287189e4f0d4c3df76@https://storage.googleapis.com/kubernetes-release/release/v1.27.0-alpha.3/bin/linux/arm64/kubelet
   - c5fad9f96ab5fe04b8927ce83ceefc4db65b032303a23b35e353276479450d2a@https://storage.googleapis.com/kubernetes-release/release/v1.27.0-alpha.3/bin/linux/arm64/kubectl
+  - b3d567bda9e2996fc1fbd9d13506bd16763d3865b5c7b0b3c4b48c6088c04481@https://artifacts.k8s.io/binaries/cloud-provider-aws/v1.27.1/linux/arm64/ecr-credential-provider-linux-arm64
   - ef17764ffd6cdcb16d76401bac1db6acc050c9b088f1be5efa0e094ea3b01df0@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.9.1/cni-plugins-linux-arm64-v0.9.1.tgz
   - c3e6a054b18b20fce06c7c3ed53f0989bb4b255c849bede446ebca955f07a9ce@https://github.com/containerd/containerd/releases/download/v1.6.20/containerd-1.6.20-linux-arm64.tar.gz
   - dbb71e737eaef454a406ce21fd021bd8f1b35afb7635016745992bbd7c17a223@https://github.com/opencontainers/runc/releases/download/v1.1.4/runc.arm64

upup/pkg/fi/cloudup/apply_cluster.go

@@ -1056,6 +1056,23 @@ func (c *ApplyClusterCmd) addFileAssets(assetBuilder *assets.AssetBuilder) error
 			c.Assets[arch] = append(c.Assets[arch], mirrors.BuildMirroredAsset(u, hash))
 		}
 
+		if c.Cluster.UsesExternalECRCredentialsProvider() {
+			binaryLocation := c.Cluster.Spec.CloudProvider.AWS.BinariesLocation
+			if binaryLocation == nil {
+				binaryLocation = fi.PtrTo("https://artifacts.k8s.io/binaries/cloud-provider-aws/v1.27.1")
+			}
+
+			k, err := url.Parse(fmt.Sprintf("%s/linux/%s/ecr-credential-provider-linux-%s", *binaryLocation, arch, arch))
+			if err != nil {
+				return err
+			}
+			u, hash, err := assetBuilder.RemapFileAndSHA(k)
+			if err != nil {
+				return err
+			}
+			c.Assets[arch] = append(c.Assets[arch], mirrors.BuildMirroredAsset(u, hash))
+		}
+
 		cniAsset, cniAssetHash, err := findCNIAssets(c.Cluster, assetBuilder, arch)
 		if err != nil {
 			return err