Merge pull request #1494 from jpculp/kubelet-cloud-provider-setting

kubelet: add setting for configuring cloudProvider

README.md

@@ -308,6 +308,7 @@ The following settings can be optionally set to customize the node labels and ta
 The following settings are optional and allow you to further configure your cluster.
 * `settings.kubernetes.cluster-domain`: The DNS domain for this cluster, allowing all Kubernetes-run containers to search this domain before the host's search domains.  Defaults to `cluster.local`.
 * `settings.kubernetes.standalone-mode`: Whether to run the kubelet in standalone mode, without connecting to an API server.  Defaults to `false`.
+* `settings.kubernetes.cloud-provider`: The cloud provider for this cluster. Defaults to `aws` for AWS variants, and `external` for other variants.
 * `settings.kubernetes.authentication-mode`: Which authentication method the kubelet should use to connect to the API server, and for incoming requests.  Defaults to `aws` for AWS variants, and `tls` for other variants.
 * `settings.kubernetes.server-tls-bootstrap`: Enables or disables server certificate bootstrap.  When enabled, the kubelet will request a certificate from the certificates.k8s.io API.  This requires an approver to approve the certificate signing requests (CSR).  Defaults to `true`.
 * `settings.kubernetes.bootstrap-token`: The token to use for [TLS bootstrapping](https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet-tls-bootstrapping/).  This is only used with the `tls` authentication mode, and is otherwise ignored.

Release.toml

@@ -40,4 +40,5 @@ version = "1.0.8"
 ]
 "(1.0.8, 1.1.0)" = [
     "migrate_v1.1.0_kubelet-server-tls-bootstrap.lz4",
+    "migrate_v1.1.0_kubelet-cloud-provider.lz4",
 ]

packages/kubernetes-1.16/kubelet-exec-start-conf

@@ -2,7 +2,7 @@
 ExecStart=
 ExecStart=/usr/bin/kubelet \
 {{~#unless settings.kubernetes.standalone-mode}}
-    --cloud-provider aws \
+    --cloud-provider {{default "external" settings.kubernetes.cloud-provider}} \
     --kubeconfig /etc/kubernetes/kubelet/kubeconfig \
 {{~#if (eq settings.kubernetes.authentication-mode "tls")}}
     --bootstrap-kubeconfig /etc/kubernetes/kubelet/bootstrap-kubeconfig \

packages/kubernetes-1.17/kubelet-exec-start-conf

@@ -2,7 +2,7 @@
 ExecStart=
 ExecStart=/usr/bin/kubelet \
 {{~#unless settings.kubernetes.standalone-mode}}
-    --cloud-provider aws \
+    --cloud-provider {{default "external" settings.kubernetes.cloud-provider}} \
     --kubeconfig /etc/kubernetes/kubelet/kubeconfig \
 {{~#if (eq settings.kubernetes.authentication-mode "tls")}}
     --bootstrap-kubeconfig /etc/kubernetes/kubelet/bootstrap-kubeconfig \

packages/kubernetes-1.18/kubelet-exec-start-conf

@@ -2,7 +2,7 @@
 ExecStart=
 ExecStart=/usr/bin/kubelet \
 {{~#unless settings.kubernetes.standalone-mode}}
-    --cloud-provider aws \
+    --cloud-provider {{default "external" settings.kubernetes.cloud-provider}} \
     --kubeconfig /etc/kubernetes/kubelet/kubeconfig \
 {{~#if (eq settings.kubernetes.authentication-mode "tls")}}
     --bootstrap-kubeconfig /etc/kubernetes/kubelet/bootstrap-kubeconfig \

packages/kubernetes-1.19/kubelet-exec-start-conf

@@ -2,7 +2,7 @@
 ExecStart=
 ExecStart=/usr/bin/kubelet \
 {{~#unless settings.kubernetes.standalone-mode}}
-    --cloud-provider aws \
+    --cloud-provider {{default "external" settings.kubernetes.cloud-provider}} \
     --kubeconfig /etc/kubernetes/kubelet/kubeconfig \
 {{~#if (eq settings.kubernetes.authentication-mode "tls")}}
     --bootstrap-kubeconfig /etc/kubernetes/kubelet/bootstrap-kubeconfig \

sources/Cargo.toml

@@ -54,6 +54,7 @@ members = [
     "api/migration/migrations/v1.0.8/admin-container-v0-7-0",
     "api/migration/migrations/v1.0.8/add-bootstrap-containers",
     "api/migration/migrations/v1.1.0/kubelet-server-tls-bootstrap",
+    "api/migration/migrations/v1.1.0/kubelet-cloud-provider",
 
     "bottlerocket-release",
 

sources/api/migration/migrations/v1.1.0/kubelet-cloud-provider/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "kubelet-cloud-provider"
+version = "0.1.0"
+authors = ["Patrick J.P. Culp <jpculp@amazon.com>"]
+license = "Apache-2.0 OR MIT"
+edition = "2018"
+publish = false
+# Don't rebuild crate just because of changes to README.
+exclude = ["README.md"]
+
+[dependencies]
+migration-helpers = { path = "../../../migration-helpers" }

sources/api/migration/migrations/v1.1.0/kubelet-cloud-provider/src/main.rs

@@ -0,0 +1,22 @@
+#![deny(rust_2018_idioms)]
+
+use migration_helpers::common_migrations::AddSettingsMigration;
+use migration_helpers::{migrate, Result};
+use std::process;
+
+/// We added a new settings for configuring kubelet, `settings.kubernetes.cloud-provider`
+fn run() -> Result<()> {
+    migrate(AddSettingsMigration(&[
+        "settings.kubernetes.cloud-provider",
+    ]))
+}
+
+// Returning a Result from main makes it print a Debug representation of the error, but with Snafu
+// we have nice Display representations of the error, so we wrap "main" (run) and print any error.
+// https://github.com/shepmaster/snafu/issues/110
+fn main() {
+    if let Err(e) = run() {
+        eprintln!("{}", e);
+        process::exit(1);
+    }
+}

sources/models/src/aws-k8s-1.19/defaults.d/50-aws-k8s.toml

@@ -59,6 +59,7 @@ cluster-domain = "cluster.local"
 standalone-mode = false
 authentication-mode = "aws"
 server-tls-bootstrap = true
+cloud-provider = "aws"
 
 # Metrics
 [settings.metrics]

sources/models/src/lib.rs

@@ -95,9 +95,10 @@ use std::net::Ipv4Addr;
 use crate::modeled_types::{
     BootstrapContainerMode, DNSDomain, ECSAgentLogLevel, ECSAttributeKey, ECSAttributeValue,
     FriendlyVersion, Identifier, KubernetesAuthenticationMode, KubernetesBootstrapToken,
-    KubernetesClusterName, KubernetesEvictionHardKey, KubernetesLabelKey, KubernetesLabelValue,
-    KubernetesQuantityValue, KubernetesReservedResourceKey, KubernetesTaintValue,
-    KubernetesThresholdValue, Lockdown, SingleLineString, SysctlKey, Url, ValidBase64,
+    KubernetesCloudProvider, KubernetesClusterName, KubernetesEvictionHardKey, KubernetesLabelKey,
+    KubernetesLabelValue, KubernetesQuantityValue, KubernetesReservedResourceKey,
+    KubernetesTaintValue, KubernetesThresholdValue, Lockdown, SingleLineString, SysctlKey, Url,
+    ValidBase64,
 };
 
 // Kubernetes static pod manifest settings
@@ -127,6 +128,7 @@ struct KubernetesSettings {
     kube_reserved: HashMap<KubernetesReservedResourceKey, KubernetesQuantityValue>,
     allowed_unsafe_sysctls: Vec<SingleLineString>,
     server_tls_bootstrap: bool,
+    cloud_provider: KubernetesCloudProvider,
 
     // Settings where we generate a value based on the runtime environment.  The user can specify a
     // value to override the generated one, but typically would not.

sources/models/src/modeled_types/kubernetes.rs

@@ -723,3 +723,48 @@ mod test_kubernetes_quantity_value {
         }
     }
 }
+
+// =^..^=   =^..^=   =^..^=   =^..^=   =^..^=   =^..^=   =^..^=   =^..^=   =^..^=
+
+/// KubernetesCloudProvider represents a string that is a valid cloud provider for the
+/// kubelet.  It stores the original string and makes it accessible through standard traits.
+#[derive(Debug, Clone, Eq, PartialEq, Hash)]
+pub struct KubernetesCloudProvider {
+    inner: String,
+}
+
+impl TryFrom<&str> for KubernetesCloudProvider {
+    type Error = error::Error;
+
+    fn try_from(input: &str) -> Result<Self, error::Error> {
+        ensure!(
+            matches!(input, "aws" | "external"),
+            error::InvalidAuthenticationMode { input }
+        );
+        Ok(KubernetesCloudProvider {
+            inner: input.to_string(),
+        })
+    }
+}
+
+string_impls_for!(KubernetesCloudProvider, "KubernetesCloudProvider");
+
+#[cfg(test)]
+mod test_kubernetes_cloud_provider {
+    use super::KubernetesCloudProvider;
+    use std::convert::TryFrom;
+
+    #[test]
+    fn good_modes() {
+        for ok in &["aws", "external"] {
+            KubernetesCloudProvider::try_from(*ok).unwrap();
+        }
+    }
+
+    #[test]
+    fn bad_modes() {
+        for err in &["", "internal"] {
+            KubernetesCloudProvider::try_from(*err).unwrap_err();
+        }
+    }
+}