diff --git a/README.md b/README.md index 4025c5fc836..2cee842a554 100644 --- a/README.md +++ b/README.md @@ -308,6 +308,7 @@ The following settings can be optionally set to customize the node labels and ta The following settings are optional and allow you to further configure your cluster. * `settings.kubernetes.cluster-domain`: The DNS domain for this cluster, allowing all Kubernetes-run containers to search this domain before the host's search domains. Defaults to `cluster.local`. * `settings.kubernetes.standalone-mode`: Whether to run the kubelet in standalone mode, without connecting to an API server. Defaults to `false`. +* `settings.kubernetes.cloud-provider`: The cloud provider for this cluster. Defaults to `aws` for AWS variants, and `external` for other variants. * `settings.kubernetes.authentication-mode`: Which authentication method the kubelet should use to connect to the API server, and for incoming requests. Defaults to `aws` for AWS variants, and `tls` for other variants. * `settings.kubernetes.server-tls-bootstrap`: Enables or disables server certificate bootstrap. When enabled, the kubelet will request a certificate from the certificates.k8s.io API. This requires an approver to approve the certificate signing requests (CSR). Defaults to `true`. * `settings.kubernetes.bootstrap-token`: The token to use for [TLS bootstrapping](https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet-tls-bootstrapping/). This is only used with the `tls` authentication mode, and is otherwise ignored. diff --git a/Release.toml b/Release.toml index 1b9782d30f1..94c4815d055 100644 --- a/Release.toml +++ b/Release.toml @@ -40,4 +40,5 @@ version = "1.0.8" ] "(1.0.8, 1.1.0)" = [ "migrate_v1.1.0_kubelet-server-tls-bootstrap.lz4", + "migrate_v1.1.0_kubelet-cloud-provider.lz4", ] diff --git a/packages/kubernetes-1.16/kubelet-exec-start-conf b/packages/kubernetes-1.16/kubelet-exec-start-conf index 1cc4d9cf246..4395e39318e 100644 --- a/packages/kubernetes-1.16/kubelet-exec-start-conf +++ b/packages/kubernetes-1.16/kubelet-exec-start-conf @@ -2,7 +2,7 @@ ExecStart= ExecStart=/usr/bin/kubelet \ {{~#unless settings.kubernetes.standalone-mode}} - --cloud-provider aws \ + --cloud-provider {{default "external" settings.kubernetes.cloud-provider}} \ --kubeconfig /etc/kubernetes/kubelet/kubeconfig \ {{~#if (eq settings.kubernetes.authentication-mode "tls")}} --bootstrap-kubeconfig /etc/kubernetes/kubelet/bootstrap-kubeconfig \ diff --git a/packages/kubernetes-1.17/kubelet-exec-start-conf b/packages/kubernetes-1.17/kubelet-exec-start-conf index 1cc4d9cf246..4395e39318e 100644 --- a/packages/kubernetes-1.17/kubelet-exec-start-conf +++ b/packages/kubernetes-1.17/kubelet-exec-start-conf @@ -2,7 +2,7 @@ ExecStart= ExecStart=/usr/bin/kubelet \ {{~#unless settings.kubernetes.standalone-mode}} - --cloud-provider aws \ + --cloud-provider {{default "external" settings.kubernetes.cloud-provider}} \ --kubeconfig /etc/kubernetes/kubelet/kubeconfig \ {{~#if (eq settings.kubernetes.authentication-mode "tls")}} --bootstrap-kubeconfig /etc/kubernetes/kubelet/bootstrap-kubeconfig \ diff --git a/packages/kubernetes-1.18/kubelet-exec-start-conf b/packages/kubernetes-1.18/kubelet-exec-start-conf index 1cc4d9cf246..4395e39318e 100644 --- a/packages/kubernetes-1.18/kubelet-exec-start-conf +++ b/packages/kubernetes-1.18/kubelet-exec-start-conf @@ -2,7 +2,7 @@ ExecStart= ExecStart=/usr/bin/kubelet \ {{~#unless settings.kubernetes.standalone-mode}} - --cloud-provider aws \ + --cloud-provider {{default "external" settings.kubernetes.cloud-provider}} \ --kubeconfig /etc/kubernetes/kubelet/kubeconfig \ {{~#if (eq settings.kubernetes.authentication-mode "tls")}} --bootstrap-kubeconfig /etc/kubernetes/kubelet/bootstrap-kubeconfig \ diff --git a/packages/kubernetes-1.19/kubelet-exec-start-conf b/packages/kubernetes-1.19/kubelet-exec-start-conf index 65a693cb49b..ee65537d040 100644 --- a/packages/kubernetes-1.19/kubelet-exec-start-conf +++ b/packages/kubernetes-1.19/kubelet-exec-start-conf @@ -2,7 +2,7 @@ ExecStart= ExecStart=/usr/bin/kubelet \ {{~#unless settings.kubernetes.standalone-mode}} - --cloud-provider aws \ + --cloud-provider {{default "external" settings.kubernetes.cloud-provider}} \ --kubeconfig /etc/kubernetes/kubelet/kubeconfig \ {{~#if (eq settings.kubernetes.authentication-mode "tls")}} --bootstrap-kubeconfig /etc/kubernetes/kubelet/bootstrap-kubeconfig \ diff --git a/sources/Cargo.lock b/sources/Cargo.lock index 0f7bac9abc2..e9bba01e998 100644 --- a/sources/Cargo.lock +++ b/sources/Cargo.lock @@ -1725,6 +1725,13 @@ dependencies = [ "winapi-build", ] +[[package]] +name = "kubelet-cloud-provider" +version = "0.1.0" +dependencies = [ + "migration-helpers", +] + [[package]] name = "kubelet-eviction-hard" version = "0.1.0" diff --git a/sources/Cargo.toml b/sources/Cargo.toml index cf298a97d33..e937a9fc52f 100644 --- a/sources/Cargo.toml +++ b/sources/Cargo.toml @@ -54,6 +54,7 @@ members = [ "api/migration/migrations/v1.0.8/admin-container-v0-7-0", "api/migration/migrations/v1.0.8/add-bootstrap-containers", "api/migration/migrations/v1.1.0/kubelet-server-tls-bootstrap", + "api/migration/migrations/v1.1.0/kubelet-cloud-provider", "bottlerocket-release", diff --git a/sources/api/migration/migrations/v1.1.0/kubelet-cloud-provider/Cargo.toml b/sources/api/migration/migrations/v1.1.0/kubelet-cloud-provider/Cargo.toml new file mode 100644 index 00000000000..4679fa9ae0f --- /dev/null +++ b/sources/api/migration/migrations/v1.1.0/kubelet-cloud-provider/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "kubelet-cloud-provider" +version = "0.1.0" +authors = ["Patrick J.P. Culp "] +license = "Apache-2.0 OR MIT" +edition = "2018" +publish = false +# Don't rebuild crate just because of changes to README. +exclude = ["README.md"] + +[dependencies] +migration-helpers = { path = "../../../migration-helpers" } diff --git a/sources/api/migration/migrations/v1.1.0/kubelet-cloud-provider/src/main.rs b/sources/api/migration/migrations/v1.1.0/kubelet-cloud-provider/src/main.rs new file mode 100644 index 00000000000..b60b64585df --- /dev/null +++ b/sources/api/migration/migrations/v1.1.0/kubelet-cloud-provider/src/main.rs @@ -0,0 +1,22 @@ +#![deny(rust_2018_idioms)] + +use migration_helpers::common_migrations::AddSettingsMigration; +use migration_helpers::{migrate, Result}; +use std::process; + +/// We added a new settings for configuring kubelet, `settings.kubernetes.cloud-provider` +fn run() -> Result<()> { + migrate(AddSettingsMigration(&[ + "settings.kubernetes.cloud-provider", + ])) +} + +// Returning a Result from main makes it print a Debug representation of the error, but with Snafu +// we have nice Display representations of the error, so we wrap "main" (run) and print any error. +// https://github.com/shepmaster/snafu/issues/110 +fn main() { + if let Err(e) = run() { + eprintln!("{}", e); + process::exit(1); + } +} diff --git a/sources/models/src/aws-k8s-1.19/defaults.d/50-aws-k8s.toml b/sources/models/src/aws-k8s-1.19/defaults.d/50-aws-k8s.toml index 7adb5f41dcc..1830b8e4c8f 100644 --- a/sources/models/src/aws-k8s-1.19/defaults.d/50-aws-k8s.toml +++ b/sources/models/src/aws-k8s-1.19/defaults.d/50-aws-k8s.toml @@ -59,6 +59,7 @@ cluster-domain = "cluster.local" standalone-mode = false authentication-mode = "aws" server-tls-bootstrap = true +cloud-provider = "aws" # Metrics [settings.metrics] diff --git a/sources/models/src/lib.rs b/sources/models/src/lib.rs index 6fab14f51e9..039106fac42 100644 --- a/sources/models/src/lib.rs +++ b/sources/models/src/lib.rs @@ -95,9 +95,10 @@ use std::net::Ipv4Addr; use crate::modeled_types::{ BootstrapContainerMode, DNSDomain, ECSAgentLogLevel, ECSAttributeKey, ECSAttributeValue, FriendlyVersion, Identifier, KubernetesAuthenticationMode, KubernetesBootstrapToken, - KubernetesClusterName, KubernetesEvictionHardKey, KubernetesLabelKey, KubernetesLabelValue, - KubernetesQuantityValue, KubernetesReservedResourceKey, KubernetesTaintValue, - KubernetesThresholdValue, Lockdown, SingleLineString, SysctlKey, Url, ValidBase64, + KubernetesCloudProvider, KubernetesClusterName, KubernetesEvictionHardKey, KubernetesLabelKey, + KubernetesLabelValue, KubernetesQuantityValue, KubernetesReservedResourceKey, + KubernetesTaintValue, KubernetesThresholdValue, Lockdown, SingleLineString, SysctlKey, Url, + ValidBase64, }; // Kubernetes static pod manifest settings @@ -127,6 +128,7 @@ struct KubernetesSettings { kube_reserved: HashMap, allowed_unsafe_sysctls: Vec, server_tls_bootstrap: bool, + cloud_provider: KubernetesCloudProvider, // Settings where we generate a value based on the runtime environment. The user can specify a // value to override the generated one, but typically would not. diff --git a/sources/models/src/modeled_types/kubernetes.rs b/sources/models/src/modeled_types/kubernetes.rs index 6263b7e5b9c..e1fd1b781c5 100644 --- a/sources/models/src/modeled_types/kubernetes.rs +++ b/sources/models/src/modeled_types/kubernetes.rs @@ -723,3 +723,48 @@ mod test_kubernetes_quantity_value { } } } + +// =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= + +/// KubernetesCloudProvider represents a string that is a valid cloud provider for the +/// kubelet. It stores the original string and makes it accessible through standard traits. +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub struct KubernetesCloudProvider { + inner: String, +} + +impl TryFrom<&str> for KubernetesCloudProvider { + type Error = error::Error; + + fn try_from(input: &str) -> Result { + ensure!( + matches!(input, "aws" | "external"), + error::InvalidAuthenticationMode { input } + ); + Ok(KubernetesCloudProvider { + inner: input.to_string(), + }) + } +} + +string_impls_for!(KubernetesCloudProvider, "KubernetesCloudProvider"); + +#[cfg(test)] +mod test_kubernetes_cloud_provider { + use super::KubernetesCloudProvider; + use std::convert::TryFrom; + + #[test] + fn good_modes() { + for ok in &["aws", "external"] { + KubernetesCloudProvider::try_from(*ok).unwrap(); + } + } + + #[test] + fn bad_modes() { + for err in &["", "internal"] { + KubernetesCloudProvider::try_from(*err).unwrap_err(); + } + } +}