diff --git a/.changelog/2233.txt b/.changelog/2233.txt new file mode 100644 index 0000000000..bb929501c9 --- /dev/null +++ b/.changelog/2233.txt @@ -0,0 +1,3 @@ +```release-note:feature +Add support for configuring graceful shutdown proxy lifecycle management settings. +``` diff --git a/acceptance/framework/config/config.go b/acceptance/framework/config/config.go index e724bf6c71..9e84064a8a 100644 --- a/acceptance/framework/config/config.go +++ b/acceptance/framework/config/config.go @@ -43,11 +43,13 @@ type TestConfig struct { DisablePeering bool - HelmChartVersion string - ConsulImage string - ConsulK8SImage string - ConsulVersion *version.Version - EnvoyImage string + HelmChartVersion string + ConsulImage string + ConsulK8SImage string + ConsulDataplaneImage string + ConsulVersion *version.Version + ConsulDataplaneVersion *version.Version + EnvoyImage string NoCleanupOnFailure bool DebugDirectory string @@ -101,6 +103,7 @@ func (t *TestConfig) HelmValuesFromConfig() (map[string]string, error) { setIfNotEmpty(helmValues, "global.image", t.ConsulImage) setIfNotEmpty(helmValues, "global.imageK8S", t.ConsulK8SImage) setIfNotEmpty(helmValues, "global.imageEnvoy", t.EnvoyImage) + setIfNotEmpty(helmValues, "global.imageConsulDataplane", t.ConsulDataplaneImage) return helmValues, nil } diff --git a/acceptance/framework/connhelper/connect_helper.go b/acceptance/framework/connhelper/connect_helper.go index c5d677ba6f..96e0fbb4d9 100644 --- a/acceptance/framework/connhelper/connect_helper.go +++ b/acceptance/framework/connhelper/connect_helper.go @@ -47,8 +47,8 @@ type ConnectHelper struct { // consulCluster is the cluster to use for the test. consulCluster consul.Cluster - // consulClient is the client used to test service mesh connectivity. - consulClient *api.Client + // ConsulClient is the client used to test service mesh connectivity. + ConsulClient *api.Client } // Setup creates a new cluster using the New*Cluster function and assigns it @@ -66,14 +66,14 @@ func (c *ConnectHelper) Setup(t *testing.T) { func (c *ConnectHelper) Install(t *testing.T) { logger.Log(t, "Installing Consul cluster") c.consulCluster.Create(t) - c.consulClient, _ = c.consulCluster.SetupConsulClient(t, c.Secure) + c.ConsulClient, _ = c.consulCluster.SetupConsulClient(t, c.Secure) } // Upgrade uses the existing Consul cluster and upgrades it using Helm values // set by the Secure, AutoEncrypt, and HelmValues fields. func (c *ConnectHelper) Upgrade(t *testing.T) { require.NotNil(t, c.consulCluster, "consulCluster must be set before calling Upgrade (Call Install first).") - require.NotNil(t, c.consulClient, "consulClient must be set before calling Upgrade (Call Install first).") + require.NotNil(t, c.ConsulClient, "ConsulClient must be set before calling Upgrade (Call Install first).") logger.Log(t, "upgrading Consul cluster") c.consulCluster.Upgrade(t, c.helmValues()) @@ -93,7 +93,7 @@ func (c *ConnectHelper) DeployClientAndServer(t *testing.T) { t.Cleanup(func() { retrier := &retry.Timer{Timeout: 30 * time.Second, Wait: 100 * time.Millisecond} retry.RunWith(retrier, t, func(r *retry.R) { - tokens, _, err := c.consulClient.ACL().TokenList(nil) + tokens, _, err := c.ConsulClient.ACL().TokenList(nil) require.NoError(r, err) for _, token := range tokens { require.NotContains(r, token.Description, StaticServerName) @@ -139,7 +139,7 @@ func (c *ConnectHelper) TestConnectionFailureWithoutIntention(t *testing.T) { // the static-client pod. func (c *ConnectHelper) CreateIntention(t *testing.T) { logger.Log(t, "creating intention") - _, _, err := c.consulClient.ConfigEntries().Set(&api.ServiceIntentionsConfigEntry{ + _, _, err := c.ConsulClient.ConfigEntries().Set(&api.ServiceIntentionsConfigEntry{ Kind: api.ServiceIntentions, Name: StaticServerName, Sources: []*api.SourceIntention{ diff --git a/acceptance/framework/flags/flags.go b/acceptance/framework/flags/flags.go index 9d9bf1d1e4..1fabcda09b 100644 --- a/acceptance/framework/flags/flags.go +++ b/acceptance/framework/flags/flags.go @@ -31,11 +31,13 @@ type TestFlags struct { flagEnableTransparentProxy bool - flagHelmChartVersion string - flagConsulImage string - flagConsulK8sImage string - flagConsulVersion string - flagEnvoyImage string + flagHelmChartVersion string + flagConsulImage string + flagConsulK8sImage string + flagConsulDataplaneImage string + flagConsulVersion string + flagConsulDataplaneVersion string + flagEnvoyImage string flagNoCleanupOnFailure bool @@ -67,7 +69,9 @@ func (t *TestFlags) init() { flag.StringVar(&t.flagConsulImage, "consul-image", "", "The Consul image to use for all tests.") flag.StringVar(&t.flagConsulK8sImage, "consul-k8s-image", "", "The consul-k8s image to use for all tests.") + flag.StringVar(&t.flagConsulDataplaneImage, "consul-dataplane-image", "", "The consul-dataplane image to use for all tests.") flag.StringVar(&t.flagConsulVersion, "consul-version", "", "The consul version used for all tests.") + flag.StringVar(&t.flagConsulDataplaneVersion, "consul-dataplane-version", "", "The consul-dataplane version used for all tests.") flag.StringVar(&t.flagHelmChartVersion, "helm-chart-version", config.HelmChartPath, "The helm chart used for all tests.") flag.StringVar(&t.flagEnvoyImage, "envoy-image", "", "The Envoy image to use for all tests.") @@ -142,6 +146,8 @@ func (t *TestFlags) TestConfigFromFlags() *config.TestConfig { // if the Version is empty consulVersion will be nil consulVersion, _ := version.NewVersion(t.flagConsulVersion) + consulDataplaneVersion, _ := version.NewVersion(t.flagConsulDataplaneVersion) + //vaultserverVersion, _ := version.NewVersion(t.flagVaultServerVersion) return &config.TestConfig{ Kubeconfig: t.flagKubeconfig, @@ -166,11 +172,13 @@ func (t *TestFlags) TestConfigFromFlags() *config.TestConfig { DisablePeering: t.flagDisablePeering, - HelmChartVersion: t.flagHelmChartVersion, - ConsulImage: t.flagConsulImage, - ConsulK8SImage: t.flagConsulK8sImage, - ConsulVersion: consulVersion, - EnvoyImage: t.flagEnvoyImage, + HelmChartVersion: t.flagHelmChartVersion, + ConsulImage: t.flagConsulImage, + ConsulK8SImage: t.flagConsulK8sImage, + ConsulDataplaneImage: t.flagConsulDataplaneImage, + ConsulVersion: consulVersion, + ConsulDataplaneVersion: consulDataplaneVersion, + EnvoyImage: t.flagEnvoyImage, NoCleanupOnFailure: t.flagNoCleanupOnFailure, DebugDirectory: tempDir, diff --git a/acceptance/tests/connect/connect_proxy_lifecycle_test.go b/acceptance/tests/connect/connect_proxy_lifecycle_test.go new file mode 100644 index 0000000000..3ec5d0a9b8 --- /dev/null +++ b/acceptance/tests/connect/connect_proxy_lifecycle_test.go @@ -0,0 +1,207 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package connect + +import ( + "context" + "fmt" + "strconv" + "strings" + "testing" + "time" + + "github.com/gruntwork-io/terratest/modules/k8s" + "github.com/hashicorp/consul-k8s/acceptance/framework/connhelper" + "github.com/hashicorp/consul-k8s/acceptance/framework/consul" + "github.com/hashicorp/consul-k8s/acceptance/framework/helpers" + "github.com/hashicorp/consul-k8s/acceptance/framework/logger" + "github.com/hashicorp/consul/sdk/testutil/retry" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type LifecycleShutdownConfig struct { + secure bool + helmValues map[string]string +} + +const ( + helmDrainListenersKey = "connectInject.sidecarProxy.lifecycle.defaultEnableShutdownDrainListeners" + helmGracePeriodSecondsKey = "connectInject.sidecarProxy.lifecycle.defaultShutdownGracePeriodSeconds" +) + +// Test the endpoints controller cleans up force-killed pods. +func TestConnectInject_ProxyLifecycleShutdown(t *testing.T) { + t.Skipf("skiping this test, will be re-added in a future commit") + cfg := suite.Config() + + for _, testCfg := range []LifecycleShutdownConfig{ + {secure: false, helmValues: map[string]string{ + helmDrainListenersKey: "true", + helmGracePeriodSecondsKey: "15", + }}, + {secure: true, helmValues: map[string]string{ + helmDrainListenersKey: "true", + helmGracePeriodSecondsKey: "15", + }}, + {secure: false, helmValues: map[string]string{ + helmDrainListenersKey: "false", + helmGracePeriodSecondsKey: "15", + }}, + {secure: true, helmValues: map[string]string{ + helmDrainListenersKey: "false", + helmGracePeriodSecondsKey: "15", + }}, + {secure: false, helmValues: map[string]string{ + helmDrainListenersKey: "false", + helmGracePeriodSecondsKey: "0", + }}, + {secure: true, helmValues: map[string]string{ + helmDrainListenersKey: "false", + helmGracePeriodSecondsKey: "0", + }}, + } { + // Determine if listeners should be expected to drain inbound connections + var drainListenersEnabled bool + var err error + val, ok := testCfg.helmValues[helmDrainListenersKey] + if ok { + drainListenersEnabled, err = strconv.ParseBool(val) + require.NoError(t, err) + } + + // Determine expected shutdown grace period + var gracePeriodSeconds int64 + val, ok = testCfg.helmValues[helmGracePeriodSecondsKey] + if ok { + gracePeriodSeconds, err = strconv.ParseInt(val, 10, 64) + require.NoError(t, err) + } else { + // Half of the helm default to speed tests up + gracePeriodSeconds = 15 + } + + name := fmt.Sprintf("secure: %t, drainListeners: %t, gracePeriodSeconds: %d", testCfg.secure, drainListenersEnabled, gracePeriodSeconds) + t.Run(name, func(t *testing.T) { + ctx := suite.Environment().DefaultContext(t) + releaseName := helpers.RandomName() + + connHelper := connhelper.ConnectHelper{ + ClusterKind: consul.Helm, + Secure: testCfg.secure, + ReleaseName: releaseName, + Ctx: ctx, + Cfg: cfg, + HelmValues: testCfg.helmValues, + } + + connHelper.Setup(t) + connHelper.Install(t) + connHelper.DeployClientAndServer(t) + + // TODO: should this move into connhelper.DeployClientAndServer? + logger.Log(t, "waiting for static-client and static-server to be registered with Consul") + retry.Run(t, func(r *retry.R) { + for _, name := range []string{ + "static-client", + "static-client-sidecar-proxy", + "static-server", + "static-server-sidecar-proxy", + } { + logger.Logf(t, "checking for %s service in Consul catalog", name) + instances, _, err := connHelper.ConsulClient.Catalog().Service(name, "", nil) + r.Check(err) + + if len(instances) != 1 { + r.Errorf("expected 1 instance of %s", name) + } + } + }) + + if testCfg.secure { + connHelper.TestConnectionFailureWithoutIntention(t) + connHelper.CreateIntention(t) + } + + connHelper.TestConnectionSuccess(t) + + // Get static-client pod name + ns := ctx.KubectlOptions(t).Namespace + pods, err := ctx.KubernetesClient(t).CoreV1().Pods(ns).List( + context.Background(), + metav1.ListOptions{ + LabelSelector: "app=static-client", + }, + ) + require.NoError(t, err) + require.Len(t, pods.Items, 1) + clientPodName := pods.Items[0].Name + + var terminationGracePeriod int64 = 60 + logger.Logf(t, "killing the %q pod with %dseconds termination grace period", clientPodName, terminationGracePeriod) + err = ctx.KubernetesClient(t).CoreV1().Pods(ns).Delete(context.Background(), clientPodName, metav1.DeleteOptions{GracePeriodSeconds: &terminationGracePeriod}) + require.NoError(t, err) + + // Exec into terminating pod, not just any static-client pod + args := []string{"exec", clientPodName, "-c", connhelper.StaticClientName, "--", "curl", "-vvvsSf"} + + if cfg.EnableTransparentProxy { + args = append(args, "http://static-server") + } else { + args = append(args, "http://localhost:1234") + } + + if gracePeriodSeconds > 0 { + // Ensure outbound requests are still successful during grace + // period. + retry.RunWith(&retry.Timer{Timeout: time.Duration(gracePeriodSeconds) * time.Second, Wait: 2 * time.Second}, t, func(r *retry.R) { + output, err := k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), args...) + require.NoError(r, err) + require.Condition(r, func() bool { + exists := false + if strings.Contains(output, "curl: (7) Failed to connect") { + exists = true + } + return !exists + }) + }) + + // If listener draining is enabled, ensure inbound + // requests are rejected during grace period. + // connHelper.TestConnectionSuccess(t) + } else { + // Ensure outbound requests fail because proxy has terminated + retry.RunWith(&retry.Timer{Timeout: time.Duration(terminationGracePeriod) * time.Second, Wait: 2 * time.Second}, t, func(r *retry.R) { + output, err := k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), args...) + require.Error(r, err) + require.Condition(r, func() bool { + exists := false + if strings.Contains(output, "curl: (7) Failed to connect") { + exists = true + } + return exists + }) + }) + } + + logger.Log(t, "ensuring pod is deregistered after termination") + retry.Run(t, func(r *retry.R) { + for _, name := range []string{ + "static-client", + "static-client-sidecar-proxy", + } { + logger.Logf(t, "checking for %s service in Consul catalog", name) + instances, _, err := connHelper.ConsulClient.Catalog().Service(name, "", nil) + r.Check(err) + + for _, instance := range instances { + if strings.Contains(instance.ServiceID, clientPodName) { + r.Errorf("%s is still registered", instance.ServiceID) + } + } + } + }) + }) + } +} diff --git a/charts/consul/templates/connect-inject-deployment.yaml b/charts/consul/templates/connect-inject-deployment.yaml index 2b52c1b81c..6f15778888 100644 --- a/charts/consul/templates/connect-inject-deployment.yaml +++ b/charts/consul/templates/connect-inject-deployment.yaml @@ -234,6 +234,19 @@ spec: -default-sidecar-proxy-cpu-request={{ $resources.requests.cpu }} \ {{- end }} -default-envoy-proxy-concurrency={{ .Values.connectInject.sidecarProxy.concurrency }} \ + {{- if .Values.connectInject.sidecarProxy.lifecycle.defaultEnabled }} + -default-enable-sidecar-proxy-lifecycle=true \ + {{- else }} + -default-enable-sidecar-proxy-lifecycle=false \ + {{- end }} + {{- if .Values.connectInject.sidecarProxy.lifecycle.defaultEnableShutdownDrainListeners }} + -default-enable-sidecar-proxy-lifecycle-shutdown-drain-listeners=true \ + {{- else }} + -default-enable-sidecar-proxy-lifecycle-shutdown-drain-listeners=false \ + {{- end }} + -default-sidecar-proxy-lifecycle-shutdown-grace-period-seconds={{ .Values.connectInject.sidecarProxy.lifecycle.defaultShutdownGracePeriodSeconds }} \ + -default-sidecar-proxy-lifecycle-graceful-port={{ .Values.connectInject.sidecarProxy.lifecycle.defaultGracefulPort }} \ + -default-sidecar-proxy-lifecycle-graceful-shutdown-path="{{ .Values.connectInject.sidecarProxy.lifecycle.defaultGracefulShutdownPath }}" \ {{- if .Values.connectInject.initContainer }} {{- $initResources := .Values.connectInject.initContainer.resources }} diff --git a/charts/consul/test/unit/connect-inject-deployment.bats b/charts/consul/test/unit/connect-inject-deployment.bats index 4c396cbb0b..8a4716889a 100755 --- a/charts/consul/test/unit/connect-inject-deployment.bats +++ b/charts/consul/test/unit/connect-inject-deployment.bats @@ -986,7 +986,7 @@ load _helpers local actual=$(echo "$cmd" | yq 'any(contains("-init-container-memory-limit=150Mi"))' | tee /dev/stderr) [ "${actual}" = "true" ] - + } @test "connectInject/Deployment: can set init container resources" { @@ -1218,6 +1218,144 @@ load _helpers [ "${actual}" = "true" ] } +#-------------------------------------------------------------------- +# sidecarProxy.lifecycle + +@test "connectInject/Deployment: by default sidecar proxy lifecycle management is enabled" { + cd `chart_dir` + local cmd=$(helm template \ + -s templates/connect-inject-deployment.yaml \ + --set 'connectInject.enabled=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].command' | tee /dev/stderr) + + local actual=$(echo "$cmd" | + yq 'any(contains("-default-enable-sidecar-proxy-lifecycle"))' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "connectInject/Deployment: sidecar proxy lifecycle management can be disabled" { + cd `chart_dir` + local cmd=$(helm template \ + -s templates/connect-inject-deployment.yaml \ + --set 'connectInject.enabled=true' \ + --set 'connectInject.sidecarProxy.lifecycle.defaultEnabled=false' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].command' | tee /dev/stderr) + + local actual=$(echo "$cmd" | + yq 'any(contains("-default-enable-sidecar-proxy-lifecycle=false"))' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "connectInject/Deployment: by default sidecar proxy lifecycle management shutdown listener draining is enabled" { + cd `chart_dir` + local cmd=$(helm template \ + -s templates/connect-inject-deployment.yaml \ + --set 'connectInject.enabled=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].command' | tee /dev/stderr) + + local actual=$(echo "$cmd" | + yq 'any(contains("-default-enable-sidecar-proxy-lifecycle-shutdown-drain-listeners"))' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "connectInject/Deployment: sidecar proxy lifecycle management shutdown listener draining can be disabled" { + cd `chart_dir` + local cmd=$(helm template \ + -s templates/connect-inject-deployment.yaml \ + --set 'connectInject.enabled=true' \ + --set 'connectInject.sidecarProxy.lifecycle.defaultEnableShutdownDrainListeners=false' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].command' | tee /dev/stderr) + + local actual=$(echo "$cmd" | + yq 'any(contains("-default-enable-sidecar-proxy-lifecycle-shutdown-drain-listeners=false"))' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "connectInject/Deployment: by default sidecar proxy lifecycle management shutdown grace period is set to 30 seconds" { + cd `chart_dir` + local cmd=$(helm template \ + -s templates/connect-inject-deployment.yaml \ + --set 'connectInject.enabled=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].command' | tee /dev/stderr) + + local actual=$(echo "$cmd" | + yq 'any(contains("-default-sidecar-proxy-lifecycle-shutdown-grace-period-seconds=30"))' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "connectInject/Deployment: sidecar proxy lifecycle management shutdown grace period can be set" { + cd `chart_dir` + local cmd=$(helm template \ + -s templates/connect-inject-deployment.yaml \ + --set 'connectInject.enabled=true' \ + --set 'connectInject.sidecarProxy.lifecycle.defaultShutdownGracePeriodSeconds=23' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].command' | tee /dev/stderr) + + local actual=$(echo "$cmd" | + yq 'any(contains("-default-sidecar-proxy-lifecycle-shutdown-grace-period-seconds=23"))' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "connectInject/Deployment: by default sidecar proxy lifecycle management port is set to 20600" { + cd `chart_dir` + local cmd=$(helm template \ + -s templates/connect-inject-deployment.yaml \ + --set 'connectInject.enabled=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].command' | tee /dev/stderr) + + local actual=$(echo "$cmd" | + yq 'any(contains("-default-sidecar-proxy-lifecycle-graceful-port=20600"))' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "connectInject/Deployment: sidecar proxy lifecycle management port can be set" { + cd `chart_dir` + local cmd=$(helm template \ + -s templates/connect-inject-deployment.yaml \ + --set 'connectInject.enabled=true' \ + --set 'connectInject.sidecarProxy.lifecycle.defaultGracefulPort=20307' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].command' | tee /dev/stderr) + + local actual=$(echo "$cmd" | + yq 'any(contains("-default-sidecar-proxy-lifecycle-graceful-port=20307"))' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "connectInject/Deployment: by default sidecar proxy lifecycle management graceful shutdown path is set to /graceful_shutdown" { + cd `chart_dir` + local cmd=$(helm template \ + -s templates/connect-inject-deployment.yaml \ + --set 'connectInject.enabled=true' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].command' | tee /dev/stderr) + + local actual=$(echo "$cmd" | + yq 'any(contains("-default-sidecar-proxy-lifecycle-graceful-shutdown-path=\"/graceful_shutdown\""))' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "connectInject/Deployment: sidecar proxy lifecycle management graceful shutdown path can be set" { + cd `chart_dir` + local cmd=$(helm template \ + -s templates/connect-inject-deployment.yaml \ + --set 'connectInject.enabled=true' \ + --set 'connectInject.sidecarProxy.lifecycle.defaultGracefulShutdownPath=/exit' \ + . | tee /dev/stderr | + yq '.spec.template.spec.containers[0].command' | tee /dev/stderr) + + local actual=$(echo "$cmd" | + yq 'any(contains("-default-sidecar-proxy-lifecycle-graceful-shutdown-path=\"/exit\""))' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + #-------------------------------------------------------------------- # priorityClassName @@ -1405,7 +1543,7 @@ load _helpers } #-------------------------------------------------------------------- -# cni +# cni @test "connectInject/Deployment: cni is disabled by default" { cd `chart_dir` @@ -2305,7 +2443,7 @@ reservedNameTest() { --set 'global.cloud.authUrl.secretName=auth-url-name' \ . [ "$status" -eq 1 ] - + [[ "$output" =~ "When either global.cloud.authUrl.secretName or global.cloud.authUrl.secretKey is defined, both must be set." ]] } @@ -2326,7 +2464,7 @@ reservedNameTest() { --set 'global.cloud.authUrl.secretKey=auth-url-key' \ . [ "$status" -eq 1 ] - + [[ "$output" =~ "When either global.cloud.authUrl.secretName or global.cloud.authUrl.secretKey is defined, both must be set." ]] } @@ -2347,7 +2485,7 @@ reservedNameTest() { --set 'global.cloud.apiHost.secretName=auth-url-name' \ . [ "$status" -eq 1 ] - + [[ "$output" =~ "When either global.cloud.apiHost.secretName or global.cloud.apiHost.secretKey is defined, both must be set." ]] } @@ -2368,7 +2506,7 @@ reservedNameTest() { --set 'global.cloud.apiHost.secretKey=auth-url-key' \ . [ "$status" -eq 1 ] - + [[ "$output" =~ "When either global.cloud.apiHost.secretName or global.cloud.apiHost.secretKey is defined, both must be set." ]] } @@ -2389,7 +2527,7 @@ reservedNameTest() { --set 'global.cloud.scadaAddress.secretName=scada-address-name' \ . [ "$status" -eq 1 ] - + [[ "$output" =~ "When either global.cloud.scadaAddress.secretName or global.cloud.scadaAddress.secretKey is defined, both must be set." ]] } @@ -2410,7 +2548,7 @@ reservedNameTest() { --set 'global.cloud.scadaAddress.secretKey=scada-address-key' \ . [ "$status" -eq 1 ] - + [[ "$output" =~ "When either global.cloud.scadaAddress.secretName or global.cloud.scadaAddress.secretKey is defined, both must be set." ]] } diff --git a/charts/consul/values.yaml b/charts/consul/values.yaml index fe6fc6eddc..175380c40d 100644 --- a/charts/consul/values.yaml +++ b/charts/consul/values.yaml @@ -77,7 +77,7 @@ global: # - name: pull-secret-name-2 # ``` # @type: array - imagePullSecrets: [ ] + imagePullSecrets: [] # The name (and tag) of the consul-k8s-control-plane Docker # image that is used for functionality such as catalog sync. @@ -252,7 +252,7 @@ global: # @type: string secretName: null - # Configuration to the Vault Secret that Kubernetes will use on + # Configuration to the Vault Secret that Kubernetes will use on # Kubernetes CRD creation, deletion, and update, to get CA certificates # used issued from vault to send webhooks to the controller. caCert: @@ -317,7 +317,7 @@ global: # Refer to [`-recursor`](https://developer.hashicorp.com/consul/docs/agent/config/cli-flags#_recursor) for more details. # If this is an empty array (the default), then Consul DNS will only resolve queries for the Consul top level domain (by default `.consul`). # @type: array - recursors: [ ] + recursors: [] # Enables [TLS](https://developer.hashicorp.com/consul/tutorials/security/tls-encryption-secure) # across the cluster to verify authenticity of the Consul servers and clients. @@ -338,13 +338,13 @@ global: # in the server certificate. This is useful when you need to access the # Consul server(s) externally, for example, if you're using the UI. # @type: array - serverAdditionalDNSSANs: [ ] + serverAdditionalDNSSANs: [] # A list of additional IP addresses to set as Subject Alternative Names (SANs) # in the server certificate. This is useful when you need to access the # Consul server(s) externally, for example, if you're using the UI. # @type: array - serverAdditionalIPSANs: [ ] + serverAdditionalIPSANs: [] # If true, `verify_outgoing`, `verify_server_hostname`, # and `verify_incoming` for internal RPC communication will be set to `true` for Consul servers and clients. @@ -411,7 +411,6 @@ global: # Configure ACLs. acls: - # If true, the Helm chart will automatically manage ACL tokens and policies # for all Consul and consul-k8s-control-plane components. # This requires Consul >= 1.4. @@ -447,7 +446,7 @@ global: # @type: string secretKey: null - # The resource requests (CPU, memory, etc.) for the server-acl-init and server-acl-init-cleanup pods. + # The resource requests (CPU, memory, etc.) for the server-acl-init and server-acl-init-cleanup pods. # This should be a YAML map corresponding to a Kubernetes # [`ResourceRequirements``](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#resourcerequirements-v1-core) # object. @@ -548,7 +547,7 @@ global: # A list of addresses of the primary mesh gateways in the form `:`. # (e.g. ["1.1.1.1:443", "2.3.4.5:443"] # @type: array - primaryGateways: [ ] + primaryGateways: [] # If you are setting `global.federation.enabled` to true and are in a secondary datacenter, # set `k8sAuthMethodHost` to the address of the Kubernetes API server of the secondary datacenter. @@ -693,7 +692,6 @@ global: # be disabled if you plan on connecting to a Consul cluster external to # the Kube cluster. server: - # If true, the chart will install all the resources necessary for a # Consul server cluster. If you're running Consul externally and want agents # within Kubernetes to join that cluster, this should probably be false. @@ -948,7 +946,7 @@ server: # with `-config-dir`. This defaults to false. # # @type: array - extraVolumes: [ ] + extraVolumes: [] # A list of sidecar containers. # Example: @@ -961,7 +959,7 @@ server: # - ... # ``` # @type: array - extraContainers: [ ] + extraContainers: [] # This value defines the [affinity](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity) # for server pods. It defaults to allowing only a single server pod on each node, which @@ -993,14 +991,14 @@ server: topologyKey: kubernetes.io/hostname # Toleration settings for server pods. This - # should be a multi-line string matching the - # [Tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) + # should be a multi-line string matching the + # [Tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) # array in a Pod spec. tolerations: "" # Pod topology spread constraints for server pods. - # This should be a multi-line YAML string matching the - # [`topologySpreadConstraints`](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/) + # This should be a multi-line YAML string matching the + # [`topologySpreadConstraints`](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/) # array in a Pod Spec. # # This requires K8S >= 1.18 (beta) or 1.19 (stable). @@ -1117,9 +1115,9 @@ server: # feature, in case kubernetes cluster is behind egress http proxies. Additionally, # it could be used to configure custom consul parameters. # @type: map - extraEnvironmentVars: { } + extraEnvironmentVars: {} - # [Enterprise Only] Values for setting up and running + # [Enterprise Only] Values for setting up and running # [snapshot agents](https://developer.hashicorp.com/consul/commands/snapshot/agent) # within the Consul clusters. They run as a sidecar with Consul servers. snapshotAgent: @@ -1169,7 +1167,7 @@ server: # @type: string caCert: null - # [Enterprise Only] Added in Consul 1.8, the audit object allow users to enable auditing + # [Enterprise Only] Added in Consul 1.8, the audit object allow users to enable auditing # and configure a sink and filters for their audit logs. Please refer to # [audit logs](https://developer.hashicorp.com/consul/docs/enterprise/audit-logging) documentation # for further information. @@ -1178,7 +1176,7 @@ server: # global.acls.manageSystemACLs must be enabled to use this feature. enabled: false - # A single entry of the sink object provides configuration for the destination to which Consul + # A single entry of the sink object provides configuration for the destination to which Consul # will log auditing events. # # Example: @@ -1193,7 +1191,7 @@ server: # rotate_duration: 24h # rotate_max_files: 15 # rotate_bytes: 25165824 - # + # # ``` # # The sink object supports the following keys: @@ -1242,7 +1240,7 @@ externalServers: # should be the same, however, they may be different if you # wish to use separate hosts for the HTTPS connections. # @type: array - hosts: [ ] + hosts: [] # The HTTPS port of the Consul servers. httpsPort: 8501 @@ -1440,7 +1438,7 @@ client: # with `-config-dir`. This defaults to false. # # @type: array - extraVolumes: [ ] + extraVolumes: [] # A list of sidecar containers. # Example: @@ -1453,7 +1451,7 @@ client: # - ... # ``` # @type: array - extraContainers: [ ] + extraContainers: [] # Toleration Settings for Client pods # This should be a multi-line string matching the Toleration array @@ -1531,7 +1529,7 @@ client: # feature, in case kubernetes cluster is behind egress http proxies. Additionally, # it could be used to configure custom consul parameters. # @type: map - extraEnvironmentVars: { } + extraEnvironmentVars: {} # This value defines the [Pod DNS policy](https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy) # for client pods to use. @@ -1625,7 +1623,6 @@ ui: # Set the port value of the UI service. port: - # HTTP port. http: 80 @@ -1636,7 +1633,6 @@ ui: # If not set and using a NodePort service, Kubernetes will automatically assign # a port. nodePort: - # HTTP node port # @type: integer http: null @@ -1689,7 +1685,7 @@ ui: # ``` # # @type: array - hosts: [ ] + hosts: [] # tls is a list of hosts and secret name in an Ingress # which tells the Ingress controller to secure the channel. @@ -1701,7 +1697,7 @@ ui: # secretName: testsecret-tls # ``` # @type: array - tls: [ ] + tls: [] # Annotations to apply to the UI ingress. # @@ -1792,7 +1788,7 @@ syncCatalog: # # Note: `k8sDenyNamespaces` takes precedence over values defined here. # @type: array - k8sAllowNamespaces: [ "*" ] + k8sAllowNamespaces: ["*"] # List of k8s namespaces that should not have their # services synced. This list takes precedence over `k8sAllowNamespaces`. @@ -1802,7 +1798,7 @@ syncCatalog: # `["namespace1", "namespace2"]`, then all k8s namespaces besides `namespace1` # and `namespace2` will be synced. # @type: array - k8sDenyNamespaces: [ "kube-system", "kube-public" ] + k8sDenyNamespaces: ["kube-system", "kube-public"] # [DEPRECATED] Use k8sAllowNamespaces and k8sDenyNamespaces instead. For # backwards compatibility, if both this and the allow/deny lists are set, @@ -2060,7 +2056,7 @@ connectInject: logLevel: null # Set the namespace to install the CNI plugin into. Overrides global namespace settings for CNI resources. - # Ex: "kube-system" + # Ex: "kube-system" # @type: string namespace: null @@ -2148,7 +2144,6 @@ connectInject: # @type: map meta: null - # Configures metrics for Consul Connect services. All values are overridable # via annotations on a per-pod basis. metrics: @@ -2243,7 +2238,7 @@ connectInject: # @type: string annotations: null - # The resource settings for connect inject pods. The defaults, are optimized for getting started worklows on developer deployments. The settings should be tweaked for production deployments. + # The resource settings for connect inject pods. The defaults, are optimized for getting started worklows on developer deployments. The settings should be tweaked for production deployments. # @type: map resources: requests: @@ -2308,7 +2303,7 @@ connectInject: # `namespaceSelector` takes precedence over both since it is applied first. # `kube-system` and `kube-public` are never injected, even if included here. # @type: array - k8sAllowNamespaces: [ "*" ] + k8sAllowNamespaces: ["*"] # List of k8s namespaces that should not allow Connect # sidecar injection. This list takes precedence over `k8sAllowNamespaces`. @@ -2321,7 +2316,7 @@ connectInject: # Note: `namespaceSelector` takes precedence over this since it is applied first. # `kube-system` and `kube-public` are never injected. # @type: array - k8sDenyNamespaces: [ ] + k8sDenyNamespaces: [] # [Enterprise Only] These settings manage the connect injector's interaction with # Consul namespaces (requires consul-ent v1.7+). @@ -2439,10 +2434,30 @@ connectInject: # Recommended production default: 100m # @type: string cpu: null + # Set default lifecycle management configuration for sidecar proxy. + # These settings can be overridden on a per-pod basis via these annotations: + # + # - `consul.hashicorp.com/enable-sidecar-proxy-lifecycle` + # - `consul.hashicorp.com/enable-sidecar-proxy-shutdown-drain-listeners` + # - `consul.hashicorp.com/sidecar-proxy-lifecycle-shutdown-grace-period-seconds` + # - `consul.hashicorp.com/sidecar-proxy-lifecycle-graceful-port` + # - `consul.hashicorp.com/sidecar-proxy-lifecycle-graceful-shutdown-path` + # @type: map + lifecycle: + # @type: boolean + defaultEnabled: true + # @type: boolean + defaultEnableShutdownDrainListeners: true + # @type: integer + defaultShutdownGracePeriodSeconds: 30 + # @type: integer + defaultGracefulPort: 20600 + # @type: string + defaultGracefulShutdownPath: "/graceful_shutdown" # The resource settings for the Connect injected init container. If null, the resources - # won't be set for the initContainer. The defaults are optimized for developer instances of - # Kubernetes, however they should be tweaked with the recommended defaults as shown below to speed up service registration times. + # won't be set for the initContainer. The defaults are optimized for developer instances of + # Kubernetes, however they should be tweaked with the recommended defaults as shown below to speed up service registration times. # @type: map initContainer: resources: @@ -2631,7 +2646,7 @@ meshGateway: tolerations: null # Pod topology spread constraints for mesh gateway pods. - # This should be a multi-line YAML string matching the + # This should be a multi-line YAML string matching the # [`topologySpreadConstraints`](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/) # array in a Pod Spec. # @@ -2707,10 +2722,10 @@ ingressGateways: # @default: [{port: 8080, port: 8443}] # @recurse: false ports: - - port: 8080 - nodePort: null - - port: 8443 - nodePort: null + - port: 8080 + nodePort: null + - port: 8443 + nodePort: null # Annotations to apply to the ingress gateway service. Annotations defined # here will be applied to all ingress gateway services in addition to any @@ -2779,7 +2794,7 @@ ingressGateways: tolerations: null # Pod topology spread constraints for ingress gateway pods. - # This should be a multi-line YAML string matching the + # This should be a multi-line YAML string matching the # [`topologySpreadConstraints`](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/) # array in a Pod Spec. # @@ -2835,7 +2850,7 @@ ingressGateways: # case of annotations where both will be applied. # @type: array gateways: - - name: ingress-gateway + - name: ingress-gateway # Configuration options for terminating gateways. Default values for all # terminating gateways are defined in `terminatingGateways.defaults`. Any of @@ -2871,7 +2886,7 @@ terminatingGateways: # path: path # secret will now mount to /consul/userconfig/my-secret/path # ``` # @type: array - extraVolumes: [ ] + extraVolumes: [] # Resource limits for all terminating gateway pods # @recurse: false @@ -2977,7 +2992,7 @@ terminatingGateways: # case of annotations where both will be applied. # @type: array gateways: - - name: terminating-gateway + - name: terminating-gateway # Configuration settings for the Consul API Gateway integration apiGateway: @@ -3017,7 +3032,7 @@ apiGateway: # @type: string nodeSelector: null - # Toleration settings for gateway pods created with the managed gateway class. + # Toleration settings for gateway pods created with the managed gateway class. # This should be a multi-line string matching the # [Tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) array in a Pod spec. # @@ -3150,7 +3165,6 @@ apiGateway: # Configuration settings for the webhook-cert-manager # `webhook-cert-manager` ensures that cert bundles are up to date for the mutating webhook. webhookCertManager: - # Toleration Settings # This should be a multi-line string matching the Toleration array # in a PodSpec. diff --git a/cli/helm/values.go b/cli/helm/values.go index e7042f85e1..1f55ecfdf3 100644 --- a/cli/helm/values.go +++ b/cli/helm/values.go @@ -408,7 +408,7 @@ type TransparentProxy struct { } type Metrics struct { - DefaultEnabled string `yaml:"defaultEnabled"` + DefaultEnabled bool `yaml:"defaultEnabled"` DefaultEnableMerging bool `yaml:"defaultEnableMerging"` DefaultMergedMetricsPort int `yaml:"defaultMergedMetricsPort"` DefaultPrometheusScrapePort int `yaml:"defaultPrometheusScrapePort"` @@ -422,12 +422,21 @@ type ACLInjectToken struct { type SidecarProxy struct { Resources Resources `yaml:"resources"` + Lifecycle Lifecycle `yaml:"lifecycle"` } type InitContainer struct { Resources Resources `yaml:"resources"` } +type Lifecycle struct { + DefaultEnabled bool `yaml:"defaultEnabled"` + DefaultEnableShutdownDrainListeners bool `yaml:"defaultEnableShutdownDrainListeners"` + DefaultShutdownGracePeriodSeconds int `yaml:"defaultShutdownGracePeriodSeconds"` + DefaultGracefulPort int `yaml:"defaultGracefulPort"` + DefaultGracefulShutdownPath string `yaml:"defaultGracefulShutdownPath"` +} + type ConnectInject struct { Enabled bool `yaml:"enabled"` Replicas int `yaml:"replicas"` diff --git a/control-plane/connect-inject/common/common.go b/control-plane/connect-inject/common/common.go index 9611797c9f..0ebf829666 100644 --- a/control-plane/connect-inject/common/common.go +++ b/control-plane/connect-inject/common/common.go @@ -9,6 +9,40 @@ import ( corev1 "k8s.io/api/core/v1" ) +// DetermineAndValidatePort behaves as follows: +// If the annotation exists, validate the port and return it. +// If the annotation does not exist, return the default port. +// If the privileged flag is true, it will allow the port to be in the +// privileged port range of 1-1023. Otherwise, it will only allow ports in the +// unprivileged range of 1024-65535. +func DetermineAndValidatePort(pod corev1.Pod, annotation string, defaultPort string, privileged bool) (string, error) { + if raw, ok := pod.Annotations[annotation]; ok && raw != "" { + port, err := PortValue(pod, raw) + if err != nil { + return "", fmt.Errorf("%s annotation value of %s is not a valid integer", annotation, raw) + } + + if privileged && (port < 1 || port > 65535) { + return "", fmt.Errorf("%s annotation value of %d is not in the valid port range 1-65535", annotation, port) + } else if !privileged && (port < 1024 || port > 65535) { + return "", fmt.Errorf("%s annotation value of %d is not in the unprivileged port range 1024-65535", annotation, port) + } + + // If the annotation exists, return the validated port. + return fmt.Sprint(port), nil + } + + // If the annotation does not exist, return the default. + if defaultPort != "" { + port, err := PortValue(pod, defaultPort) + if err != nil { + return "", fmt.Errorf("%s is not a valid port on the pod %s", defaultPort, pod.Name) + } + return fmt.Sprint(port), nil + } + return "", nil +} + // PortValue returns the port of the container for the string value passed // in as an argument on the provided pod. func PortValue(pod corev1.Pod, value string) (int32, error) { diff --git a/control-plane/connect-inject/common/common_test.go b/control-plane/connect-inject/common/common_test.go index e43ccf8255..155f6b892d 100644 --- a/control-plane/connect-inject/common/common_test.go +++ b/control-plane/connect-inject/common/common_test.go @@ -3,10 +3,153 @@ package common import ( "testing" + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants" + "github.com/hashicorp/consul-k8s/control-plane/namespaces" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +func TestCommonDetermineAndValidatePort(t *testing.T) { + cases := []struct { + Name string + Pod func(*corev1.Pod) *corev1.Pod + Annotation string + Privileged bool + DefaultPort string + Expected string + Err string + }{ + { + Name: "Valid annotation", + Pod: func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations["consul.hashicorp.com/test-annotation-port"] = "1234" + return pod + }, + Annotation: "consul.hashicorp.com/test-annotation-port", + Privileged: false, + Expected: "1234", + Err: "", + }, + { + Name: "Uses default when there's no annotation", + Pod: func(pod *corev1.Pod) *corev1.Pod { + return pod + }, + Annotation: "consul.hashicorp.com/test-annotation-port", + Privileged: false, + DefaultPort: "4321", + Expected: "4321", + Err: "", + }, + { + Name: "Gets the value of the named default port when there's no annotation", + Pod: func(pod *corev1.Pod) *corev1.Pod { + pod.Spec.Containers[0].Ports = []corev1.ContainerPort{ + { + Name: "web-port", + ContainerPort: 2222, + }, + } + return pod + }, + Annotation: "consul.hashicorp.com/test-annotation-port", + Privileged: false, + DefaultPort: "web-port", + Expected: "2222", + Err: "", + }, + { + Name: "Errors if the named default port doesn't exist on the pod", + Pod: func(pod *corev1.Pod) *corev1.Pod { + return pod + }, + Annotation: "consul.hashicorp.com/test-annotation-port", + Privileged: false, + DefaultPort: "web-port", + Expected: "", + Err: "web-port is not a valid port on the pod minimal", + }, + { + Name: "Gets the value of the named port", + Pod: func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations["consul.hashicorp.com/test-annotation-port"] = "web-port" + pod.Spec.Containers[0].Ports = []corev1.ContainerPort{ + { + Name: "web-port", + ContainerPort: 2222, + }, + } + return pod + }, + Annotation: "consul.hashicorp.com/test-annotation-port", + Privileged: false, + DefaultPort: "4321", + Expected: "2222", + Err: "", + }, + { + Name: "Invalid annotation (not an integer)", + Pod: func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations["consul.hashicorp.com/test-annotation-port"] = "not-an-int" + return pod + }, + Annotation: "consul.hashicorp.com/test-annotation-port", + Privileged: false, + Expected: "", + Err: "consul.hashicorp.com/test-annotation-port annotation value of not-an-int is not a valid integer", + }, + { + Name: "Invalid annotation (integer not in port range)", + Pod: func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations["consul.hashicorp.com/test-annotation-port"] = "100000" + return pod + }, + Annotation: "consul.hashicorp.com/test-annotation-port", + Privileged: true, + Expected: "", + Err: "consul.hashicorp.com/test-annotation-port annotation value of 100000 is not in the valid port range 1-65535", + }, + { + Name: "Invalid annotation (integer not in unprivileged port range)", + Pod: func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations["consul.hashicorp.com/test-annotation-port"] = "22" + return pod + }, + Annotation: "consul.hashicorp.com/test-annotation-port", + Privileged: false, + Expected: "", + Err: "consul.hashicorp.com/test-annotation-port annotation value of 22 is not in the unprivileged port range 1024-65535", + }, + { + Name: "Privileged ports allowed", + Pod: func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations["consul.hashicorp.com/test-annotation-port"] = "22" + return pod + }, + Annotation: "consul.hashicorp.com/test-annotation-port", + Privileged: true, + Expected: "22", + Err: "", + }, + } + + for _, tt := range cases { + t.Run(tt.Name, func(t *testing.T) { + require := require.New(t) + + actual, err := DetermineAndValidatePort(*tt.Pod(minimal()), tt.Annotation, tt.DefaultPort, tt.Privileged) + + if tt.Err == "" { + require.NoError(err) + require.Equal(tt.Expected, actual) + } else { + require.EqualError(err, tt.Err) + } + }) + } +} + func TestPortValue(t *testing.T) { cases := []struct { Name string @@ -90,3 +233,26 @@ func TestPortValue(t *testing.T) { }) } } + +func minimal() *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespaces.DefaultNamespace, + Name: "minimal", + Annotations: map[string]string{ + constants.AnnotationService: "foo", + }, + }, + + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "web", + }, + { + Name: "web-side", + }, + }, + }, + } +} diff --git a/control-plane/connect-inject/constants/annotations_and_labels.go b/control-plane/connect-inject/constants/annotations_and_labels.go index 637e028202..8767de33b3 100644 --- a/control-plane/connect-inject/constants/annotations_and_labels.go +++ b/control-plane/connect-inject/constants/annotations_and_labels.go @@ -97,6 +97,13 @@ const ( AnnotationSidecarProxyMemoryLimit = "consul.hashicorp.com/sidecar-proxy-memory-limit" AnnotationSidecarProxyMemoryRequest = "consul.hashicorp.com/sidecar-proxy-memory-request" + // annotations for sidecar proxy lifecycle configuration. + AnnotationEnableSidecarProxyLifecycle = "consul.hashicorp.com/enable-sidecar-proxy-lifecycle" + AnnotationEnableSidecarProxyLifecycleShutdownDrainListeners = "consul.hashicorp.com/enable-sidecar-proxy-lifecycle-shutdown-drain-listeners" + AnnotationSidecarProxyLifecycleShutdownGracePeriodSeconds = "consul.hashicorp.com/sidecar-proxy-lifecycle-shutdown-grace-period-seconds" + AnnotationSidecarProxyLifecycleGracefulPort = "consul.hashicorp.com/sidecar-proxy-lifecycle-graceful-port" + AnnotationSidecarProxyLifecycleGracefulShutdownPath = "consul.hashicorp.com/sidecar-proxy-lifecycle-graceful-shutdown-path" + // annotations for sidecar volumes. AnnotationConsulSidecarUserVolume = "consul.hashicorp.com/consul-sidecar-user-volume" AnnotationConsulSidecarUserVolumeMount = "consul.hashicorp.com/consul-sidecar-user-volume-mount" diff --git a/control-plane/connect-inject/constants/constants.go b/control-plane/connect-inject/constants/constants.go index e371677629..673a64eabc 100644 --- a/control-plane/connect-inject/constants/constants.go +++ b/control-plane/connect-inject/constants/constants.go @@ -15,4 +15,10 @@ const ( // MetaKeyPodName is the meta key name for Kubernetes pod name used for the Consul services. MetaKeyPodName = "pod-name" + + // DefaultGracefulPort is the default port that consul-dataplane uses for graceful shutdown. + DefaultGracefulPort = 20600 + + // DefaultGracefulShutdownPath is the default path that consul-dataplane uses for graceful shutdown. + DefaultGracefulShutdownPath = "/graceful_shutdown" ) diff --git a/control-plane/connect-inject/lifecycle/lifecycle_configuration.go b/control-plane/connect-inject/lifecycle/lifecycle_configuration.go new file mode 100644 index 0000000000..651d4eecae --- /dev/null +++ b/control-plane/connect-inject/lifecycle/lifecycle_configuration.go @@ -0,0 +1,95 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package lifecycle + +import ( + "fmt" + "strconv" + + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/common" + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants" + corev1 "k8s.io/api/core/v1" +) + +// Config represents configuration common to connect-inject components related to proxy lifecycle management. +type Config struct { + DefaultEnableProxyLifecycle bool + DefaultEnableShutdownDrainListeners bool + DefaultShutdownGracePeriodSeconds int + DefaultGracefulPort string + DefaultGracefulShutdownPath string +} + +// EnableProxyLifecycle returns whether proxy lifecycle management is enabled either via the default value in the meshWebhook, or if it's been +// overridden via the annotation. +func (lc Config) EnableProxyLifecycle(pod corev1.Pod) (bool, error) { + enabled := lc.DefaultEnableProxyLifecycle + if raw, ok := pod.Annotations[constants.AnnotationEnableSidecarProxyLifecycle]; ok && raw != "" { + enableProxyLifecycle, err := strconv.ParseBool(raw) + if err != nil { + return false, fmt.Errorf("%s annotation value of %s was invalid: %s", constants.AnnotationEnableSidecarProxyLifecycle, raw, err) + } + enabled = enableProxyLifecycle + } + return enabled, nil +} + +// EnableShutdownDrainListeners returns whether proxy listener draining during shutdown is enabled either via the default value in the meshWebhook, or if it's been +// overridden via the annotation. +func (lc Config) EnableShutdownDrainListeners(pod corev1.Pod) (bool, error) { + enabled := lc.DefaultEnableShutdownDrainListeners + if raw, ok := pod.Annotations[constants.AnnotationEnableSidecarProxyLifecycleShutdownDrainListeners]; ok && raw != "" { + enableShutdownDrainListeners, err := strconv.ParseBool(raw) + if err != nil { + return false, fmt.Errorf("%s annotation value of %s was invalid: %s", constants.AnnotationEnableSidecarProxyLifecycleShutdownDrainListeners, raw, err) + } + enabled = enableShutdownDrainListeners + } + return enabled, nil +} + +// ShutdownGracePeriodSeconds returns how long the sidecar proxy should wait before shutdown, either via the default value in the meshWebhook, or if it's been +// overridden via the annotation. +func (lc Config) ShutdownGracePeriodSeconds(pod corev1.Pod) (int, error) { + shutdownGracePeriodSeconds := lc.DefaultShutdownGracePeriodSeconds + if shutdownGracePeriodSecondsAnnotation, ok := pod.Annotations[constants.AnnotationSidecarProxyLifecycleShutdownGracePeriodSeconds]; ok { + val, err := strconv.ParseUint(shutdownGracePeriodSecondsAnnotation, 10, 64) + if err != nil { + return 0, fmt.Errorf("unable to parse annotation %q: %w", constants.AnnotationSidecarProxyLifecycleShutdownGracePeriodSeconds, err) + } + shutdownGracePeriodSeconds = int(val) + } + return shutdownGracePeriodSeconds, nil +} + +// GracefulPort returns the port on which consul-dataplane should serve the proxy lifecycle management HTTP endpoints, either via the default value in the meshWebhook, or +// if it's been overridden via the annotation. It also validates the port is in the unprivileged port range. +func (lc Config) GracefulPort(pod corev1.Pod) (int, error) { + anno, err := common.DetermineAndValidatePort(pod, constants.AnnotationSidecarProxyLifecycleGracefulPort, lc.DefaultGracefulPort, false) + if err != nil { + return 0, err + } + + if anno == "" { + return constants.DefaultGracefulPort, nil + } + + port, _ := strconv.Atoi(anno) + + return port, nil +} + +// GracefulShutdownPath returns the path on which consul-dataplane should serve the graceful shutdown HTTP endpoint, either via the default value in the meshWebhook, or +// if it's been overridden via the annotation. +func (lc Config) GracefulShutdownPath(pod corev1.Pod) string { + if raw, ok := pod.Annotations[constants.AnnotationSidecarProxyLifecycleGracefulShutdownPath]; ok && raw != "" { + return raw + } + + if lc.DefaultGracefulShutdownPath == "" { + return constants.DefaultGracefulShutdownPath + } + + return lc.DefaultGracefulShutdownPath +} diff --git a/control-plane/connect-inject/lifecycle/lifecycle_configuration_test.go b/control-plane/connect-inject/lifecycle/lifecycle_configuration_test.go new file mode 100644 index 0000000000..64157a3d55 --- /dev/null +++ b/control-plane/connect-inject/lifecycle/lifecycle_configuration_test.go @@ -0,0 +1,351 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package lifecycle + +import ( + "testing" + + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants" + "github.com/hashicorp/consul-k8s/control-plane/namespaces" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestLifecycleConfig_EnableSidecarProxyLifecycle(t *testing.T) { + cases := []struct { + Name string + Pod func(*corev1.Pod) *corev1.Pod + LifecycleConfig Config + Expected bool + Err string + }{ + { + Name: "Sidecar proxy lifecycle management enabled via meshWebhook", + Pod: func(pod *corev1.Pod) *corev1.Pod { + return pod + }, + LifecycleConfig: Config{ + DefaultEnableProxyLifecycle: true, + }, + Expected: true, + Err: "", + }, + { + Name: "Sidecar proxy lifecycle management enabled via annotation", + Pod: func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations[constants.AnnotationEnableSidecarProxyLifecycle] = "true" + return pod + }, + LifecycleConfig: Config{ + DefaultEnableProxyLifecycle: false, + }, + Expected: true, + Err: "", + }, + { + Name: "Sidecar proxy lifecycle management configured via invalid annotation", + Pod: func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations[constants.AnnotationEnableSidecarProxyLifecycle] = "not-a-bool" + return pod + }, + LifecycleConfig: Config{ + DefaultEnableProxyLifecycle: false, + }, + Expected: false, + Err: "consul.hashicorp.com/enable-sidecar-proxy-lifecycle annotation value of not-a-bool was invalid: strconv.ParseBool: parsing \"not-a-bool\": invalid syntax", + }, + } + + for _, tt := range cases { + t.Run(tt.Name, func(t *testing.T) { + require := require.New(t) + lc := tt.LifecycleConfig + + actual, err := lc.EnableProxyLifecycle(*tt.Pod(minimal())) + + if tt.Err == "" { + require.Equal(tt.Expected, actual) + require.NoError(err) + } else { + require.EqualError(err, tt.Err) + } + }) + } +} + +func TestLifecycleConfig_ShutdownDrainListeners(t *testing.T) { + cases := []struct { + Name string + Pod func(*corev1.Pod) *corev1.Pod + LifecycleConfig Config + Expected bool + Err string + }{ + { + Name: "Sidecar proxy shutdown listener draining enabled via meshWebhook", + Pod: func(pod *corev1.Pod) *corev1.Pod { + return pod + }, + LifecycleConfig: Config{ + DefaultEnableShutdownDrainListeners: true, + }, + Expected: true, + Err: "", + }, + { + Name: "Sidecar proxy shutdown listener draining enabled via annotation", + Pod: func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations[constants.AnnotationEnableSidecarProxyLifecycleShutdownDrainListeners] = "true" + return pod + }, + LifecycleConfig: Config{ + DefaultEnableShutdownDrainListeners: false, + }, + Expected: true, + Err: "", + }, + { + Name: "Sidecar proxy shutdown listener draining configured via invalid annotation", + Pod: func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations[constants.AnnotationEnableSidecarProxyLifecycleShutdownDrainListeners] = "not-a-bool" + return pod + }, + Expected: false, + Err: "consul.hashicorp.com/enable-sidecar-proxy-lifecycle-shutdown-drain-listeners annotation value of not-a-bool was invalid: strconv.ParseBool: parsing \"not-a-bool\": invalid syntax", + }, + } + + for _, tt := range cases { + t.Run(tt.Name, func(t *testing.T) { + require := require.New(t) + lc := tt.LifecycleConfig + + actual, err := lc.EnableShutdownDrainListeners(*tt.Pod(minimal())) + + if tt.Err == "" { + require.Equal(tt.Expected, actual) + require.NoError(err) + } else { + require.EqualError(err, tt.Err) + } + }) + } +} + +func TestLifecycleConfig_ShutdownGracePeriodSeconds(t *testing.T) { + cases := []struct { + Name string + Pod func(*corev1.Pod) *corev1.Pod + LifecycleConfig Config + Expected int + Err string + }{ + { + Name: "Sidecar proxy shutdown grace period set via meshWebhook", + Pod: func(pod *corev1.Pod) *corev1.Pod { + return pod + }, + LifecycleConfig: Config{ + DefaultShutdownGracePeriodSeconds: 10, + }, + Expected: 10, + Err: "", + }, + { + Name: "Sidecar proxy shutdown grace period set via annotation", + Pod: func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations[constants.AnnotationSidecarProxyLifecycleShutdownGracePeriodSeconds] = "20" + return pod + }, + LifecycleConfig: Config{ + DefaultShutdownGracePeriodSeconds: 10, + }, + Expected: 20, + Err: "", + }, + { + Name: "Sidecar proxy shutdown grace period configured via invalid annotation, negative number", + Pod: func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations[constants.AnnotationSidecarProxyLifecycleShutdownGracePeriodSeconds] = "-1" + return pod + }, + Err: "unable to parse annotation \"consul.hashicorp.com/sidecar-proxy-lifecycle-shutdown-grace-period-seconds\": strconv.ParseUint: parsing \"-1\": invalid syntax", + }, + { + Name: "Sidecar proxy shutdown grace period configured via invalid annotation, not-parseable string", + Pod: func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations[constants.AnnotationSidecarProxyLifecycleShutdownGracePeriodSeconds] = "not-int" + return pod + }, + Err: "unable to parse annotation \"consul.hashicorp.com/sidecar-proxy-lifecycle-shutdown-grace-period-seconds\": strconv.ParseUint: parsing \"not-int\": invalid syntax", + }, + } + + for _, tt := range cases { + t.Run(tt.Name, func(t *testing.T) { + require := require.New(t) + lc := tt.LifecycleConfig + + actual, err := lc.ShutdownGracePeriodSeconds(*tt.Pod(minimal())) + + if tt.Err == "" { + require.Equal(tt.Expected, actual) + require.NoError(err) + } else { + require.EqualError(err, tt.Err) + } + }) + } +} + +func TestLifecycleConfig_GracefulPort(t *testing.T) { + cases := []struct { + Name string + Pod func(*corev1.Pod) *corev1.Pod + LifecycleConfig Config + Expected int + Err string + }{ + { + Name: "Sidecar proxy lifecycle graceful port set to default", + Pod: func(pod *corev1.Pod) *corev1.Pod { + return pod + }, + Expected: constants.DefaultGracefulPort, + Err: "", + }, + { + Name: "Sidecar proxy lifecycle graceful port set via meshWebhook", + Pod: func(pod *corev1.Pod) *corev1.Pod { + return pod + }, + LifecycleConfig: Config{ + DefaultGracefulPort: "3000", + }, + Expected: 3000, + Err: "", + }, + { + Name: "Sidecar proxy lifecycle graceful port set via annotation", + Pod: func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations[constants.AnnotationSidecarProxyLifecycleGracefulPort] = "9000" + return pod + }, + LifecycleConfig: Config{ + DefaultGracefulPort: "3000", + }, + Expected: 9000, + Err: "", + }, + { + Name: "Sidecar proxy lifecycle graceful port configured via invalid annotation, negative number", + Pod: func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations[constants.AnnotationSidecarProxyLifecycleGracefulPort] = "-1" + return pod + }, + Err: "consul.hashicorp.com/sidecar-proxy-lifecycle-graceful-port annotation value of -1 is not in the unprivileged port range 1024-65535", + }, + { + Name: "Sidecar proxy lifecycle graceful port configured via invalid annotation, not-parseable string", + Pod: func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations[constants.AnnotationSidecarProxyLifecycleGracefulPort] = "not-int" + return pod + }, + Err: "consul.hashicorp.com/sidecar-proxy-lifecycle-graceful-port annotation value of not-int is not a valid integer", + }, + } + + for _, tt := range cases { + t.Run(tt.Name, func(t *testing.T) { + require := require.New(t) + lc := tt.LifecycleConfig + + actual, err := lc.GracefulPort(*tt.Pod(minimal())) + + if tt.Err == "" { + require.Equal(tt.Expected, actual) + require.NoError(err) + } else { + require.EqualError(err, tt.Err) + } + }) + } +} + +func TestLifecycleConfig_GracefulShutdownPath(t *testing.T) { + cases := []struct { + Name string + Pod func(*corev1.Pod) *corev1.Pod + LifecycleConfig Config + Expected string + Err string + }{ + { + Name: "Sidecar proxy lifecycle graceful shutdown path defaults to /graceful_shutdown", + Pod: func(pod *corev1.Pod) *corev1.Pod { + return pod + }, + Expected: "/graceful_shutdown", + Err: "", + }, + { + Name: "Sidecar proxy lifecycle graceful shutdown path set via meshWebhook", + Pod: func(pod *corev1.Pod) *corev1.Pod { + return pod + }, + LifecycleConfig: Config{ + DefaultGracefulShutdownPath: "/quit", + }, + Expected: "/quit", + Err: "", + }, + { + Name: "Sidecar proxy lifecycle graceful port set via annotation", + Pod: func(pod *corev1.Pod) *corev1.Pod { + pod.Annotations[constants.AnnotationSidecarProxyLifecycleGracefulShutdownPath] = "/custom-shutdown-path" + return pod + }, + LifecycleConfig: Config{ + DefaultGracefulShutdownPath: "/quit", + }, + Expected: "/custom-shutdown-path", + Err: "", + }, + } + + for _, tt := range cases { + t.Run(tt.Name, func(t *testing.T) { + require := require.New(t) + lc := tt.LifecycleConfig + + actual := lc.GracefulShutdownPath(*tt.Pod(minimal())) + + require.Equal(tt.Expected, actual) + }) + } +} + +func minimal() *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespaces.DefaultNamespace, + Name: "minimal", + Annotations: map[string]string{ + constants.AnnotationService: "foo", + }, + }, + + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "web", + }, + { + Name: "web-side", + }, + }, + }, + } +} diff --git a/control-plane/connect-inject/metrics/metrics_configuration.go b/control-plane/connect-inject/metrics/metrics_configuration.go index 651fb87184..09f01a4c87 100644 --- a/control-plane/connect-inject/metrics/metrics_configuration.go +++ b/control-plane/connect-inject/metrics/metrics_configuration.go @@ -95,13 +95,13 @@ func (mc Config) EnableMetricsMerging(pod corev1.Pod) (bool, error) { // MergedMetricsPort returns the port to run the merged metrics server on, either via the default value in the meshWebhook, // or if it's been overridden via the annotation. It also validates the port is in the unprivileged port range. func (mc Config) MergedMetricsPort(pod corev1.Pod) (string, error) { - return determineAndValidatePort(pod, constants.AnnotationMergedMetricsPort, mc.DefaultMergedMetricsPort, false) + return common.DetermineAndValidatePort(pod, constants.AnnotationMergedMetricsPort, mc.DefaultMergedMetricsPort, false) } // PrometheusScrapePort returns the port for Prometheus to scrape from, either via the default value in the meshWebhook, or // if it's been overridden via the annotation. It also validates the port is in the unprivileged port range. func (mc Config) PrometheusScrapePort(pod corev1.Pod) (string, error) { - return determineAndValidatePort(pod, constants.AnnotationPrometheusScrapePort, mc.DefaultPrometheusScrapePort, false) + return common.DetermineAndValidatePort(pod, constants.AnnotationPrometheusScrapePort, mc.DefaultPrometheusScrapePort, false) } // PrometheusScrapePath returns the path for Prometheus to scrape from, either via the default value in the meshWebhook, or @@ -130,14 +130,14 @@ func (mc Config) ServiceMetricsPort(pod corev1.Pod) (string, error) { // written their service in such a way that it expects to be able to use // privileged ports. So, the port metrics are exposed on the service can // be privileged. - return determineAndValidatePort(pod, constants.AnnotationServiceMetricsPort, raw, true) + return common.DetermineAndValidatePort(pod, constants.AnnotationServiceMetricsPort, raw, true) } // If the annotationPort is not set, the serviceMetrics port will be 0 // unless overridden by the service-metrics-port annotation. If the service // metrics port is 0, the consul sidecar will not run a merged metrics // server. - return determineAndValidatePort(pod, constants.AnnotationServiceMetricsPort, "0", true) + return common.DetermineAndValidatePort(pod, constants.AnnotationServiceMetricsPort, "0", true) } // ServiceMetricsPath returns a default of /metrics, or overrides @@ -177,37 +177,3 @@ func (mc Config) ShouldRunMergedMetricsServer(pod corev1.Pod) (bool, error) { } return false, nil } - -// determineAndValidatePort behaves as follows: -// If the annotation exists, validate the port and return it. -// If the annotation does not exist, return the default port. -// If the privileged flag is true, it will allow the port to be in the -// privileged port range of 1-1023. Otherwise, it will only allow ports in the -// unprivileged range of 1024-65535. -func determineAndValidatePort(pod corev1.Pod, annotation string, defaultPort string, privileged bool) (string, error) { - if raw, ok := pod.Annotations[annotation]; ok && raw != "" { - port, err := common.PortValue(pod, raw) - if err != nil { - return "", fmt.Errorf("%s annotation value of %s is not a valid integer", annotation, raw) - } - - if privileged && (port < 1 || port > 65535) { - return "", fmt.Errorf("%s annotation value of %d is not in the valid port range 1-65535", annotation, port) - } else if !privileged && (port < 1024 || port > 65535) { - return "", fmt.Errorf("%s annotation value of %d is not in the unprivileged port range 1024-65535", annotation, port) - } - - // If the annotation exists, return the validated port. - return fmt.Sprint(port), nil - } - - // If the annotation does not exist, return the default. - if defaultPort != "" { - port, err := common.PortValue(pod, defaultPort) - if err != nil { - return "", fmt.Errorf("%s is not a valid port on the pod %s", defaultPort, pod.Name) - } - return fmt.Sprint(port), nil - } - return "", nil -} diff --git a/control-plane/connect-inject/metrics/metrics_configuration_test.go b/control-plane/connect-inject/metrics/metrics_configuration_test.go index 2f41b05744..ea232576b5 100644 --- a/control-plane/connect-inject/metrics/metrics_configuration_test.go +++ b/control-plane/connect-inject/metrics/metrics_configuration_test.go @@ -304,149 +304,6 @@ func TestMetricsConfigShouldRunMergedMetricsServer(t *testing.T) { } } -// Tests determineAndValidatePort, which in turn tests the -// PrometheusScrapePort() and MergedMetricsPort() functions because their logic -// is just to call out to determineAndValidatePort(). -func TestMetricsConfigDetermineAndValidatePort(t *testing.T) { - cases := []struct { - Name string - Pod func(*corev1.Pod) *corev1.Pod - Annotation string - Privileged bool - DefaultPort string - Expected string - Err string - }{ - { - Name: "Valid annotation", - Pod: func(pod *corev1.Pod) *corev1.Pod { - pod.Annotations["consul.hashicorp.com/test-annotation-port"] = "1234" - return pod - }, - Annotation: "consul.hashicorp.com/test-annotation-port", - Privileged: false, - Expected: "1234", - Err: "", - }, - { - Name: "Uses default when there's no annotation", - Pod: func(pod *corev1.Pod) *corev1.Pod { - return pod - }, - Annotation: "consul.hashicorp.com/test-annotation-port", - Privileged: false, - DefaultPort: "4321", - Expected: "4321", - Err: "", - }, - { - Name: "Gets the value of the named default port when there's no annotation", - Pod: func(pod *corev1.Pod) *corev1.Pod { - pod.Spec.Containers[0].Ports = []corev1.ContainerPort{ - { - Name: "web-port", - ContainerPort: 2222, - }, - } - return pod - }, - Annotation: "consul.hashicorp.com/test-annotation-port", - Privileged: false, - DefaultPort: "web-port", - Expected: "2222", - Err: "", - }, - { - Name: "Errors if the named default port doesn't exist on the pod", - Pod: func(pod *corev1.Pod) *corev1.Pod { - return pod - }, - Annotation: "consul.hashicorp.com/test-annotation-port", - Privileged: false, - DefaultPort: "web-port", - Expected: "", - Err: "web-port is not a valid port on the pod minimal", - }, - { - Name: "Gets the value of the named port", - Pod: func(pod *corev1.Pod) *corev1.Pod { - pod.Annotations["consul.hashicorp.com/test-annotation-port"] = "web-port" - pod.Spec.Containers[0].Ports = []corev1.ContainerPort{ - { - Name: "web-port", - ContainerPort: 2222, - }, - } - return pod - }, - Annotation: "consul.hashicorp.com/test-annotation-port", - Privileged: false, - DefaultPort: "4321", - Expected: "2222", - Err: "", - }, - { - Name: "Invalid annotation (not an integer)", - Pod: func(pod *corev1.Pod) *corev1.Pod { - pod.Annotations["consul.hashicorp.com/test-annotation-port"] = "not-an-int" - return pod - }, - Annotation: "consul.hashicorp.com/test-annotation-port", - Privileged: false, - Expected: "", - Err: "consul.hashicorp.com/test-annotation-port annotation value of not-an-int is not a valid integer", - }, - { - Name: "Invalid annotation (integer not in port range)", - Pod: func(pod *corev1.Pod) *corev1.Pod { - pod.Annotations["consul.hashicorp.com/test-annotation-port"] = "100000" - return pod - }, - Annotation: "consul.hashicorp.com/test-annotation-port", - Privileged: true, - Expected: "", - Err: "consul.hashicorp.com/test-annotation-port annotation value of 100000 is not in the valid port range 1-65535", - }, - { - Name: "Invalid annotation (integer not in unprivileged port range)", - Pod: func(pod *corev1.Pod) *corev1.Pod { - pod.Annotations["consul.hashicorp.com/test-annotation-port"] = "22" - return pod - }, - Annotation: "consul.hashicorp.com/test-annotation-port", - Privileged: false, - Expected: "", - Err: "consul.hashicorp.com/test-annotation-port annotation value of 22 is not in the unprivileged port range 1024-65535", - }, - { - Name: "Privileged ports allowed", - Pod: func(pod *corev1.Pod) *corev1.Pod { - pod.Annotations["consul.hashicorp.com/test-annotation-port"] = "22" - return pod - }, - Annotation: "consul.hashicorp.com/test-annotation-port", - Privileged: true, - Expected: "22", - Err: "", - }, - } - - for _, tt := range cases { - t.Run(tt.Name, func(t *testing.T) { - require := require.New(t) - - actual, err := determineAndValidatePort(*tt.Pod(minimal()), tt.Annotation, tt.DefaultPort, tt.Privileged) - - if tt.Err == "" { - require.NoError(err) - require.Equal(tt.Expected, actual) - } else { - require.EqualError(err, tt.Err) - } - }) - } -} - // Tests MergedMetricsServerConfiguration happy path and error case not covered by other Config tests. func TestMetricsConfigMergedMetricsServerConfiguration(t *testing.T) { cases := []struct { diff --git a/control-plane/connect-inject/webhook/consul_dataplane_sidecar.go b/control-plane/connect-inject/webhook/consul_dataplane_sidecar.go index 9d02a26fa4..5efc19f4e6 100644 --- a/control-plane/connect-inject/webhook/consul_dataplane_sidecar.go +++ b/control-plane/connect-inject/webhook/consul_dataplane_sidecar.go @@ -244,6 +244,45 @@ func (w *MeshWebhook) getContainerSidecarArgs(namespace corev1.Namespace, mpi mu args = append(args, fmt.Sprintf("-envoy-admin-bind-port=%d", 19000+mpi.serviceIndex)) } + // The consul-dataplane HTTP listener always starts for graceful shutdown. To avoid port conflicts, the + // graceful port always needs to be set + gracefulPort, err := w.LifecycleConfig.GracefulPort(pod) + if err != nil { + return nil, fmt.Errorf("unable to determine proxy lifecycle graceful port: %w", err) + } + + // To avoid conflicts + if mpi.serviceName != "" { + gracefulPort = gracefulPort + mpi.serviceIndex + } + args = append(args, fmt.Sprintf("-graceful-port=%d", gracefulPort)) + + enableProxyLifecycle, err := w.LifecycleConfig.EnableProxyLifecycle(pod) + if err != nil { + return nil, fmt.Errorf("unable to determine if proxy lifecycle management is enabled: %w", err) + } + if enableProxyLifecycle { + shutdownDrainListeners, err := w.LifecycleConfig.EnableShutdownDrainListeners(pod) + if err != nil { + return nil, fmt.Errorf("unable to determine if proxy lifecycle shutdown listener draining is enabled: %w", err) + } + if shutdownDrainListeners { + args = append(args, "-shutdown-drain-listeners") + } + + shutdownGracePeriodSeconds, err := w.LifecycleConfig.ShutdownGracePeriodSeconds(pod) + if err != nil { + return nil, fmt.Errorf("unable to determine proxy lifecycle shutdown grace period: %w", err) + } + args = append(args, fmt.Sprintf("-shutdown-grace-period-seconds=%d", shutdownGracePeriodSeconds)) + + gracefulShutdownPath := w.LifecycleConfig.GracefulShutdownPath(pod) + if err != nil { + return nil, fmt.Errorf("unable to determine proxy lifecycle graceful shutdown path: %w", err) + } + args = append(args, fmt.Sprintf("-graceful-shutdown-path=%s", gracefulShutdownPath)) + } + // Set a default scrape path that can be overwritten by the annotation. prometheusScrapePath := w.MetricsConfig.PrometheusScrapePath(pod) args = append(args, "-telemetry-prom-scrape-path="+prometheusScrapePath) diff --git a/control-plane/connect-inject/webhook/consul_dataplane_sidecar_test.go b/control-plane/connect-inject/webhook/consul_dataplane_sidecar_test.go index 05de3a0e80..a65f49228d 100644 --- a/control-plane/connect-inject/webhook/consul_dataplane_sidecar_test.go +++ b/control-plane/connect-inject/webhook/consul_dataplane_sidecar_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants" + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/lifecycle" "github.com/hashicorp/consul-k8s/control-plane/consul" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" @@ -25,20 +26,20 @@ func TestHandlerConsulDataplaneSidecar(t *testing.T) { }{ "default": { webhookSetupFunc: nil, - additionalExpCmdArgs: " -tls-disabled -telemetry-prom-scrape-path=/metrics", + additionalExpCmdArgs: " -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", }, "with custom gRPC port": { webhookSetupFunc: func(w *MeshWebhook) { w.ConsulConfig.GRPCPort = 8602 }, - additionalExpCmdArgs: " -tls-disabled -telemetry-prom-scrape-path=/metrics", + additionalExpCmdArgs: " -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", }, "with ACLs": { webhookSetupFunc: func(w *MeshWebhook) { w.AuthMethod = "test-auth-method" }, additionalExpCmdArgs: " -credential-type=login -login-auth-method=test-auth-method -login-bearer-token-path=/var/run/secrets/kubernetes.io/serviceaccount/token " + - "-login-meta=pod=k8snamespace/test-pod -tls-disabled -telemetry-prom-scrape-path=/metrics", + "-login-meta=pod=k8snamespace/test-pod -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", }, "with ACLs and namespace mirroring": { webhookSetupFunc: func(w *MeshWebhook) { @@ -47,7 +48,7 @@ func TestHandlerConsulDataplaneSidecar(t *testing.T) { w.EnableK8SNSMirroring = true }, additionalExpCmdArgs: " -credential-type=login -login-auth-method=test-auth-method -login-bearer-token-path=/var/run/secrets/kubernetes.io/serviceaccount/token " + - "-login-meta=pod=k8snamespace/test-pod -login-namespace=default -service-namespace=k8snamespace -tls-disabled -telemetry-prom-scrape-path=/metrics", + "-login-meta=pod=k8snamespace/test-pod -login-namespace=default -service-namespace=k8snamespace -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", }, "with ACLs and single destination namespace": { webhookSetupFunc: func(w *MeshWebhook) { @@ -56,7 +57,7 @@ func TestHandlerConsulDataplaneSidecar(t *testing.T) { w.ConsulDestinationNamespace = "test-ns" }, additionalExpCmdArgs: " -credential-type=login -login-auth-method=test-auth-method -login-bearer-token-path=/var/run/secrets/kubernetes.io/serviceaccount/token " + - "-login-meta=pod=k8snamespace/test-pod -login-namespace=test-ns -service-namespace=test-ns -tls-disabled -telemetry-prom-scrape-path=/metrics", + "-login-meta=pod=k8snamespace/test-pod -login-namespace=test-ns -service-namespace=test-ns -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", }, "with ACLs and partitions": { webhookSetupFunc: func(w *MeshWebhook) { @@ -64,7 +65,7 @@ func TestHandlerConsulDataplaneSidecar(t *testing.T) { w.ConsulPartition = "test-part" }, additionalExpCmdArgs: " -credential-type=login -login-auth-method=test-auth-method -login-bearer-token-path=/var/run/secrets/kubernetes.io/serviceaccount/token " + - "-login-meta=pod=k8snamespace/test-pod -login-partition=test-part -service-partition=test-part -tls-disabled -telemetry-prom-scrape-path=/metrics", + "-login-meta=pod=k8snamespace/test-pod -login-partition=test-part -service-partition=test-part -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", }, "with TLS and CA cert provided": { webhookSetupFunc: func(w *MeshWebhook) { @@ -72,28 +73,28 @@ func TestHandlerConsulDataplaneSidecar(t *testing.T) { w.ConsulTLSServerName = "server.dc1.consul" w.ConsulCACert = "consul-ca-cert" }, - additionalExpCmdArgs: " -tls-server-name=server.dc1.consul -ca-certs=/consul/connect-inject/consul-ca.pem -telemetry-prom-scrape-path=/metrics", + additionalExpCmdArgs: " -tls-server-name=server.dc1.consul -ca-certs=/consul/connect-inject/consul-ca.pem -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", }, "with TLS and no CA cert provided": { webhookSetupFunc: func(w *MeshWebhook) { w.TLSEnabled = true w.ConsulTLSServerName = "server.dc1.consul" }, - additionalExpCmdArgs: " -tls-server-name=server.dc1.consul -telemetry-prom-scrape-path=/metrics", + additionalExpCmdArgs: " -tls-server-name=server.dc1.consul -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", }, "with single destination namespace": { webhookSetupFunc: func(w *MeshWebhook) { w.EnableNamespaces = true w.ConsulDestinationNamespace = "consul-namespace" }, - additionalExpCmdArgs: " -service-namespace=consul-namespace -tls-disabled -telemetry-prom-scrape-path=/metrics", + additionalExpCmdArgs: " -service-namespace=consul-namespace -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", }, "with namespace mirroring": { webhookSetupFunc: func(w *MeshWebhook) { w.EnableNamespaces = true w.EnableK8SNSMirroring = true }, - additionalExpCmdArgs: " -service-namespace=k8snamespace -tls-disabled -telemetry-prom-scrape-path=/metrics", + additionalExpCmdArgs: " -service-namespace=k8snamespace -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", }, "with namespace mirroring prefix": { webhookSetupFunc: func(w *MeshWebhook) { @@ -101,38 +102,38 @@ func TestHandlerConsulDataplaneSidecar(t *testing.T) { w.EnableK8SNSMirroring = true w.K8SNSMirroringPrefix = "foo-" }, - additionalExpCmdArgs: " -service-namespace=foo-k8snamespace -tls-disabled -telemetry-prom-scrape-path=/metrics", + additionalExpCmdArgs: " -service-namespace=foo-k8snamespace -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", }, "with partitions": { webhookSetupFunc: func(w *MeshWebhook) { w.ConsulPartition = "partition-1" }, - additionalExpCmdArgs: " -service-partition=partition-1 -tls-disabled -telemetry-prom-scrape-path=/metrics", + additionalExpCmdArgs: " -service-partition=partition-1 -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", }, "with different log level": { webhookSetupFunc: func(w *MeshWebhook) { w.LogLevel = "debug" }, - additionalExpCmdArgs: " -tls-disabled -telemetry-prom-scrape-path=/metrics", + additionalExpCmdArgs: " -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", }, "with different log level and log json": { webhookSetupFunc: func(w *MeshWebhook) { w.LogLevel = "debug" w.LogJSON = true }, - additionalExpCmdArgs: " -tls-disabled -telemetry-prom-scrape-path=/metrics", + additionalExpCmdArgs: " -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", }, "skip server watch enabled": { webhookSetupFunc: func(w *MeshWebhook) { w.SkipServerWatch = true }, - additionalExpCmdArgs: " -server-watch-disabled=true -tls-disabled -telemetry-prom-scrape-path=/metrics", + additionalExpCmdArgs: " -server-watch-disabled=true -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/metrics", }, "custom prometheus scrape path": { webhookSetupFunc: func(w *MeshWebhook) { w.MetricsConfig.DefaultPrometheusScrapePath = "/scrape-path" // Simulate what would be passed as a flag }, - additionalExpCmdArgs: " -tls-disabled -telemetry-prom-scrape-path=/scrape-path", + additionalExpCmdArgs: " -tls-disabled -graceful-port=20600 -telemetry-prom-scrape-path=/scrape-path", }, } @@ -619,18 +620,18 @@ func TestHandlerConsulDataplaneSidecar_Multiport(t *testing.T) { } expArgs := []string{ "-addresses 1.1.1.1 -grpc-port=8502 -proxy-service-id-path=/consul/connect-inject/proxyid-web " + - "-log-level=info -log-json=false -envoy-concurrency=0 -tls-disabled -envoy-admin-bind-port=19000 -telemetry-prom-scrape-path=/metrics -- --base-id 0", + "-log-level=info -log-json=false -envoy-concurrency=0 -tls-disabled -envoy-admin-bind-port=19000 -graceful-port=20600 -telemetry-prom-scrape-path=/metrics -- --base-id 0", "-addresses 1.1.1.1 -grpc-port=8502 -proxy-service-id-path=/consul/connect-inject/proxyid-web-admin " + - "-log-level=info -log-json=false -envoy-concurrency=0 -tls-disabled -envoy-admin-bind-port=19001 -telemetry-prom-scrape-path=/metrics -- --base-id 1", + "-log-level=info -log-json=false -envoy-concurrency=0 -tls-disabled -envoy-admin-bind-port=19001 -graceful-port=20601 -telemetry-prom-scrape-path=/metrics -- --base-id 1", } if aclsEnabled { expArgs = []string{ "-addresses 1.1.1.1 -grpc-port=8502 -proxy-service-id-path=/consul/connect-inject/proxyid-web " + "-log-level=info -log-json=false -envoy-concurrency=0 -credential-type=login -login-auth-method=test-auth-method " + - "-login-bearer-token-path=/var/run/secrets/kubernetes.io/serviceaccount/token -login-meta=pod=k8snamespace/test-pod -tls-disabled -envoy-admin-bind-port=19000 -telemetry-prom-scrape-path=/metrics -- --base-id 0", + "-login-bearer-token-path=/var/run/secrets/kubernetes.io/serviceaccount/token -login-meta=pod=k8snamespace/test-pod -tls-disabled -envoy-admin-bind-port=19000 -graceful-port=20600 -telemetry-prom-scrape-path=/metrics -- --base-id 0", "-addresses 1.1.1.1 -grpc-port=8502 -proxy-service-id-path=/consul/connect-inject/proxyid-web-admin " + "-log-level=info -log-json=false -envoy-concurrency=0 -credential-type=login -login-auth-method=test-auth-method " + - "-login-bearer-token-path=/consul/serviceaccount-web-admin/token -login-meta=pod=k8snamespace/test-pod -tls-disabled -envoy-admin-bind-port=19001 -telemetry-prom-scrape-path=/metrics -- --base-id 1", + "-login-bearer-token-path=/consul/serviceaccount-web-admin/token -login-meta=pod=k8snamespace/test-pod -tls-disabled -envoy-admin-bind-port=19001 -graceful-port=20601 -telemetry-prom-scrape-path=/metrics -- --base-id 1", } } expSAVolumeMounts := []corev1.VolumeMount{ @@ -1296,6 +1297,156 @@ func TestHandlerConsulDataplaneSidecar_Metrics(t *testing.T) { } } +func TestHandlerConsulDataplaneSidecar_Lifecycle(t *testing.T) { + gracefulShutdownSeconds := 10 + gracefulPort := "20307" + gracefulShutdownPath := "/exit" + + cases := []struct { + name string + webhook MeshWebhook + annotations map[string]string + expCmdArgs string + expErr string + }{ + { + name: "no defaults, no annotations", + webhook: MeshWebhook{}, + annotations: nil, + expCmdArgs: "", + }, + { + name: "all defaults, no annotations", + webhook: MeshWebhook{ + LifecycleConfig: lifecycle.Config{ + DefaultEnableProxyLifecycle: true, + DefaultEnableShutdownDrainListeners: true, + DefaultShutdownGracePeriodSeconds: gracefulShutdownSeconds, + DefaultGracefulPort: gracefulPort, + DefaultGracefulShutdownPath: gracefulShutdownPath, + }, + }, + annotations: nil, + expCmdArgs: "graceful-port=20307 -shutdown-drain-listeners -shutdown-grace-period-seconds=10 -graceful-shutdown-path=/exit", + }, + { + name: "no defaults, all annotations", + webhook: MeshWebhook{}, + annotations: map[string]string{ + constants.AnnotationEnableSidecarProxyLifecycle: "true", + constants.AnnotationEnableSidecarProxyLifecycleShutdownDrainListeners: "true", + constants.AnnotationSidecarProxyLifecycleShutdownGracePeriodSeconds: fmt.Sprint(gracefulShutdownSeconds), + constants.AnnotationSidecarProxyLifecycleGracefulPort: gracefulPort, + constants.AnnotationSidecarProxyLifecycleGracefulShutdownPath: gracefulShutdownPath, + }, + expCmdArgs: "-graceful-port=20307 -shutdown-drain-listeners -shutdown-grace-period-seconds=10 -graceful-shutdown-path=/exit", + }, + { + name: "annotations override defaults", + webhook: MeshWebhook{ + LifecycleConfig: lifecycle.Config{ + DefaultEnableProxyLifecycle: false, + DefaultEnableShutdownDrainListeners: true, + DefaultShutdownGracePeriodSeconds: gracefulShutdownSeconds, + DefaultGracefulPort: gracefulPort, + DefaultGracefulShutdownPath: gracefulShutdownPath, + }, + }, + annotations: map[string]string{ + constants.AnnotationEnableSidecarProxyLifecycle: "true", + constants.AnnotationEnableSidecarProxyLifecycleShutdownDrainListeners: "false", + constants.AnnotationSidecarProxyLifecycleShutdownGracePeriodSeconds: fmt.Sprint(gracefulShutdownSeconds + 5), + constants.AnnotationSidecarProxyLifecycleGracefulPort: "20317", + constants.AnnotationSidecarProxyLifecycleGracefulShutdownPath: "/foo", + }, + expCmdArgs: "-graceful-port=20317 -shutdown-grace-period-seconds=15 -graceful-shutdown-path=/foo", + }, + { + name: "lifecycle disabled, no annotations", + webhook: MeshWebhook{ + LifecycleConfig: lifecycle.Config{ + DefaultEnableProxyLifecycle: false, + DefaultEnableShutdownDrainListeners: true, + DefaultShutdownGracePeriodSeconds: gracefulShutdownSeconds, + DefaultGracefulPort: gracefulPort, + DefaultGracefulShutdownPath: gracefulShutdownPath, + }, + }, + annotations: nil, + expCmdArgs: "-graceful-port=20307", + }, + { + name: "lifecycle enabled, defaults omited, no annotations", + webhook: MeshWebhook{ + LifecycleConfig: lifecycle.Config{ + DefaultEnableProxyLifecycle: true, + }, + }, + annotations: nil, + expCmdArgs: "", + }, + { + name: "annotations disable lifecycle default", + webhook: MeshWebhook{ + LifecycleConfig: lifecycle.Config{ + DefaultEnableProxyLifecycle: true, + DefaultEnableShutdownDrainListeners: true, + DefaultShutdownGracePeriodSeconds: gracefulShutdownSeconds, + DefaultGracefulPort: gracefulPort, + DefaultGracefulShutdownPath: gracefulShutdownPath, + }, + }, + annotations: map[string]string{ + constants.AnnotationEnableSidecarProxyLifecycle: "false", + }, + expCmdArgs: "-graceful-port=20307", + }, + { + name: "annotations skip graceful shutdown", + webhook: MeshWebhook{ + LifecycleConfig: lifecycle.Config{ + DefaultEnableProxyLifecycle: false, + DefaultEnableShutdownDrainListeners: true, + DefaultShutdownGracePeriodSeconds: gracefulShutdownSeconds, + }, + }, + annotations: map[string]string{ + constants.AnnotationEnableSidecarProxyLifecycle: "false", + constants.AnnotationEnableSidecarProxyLifecycleShutdownDrainListeners: "false", + constants.AnnotationSidecarProxyLifecycleShutdownGracePeriodSeconds: "0", + }, + expCmdArgs: "", + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + c.webhook.ConsulConfig = &consul.Config{HTTPPort: 8500, GRPCPort: 8502} + require := require.New(t) + pod := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: c.annotations, + }, + + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "web", + }, + }, + }, + } + container, err := c.webhook.consulDataplaneSidecar(testNS, pod, multiPortInfo{}) + if c.expErr != "" { + require.NotNil(err) + require.Contains(err.Error(), c.expErr) + } else { + require.NoError(err) + require.Contains(strings.Join(container.Args, " "), c.expCmdArgs) + } + }) + } +} + // boolPtr returns pointer to b. func boolPtr(b bool) *bool { return &b diff --git a/control-plane/connect-inject/webhook/mesh_webhook.go b/control-plane/connect-inject/webhook/mesh_webhook.go index 7ee9e0a8ea..c8d412184b 100644 --- a/control-plane/connect-inject/webhook/mesh_webhook.go +++ b/control-plane/connect-inject/webhook/mesh_webhook.go @@ -14,6 +14,7 @@ import ( "github.com/go-logr/logr" "github.com/hashicorp/consul-k8s/control-plane/connect-inject/common" "github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants" + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/lifecycle" "github.com/hashicorp/consul-k8s/control-plane/connect-inject/metrics" "github.com/hashicorp/consul-k8s/control-plane/consul" "github.com/hashicorp/consul-k8s/control-plane/namespaces" @@ -148,6 +149,10 @@ type MeshWebhook struct { DefaultProxyMemoryRequest resource.Quantity DefaultProxyMemoryLimit resource.Quantity + // LifecycleConfig contains proxy lifecycle management configuration from the inject-connect command and has methods to determine whether + // configuration should come from the default flags or annotations. The meshWebhook uses this to configure container sidecar proxy args. + LifecycleConfig lifecycle.Config + // Default Envoy concurrency flag, this is the number of worker threads to be used by the proxy. DefaultEnvoyProxyConcurrency int @@ -303,6 +308,7 @@ func (w *MeshWebhook) Handle(ctx context.Context, req admission.Request) admissi w.Log.Error(err, "error configuring injection sidecar container", "request name", req.Name) return admission.Errored(http.StatusInternalServerError, fmt.Errorf("error configuring injection sidecar container: %s", err)) } + // TODO: invert to start the Envoy sidecar before the application container pod.Spec.Containers = append(pod.Spec.Containers, envoySidecar) } else { // For multi port pods, check for unsupported cases, mount all relevant service account tokens, and mount an init @@ -373,6 +379,8 @@ func (w *MeshWebhook) Handle(ctx context.Context, req admission.Request) admissi w.Log.Error(err, "error configuring injection sidecar container", "request name", req.Name) return admission.Errored(http.StatusInternalServerError, fmt.Errorf("error configuring injection sidecar container: %s", err)) } + // TODO: invert to start the Envoy sidecar container before the + // application container pod.Spec.Containers = append(pod.Spec.Containers, envoySidecar) } } diff --git a/control-plane/subcommand/inject-connect/command.go b/control-plane/subcommand/inject-connect/command.go index 5cbd2abdee..f3559e2679 100644 --- a/control-plane/subcommand/inject-connect/command.go +++ b/control-plane/subcommand/inject-connect/command.go @@ -14,8 +14,10 @@ import ( apicommon "github.com/hashicorp/consul-k8s/control-plane/api/common" "github.com/hashicorp/consul-k8s/control-plane/api/v1alpha1" + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants" "github.com/hashicorp/consul-k8s/control-plane/connect-inject/controllers/endpoints" "github.com/hashicorp/consul-k8s/control-plane/connect-inject/controllers/peering" + "github.com/hashicorp/consul-k8s/control-plane/connect-inject/lifecycle" "github.com/hashicorp/consul-k8s/control-plane/connect-inject/metrics" "github.com/hashicorp/consul-k8s/control-plane/connect-inject/webhook" "github.com/hashicorp/consul-k8s/control-plane/controller" @@ -78,6 +80,13 @@ type Command struct { flagDefaultSidecarProxyMemoryRequest string flagDefaultEnvoyProxyConcurrency int + // Proxy lifecycle settings. + flagDefaultEnableSidecarProxyLifecycle bool + flagDefaultEnableSidecarProxyLifecycleShutdownDrainListeners bool + flagDefaultSidecarProxyLifecycleShutdownGracePeriodSeconds int + flagDefaultSidecarProxyLifecycleGracefulPort string + flagDefaultSidecarProxyLifecycleGracefulShutdownPath string + // Metrics settings. flagDefaultEnableMetrics bool flagEnableGatewayMetrics bool @@ -205,6 +214,13 @@ func (c *Command) init() { c.flagSet.StringVar(&c.flagDefaultSidecarProxyMemoryRequest, "default-sidecar-proxy-memory-request", "", "Default sidecar proxy memory request.") c.flagSet.StringVar(&c.flagDefaultSidecarProxyMemoryLimit, "default-sidecar-proxy-memory-limit", "", "Default sidecar proxy memory limit.") + // Proxy lifecycle setting flags. + c.flagSet.BoolVar(&c.flagDefaultEnableSidecarProxyLifecycle, "default-enable-sidecar-proxy-lifecycle", false, "Default for enabling sidecar proxy lifecycle management.") + c.flagSet.BoolVar(&c.flagDefaultEnableSidecarProxyLifecycleShutdownDrainListeners, "default-enable-sidecar-proxy-lifecycle-shutdown-drain-listeners", false, "Default for enabling sidecar proxy listener draining of inbound connections during shutdown.") + c.flagSet.IntVar(&c.flagDefaultSidecarProxyLifecycleShutdownGracePeriodSeconds, "default-sidecar-proxy-lifecycle-shutdown-grace-period-seconds", 0, "Default sidecar proxy shutdown grace period in seconds.") + c.flagSet.StringVar(&c.flagDefaultSidecarProxyLifecycleGracefulPort, "default-sidecar-proxy-lifecycle-graceful-port", strconv.Itoa(constants.DefaultGracefulPort), "Default port for sidecar proxy lifecycle management HTTP endpoints.") + c.flagSet.StringVar(&c.flagDefaultSidecarProxyLifecycleGracefulShutdownPath, "default-sidecar-proxy-lifecycle-graceful-shutdown-path", "/graceful_shutdown", "Default sidecar proxy lifecycle management graceful shutdown path.") + // Metrics setting flags. c.flagSet.BoolVar(&c.flagDefaultEnableMetrics, "default-enable-metrics", false, "Default for enabling connect service metrics.") c.flagSet.BoolVar(&c.flagEnableGatewayMetrics, "enable-gateway-metrics", false, "Allows enabling Consul gateway metrics.") @@ -407,6 +423,14 @@ func (c *Command) Run(args []string) int { return 1 } + lifecycleConfig := lifecycle.Config{ + DefaultEnableProxyLifecycle: c.flagDefaultEnableSidecarProxyLifecycle, + DefaultEnableShutdownDrainListeners: c.flagDefaultEnableSidecarProxyLifecycleShutdownDrainListeners, + DefaultShutdownGracePeriodSeconds: c.flagDefaultSidecarProxyLifecycleShutdownGracePeriodSeconds, + DefaultGracefulPort: c.flagDefaultSidecarProxyLifecycleGracefulPort, + DefaultGracefulShutdownPath: c.flagDefaultSidecarProxyLifecycleGracefulShutdownPath, + } + metricsConfig := metrics.Config{ DefaultEnableMetrics: c.flagDefaultEnableMetrics, EnableGatewayMetrics: c.flagEnableGatewayMetrics, @@ -622,6 +646,7 @@ func (c *Command) Run(args []string) int { DefaultProxyMemoryRequest: sidecarProxyMemoryRequest, DefaultProxyMemoryLimit: sidecarProxyMemoryLimit, DefaultEnvoyProxyConcurrency: c.flagDefaultEnvoyProxyConcurrency, + LifecycleConfig: lifecycleConfig, MetricsConfig: metricsConfig, InitContainerResources: initResources, ConsulPartition: c.consul.Partition,