Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

lifecycle-sidecar and connect inject init container have mandatory resource requirements #289

Closed
wants to merge 6 commits into from
Closed

lifecycle-sidecar and connect inject init container have mandatory resource requirements #289

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
4b01769
make resource settings mandatory for lifecycle sidecar and connect in…
kschoche Jul 6, 2020
c8e5f0b
update go fmt
kschoche Jul 6, 2020
33142b8
Fix inject-connect tests
kschoche Jul 7, 2020
a63d50f
go fmt
kschoche Jul 7, 2020
27379c3
fix tests, add validation tests
kschoche Jul 7, 2020
f9414ab
review comments and changes
kschoche Jul 8, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 4 additions & 5 deletions connect-inject/container_init.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (
"text/template"

corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
)

const (
Expand Down Expand Up @@ -202,12 +201,12 @@ func (h *Handler) containerInit(pod *corev1.Pod, k8sNamespace string) (corev1.Co

resources := corev1.ResourceRequirements{
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse(initContainerCPULimit),
kschoche marked this conversation as resolved.
Show resolved Hide resolved
corev1.ResourceMemory: resource.MustParse(initContainerMemoryLimit),
corev1.ResourceCPU: h.InitCopyContainerCPULimit,
corev1.ResourceMemory: h.InitCopyContainerMemoryLimit,
},
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse(initContainerCPURequest),
corev1.ResourceMemory: resource.MustParse(initContainerMemoryRequest),
corev1.ResourceCPU: h.InitCopyContainerCPURequest,
corev1.ResourceMemory: h.InitCopyContainerMemoryRequest,
},
}

Expand Down
9 changes: 4 additions & 5 deletions connect-inject/container_init_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import (

"github.com/stretchr/testify/require"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

Expand Down Expand Up @@ -1508,12 +1507,12 @@ func TestHandlerContainerInit_Resources(t *testing.T) {
require.NoError(err)
require.Equal(corev1.ResourceRequirements{
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse(initContainerCPULimit),
corev1.ResourceMemory: resource.MustParse(initContainerMemoryLimit),
corev1.ResourceCPU: h.InitCopyContainerCPULimit,
kschoche marked this conversation as resolved.
Show resolved Hide resolved
corev1.ResourceMemory: h.InitCopyContainerMemoryLimit,
},
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse(initContainerCPURequest),
corev1.ResourceMemory: resource.MustParse(initContainerMemoryRequest),
corev1.ResourceCPU: h.InitCopyContainerCPURequest,
corev1.ResourceMemory: h.InitCopyContainerMemoryRequest,
},
}, container.Resources)
}
Expand Down
14 changes: 13 additions & 1 deletion connect-inject/handler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import (
"net/http"
"strconv"

"github.com/deckarep/golang-set"
mapset "github.com/deckarep/golang-set"
kschoche marked this conversation as resolved.
Show resolved Hide resolved
"github.com/hashicorp/consul/api"
"github.com/hashicorp/go-hclog"
"github.com/mattbaird/jsonpatch"
Expand Down Expand Up @@ -175,6 +175,18 @@ type Handler struct {
DefaultProxyMemoryRequest resource.Quantity
DefaultProxyMemoryLimit resource.Quantity

// Resource settings for Init Copy Container
kschoche marked this conversation as resolved.
Show resolved Hide resolved
InitCopyContainerCPULimit resource.Quantity
InitCopyContainerCPURequest resource.Quantity
InitCopyContainerMemoryLimit resource.Quantity
InitCopyContainerMemoryRequest resource.Quantity

// Resource settings for lifecycle sidecar
LifecycleSidecarCPULimit resource.Quantity
LifecycleSidecarCPURequest resource.Quantity
LifecycleSidecarMemoryLimit resource.Quantity
LifecycleSidecarMemoryRequest resource.Quantity
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As discussed, in the refactor I think we could have

LifecycleSidecarResources corev1.ResourceRequirements


// Log
Log hclog.Logger
}
Expand Down
18 changes: 5 additions & 13 deletions connect-inject/lifecycle_sidecar.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,17 +1,9 @@
package connectinject

import (
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"

"strings"
)

const (
lifecycleContainerCPULimit = "10m"
lifecycleContainerCPURequest = "10m"
lifecycleContainerMemoryLimit = "25Mi"
lifecycleContainerMemoryRequest = "25Mi"
corev1 "k8s.io/api/core/v1"
)

func (h *Handler) lifecycleSidecar(pod *corev1.Pod) corev1.Container {
Expand Down Expand Up @@ -63,12 +55,12 @@ func (h *Handler) lifecycleSidecar(pod *corev1.Pod) corev1.Container {

resources := corev1.ResourceRequirements{
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse(lifecycleContainerCPULimit),
corev1.ResourceMemory: resource.MustParse(lifecycleContainerMemoryLimit),
corev1.ResourceCPU: h.LifecycleSidecarCPULimit,
corev1.ResourceMemory: h.LifecycleSidecarMemoryLimit,
},
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse(lifecycleContainerCPURequest),
corev1.ResourceMemory: resource.MustParse(lifecycleContainerMemoryRequest),
corev1.ResourceCPU: h.LifecycleSidecarCPURequest,
corev1.ResourceMemory: h.LifecycleSidecarMemoryRequest,
},
}

Expand Down
28 changes: 22 additions & 6 deletions connect-inject/lifecycle_sidecar_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,20 @@
package connectinject

import (
"testing"

"github.com/hashicorp/go-hclog"
"github.com/stretchr/testify/require"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"testing"
)

const (
kschoche marked this conversation as resolved.
Show resolved Hide resolved
lifecycleContainerCPULimit = "10m"
lifecycleContainerCPURequest = "10m"
kschoche marked this conversation as resolved.
Show resolved Hide resolved
lifecycleContainerMemoryLimit = "25Mi"
lifecycleContainerMemoryRequest = "25Mi"
)

// NOTE: This is tested here rather than in handler_test because doing it there
Expand All @@ -16,8 +24,12 @@ import (
// Test that the lifecycle sidecar is as expected.
func TestLifecycleSidecar_Default(t *testing.T) {
handler := Handler{
Log: hclog.Default().Named("handler"),
ImageConsulK8S: "hashicorp/consul-k8s:9.9.9",
Log: hclog.Default().Named("handler"),
ImageConsulK8S: "hashicorp/consul-k8s:9.9.9",
LifecycleSidecarMemoryRequest: resource.MustParse(lifecycleContainerMemoryRequest),
LifecycleSidecarMemoryLimit: resource.MustParse(lifecycleContainerMemoryLimit),
LifecycleSidecarCPURequest: resource.MustParse(lifecycleContainerCPURequest),
LifecycleSidecarCPULimit: resource.MustParse(lifecycleContainerCPULimit),
}
container := handler.lifecycleSidecar(&corev1.Pod{
Spec: corev1.PodSpec{
Expand Down Expand Up @@ -128,9 +140,13 @@ func TestLifecycleSidecar_SyncPeriodAnnotation(t *testing.T) {
// and that the CA is provided
func TestLifecycleSidecar_TLS(t *testing.T) {
handler := Handler{
Log: hclog.Default().Named("handler"),
ImageConsulK8S: "hashicorp/consul-k8s:9.9.9",
ConsulCACert: "consul-ca-cert",
Log: hclog.Default().Named("handler"),
ImageConsulK8S: "hashicorp/consul-k8s:9.9.9",
ConsulCACert: "consul-ca-cert",
LifecycleSidecarMemoryRequest: resource.MustParse(lifecycleContainerMemoryRequest),
LifecycleSidecarMemoryLimit: resource.MustParse(lifecycleContainerMemoryLimit),
LifecycleSidecarCPURequest: resource.MustParse(lifecycleContainerCPURequest),
LifecycleSidecarCPULimit: resource.MustParse(lifecycleContainerCPULimit),
}
container := handler.lifecycleSidecar(&corev1.Pod{
Spec: corev1.PodSpec{
Expand Down
158 changes: 128 additions & 30 deletions subcommand/inject-connect/command.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ import (
"sync/atomic"
"time"

"github.com/deckarep/golang-set"
"github.com/hashicorp/consul-k8s/connect-inject"
mapset "github.com/deckarep/golang-set"
kschoche marked this conversation as resolved.
Show resolved Hide resolved
connectinject "github.com/hashicorp/consul-k8s/connect-inject"
"github.com/hashicorp/consul-k8s/helper/cert"
"github.com/hashicorp/consul/api"
"github.com/hashicorp/consul/command/flags"
Expand Down Expand Up @@ -58,6 +58,18 @@ type Command struct {
flagDefaultSidecarProxyMemoryLimit string
flagDefaultSidecarProxyMemoryRequest string

// Lifecycle sidecar resource settings.
flagLifecycleSidecarCPULimit string
flagLifecycleSidecarCPURequest string
flagLifecycleSidecarMemoryLimit string
flagLifecycleSidecarMemoryRequest string

// Init copy container resource settings.
flagInitCopyContainerCPULimit string
flagInitCopyContainerCPURequest string
flagInitCopyContainerMemoryLimit string
flagInitCopyContainerMemoryRequest string

flagSet *flag.FlagSet
http *flags.HTTPFlags

Expand Down Expand Up @@ -117,6 +129,16 @@ func (c *Command) init() {
c.flagSet.StringVar(&c.flagDefaultSidecarProxyCPURequest, "default-sidecar-proxy-cpu-request", "", "Default sidecar proxy CPU request.")
c.flagSet.StringVar(&c.flagDefaultSidecarProxyMemoryLimit, "default-sidecar-proxy-memory-limit", "", "Default sidecar proxy memory limit.")
c.flagSet.StringVar(&c.flagDefaultSidecarProxyMemoryRequest, "default-sidecar-proxy-memory-request", "", "Default sidecar proxy memory request.")
// Init container resource setting flags.
c.flagSet.StringVar(&c.flagInitCopyContainerCPULimit, "init-copy-container-cpu-limit", "", "Init copy container CPU limit.")
kschoche marked this conversation as resolved.
Show resolved Hide resolved
c.flagSet.StringVar(&c.flagInitCopyContainerCPURequest, "init-copy-container-cpu-request", "", "Init copy container CPU request.")
c.flagSet.StringVar(&c.flagInitCopyContainerMemoryLimit, "init-copy-container-memory-limit", "", "Init copy container memory limit.")
c.flagSet.StringVar(&c.flagInitCopyContainerMemoryRequest, "init-copy-container-memory-request", "", "Init copy container memory request.")
// Lifecycle sidecar resource setting flags.
c.flagSet.StringVar(&c.flagLifecycleSidecarCPULimit, "lifecycle-sidecar-cpu-limit", "", "Lifecycle sidecar CPU limit.")
c.flagSet.StringVar(&c.flagLifecycleSidecarCPURequest, "lifecycle-sidecar-cpu-request", "", "Lifecycle sidecar CPU request.")
c.flagSet.StringVar(&c.flagLifecycleSidecarMemoryLimit, "lifecycle-sidecar-memory-limit", "", "Lifecycle sidecar memory limit.")
c.flagSet.StringVar(&c.flagLifecycleSidecarMemoryRequest, "lifecycle-sidecar-memory-request", "", "Lifecycle sidecar memory request.")

c.http = &flags.HTTPFlags{}
flags.Merge(c.flagSet, c.http.ClientFlags())
Expand All @@ -136,44 +158,112 @@ func (c *Command) Run(args []string) int {
return 1
}

var cpuLimit, cpuRequest, memoryLimit, memoryRequest resource.Quantity
// Cpu/Memory requests and limits for the initCopyContainers which copy
// the consul binary to shared volume
var initCopyContainerCpuLimit, initCopyContainerCpuRequest, initCopyContainerMemoryLimit, initCopyContainerMemoryRequest resource.Quantity
// lifecycle sidecar container resources
var lifecycleSidecarCpuLimit, lifecycleSidecarCpuRequest, lifecycleSidecarMemoryLimit, lifecycleSidecarMemoryRequest resource.Quantity
// sidecar proxy container resources
var sidecarProxyCpuLimit, sidecarProxyCpuRequest, sidecarProxyMemoryLimit, sidecarProxyMemoryRequest resource.Quantity
var err error

if c.flagLifecycleSidecarCPULimit == "" || c.flagLifecycleSidecarCPURequest == "" {
c.UI.Error("-lifecycle-sidecar-cpu-limit && -lifecycle-sidecar-cpu-request must be set")
return 1
}
if c.flagLifecycleSidecarMemoryLimit == "" || c.flagLifecycleSidecarMemoryRequest == "" {
c.UI.Error("-lifecycle-sidecar-memory-limit && -lifecycle-sidecar-memory-request must be set")
return 1
}
if c.flagInitCopyContainerCPULimit == "" || c.flagInitCopyContainerCPURequest == "" {
c.UI.Error("-init-copy-container-cpu-limit && -init-copy-container-cpu-request must be set")
return 1
}
if c.flagInitCopyContainerMemoryLimit == "" || c.flagInitCopyContainerMemoryRequest == "" {
c.UI.Error("-init-copy-container-memory-limit && -init-copy-container-memory-request must be set")
return 1
}

// Parse the initCopyContainer resources
initCopyContainerCpuLimit, err = resource.ParseQuantity(c.flagInitCopyContainerCPULimit)
kschoche marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
c.UI.Error(fmt.Sprintf("-init-copy-container-cpu-limit is invalid: %s", err))
return 1
}
initCopyContainerCpuRequest, err = resource.ParseQuantity(c.flagInitCopyContainerCPURequest)
if err != nil {
c.UI.Error(fmt.Sprintf("-init-copy-container-cpu-request is invalid: %s", err))
return 1
}
initCopyContainerMemoryLimit, err = resource.ParseQuantity(c.flagInitCopyContainerMemoryLimit)
if err != nil {
c.UI.Error(fmt.Sprintf("-init-copy-container-memory-limit is invalid: %s", err))
return 1
}
initCopyContainerMemoryRequest, err = resource.ParseQuantity(c.flagInitCopyContainerMemoryRequest)
if err != nil {
c.UI.Error(fmt.Sprintf("-init-copy-container-memory-request is invalid: %s", err))
return 1
}

// Parse the lifecycle sidecar resources
lifecycleSidecarCpuLimit, err = resource.ParseQuantity(c.flagLifecycleSidecarCPULimit)
if err != nil {
c.UI.Error(fmt.Sprintf("-lifecycle-sidecar-cpu-limit is invalid: %s", err))
return 1
}
lifecycleSidecarCpuRequest, err = resource.ParseQuantity(c.flagLifecycleSidecarCPURequest)
if err != nil {
c.UI.Error(fmt.Sprintf("-lifecycle-sidecar-cpu-request is invalid: %s", err))
return 1
}
lifecycleSidecarMemoryLimit, err = resource.ParseQuantity(c.flagLifecycleSidecarMemoryLimit)
if err != nil {
c.UI.Error(fmt.Sprintf("-lifecycle-sidecar-memory-limit is invalid: %s", err))
return 1
}
initCopyContainerMemoryRequest, err = resource.ParseQuantity(c.flagLifecycleSidecarMemoryRequest)
if err != nil {
c.UI.Error(fmt.Sprintf("-lifecycle-sidecar-memory-request is invalid: %s", err))
return 1
}
kschoche marked this conversation as resolved.
Show resolved Hide resolved

if c.flagDefaultSidecarProxyCPULimit != "" {
cpuLimit, err = resource.ParseQuantity(c.flagDefaultSidecarProxyCPULimit)
sidecarProxyCpuLimit, err = resource.ParseQuantity(c.flagDefaultSidecarProxyCPULimit)
if err != nil {
c.UI.Error(fmt.Sprintf("-default-sidecar-proxy-cpu-limit is invalid: %s", err))
return 1
}
}
if c.flagDefaultSidecarProxyCPURequest != "" {
cpuRequest, err = resource.ParseQuantity(c.flagDefaultSidecarProxyCPURequest)
sidecarProxyCpuRequest, err = resource.ParseQuantity(c.flagDefaultSidecarProxyCPURequest)
if err != nil {
c.UI.Error(fmt.Sprintf("-default-sidecar-proxy-cpu-request is invalid: %s", err))
return 1
}
}
if cpuLimit.Value() != 0 && cpuRequest.Cmp(cpuLimit) > 0 {
if sidecarProxyCpuLimit.Value() != 0 && sidecarProxyCpuRequest.Cmp(sidecarProxyCpuLimit) > 0 {
c.UI.Error(fmt.Sprintf(
"request must be <= limit: -default-sidecar-proxy-cpu-request value of %q is greater than the -default-sidecar-proxy-cpu-limit value of %q",
c.flagDefaultSidecarProxyCPURequest, c.flagDefaultSidecarProxyCPULimit))
Comment on lines 252 to 253
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we need to add a similar validation for other resource flags too.

return 1
}

if c.flagDefaultSidecarProxyMemoryLimit != "" {
memoryLimit, err = resource.ParseQuantity(c.flagDefaultSidecarProxyMemoryLimit)
sidecarProxyMemoryLimit, err = resource.ParseQuantity(c.flagDefaultSidecarProxyMemoryLimit)
if err != nil {
c.UI.Error(fmt.Sprintf("-default-sidecar-proxy-memory-limit is invalid: %s", err))
return 1
}
}
if c.flagDefaultSidecarProxyMemoryRequest != "" {
memoryRequest, err = resource.ParseQuantity(c.flagDefaultSidecarProxyMemoryRequest)
sidecarProxyMemoryRequest, err = resource.ParseQuantity(c.flagDefaultSidecarProxyMemoryRequest)
if err != nil {
c.UI.Error(fmt.Sprintf("-default-sidecar-proxy-memory-request is invalid: %s", err))
return 1
}
}
if memoryLimit.Value() != 0 && memoryRequest.Cmp(memoryLimit) > 0 {
if sidecarProxyMemoryLimit.Value() != 0 && sidecarProxyMemoryRequest.Cmp(sidecarProxyMemoryLimit) > 0 {
c.UI.Error(fmt.Sprintf(
"request must be <= limit: -default-sidecar-proxy-memory-request value of %q is greater than the -default-sidecar-proxy-memory-limit value of %q",
c.flagDefaultSidecarProxyMemoryRequest, c.flagDefaultSidecarProxyMemoryLimit))
Expand Down Expand Up @@ -256,27 +346,35 @@ func (c *Command) Run(args []string) int {

// Build the HTTP handler and server
injector := connectinject.Handler{
ConsulClient: c.consulClient,
ImageConsul: c.flagConsulImage,
ImageEnvoy: c.flagEnvoyImage,
ImageConsulK8S: c.flagConsulK8sImage,
RequireAnnotation: !c.flagDefaultInject,
AuthMethod: c.flagACLAuthMethod,
WriteServiceDefaults: c.flagWriteServiceDefaults,
DefaultProtocol: c.flagDefaultProtocol,
ConsulCACert: string(consulCACert),
DefaultProxyCPULimit: cpuLimit,
DefaultProxyCPURequest: cpuRequest,
DefaultProxyMemoryLimit: memoryLimit,
DefaultProxyMemoryRequest: memoryRequest,
EnableNamespaces: c.flagEnableNamespaces,
AllowK8sNamespacesSet: allowSet,
DenyK8sNamespacesSet: denySet,
ConsulDestinationNamespace: c.flagConsulDestinationNamespace,
EnableK8SNSMirroring: c.flagEnableK8SNSMirroring,
K8SNSMirroringPrefix: c.flagK8SNSMirroringPrefix,
CrossNamespaceACLPolicy: c.flagCrossNamespaceACLPolicy,
Log: hclog.Default().Named("handler"),
ConsulClient: c.consulClient,
ImageConsul: c.flagConsulImage,
ImageEnvoy: c.flagEnvoyImage,
ImageConsulK8S: c.flagConsulK8sImage,
RequireAnnotation: !c.flagDefaultInject,
AuthMethod: c.flagACLAuthMethod,
WriteServiceDefaults: c.flagWriteServiceDefaults,
DefaultProtocol: c.flagDefaultProtocol,
ConsulCACert: string(consulCACert),
DefaultProxyCPULimit: sidecarProxyCpuLimit,
DefaultProxyCPURequest: sidecarProxyCpuRequest,
DefaultProxyMemoryLimit: sidecarProxyMemoryLimit,
DefaultProxyMemoryRequest: sidecarProxyMemoryRequest,
InitCopyContainerCPULimit: initCopyContainerCpuLimit,
InitCopyContainerCPURequest: initCopyContainerCpuRequest,
InitCopyContainerMemoryLimit: initCopyContainerMemoryLimit,
InitCopyContainerMemoryRequest: initCopyContainerMemoryRequest,
LifecycleSidecarCPULimit: lifecycleSidecarCpuLimit,
LifecycleSidecarCPURequest: lifecycleSidecarCpuRequest,
LifecycleSidecarMemoryLimit: lifecycleSidecarMemoryLimit,
LifecycleSidecarMemoryRequest: lifecycleSidecarMemoryRequest,
EnableNamespaces: c.flagEnableNamespaces,
AllowK8sNamespacesSet: allowSet,
DenyK8sNamespacesSet: denySet,
ConsulDestinationNamespace: c.flagConsulDestinationNamespace,
EnableK8SNSMirroring: c.flagEnableK8SNSMirroring,
K8SNSMirroringPrefix: c.flagK8SNSMirroringPrefix,
CrossNamespaceACLPolicy: c.flagCrossNamespaceACLPolicy,
Log: hclog.Default().Named("handler"),
}
mux := http.NewServeMux()
mux.HandleFunc("/mutate", injector.Handle)
Expand Down
Loading