Skip to content

Commit

Permalink
support overriding load balancer health check protocol (#147)
Browse files Browse the repository at this point in the history 
* support override for health check protocol

Co-Authored-By: andrewsykim <kim.andrewsy@gmail.com>
  • Loading branch information
@andrewsykim
andrewsykim committed Nov 22, 2018
1 parent 4371b4b commit 5ef072a
Show file tree
Hide file tree
Showing 4 changed files with 224 additions and 6 deletions.
34 changes: 32 additions & 2 deletions cloud-controller-manager/do/loadbalancers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,11 @@ const (
// for DO load balancers. Defaults to '/'.
annDOHealthCheckPath = "service.beta.kubernetes.io/do-loadbalancer-healthcheck-path"

// annDOHealthCheckProtocol is the annotation used to specify the health check protocol
// for DO load balancers. Defaults to the protocol used in
// 'service.beta.kubernetes.io/do-loadbalancer-protocol'.
annDOHealthCheckProtocol = "service.beta.kubernetes.io/do-loadbalancer-healthcheck-protocol"

// annDOTLSPorts is the annotation used to specify which ports of the loadbalancer
// should use the https protocol. This is a comma separated list of ports
// (e.g. 443,6443,7443).
Expand Down Expand Up @@ -358,16 +363,27 @@ func (l *loadbalancers) waitActive(lbID string) (*godo.LoadBalancer, error) {
// Balancers can only take one node port so we choose the first node port for
// health checking.
func buildHealthCheck(service *v1.Service) (*godo.HealthCheck, error) {
protocol, err := getProtocol(service)
healthCheckProtocol, err := healthCheckProtocol(service)
if err != nil {
return nil, err
}

// if no health check protocol is specified, use the protocol used for
// load balancer traffic.
if healthCheckProtocol == "" {
protocol, err := getProtocol(service)
if err != nil {
return nil, err
}

healthCheckProtocol = protocol
}

healthCheckPath := healthCheckPath(service)
port := service.Spec.Ports[0].NodePort

return &godo.HealthCheck{
Protocol: protocol,
Protocol: healthCheckProtocol,
Port: int(port),
Path: healthCheckPath,
CheckIntervalSeconds: 3,
Expand Down Expand Up @@ -482,6 +498,20 @@ func getProtocol(service *v1.Service) (string, error) {
return protocol, nil
}

// healthCheckProtocol returns the health check protocol as specified in the service
func healthCheckProtocol(service *v1.Service) (string, error) {
protocol, ok := service.Annotations[annDOHealthCheckProtocol]
if !ok {
return "", nil
}

if protocol != "tcp" && protocol != "http" {
return "", fmt.Errorf("invalid protocol: %q specified in annotation: %q", protocol, annDOProtocol)
}

return protocol, nil
}

// getHealthCheckPath returns the desired path for health checking
// health check path should default to / if not specified
func healthCheckPath(service *v1.Service) string {
Expand Down
144 changes: 144 additions & 0 deletions cloud-controller-manager/do/loadbalancers_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -926,6 +926,38 @@ func Test_buildHealthCheck(t *testing.T) {
},
nil,
},
{
"http health check using protocol override",
&v1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
UID: "abc123",
Annotations: map[string]string{
annDOProtocol: "tcp",
annDOHealthCheckProtocol: "http",
},
},
Spec: v1.ServiceSpec{
Ports: []v1.ServicePort{
{
Name: "test",
Protocol: "TCP",
Port: int32(80),
NodePort: int32(30000),
},
},
},
},
&godo.HealthCheck{
Protocol: "http",
Port: 30000,
CheckIntervalSeconds: 3,
ResponseTimeoutSeconds: 5,
HealthyThreshold: 5,
UnhealthyThreshold: 3,
},
nil,
},
{
"http health check with path",
&v1.Service{
Expand Down Expand Up @@ -983,6 +1015,30 @@ func Test_buildHealthCheck(t *testing.T) {
nil,
fmt.Errorf("invalid protocol: %q specified in annotation: %q", "invalid", annDOProtocol),
},
{
"invalid health check using protocol override",
&v1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
UID: "abc123",
Annotations: map[string]string{
annDOHealthCheckProtocol: "invalid",
},
},
Spec: v1.ServiceSpec{
Ports: []v1.ServicePort{
{
Name: "test",
Protocol: "TCP",
Port: int32(80),
NodePort: int32(30000),
},
},
},
},
nil,
fmt.Errorf("invalid protocol: %q specified in annotation: %q", "invalid", annDOProtocol),
},
}

for _, test := range testcases {
Expand Down Expand Up @@ -1277,6 +1333,94 @@ func Test_buildLoadBalancerRequest(t *testing.T) {
},
nil,
},
{
"successful load balancer request with custom health checks",
func(ctx context.Context, opt *godo.ListOptions) ([]godo.Droplet, *godo.Response, error) {
return []godo.Droplet{
{
ID: 100,
Name: "node-1",
},
{
ID: 101,
Name: "node-2",
},
{
ID: 102,
Name: "node-3",
},
}, newFakeOKResponse(), nil
},
&v1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
UID: "foobar123",
Annotations: map[string]string{
annDOProtocol: "tcp",
annDOHealthCheckPath: "/health",
annDOHealthCheckProtocol: "http",
},
},
Spec: v1.ServiceSpec{
Ports: []v1.ServicePort{
{
Name: "test",
Protocol: "TCP",
Port: int32(80),
NodePort: int32(30000),
},
},
},
},
[]*v1.Node{
{
ObjectMeta: metav1.ObjectMeta{
Name: "node-1",
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "node-2",
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "node-3",
},
},
},
&godo.LoadBalancerRequest{
// cloudprovider.GetLoadBalancer name uses 'a' + service.UID
// as loadbalancer name
Name: "afoobar123",
DropletIDs: []int{100, 101, 102},
Region: "nyc3",
ForwardingRules: []godo.ForwardingRule{
{
EntryProtocol: "tcp",
EntryPort: 80,
TargetProtocol: "tcp",
TargetPort: 30000,
CertificateID: "",
TlsPassthrough: false,
},
},
HealthCheck: &godo.HealthCheck{
Protocol: "http",
Port: 30000,
Path: "/health",
CheckIntervalSeconds: 3,
ResponseTimeoutSeconds: 5,
HealthyThreshold: 5,
UnhealthyThreshold: 3,
},
Algorithm: "round_robin",
StickySessions: &godo.StickySessions{
Type: "none",
},
},
nil,
},
{
"successful load balancer request using least_connections algorithm",
func(ctx context.Context, opt *godo.ListOptions) ([]godo.Droplet, *godo.Response, error) {
Expand Down
16 changes: 12 additions & 4 deletions docs/controllers/services/annotations.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,15 @@ DigitalOcean cloud controller manager watches for Services of type `LoadBalancer

The default protocol for DigitalOcean Load Balancers. Ports specified in the annotation `service.beta.kubernetes.io/do-loadbalancer-tls-ports` will be overwritten to https. Options are `tcp`, `http` and `https`. Defaults to `tcp`.

### service.beta.kubernetes.io/do-loadbalancer-tls-ports
### service.beta.kubernetes.io/do-loadbalancer-healthcheck-path

The path used to check if a backend droplet is healthy. Defaults to "/".

### service.beta.kubernetes.io/do-loadbalancer-healthcheck-protocol

The health check protocol to use to check if a backend droplet is healthy. Defaults to the protocol used in `service.beta.kubernetes.io/do-loadbalancer-protocol`. Options are `tcp` and `http`.

### service.beta.kubernetes.io/do-loadbalancer-tls-ports

Specify which ports of the loadbalancer should use the https protocol. This is a comma separated list of ports (e.g. 443,6443,7443).

Expand All @@ -18,17 +26,17 @@ Specify whether the DigitalOcean Load Balancer should pass encrypted data to bac

Specifies the certificate ID used for https. This annotation is required if `service.beta.kubernetes.io/do-loadbalancer-tls-ports` is used. To list available certificates and their IDs, use `doctl compute certificate list` or find it in the [control panel](https://cloud.digitalocean.com/account/security).

### service.beta.kubernetes.io/do-loadbalancer-algorithm
### service.beta.kubernetes.io/do-loadbalancer-algorithm

Specifies which algorithm the Load Balancer should use. Options are `round_robin`, `least_connections`. Defaults to `round_robin`.

### service.beta.kubernetes.io/do-loadbalancer-sticky-sessions-type

Specifies which stick session type the loadbalancer should use. Options are `none` or `cookies`.

### service.beta.kubernetes.io/do-loadbalancer-sticky-sessions-cookie-ttl
### service.beta.kubernetes.io/do-loadbalancer-sticky-sessions-cookie-ttl

Specifies the TTL of cookies used for loadbalancer sticky sessions. This annotation is required if `service.beta.kubernetes.io/do-loadbalancer-sticky-sessions-type` is set.
Specifies the TTL of cookies used for loadbalancer sticky sessions. This annotation is required if `service.beta.kubernetes.io/do-loadbalancer-sticky-sessions-type` is set.

### service.beta.kubernetes.io/do-loadbalancer-redirect-http-to-https

Expand Down
36 changes: 36 additions & 0 deletions docs/controllers/services/examples/http-nginx-with-healthcheck-protocol.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
---
kind: Service
apiVersion: v1
metadata:
name: http-lb
annotations:
service.beta.kubernetes.io/do-loadbalancer-protocol: "http"
service.beta.kubernetes.io/do-loadbalancer-healthcheck-protocol: "tcp"
spec:
type: LoadBalancer
selector:
app: nginx-example
ports:
- name: http
protocol: TCP
port: 80
targetPort: 80

---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: nginx-example
spec:
replicas: 2
template:
metadata:
labels:
app: nginx-example
spec:
containers:
- name: nginx
image: nginx
ports:
- containerPort: 80
protocol: TCP

0 comments on commit 5ef072a

Please sign in to comment.