Skip to content

feat(loadbalancer): add health check port annotation #195

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

feat(loadbalancer): add health check port annotation #195

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions docs/loadbalancer-annotations.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,12 @@ The default value is `5s`. The duration are go's time.Duration (ex: `1s`, `2m`,
This is the annotation to set the number of consecutive unsuccessful health checks, after wich the server will be considered dead.
The default value is `5`.

### `service.beta.kubernetes.io/scw-loadbalancer-health-check-port`
This is the annotation to explicitly define the port used for health checks.
It is possible to set a single port for all backends like `18080` or per port like `80:10080;443:10443`.
The port must be a valid TCP/UDP port (1-65535).
If not set, the service port is used as the health check port.

### `service.beta.kubernetes.io/scw-loadbalancer-health-check-http-uri`
This is the annotation to set the URI that is used by the `http` health check.
It is possible to set the uri per port, like `80:/;443,8443:mydomain.tld/healthz`.
Expand Down
10 changes: 7 additions & 3 deletions scaleway/loadbalancers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1158,9 +1158,13 @@ func servicePortToBackend(service *v1.Service, loadbalancer *scwlb.LB, port v1.S
return nil, err
}

healthCheck := &scwlb.HealthCheck{
Port: port.NodePort,
healthCheck := &scwlb.HealthCheck{}

healthCheckPort, err := getHealthCheckPort(service, port.NodePort)
if err != nil {
return nil, err
}
healthCheck.Port = healthCheckPort

healthCheckDelay, err := getHealthCheckDelay(service)
if err != nil {
Expand Down Expand Up @@ -1649,7 +1653,7 @@ func (l *loadbalancers) updateBackend(service *v1.Service, loadbalancer *scwlb.L
if _, err := l.api.UpdateHealthCheck(&scwlb.ZonedAPIUpdateHealthCheckRequest{
Zone: loadbalancer.Zone,
BackendID: backend.ID,
Port: backend.ForwardPort,
Port: backend.HealthCheck.Port,
CheckDelay: backend.HealthCheck.CheckDelay,
CheckTimeout: backend.HealthCheck.CheckTimeout,
CheckMaxRetries: backend.HealthCheck.CheckMaxRetries,
Expand Down
38 changes: 38 additions & 0 deletions scaleway/loadbalancers_annotations.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,12 @@ const (
// NB: Required when setting service.beta.kubernetes.io/scw-loadbalancer-health-check-type to "pgsql"
serviceAnnotationLoadBalancerHealthCheckPgsqlUser = "service.beta.kubernetes.io/scw-loadbalancer-health-check-pgsql-user"

// serviceAnnotationLoadBalancerHealthCheckPort is the annotation to explicitly define the port used for health checks
// It is possible to set a single port for all backends like "18080" or per port like "80:10080;443:10443"
// The port must be a valid TCP/UDP port (1-65535)
// If not set, the service port is used as the health check port
serviceAnnotationLoadBalancerHealthCheckPort = "service.beta.kubernetes.io/scw-loadbalancer-health-check-port"

// serviceAnnotationLoadBalancerSendProxyV2 is the annotation that enables PROXY protocol version 2 (must be supported by backend servers)
// The default value is "false" and the possible values are "false" or "true"
// or a comma delimited list of the service port on which to apply the proxy protocol (for instance "80,443")
Expand Down Expand Up @@ -629,6 +635,38 @@ func getHealthCheckTransientCheckDelay(service *v1.Service) (*scw.Duration, erro
}, nil
}

// getHealthCheckPort returns the port to use for health checks.
// It supports per-port configuration with the format "80:10080;443:10443" or a single port like "18080".
// If the annotation is not set, it returns the provided nodePort as default.
func getHealthCheckPort(service *v1.Service, nodePort int32) (int32, error) {
annotation, ok := service.Annotations[serviceAnnotationLoadBalancerHealthCheckPort]
if !ok {
return nodePort, nil
}

portStr, err := getValueForPort(service, nodePort, annotation)
if err != nil {
klog.Errorf("could not get value for annotation %s and port %d", serviceAnnotationLoadBalancerHealthCheckPort, nodePort)
return 0, err
}

if portStr == "" {
return nodePort, nil
}

port, err := strconv.ParseInt(portStr, 10, 32)
if err != nil {
klog.Errorf("invalid value for annotation %s: %s is not a valid port number", serviceAnnotationLoadBalancerHealthCheckPort, portStr)
return 0, errLoadBalancerInvalidAnnotation
}
if port < 1 || port > 65535 {
klog.Errorf("invalid value for annotation %s: port %d is out of range (1-65535)", serviceAnnotationLoadBalancerHealthCheckPort, port)
return 0, errLoadBalancerInvalidAnnotation
}

return int32(port), nil
}

func getForceInternalIP(service *v1.Service) bool {
forceInternalIP, ok := service.Annotations[serviceAnnotationLoadBalancerForceInternalIP]
if !ok {
Expand Down
172 changes: 172 additions & 0 deletions scaleway/loadbalancers_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -283,6 +283,178 @@ func TestGetValueForPort(t *testing.T) {
}
}

func TestGetHealthCheckPort(t *testing.T) {
testCases := []struct {
name string
svc *v1.Service
nodePort int32
result int32
errMessage string
}{
{
name: "no annotation, returns nodePort",
svc: &v1.Service{
Spec: v1.ServiceSpec{
Ports: []v1.ServicePort{
{
NodePort: 30080,
Port: 80,
},
},
},
},
nodePort: 30080,
result: 30080,
errMessage: "",
},
{
name: "minimum valid port",
svc: &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
"service.beta.kubernetes.io/scw-loadbalancer-health-check-port": "1",
},
},
Spec: v1.ServiceSpec{
Ports: []v1.ServicePort{
{
NodePort: 30080,
Port: 80,
},
},
},
},
nodePort: 30080,
result: 1,
errMessage: "",
},
{
name: "maximum valid port",
svc: &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
"service.beta.kubernetes.io/scw-loadbalancer-health-check-port": "65535",
},
},
Spec: v1.ServiceSpec{
Ports: []v1.ServicePort{
{
NodePort: 30080,
Port: 80,
},
},
},
},
nodePort: 30080,
result: 65535,
errMessage: "",
},
// Error cases
{
name: "port too low (0)",
svc: &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
"service.beta.kubernetes.io/scw-loadbalancer-health-check-port": "0",
},
},
Spec: v1.ServiceSpec{
Ports: []v1.ServicePort{
{
NodePort: 30080,
Port: 80,
},
},
},
},
nodePort: 30080,
result: 0,
errMessage: "load balancer invalid annotation",
},
{
name: "port too high (65536)",
svc: &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
"service.beta.kubernetes.io/scw-loadbalancer-health-check-port": "65536",
},
},
Spec: v1.ServiceSpec{
Ports: []v1.ServicePort{
{
NodePort: 30080,
Port: 80,
},
},
},
},
nodePort: 30080,
result: 0,
errMessage: "load balancer invalid annotation",
},
{
name: "negative port",
svc: &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
"service.beta.kubernetes.io/scw-loadbalancer-health-check-port": "-1",
},
},
Spec: v1.ServiceSpec{
Ports: []v1.ServicePort{
{
NodePort: 30080,
Port: 80,
},
},
},
},
nodePort: 30080,
result: 0,
errMessage: "load balancer invalid annotation",
},
{
name: "non-numeric value",
svc: &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
"service.beta.kubernetes.io/scw-loadbalancer-health-check-port": "not-a-number",
},
},
Spec: v1.ServiceSpec{
Ports: []v1.ServicePort{
{
NodePort: 30080,
Port: 80,
},
},
},
},
nodePort: 30080,
result: 0,
errMessage: "load balancer invalid annotation",
},
}

for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
result, err := getHealthCheckPort(tc.svc, tc.nodePort)
if result != tc.result {
t.Errorf("getHealthCheckPort: got %d, expected %d", result, tc.result)
}
if err == nil && tc.errMessage != "" {
t.Errorf("getHealthCheckPort: expected error %q, got nil", tc.errMessage)
}
if err != nil && tc.errMessage == "" {
t.Errorf("getHealthCheckPort: unexpected error %v", err)
}
if err != nil && tc.errMessage != "" && err.Error() != tc.errMessage {
t.Errorf("getHealthCheckPort: got error %q, expected %q", err.Error(), tc.errMessage)
}
})
}
}

func TestFilterNodes(t *testing.T) {
service := &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Expand Down