Skip to content

Commit

Permalink
add the new metric
Browse files Browse the repository at this point in the history
Signed-off-by: Mustafa Abdelrahman <mustafa.abdelrahman@zalando.de>
  • Loading branch information
MustafaSaber committed May 10, 2024
1 parent 7b84f67 commit d22935c
Showing 1 changed file with 31 additions and 32 deletions.
63 changes: 31 additions & 32 deletions proxy/healthy_endpoints_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"math"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"

Expand Down Expand Up @@ -81,7 +82,7 @@ func setupProxyWithCustomEndpointRegisty(t *testing.T, doc string, endpointRegis
return m, ps
}

func setupServices(t *testing.T, healthy, unhealthy int) []string {
func setupServices(t *testing.T, healthy, unhealthy int) string {
services := []string{}
for i := 0; i < healthy; i++ {
service := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
Expand All @@ -98,27 +99,16 @@ func setupServices(t *testing.T, healthy, unhealthy int) []string {
services = append(services, service.URL)
t.Cleanup(service.Close)
}
return services
}

func formatServicesString(services []string) string {
backends := ""
for i, service := range services {
backends += fmt.Sprintf(`"%s"`, service)
if i < len(services)-1 {
backends += ", "
}
}
return backends
return fmt.Sprint(`"`, strings.Join(services, `", "`), `"`)
}

func TestPHCWithoutRequests(t *testing.T) {
services := setupServices(t, 3, 0)
servicesString := setupServices(t, 3, 0)

for _, algorithm := range []string{"random", "consistentHash", "roundRobin", "powerOfRandomNChoices"} {
t.Run(algorithm, func(t *testing.T) {
_, ps := setupProxy(t, fmt.Sprintf(`* -> <%s, "%s", "%s", "%s">`,
algorithm, services[0], services[1], services[2]))
_, ps := setupProxy(t, fmt.Sprintf(`* -> <%s, %s>`,
algorithm, servicesString))
rsp := sendGetRequest(t, ps, 0)
assert.Equal(t, http.StatusOK, rsp.StatusCode)
rsp.Body.Close()
Expand All @@ -129,8 +119,8 @@ func TestPHCWithoutRequests(t *testing.T) {
}

t.Run("consistent hash with balance factor", func(t *testing.T) {
_, ps := setupProxy(t, fmt.Sprintf(`* -> consistentHashBalanceFactor(1.25) -> <consistentHash, "%s", "%s", "%s">`,
services[0], services[1], services[2]))
_, ps := setupProxy(t, fmt.Sprintf(`* -> consistentHashBalanceFactor(1.25) -> <consistentHash, %s>`,
servicesString))
rsp := sendGetRequest(t, ps, 0)
assert.Equal(t, http.StatusOK, rsp.StatusCode)
rsp.Body.Close()
Expand All @@ -141,10 +131,10 @@ func TestPHCWithoutRequests(t *testing.T) {
}

func TestPHCForSingleHealthyEndpoint(t *testing.T) {
service := setupServices(t, 1, 0)[0]
servicesString := setupServices(t, 1, 0)
endpointRegistry := defaultEndpointRegistry()

doc := fmt.Sprintf(`* -> "%s"`, service)
doc := fmt.Sprintf(`* -> %s`, servicesString)
tp, err := newTestProxyWithParams(doc, Params{
EnablePassiveHealthCheck: true,
EndpointRegistry: endpointRegistry,
Expand All @@ -162,6 +152,7 @@ func TestPHCForSingleHealthyEndpoint(t *testing.T) {
}

func TestPHC(t *testing.T) {
rpsFactor := 0.0
for _, tt := range []struct {
name string
healthy, unhealthy int
Expand All @@ -171,15 +162,20 @@ func TestPHC(t *testing.T) {
{"multiple healthy and one unhealthy", 2, 1},
{"multiple healthy and multiple unhealthy", 2, 2},
} {
if (tt.healthy != 0 && tt.unhealthy == 0) || (tt.healthy == 0 && tt.unhealthy != 0) {
t.Logf("test %s has no mitigations", tt.name)
rpsFactor = 0.0
} else {
t.Logf("test %s has mitigations", tt.name)
rpsFactor = 1.0
}
t.Run(tt.name, func(t *testing.T) {
services := setupServices(t, tt.healthy, tt.unhealthy)
servicesString := formatServicesString(services)
servicesString := setupServices(t, tt.healthy, tt.unhealthy)
for _, algorithm := range []string{"random", "roundRobin", "powerOfRandomNChoices"} {
t.Run(algorithm, func(t *testing.T) {
mockMetrics, ps := setupProxy(t, fmt.Sprintf(`* -> backendTimeout("5ms") -> consistentHashKey("${request.header.ConsistentHashKey}") -> <%s, %s>`,
mockMetrics, ps := setupProxy(t, fmt.Sprintf(`* -> backendTimeout("20ms") -> <%s, %s>`,
algorithm, servicesString))

va := fireVegeta(t, ps, 3000, 1*time.Second, 5*time.Second)
va := fireVegeta(t, ps, 5000, 1*time.Second, 5*time.Second)

count200, ok := va.CountStatus(200)
assert.True(t, ok)
Expand All @@ -201,42 +197,45 @@ func TestPHC(t *testing.T) {
assert.InDelta(t, float64(count504), failedRequests, 5)
assert.InDelta(t, 0, float64(failedRequests), 0.3*float64(va.TotalRequests()))
mockMetrics.WithCounters(func(counters map[string]int64) {
assert.InDelta(t, float64(tt.unhealthy)*float64(va.TotalRequests()), float64(counters["passive-health-check.endpoints.dropped"]), 0.6*float64(va.TotalRequests()))
assert.InDelta(t, float64(tt.unhealthy)*float64(va.TotalRequests()), float64(counters["passive-health-check.endpoints.dropped"]), 0.3*float64(tt.unhealthy)*float64(va.TotalRequests()))
assert.InDelta(t, float64(va.TotalRequests())*rpsFactor, float64(counters["passive-health-check.requests.passed"]), 0.3*float64(va.TotalRequests())) // allow 30% error
})
})
}

t.Run("consistentHash", func(t *testing.T) {
endpointRegistry := routing.NewEndpointRegistry(routing.RegistryOptions{
consistantHashCustomEndpointRegistry := routing.NewEndpointRegistry(routing.RegistryOptions{
PassiveHealthCheckEnabled: true,
StatsResetPeriod: 1 * time.Second,
MinRequests: 1, // with 2 test case fails on github actions
MaxHealthCheckDropProbability: 0.95,
MinHealthCheckDropProbability: 0.01,
})
mockMetrics, ps := setupProxyWithCustomEndpointRegisty(t, fmt.Sprintf(`* -> backendTimeout("5ms") -> consistentHashKey("${request.header.ConsistentHashKey}") -> <consistentHash, %s>`,
servicesString), endpointRegistry)
servicesString), consistantHashCustomEndpointRegistry)
failedReqs := sendGetRequests(t, ps)
assert.InDelta(t, 0, failedReqs, 0.2*float64(nRequests))
mockMetrics.WithCounters(func(counters map[string]int64) {
assert.InDelta(t, float64(tt.unhealthy*nRequests), float64(counters["passive-health-check.endpoints.dropped"]), 0.3*float64(tt.unhealthy*nRequests))
assert.InDelta(t, float64(tt.unhealthy*nRequests), float64(counters["passive-health-check.endpoints.dropped"]), 0.3*float64(tt.unhealthy)*float64(nRequests))
assert.InDelta(t, float64(nRequests)*rpsFactor, float64(counters["passive-health-check.requests.passed"]), 0.3*float64(nRequests)) // allow 30% error
})
})

t.Run("consistent hash with balance factor", func(t *testing.T) {
endpointRegistery := routing.NewEndpointRegistry(routing.RegistryOptions{
consistantHashCustomEndpointRegistry := routing.NewEndpointRegistry(routing.RegistryOptions{
PassiveHealthCheckEnabled: true,
StatsResetPeriod: 1 * time.Second,
MinRequests: 1, // with 2 test case fails on github actions
MaxHealthCheckDropProbability: 0.95,
MinHealthCheckDropProbability: 0.01,
})
mockMetrics, ps := setupProxyWithCustomEndpointRegisty(t, fmt.Sprintf(`* -> backendTimeout("5ms") -> consistentHashKey("${request.header.ConsistentHashKey}") -> consistentHashBalanceFactor(1.25) -> <consistentHash, %s>`,
servicesString), endpointRegistery)
servicesString), consistantHashCustomEndpointRegistry)
failedReqs := sendGetRequests(t, ps)
assert.InDelta(t, 0, failedReqs, 0.2*float64(nRequests))
mockMetrics.WithCounters(func(counters map[string]int64) {
assert.InDelta(t, float64(tt.unhealthy*nRequests), float64(counters["passive-health-check.endpoints.dropped"]), 0.3*float64(nRequests)*float64(tt.unhealthy))
assert.InDelta(t, float64(tt.unhealthy*nRequests), float64(counters["passive-health-check.endpoints.dropped"]), 0.3*float64(tt.unhealthy)*float64(nRequests))
assert.InDelta(t, float64(nRequests)*rpsFactor, float64(counters["passive-health-check.requests.passed"]), 0.3*float64(nRequests)) // allow 30% error
})
})
})
Expand Down

0 comments on commit d22935c

Please sign in to comment.