From 96b1545048e01003dcbf90029d70137a93d0d196 Mon Sep 17 00:00:00 2001 From: bwplotka Date: Fri, 21 Oct 2022 13:46:34 +0200 Subject: [PATCH] e2e: Adding test for querier with two stores loadbalancing across them. Signed-off-by: bwplotka --- go.mod | 2 +- go.sum | 4 +- test/e2e/query_lb_test.go | 104 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 3 deletions(-) create mode 100644 test/e2e/query_lb_test.go diff --git a/go.mod b/go.mod index bee3e97fe7..d0b1088dd8 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/chromedp/chromedp v0.8.2 github.com/davecgh/go-spew v1.1.1 github.com/dustin/go-humanize v1.0.0 - github.com/efficientgo/e2e v0.13.1-0.20220923082810-8fa9daa8af8a + github.com/efficientgo/e2e v0.13.2-0.20221003194337-cbc7a9c8405f github.com/efficientgo/tools/extkingpin v0.0.0-20220817170617-6c25e3b627dd github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb github.com/fatih/structtag v1.2.0 diff --git a/go.sum b/go.sum index 97fc0d0411..7ab0a6582a 100644 --- a/go.sum +++ b/go.sum @@ -248,8 +248,8 @@ github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ= github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q= github.com/efficientgo/core v1.0.0-rc.0 h1:jJoA0N+C4/knWYVZ6GrdHOtDyrg8Y/TR4vFpTaqTsqs= github.com/efficientgo/core v1.0.0-rc.0/go.mod h1:kQa0V74HNYMfuJH6jiPiwNdpWXl4xd/K4tzlrcvYDQI= -github.com/efficientgo/e2e v0.13.1-0.20220923082810-8fa9daa8af8a h1:cnJajqeh/HjvJLhI3wPvWG9OQ4gU79+4pELRD5Pkih8= -github.com/efficientgo/e2e v0.13.1-0.20220923082810-8fa9daa8af8a/go.mod h1:Hi+sz0REtlhVZ8zcdeTC3j6LUEEpJpPtNjOaOKuNcgI= +github.com/efficientgo/e2e v0.13.2-0.20221003194337-cbc7a9c8405f h1:kS5jX3et5GqgKDTjZZxYuBmJdCQTkuX2Ss57blDaL/Y= +github.com/efficientgo/e2e v0.13.2-0.20221003194337-cbc7a9c8405f/go.mod h1:Hi+sz0REtlhVZ8zcdeTC3j6LUEEpJpPtNjOaOKuNcgI= github.com/efficientgo/tools/core v0.0.0-20220817170617-6c25e3b627dd h1:svR6KxSP1xiPw10RN4Pd7g6BAVkEcNN628PAqZH31mM= github.com/efficientgo/tools/core v0.0.0-20220817170617-6c25e3b627dd/go.mod h1:OmVcnJopJL8d3X3sSXTiypGoUSgFq1aDGmlrdi9dn/M= github.com/efficientgo/tools/extkingpin v0.0.0-20220817170617-6c25e3b627dd h1:VaYzzXeUbC5fVheskcKVNOyJMEYD+HgrJNzIAg/mRIM= diff --git a/test/e2e/query_lb_test.go b/test/e2e/query_lb_test.go new file mode 100644 index 0000000000..677fd26f8f --- /dev/null +++ b/test/e2e/query_lb_test.go @@ -0,0 +1,104 @@ +package e2e + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/efficientgo/e2e" + e2edb "github.com/efficientgo/e2e/db" + e2einteractive "github.com/efficientgo/e2e/interactive" + e2emon "github.com/efficientgo/e2e/monitoring" + "github.com/thanos-io/objstore/client" + "github.com/thanos-io/objstore/providers/filesystem" + "github.com/thanos-io/thanos/pkg/testutil" + tracingclient "github.com/thanos-io/thanos/pkg/tracing/client" + "github.com/thanos-io/thanos/pkg/tracing/jaeger" + "gopkg.in/yaml.v2" +) + +func marshal(t testing.TB, i interface{}) []byte { + t.Helper() + + b, err := yaml.Marshal(i) + testutil.Ok(t, err) + + return b +} + +// TestQuery_WithStores_Loadbalancing is testing. +// * Create & Start Querier +// * Prepare "object storage" (test hack: It can just filesystem). +// - Create one TSDB block. +// +// * Create & Start 2x Stores +// * Connect Querier with Stores (tricky - there is no way of marking store as LB...) +// * Assertion: Monitor the traffic distribution. +func TestQuery_WithStores_Loadbalancing(t *testing.T) { + pwd, err := os.Getwd() + testutil.Ok(t, err) + + // Create a local dir that will be shared with containers with TSDB blocks we need. + // TODO(bwplotka): Create a block here (e.g using thanosbench). + bktDir := filepath.Join(pwd, "tsdb/bucket") + e, err := e2e.New( + e2e.WithVolumes( + fmt.Sprintf("%v:%v:z", filepath.Join(pwd, "tsdb"), filepath.Join(pwd, "tsdb"))), + ) + testutil.Ok(t, err) + t.Cleanup(e.Close) + + // Start monitoring. + mon, err := e2emon.Start(e) + testutil.Ok(t, err) + testutil.Ok(t, mon.OpenUserInterfaceInBrowser()) + + // Start tracing. + j := e.Runnable("tracing").WithPorts(map[string]int{"http-front": 16686, "jaeger.thrift": 14268}).Init(e2e.StartOptions{Image: "jaegertracing/all-in-one:1.25"}) + testutil.Ok(t, e2e.StartAndWaitReady(j)) + //testutil.Ok(t, e2einteractive.OpenInBrowser("http://"+j.Endpoint("http-front"))) + + jaegerConfig, err := yaml.Marshal(tracingclient.TracingConfig{ + Type: tracingclient.Jaeger, + Config: jaeger.Config{ + ServiceName: "thanos", + SamplerType: "const", + SamplerParam: 1, + Endpoint: "http://" + j.InternalEndpoint("jaeger.thrift") + "/api/traces", + }, + }) + testutil.Ok(t, err) + + const thanosImage = "thanos:latest" // Run 'make thanos' in thanos root to recreate it. + store1 := e2edb.NewThanosStore(e, "store1", marshal(t, client.BucketConfig{ + Type: client.FILESYSTEM, + Config: filesystem.Config{ + Directory: bktDir, + }, + }), e2edb.WithImage(thanosImage), e2edb.WithFlagOverride(map[string]string{ + "--tracing.config": string(jaegerConfig), + })) + store2 := e2edb.NewThanosStore(e, "store2", marshal(t, client.BucketConfig{ + Type: client.FILESYSTEM, + Config: filesystem.Config{ + Directory: bktDir, + }, + }), e2edb.WithImage(thanosImage), e2edb.WithFlagOverride(map[string]string{ + "--tracing.config": string(jaegerConfig), + })) + querier := e2edb.NewThanosQuerier(e, "query", []string{ + // TODO(bwplotka): Play with loadbalancing to ensure half of requests goes to store1 and half to store2. + store1.InternalEndpoint("grpc"), + store2.InternalEndpoint("grpc"), + }, e2edb.WithImage(thanosImage), e2edb.WithFlagOverride(map[string]string{ + "--tracing.config": string(jaegerConfig), + })) + testutil.Ok(t, e2e.StartAndWaitReady(store1, store2, querier)) + + testutil.Ok(t, e2einteractive.OpenInBrowser("http://"+querier.Endpoint("http"))) + + // Once done, wait for user input so user can explore + // the results in Prometheus UI and logs. + testutil.Ok(t, e2einteractive.RunUntilEndpointHit()) +}