diff --git a/test/e2e/configs.go b/test/e2e/configs.go index 25ff20fdf..ce7ca6a6c 100644 --- a/test/e2e/configs.go +++ b/test/e2e/configs.go @@ -18,6 +18,7 @@ type testType string const ( metrics testType = "metrics" + rules testType = "rules" logs testType = "logs" tenants testType = "tenants" interactive testType = "interactive" @@ -29,13 +30,16 @@ const ( certsContainerPath = dockerLocalSharedDir + "/" + certsSharedDir configsContainerPath = dockerLocalSharedDir + "/" + configSharedDir - envMetricsName = "e2e_metrics_read_write" - envLogsName = "e2e_logs_read_write_tail" - envTenantsName = "e2e_tenants" - envInteractive = "e2e_interactive" + envMetricsName = "e2e_metrics_read_write" + envRulesAPIName = "e2e_rules_api" + envLogsName = "e2e_logs_read_write_tail" + envTenantsName = "e2e_tenants" + envInteractive = "e2e_interactive" defaultTenantID = "1610b0c3-c509-4592-a256-a1871353dbfa" mtlsTenantID = "845cdfd9-f936-443c-979c-2ee7dc91f646" + + defaultTenantName = "test-oidc" ) const tenantsYamlTpl = ` @@ -166,3 +170,74 @@ func createDexYAML( ) testutil.Ok(t, err) } + +const rulesYAMLTpl = ` +type: S3 +config: + bucket: %s + endpoint: %s + access_key: %s + insecure: true + secret_key: %s +` + +func createRulesYAML( + t *testing.T, + e e2e.Environment, + bucket, endpoint, accessKey, secretKey string, +) { + yamlContent := []byte(fmt.Sprintf( + rulesYAMLTpl, + bucket, + endpoint, + accessKey, + secretKey, + )) + + err := ioutil.WriteFile( + filepath.Join(e.SharedDir(), configSharedDir, "rules-objstore.yaml"), + yamlContent, + os.FileMode(0755), + ) + testutil.Ok(t, err) +} + +const recordingRuleYamlTpl = ` +groups: + - name: example + rules: + - record: job:http_inprogress_requests:sum + expr: sum by (job) (http_inprogress_requests) +` + +const alertingRuleYamlTpl = ` +groups: +- name: example + rules: + - alert: HighRequestLatency + expr: job:request_latency_seconds:mean5m{job="myjob"} > 0.5 + for: 10m + labels: + severity: page + annotations: + summary: High request latency +` +const recordAndAlertingRulesYamlTpl = ` +groups: +- name: node_rules + rules: + - record: job:up:avg + expr: avg without(instance)(up{job="node"}) + - alert: ManyInstancesDown + expr: job:up:avg{job="node"} < 0.5 +` + +const invalidRulesYamlTpl = ` +invalid: +- name: testing + invalid_rules: + - rule1: job:up:avg + expr: avg without(instance)(up{job="node"}) + - rule2: ManyInstancesDown + expr: job:up:avg{job="node"} < 0.5 +` diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go index 7429b9793..3c6c66486 100644 --- a/test/e2e/helpers.go +++ b/test/e2e/helpers.go @@ -87,6 +87,8 @@ func getContainerName(t *testing.T, tt testType, serviceName string) string { return envLogsName + "-" + serviceName case metrics: return envMetricsName + "-" + serviceName + case rules: + return envRulesAPIName + "-" + serviceName case tenants: return envTenantsName + "-" + serviceName case interactive: @@ -116,3 +118,13 @@ func assertResponse(t *testing.T, response string, expected string) { fmt.Sprintf("failed to assert that the response '%s' contains '%s'", response, expected), ) } + +type tokenRoundTripper struct { + rt http.RoundTripper + token string +} + +func (rt *tokenRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { + r.Header.Add("Authorization", "bearer "+rt.token) + return rt.rt.RoundTrip(r) +} diff --git a/test/e2e/interactive_test.go b/test/e2e/interactive_test.go index 59bf265f3..9cf6b5891 100644 --- a/test/e2e/interactive_test.go +++ b/test/e2e/interactive_test.go @@ -34,8 +34,8 @@ func TestInteractiveSetup(t *testing.T) { up, err := newUpRun( e, "up-metrics-read-write", metrics, - "https://"+api.InternalEndpoint("https")+"/api/metrics/v1/test-oidc/api/v1/query", - "https://"+api.InternalEndpoint("https")+"/api/metrics/v1/test-oidc/api/v1/receive", + "https://"+api.InternalEndpoint("https")+"/api/metrics/v1/"+defaultTenantName+"/api/v1/query", + "https://"+api.InternalEndpoint("https")+"/api/metrics/v1/"+defaultTenantName+"/api/v1/receive", withToken(token), withRunParameters(&runParams{period: "5000ms", threshold: "1", latency: "10s", duration: "0"}), ) diff --git a/test/e2e/logs_test.go b/test/e2e/logs_test.go index c4f1baa3e..455103131 100644 --- a/test/e2e/logs_test.go +++ b/test/e2e/logs_test.go @@ -93,8 +93,8 @@ func TestLogsReadWriteAndTail(t *testing.T) { t.Run("logs-tail", func(t *testing.T) { up, err := newUpRun( e, "up-logs-tail", logs, - "https://"+api.InternalEndpoint("https")+"/api/logs/v1/test-oidc/loki/api/v1/query", - "https://"+api.InternalEndpoint("https")+"/api/logs/v1/test-oidc/loki/api/v1/push", + "https://"+api.InternalEndpoint("https")+"/api/logs/v1/"+defaultTenantName+"/loki/api/v1/query", + "https://"+api.InternalEndpoint("https")+"/api/logs/v1/"+defaultTenantName+"/loki/api/v1/push", withToken(token), withRunParameters(&runParams{initialDelay: "0s", period: "250ms", threshold: "1", latency: "10s", duration: "0"}), ) @@ -112,7 +112,7 @@ func TestLogsReadWriteAndTail(t *testing.T) { d := websocket.Dialer{TLSClientConfig: getTLSClientConfig(t, e)} conn, _, err := d.Dial( - "wss://"+api.Endpoint("https")+"/api/logs/v1/test-oidc/loki/api/v1/tail?query=%7B_id%3D%22test%22%7D", + "wss://"+api.Endpoint("https")+"/api/logs/v1/"+defaultTenantName+"/loki/api/v1/tail?query=%7B_id%3D%22test%22%7D", http.Header{ "Authorization": []string{"Bearer " + token}, "X-Scope-OrgID": []string{defaultTenantID}, diff --git a/test/e2e/metrics_test.go b/test/e2e/metrics_test.go index 9ceb178b1..ea142a5e1 100644 --- a/test/e2e/metrics_test.go +++ b/test/e2e/metrics_test.go @@ -39,8 +39,8 @@ func TestMetricsReadAndWrite(t *testing.T) { t.Run("metrics-read-write", func(t *testing.T) { up, err := newUpRun( e, "up-metrics-read-write", metrics, - "https://"+api.InternalEndpoint("https")+"/api/metrics/v1/test-oidc/api/v1/query", - "https://"+api.InternalEndpoint("https")+"/api/metrics/v1/test-oidc/api/v1/receive", + "https://"+api.InternalEndpoint("https")+"/api/metrics/v1/"+defaultTenantName+"/api/v1/query", + "https://"+api.InternalEndpoint("https")+"/api/metrics/v1/"+defaultTenantName+"/api/v1/receive", withToken(token), withRunParameters(&runParams{period: "500ms", threshold: "1", latency: "10s", duration: "0"}), ) @@ -106,7 +106,7 @@ func TestMetricsReadAndWrite(t *testing.T) { } apiTest, err := promapi.NewClient(promapi.Config{ - Address: "https://" + api.Endpoint("https") + "/api/metrics/v1/test-oidc", + Address: "https://" + api.Endpoint("https") + "/api/metrics/v1/"+defaultTenantName, RoundTripper: &tokenRoundTripper{rt: tr, token: token}, }) testutil.Ok(t, err) @@ -191,13 +191,3 @@ func TestMetricsReadAndWrite(t *testing.T) { }) }) } - -type tokenRoundTripper struct { - rt http.RoundTripper - token string -} - -func (rt *tokenRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { - r.Header.Add("Authorization", "bearer "+rt.token) - return rt.rt.RoundTrip(r) -} diff --git a/test/e2e/rules_test.go b/test/e2e/rules_test.go new file mode 100644 index 000000000..8cdf5b1f5 --- /dev/null +++ b/test/e2e/rules_test.go @@ -0,0 +1,173 @@ +// +build integration + +package e2e + +import ( + "bytes" + "io/ioutil" + "net/http" + "testing" + + "github.com/efficientgo/e2e" + "github.com/efficientgo/tools/core/pkg/testutil" +) + +func TestRulesAPI(t *testing.T) { + t.Parallel() + + e, err := e2e.NewDockerEnvironment(envRulesAPIName) + testutil.Ok(t, err) + t.Cleanup(e.Close) + + prepareConfigsAndCerts(t, rules, e) + _, token, rateLimiterAddr := startBaseServices(t, e, rules) + rulesEndpoint := startServicesForRules(t, e) + + api, err := newObservatoriumAPIService( + e, + withRulesEndpoint("http://"+rulesEndpoint), + withRateLimiter(rateLimiterAddr), + ) + testutil.Ok(t, err) + testutil.Ok(t, e2e.StartAndWaitReady(api)) + + rulesEndpointURL := "https://" + api.Endpoint("https") + "/api/metrics/v1/"+defaultTenantName+"/api/v1/rules/raw" + tr := &http.Transport{ + TLSClientConfig: getTLSClientConfig(t, e), + } + + client := &http.Client{ + Transport: &tokenRoundTripper{rt: tr, token: token}, + } + + t.Run("write-then-read-recording-rules", func(t *testing.T) { + // Try to list rules + r, err := http.NewRequest( + http.MethodGet, + rulesEndpointURL, + nil, + ) + testutil.Ok(t, err) + + res, err := client.Do(r) + testutil.Ok(t, err) + testutil.Equals(t, http.StatusNotFound, res.StatusCode) + + // Set a file containing a recording rule + recordingRule := []byte(recordingRuleYamlTpl) + r, err = http.NewRequest( + http.MethodPut, + rulesEndpointURL, + bytes.NewReader(recordingRule), + ) + testutil.Ok(t, err) + + res, err = client.Do(r) + testutil.Ok(t, err) + testutil.Equals(t, http.StatusOK, res.StatusCode) + + // Check if recording rule is listed + r, err = http.NewRequest( + http.MethodGet, + rulesEndpointURL, + nil, + ) + testutil.Ok(t, err) + + res, err = client.Do(r) + defer res.Body.Close() + + testutil.Ok(t, err) + testutil.Equals(t, http.StatusOK, res.StatusCode) + + body, err := ioutil.ReadAll(res.Body) + bodyStr := string(body) + + assertResponse(t, bodyStr, "sum by (job) (http_inprogress_requests)") + assertResponse(t, bodyStr, "tenant_id: "+defaultTenantID) + }) + + t.Run("write-then-read-alerting-rules", func(t *testing.T) { + // Set a file containing an alerting rule + alertingRule := []byte(alertingRuleYamlTpl) + r, err := http.NewRequest( + http.MethodPut, + rulesEndpointURL, + bytes.NewReader(alertingRule), + ) + testutil.Ok(t, err) + + res, err := client.Do(r) + testutil.Ok(t, err) + testutil.Equals(t, http.StatusOK, res.StatusCode) + + // Check if the alerting rule is listed + r, err = http.NewRequest( + http.MethodGet, + rulesEndpointURL, + nil, + ) + testutil.Ok(t, err) + + res, err = client.Do(r) + defer res.Body.Close() + + testutil.Ok(t, err) + testutil.Equals(t, http.StatusOK, res.StatusCode) + + body, err := ioutil.ReadAll(res.Body) + bodyStr := string(body) + assertResponse(t, bodyStr, "alert: HighRequestLatency") + assertResponse(t, bodyStr, "tenant_id: "+defaultTenantID) + }) + + t.Run("write-then-read-recording-and-alerting-rules", func(t *testing.T) { + // Set a file containing both recording and alerting rules + recordAndAlertingRules := []byte(recordAndAlertingRulesYamlTpl) + r, err := http.NewRequest( + http.MethodPut, + rulesEndpointURL, + bytes.NewReader(recordAndAlertingRules), + ) + testutil.Ok(t, err) + + res, err := client.Do(r) + testutil.Ok(t, err) + testutil.Equals(t, http.StatusOK, res.StatusCode) + + // Check if both recording and alerting rules are listed + r, err = http.NewRequest( + http.MethodGet, + rulesEndpointURL, + nil, + ) + testutil.Ok(t, err) + + res, err = client.Do(r) + defer res.Body.Close() + + testutil.Ok(t, err) + testutil.Equals(t, http.StatusOK, res.StatusCode) + + body, err := ioutil.ReadAll(res.Body) + bodyStr := string(body) + assertResponse(t, bodyStr, "record: job:up:avg") + assertResponse(t, bodyStr, "alert: ManyInstancesDown") + assertResponse(t, bodyStr, "tenant_id: "+defaultTenantID) + }) + t.Run("write-invalid-rules", func(t *testing.T) { + // Set an invalid rules file + invalidRules := []byte(invalidRulesYamlTpl) + r, err := http.NewRequest( + http.MethodPut, + rulesEndpointURL, + bytes.NewReader(invalidRules), + ) + testutil.Ok(t, err) + + res, err := client.Do(r) + testutil.Ok(t, err) + testutil.Equals(t, http.StatusBadRequest, res.StatusCode) + }) +} + diff --git a/test/e2e/services.go b/test/e2e/services.go index 2fab2dcdb..0e9ef1a5c 100644 --- a/test/e2e/services.go +++ b/test/e2e/services.go @@ -25,6 +25,7 @@ const ( dexImage = "dexidp/dex:v2.30.0" opaImage = "openpolicyagent/opa:0.31.0" gubernatorImage = "thrawn01/gubernator:1.0.0-rc.8" + rulesObjectStoreImage = "quay.io/observatorium/rules-objstore:main-2022-01-19-8650540" logLevelError = "error" logLevelDebug = "debug" @@ -49,6 +50,19 @@ func startServicesForMetrics(t *testing.T, e e2e.Environment) ( thanosQuery.Endpoint("http") } +func startServicesForRules(t *testing.T, e e2e.Environment) (metricsRulesEndpoint string) { + // Create S3 replacement for rules backend + bucket := "obs_rules_test" + m := e2edb.NewMinio(e, "rules-minio", bucket) + testutil.Ok(t, e2e.StartAndWaitReady(m)) + + createRulesYAML(t, e, bucket, m.InternalEndpoint(e2edb.AccessPortName), e2edb.MinioAccessKey, e2edb.MinioSecretKey) + rulesBackend := newRulesBackendService(e) + testutil.Ok(t, e2e.StartAndWaitReady(rulesBackend)) + + return rulesBackend.InternalEndpoint("http") +} + func startServicesForLogs(t *testing.T, e e2e.Environment) ( logsEndpoint string, logsExtEndpoint string, @@ -174,6 +188,27 @@ func newLokiService(e e2e.Environment) *e2e.InstrumentedRunnable { ) } +func newRulesBackendService(e e2e.Environment) *e2e.InstrumentedRunnable { + ports := map[string]int{"http": 8080, "internal": 8081} + + args := e2e.BuildArgs(map[string]string{ + "--log.level": logLevelDebug, + "--web.listen": ":" + strconv.Itoa(ports["http"]), + "--web.internal.listen": ":" + strconv.Itoa(ports["internal"]), + "--web.healthchecks.url": "http://127.0.0.1:" + strconv.Itoa(ports["http"]), + "--objstore.config-file": filepath.Join(configsContainerPath, "rules-objstore.yaml"), + }) + + return e2e.NewInstrumentedRunnable(e, "rules_objstore", ports, "internal").Init( + e2e.StartOptions{ + Image: rulesObjectStoreImage, + Command: e2e.NewCommand("", args...), + Readiness: e2e.NewHTTPReadinessProbe("internal", "/ready", 200, 200), + User: strconv.Itoa(os.Getuid()), + }, + ) +} + func newOPAService(e e2e.Environment) *e2e.InstrumentedRunnable { ports := map[string]int{"http": 8181} @@ -197,6 +232,7 @@ type apiOptions struct { logsEndpoint string metricsReadEndpoint string metricsWriteEndpoint string + metricsRulesEndpoint string ratelimiterAddr string } @@ -215,6 +251,12 @@ func withMetricsEndpoints(readEndpoint string, writeEndpoint string) apiOption { } } +func withRulesEndpoint(rulesEndpoint string) apiOption { + return func(o *apiOptions) { + o.metricsRulesEndpoint = rulesEndpoint + } +} + func withRateLimiter(addr string) apiOption { return func(o *apiOptions) { o.ratelimiterAddr = addr @@ -252,6 +294,10 @@ func newObservatoriumAPIService( args = append(args, "--metrics.write.endpoint="+opts.metricsWriteEndpoint) } + if opts.metricsRulesEndpoint != "" { + args = append(args, "--metrics.rules.endpoint="+opts.metricsRulesEndpoint) + } + if opts.logsEndpoint != "" { args = append(args, "--logs.read.endpoint="+opts.logsEndpoint) args = append(args, "--logs.tail.endpoint="+opts.logsEndpoint) diff --git a/test/e2e/tenants_test.go b/test/e2e/tenants_test.go index e0dd8069a..2a08d474b 100644 --- a/test/e2e/tenants_test.go +++ b/test/e2e/tenants_test.go @@ -39,7 +39,7 @@ func TestTenantsRetryAuthenticationProviderRegistration(t *testing.T) { []string{"observatorium_api_tenants_failed_registrations"}, e2e.WaitMissingMetrics(), e2e.WithLabelMatchers( - matchers.MustNewMatcher(matchers.MatchEqual, "tenant", "test-oidc"), + matchers.MustNewMatcher(matchers.MatchEqual, "tenant", defaultTenantName), matchers.MustNewMatcher(matchers.MatchEqual, "provider", "oidc"), ), )) @@ -62,8 +62,8 @@ func TestTenantsRetryAuthenticationProviderRegistration(t *testing.T) { up, err := newUpRun( e, "up-tenants", metrics, - "https://"+api.InternalEndpoint("https")+"/api/metrics/v1/test-oidc/api/v1/query", - "https://"+api.InternalEndpoint("https")+"/api/metrics/v1/test-oidc/api/v1/receive", + "https://"+api.InternalEndpoint("https")+"/api/metrics/v1/"+defaultTenantName+"/api/v1/query", + "https://"+api.InternalEndpoint("https")+"/api/metrics/v1/"+defaultTenantName+"/api/v1/receive", withToken(token), withRunParameters(&runParams{initialDelay: "100ms", period: "300ms", threshold: "1", latency: "5s", duration: "0"}), )