From ce0d84711c3e075c267ea9d620ad2058fa81d8b3 Mon Sep 17 00:00:00 2001 From: Daniel Dias Date: Fri, 26 May 2023 16:09:03 -0300 Subject: [PATCH 1/3] wip - fix test formatting --- cli/actions/run_test_action.go | 11 +++--- cli/cmd/test_run_cmd.go | 2 +- cli/formatters/test_run.go | 9 ++++- ...test-with-assertion-with-missing-span.yaml | 22 ++++++++++++ ...n_test_with_assertions_missingspan_test.go | 36 +++++++++++++++++++ 5 files changed, 73 insertions(+), 7 deletions(-) create mode 100644 testing/cli-e2etest/testscenarios/test/resources/test-with-assertion-with-missing-span.yaml create mode 100644 testing/cli-e2etest/testscenarios/test/run_test_with_assertions_missingspan_test.go diff --git a/cli/actions/run_test_action.go b/cli/actions/run_test_action.go index d507a0d8ef..791d63bb87 100644 --- a/cli/actions/run_test_action.go +++ b/cli/actions/run_test_action.go @@ -35,6 +35,7 @@ type runTestAction struct { logger *zap.Logger client *openapi.APIClient environmentActions environmentsActions + cliExit func(int) } var _ Action[RunResourceArgs] = &runTestAction{} @@ -48,8 +49,8 @@ type runDefParams struct { EnvironmentVariables map[string]string } -func NewRunTestAction(config config.Config, logger *zap.Logger, client *openapi.APIClient, environmentActions environmentsActions) runTestAction { - return runTestAction{config, logger, client, environmentActions} +func NewRunTestAction(config config.Config, logger *zap.Logger, client *openapi.APIClient, environmentActions environmentsActions, cliExit func(int)) runTestAction { + return runTestAction{config, logger, client, environmentActions, cliExit} } func (a runTestAction) Run(ctx context.Context, args RunResourceArgs) error { @@ -349,7 +350,7 @@ func (a runTestAction) testRun(ctx context.Context, test openapi.Test, runID int allPassed := tro.Run.Result.GetAllPassed() if params.WaitForResult && !allPassed { // It failed, so we have to return an error status - os.Exit(1) + a.cliExit(1) } return nil @@ -385,13 +386,13 @@ func (a runTestAction) transactionRun(ctx context.Context, transaction openapi.T if params.WaitForResult { if utils.RunStateIsFailed(tro.Run.GetState()) { // It failed, so we have to return an error status - os.Exit(1) + a.cliExit(1) } for _, step := range tro.Run.Steps { if !step.Result.GetAllPassed() { // if any test doesn't pass, fail the transaction execution - os.Exit(1) + a.cliExit(1) } } } diff --git a/cli/cmd/test_run_cmd.go b/cli/cmd/test_run_cmd.go index d58bbcf0b8..6cdb907e09 100644 --- a/cli/cmd/test_run_cmd.go +++ b/cli/cmd/test_run_cmd.go @@ -31,7 +31,7 @@ var testRunCmd = &cobra.Command{ environmentOptions := append(baseOptions, actions.WithClient(utils.GetResourceAPIClient("environments", cliConfig))) environmentActions := actions.NewEnvironmentsActions(environmentOptions...) - runTestAction := actions.NewRunTestAction(cliConfig, cliLogger, client, environmentActions) + runTestAction := actions.NewRunTestAction(cliConfig, cliLogger, client, environmentActions, ExitCLI) actionArgs := actions.RunResourceArgs{ DefinitionFile: runTestFileDefinition, EnvID: runTestEnvID, diff --git a/cli/formatters/test_run.go b/cli/formatters/test_run.go index be56ddf014..190dac0cc2 100644 --- a/cli/formatters/test_run.go +++ b/cli/formatters/test_run.go @@ -111,6 +111,10 @@ func (f testRun) formatSuccessfulTest(test openapi.Test, run openapi.TestRun) st buffer.WriteString(message) for i, specResult := range run.Result.Results { + if len(test.Specs) <= i { + break // guard clause: this means that the server sent more results than specs + } + title := f.getTestSpecTitle(test.Specs[i].GetName(), specResult) message := f.formatMessage("\t%s %s\n", PASSED_TEST_ICON, title) message = f.getColoredText(true, message) @@ -145,7 +149,6 @@ func (f testRun) formatFailedTest(test openapi.Test, run openapi.TestRun) string allPassed := true for _, result := range specResult.Results { - for _, spanResult := range result.SpanResults { // meta assertions such as tracetest.selected_spans.count don't have a spanID, // so they will be treated differently. To overcome them, we will place all @@ -182,6 +185,10 @@ func (f testRun) formatFailedTest(test openapi.Test, run openapi.TestRun) string } } + if len(test.Specs) <= i { + break // guard clause: this means that the server sent more results than specs + } + title := f.getTestSpecTitle(test.Specs[i].GetName(), specResult) icon := f.getStateIcon(allPassed) message := f.formatMessage("\t%s %s\n", icon, title) diff --git a/testing/cli-e2etest/testscenarios/test/resources/test-with-assertion-with-missing-span.yaml b/testing/cli-e2etest/testscenarios/test/resources/test-with-assertion-with-missing-span.yaml new file mode 100644 index 0000000000..f57a9ecad4 --- /dev/null +++ b/testing/cli-e2etest/testscenarios/test/resources/test-with-assertion-with-missing-span.yaml @@ -0,0 +1,22 @@ +type: Test +spec: + id: aZobhTwVg + name: Test with Assertion with Missing Span + trigger: + type: http + httpRequest: + url: http://localhost:11633/api/tests + method: GET + headers: + - key: Content-Type + value: application/json + specs: + - name: It should call /api/tests on HTTP + selector: span[tracetest.span.type="http" name="GET /api/tests" http.target="/api/tests" + http.method="GET"] + assertions: + - attr:http.target = "/api/tests" + - name: It should not have grpc spans + selector: span[tracetest.span.type="grpc"] + assertions: + - attr:name = unknown diff --git a/testing/cli-e2etest/testscenarios/test/run_test_with_assertions_missingspan_test.go b/testing/cli-e2etest/testscenarios/test/run_test_with_assertions_missingspan_test.go new file mode 100644 index 0000000000..b6f49b5ce9 --- /dev/null +++ b/testing/cli-e2etest/testscenarios/test/run_test_with_assertions_missingspan_test.go @@ -0,0 +1,36 @@ +package test + +import ( + "fmt" + "testing" + + "github.com/kubeshop/tracetest/cli-e2etest/environment" + "github.com/kubeshop/tracetest/cli-e2etest/helpers" + "github.com/kubeshop/tracetest/cli-e2etest/tracetestcli" + "github.com/stretchr/testify/require" +) + +func TestRunWithAssertionMissingSpan(t *testing.T) { + // instantiate require with testing helper + require := require.New(t) + + // setup isolated e2e environment + env := environment.CreateAndStart(t) + defer env.Close(t) + + cliConfig := env.GetCLIConfigPath(t) + + // Given I am a Tracetest CLI user + // And I have my server recently created + + // When I try run a test that has a selector that will find a span + // And a selector that will find no spans + // Then it run, fail and return only the assertion to the selector with spans + testWithAssertionWithNoSpan := env.GetTestResourcePath(t, "test-with-assertion-with-missing-span") + + command := fmt.Sprintf("test run -w -d %s", testWithAssertionWithNoSpan) + result := tracetestcli.Exec(t, command, tracetestcli.WithCLIConfig(cliConfig)) + helpers.RequireExitCodeEqual(t, result, 1) // test failed + require.Contains(result.StdOut, "It should call /api/tests on HTTP") + require.NotContains(result.StdOut, "It should not have grpc spans") +} From 8372c99fe912de99965845421dd88c911382e76b Mon Sep 17 00:00:00 2001 From: Daniel Dias Date: Mon, 29 May 2023 22:09:11 -0300 Subject: [PATCH 2/3] Fixed transaction query with problem --- server/testdb/runs.go | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/server/testdb/runs.go b/server/testdb/runs.go index aa1565e072..7976b80877 100644 --- a/server/testdb/runs.go +++ b/server/testdb/runs.go @@ -334,12 +334,12 @@ SELECT "environment", -- transaction run - transaction_run.transaction_run_id, - transaction_run.transaction_run_transaction_id -FROM test_runs -LEFT OUTER JOIN - transaction_run_steps transaction_run -ON transaction_run.test_run_id = id AND transaction_run.test_run_test_id = test_id + transaction_run_steps.transaction_run_id, + transaction_run_steps.transaction_run_transaction_id +FROM + test_runs + LEFT OUTER JOIN transaction_run_steps + ON transaction_run_steps.test_run_id = test_runs.id AND transaction_run_steps.test_run_test_id = test_runs.test_id ` func (td *postgresDB) GetRun(ctx context.Context, testID id.ID, runID int) (model.Run, error) { @@ -554,11 +554,12 @@ func readRunRow(row scanner) (model.Run, error) { } func (td *postgresDB) GetTransactionRunSteps(ctx context.Context, tr tests.TransactionRun) ([]model.Run, error) { - // stmt, err := td.db.Prepare(selectRunQuery + " WHERE id = $1 AND test_id = $2") - stmt, err := td.db.Prepare(selectRunQuery + ` INNER JOIN - transaction_run_steps trs ON - test_runs.id = trs.test_run_id AND test_runs.test_id = trs.test_run_test_id - WHERE trs.transaction_run_id = $1 AND trs.transaction_run_transaction_id = $2`) + query := selectRunQuery + ` +WHERE transaction_run_steps.transaction_run_id = $1 AND transaction_run_steps.transaction_run_transaction_id = $2 +ORDER BY test_runs.completed_at ASC +` + + stmt, err := td.db.Prepare(query) if err != nil { return []model.Run{}, fmt.Errorf("prepare: %w", err) } From 13ebe9d73911160e634cefaf9baf8679d04db53e Mon Sep 17 00:00:00 2001 From: Daniel Dias Date: Mon, 29 May 2023 22:46:28 -0300 Subject: [PATCH 3/3] Removing unused CLI test --- ...test-with-assertion-with-missing-span.yaml | 22 ------------ ...n_test_with_assertions_missingspan_test.go | 36 ------------------- 2 files changed, 58 deletions(-) delete mode 100644 testing/cli-e2etest/testscenarios/test/resources/test-with-assertion-with-missing-span.yaml delete mode 100644 testing/cli-e2etest/testscenarios/test/run_test_with_assertions_missingspan_test.go diff --git a/testing/cli-e2etest/testscenarios/test/resources/test-with-assertion-with-missing-span.yaml b/testing/cli-e2etest/testscenarios/test/resources/test-with-assertion-with-missing-span.yaml deleted file mode 100644 index f57a9ecad4..0000000000 --- a/testing/cli-e2etest/testscenarios/test/resources/test-with-assertion-with-missing-span.yaml +++ /dev/null @@ -1,22 +0,0 @@ -type: Test -spec: - id: aZobhTwVg - name: Test with Assertion with Missing Span - trigger: - type: http - httpRequest: - url: http://localhost:11633/api/tests - method: GET - headers: - - key: Content-Type - value: application/json - specs: - - name: It should call /api/tests on HTTP - selector: span[tracetest.span.type="http" name="GET /api/tests" http.target="/api/tests" - http.method="GET"] - assertions: - - attr:http.target = "/api/tests" - - name: It should not have grpc spans - selector: span[tracetest.span.type="grpc"] - assertions: - - attr:name = unknown diff --git a/testing/cli-e2etest/testscenarios/test/run_test_with_assertions_missingspan_test.go b/testing/cli-e2etest/testscenarios/test/run_test_with_assertions_missingspan_test.go deleted file mode 100644 index b6f49b5ce9..0000000000 --- a/testing/cli-e2etest/testscenarios/test/run_test_with_assertions_missingspan_test.go +++ /dev/null @@ -1,36 +0,0 @@ -package test - -import ( - "fmt" - "testing" - - "github.com/kubeshop/tracetest/cli-e2etest/environment" - "github.com/kubeshop/tracetest/cli-e2etest/helpers" - "github.com/kubeshop/tracetest/cli-e2etest/tracetestcli" - "github.com/stretchr/testify/require" -) - -func TestRunWithAssertionMissingSpan(t *testing.T) { - // instantiate require with testing helper - require := require.New(t) - - // setup isolated e2e environment - env := environment.CreateAndStart(t) - defer env.Close(t) - - cliConfig := env.GetCLIConfigPath(t) - - // Given I am a Tracetest CLI user - // And I have my server recently created - - // When I try run a test that has a selector that will find a span - // And a selector that will find no spans - // Then it run, fail and return only the assertion to the selector with spans - testWithAssertionWithNoSpan := env.GetTestResourcePath(t, "test-with-assertion-with-missing-span") - - command := fmt.Sprintf("test run -w -d %s", testWithAssertionWithNoSpan) - result := tracetestcli.Exec(t, command, tracetestcli.WithCLIConfig(cliConfig)) - helpers.RequireExitCodeEqual(t, result, 1) // test failed - require.Contains(result.StdOut, "It should call /api/tests on HTTP") - require.NotContains(result.StdOut, "It should not have grpc spans") -}