diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml index e4aa0fd58b..adada0bc1f 100644 --- a/.github/workflows/pull-request.yaml +++ b/.github/workflows/pull-request.yaml @@ -243,7 +243,7 @@ jobs: ./scripts/wait-for-port.sh 11633 ./dist/tracetest configure -g --endpoint http://localhost:11633 - ./dist/tracetest test run -d examples/${{ matrix.example_dir }}/tests/list-tests.yaml --wait-for-result || (cat /tmp/docker-log; exit 1) + ./dist/tracetest run test -f examples/${{ matrix.example_dir }}/tests/list-tests.yaml || (cat /tmp/docker-log; exit 1) smoke-test-cli: name: CLI smoke tests diff --git a/cli/actions/run_test_action.go b/cli/actions/run_test_action.go deleted file mode 100644 index e267c17241..0000000000 --- a/cli/actions/run_test_action.go +++ /dev/null @@ -1,737 +0,0 @@ -package actions - -import ( - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "os" - "strconv" - "sync" - "time" - - cienvironment "github.com/cucumber/ci-environment/go" - "github.com/goccy/go-yaml" - "github.com/kubeshop/tracetest/cli/config" - "github.com/kubeshop/tracetest/cli/formatters" - "github.com/kubeshop/tracetest/cli/openapi" - "github.com/kubeshop/tracetest/cli/pkg/fileutil" - "github.com/kubeshop/tracetest/cli/pkg/resourcemanager" - "github.com/kubeshop/tracetest/cli/ui" - "github.com/kubeshop/tracetest/cli/utils" - "github.com/kubeshop/tracetest/cli/variable" - "go.uber.org/zap" -) - -type RunResourceArgs struct { - DefinitionFile string - EnvID string - WaitForResult bool - JUnit string -} - -func (args RunResourceArgs) Validate() error { - if args.DefinitionFile == "" { - return fmt.Errorf("you must specify a definition file to run a test") - } - - if args.JUnit != "" && !args.WaitForResult { - return fmt.Errorf("--junit option requires --wait-for-result") - } - - return nil -} - -type runTestAction struct { - config config.Config - logger *zap.Logger - openapiClient *openapi.APIClient - environments resourcemanager.Client - tests resourcemanager.Client - transactions resourcemanager.Client - yamlFormat resourcemanager.Format - jsonFormat resourcemanager.Format - cliExit func(int) -} - -func NewRunTestAction( - config config.Config, - logger *zap.Logger, - openapiClient *openapi.APIClient, - tests resourcemanager.Client, - transactions resourcemanager.Client, - environments resourcemanager.Client, - cliExit func(int), -) runTestAction { - - yamlFormat, err := resourcemanager.Formats.Get(resourcemanager.FormatYAML) - if err != nil { - panic(fmt.Errorf("could not get yaml format: %w", err)) - } - - jsonFormat, err := resourcemanager.Formats.Get(resourcemanager.FormatJSON) - if err != nil { - panic(fmt.Errorf("could not get json format: %w", err)) - } - - return runTestAction{ - config, - logger, - openapiClient, - environments, - tests, - transactions, - yamlFormat, - jsonFormat, - cliExit, - } -} - -func (a runTestAction) Run(ctx context.Context, args RunResourceArgs) error { - if err := args.Validate(); err != nil { - return err - } - - a.logger.Debug( - "Running test from definition", - zap.String("definitionFile", args.DefinitionFile), - zap.String("environment", args.EnvID), - zap.Bool("waitForResults", args.WaitForResult), - zap.String("junit", args.JUnit), - ) - - f, err := fileutil.Read(args.DefinitionFile) - if err != nil { - return fmt.Errorf("cannot read definition file %s: %w", args.DefinitionFile, err) - } - df := defFile{f} - a.logger.Debug("Definition file read", zap.String("absolutePath", df.AbsPath())) - - envID, err := a.resolveEnvID(ctx, args.EnvID) - if err != nil { - return fmt.Errorf("cannot resolve environment id: %w", err) - } - a.logger.Debug("env resolved", zap.String("ID", envID)) - - df, err = a.apply(ctx, df) - if err != nil { - return fmt.Errorf("cannot apply definition file: %w", err) - } - - a.logger.Debug("Definition file applied", zap.String("updated", string(df.Contents()))) - - var run runResult - var envVars []envVar - - // iterate until we have all env vars, - // or the server returns an actual error - for { - run, err = a.run(ctx, df, envID, envVars) - if err == nil { - break - } - missingEnvVarsErr, ok := err.(missingEnvVarsError) - if !ok { - // actual error, return - return fmt.Errorf("cannot run test: %w", err) - } - - // missing vars error - envVars = a.askForMissingVars([]envVar(missingEnvVarsErr)) - } - - if !args.WaitForResult { - fmt.Println(a.formatResult(run, false)) - a.cliExit(0) - } - - result, err := a.waitForResult(ctx, run) - if err != nil { - return fmt.Errorf("cannot wait for test result: %w", err) - } - - fmt.Println(a.formatResult(result, true)) - a.cliExit(a.exitCode(result)) - - if args.JUnit != "" { - err := a.writeJUnitReport(ctx, result, args.JUnit) - if err != nil { - return fmt.Errorf("cannot write junit report: %w", err) - } - } - - return nil -} - -func (a runTestAction) exitCode(res runResult) int { - switch res.ResourceType { - case "Test": - if !res.Run.(openapi.TestRun).Result.GetAllPassed() { - return 1 - } - case "Transaction": - for _, step := range res.Run.(openapi.TransactionRun).Steps { - if !step.Result.GetAllPassed() { - return 1 - } - } - } - return 0 -} - -func (a runTestAction) resolveEnvID(ctx context.Context, envID string) (string, error) { - if !fileutil.IsFilePath(envID) { - a.logger.Debug("envID is not a file path", zap.String("envID", envID)) - return envID, nil - } - - f, err := fileutil.Read(envID) - if err != nil { - return "", fmt.Errorf("cannot read environment file %s: %w", envID, err) - } - - a.logger.Debug("envID is a file path", zap.String("filePath", envID), zap.Any("file", f)) - updatedEnv, err := a.environments.Apply(ctx, f, a.yamlFormat) - if err != nil { - return "", fmt.Errorf("could not read environment file: %w", err) - } - - var env openapi.EnvironmentResource - err = yaml.Unmarshal([]byte(updatedEnv), &env) - if err != nil { - a.logger.Error("error parsing json", zap.String("content", updatedEnv), zap.Error(err)) - return "", fmt.Errorf("could not unmarshal environment json: %w", err) - } - - return env.Spec.GetId(), nil -} - -func (a runTestAction) injectLocalEnvVars(ctx context.Context, df defFile) (defFile, error) { - variableInjector := variable.NewInjector(variable.WithVariableProvider( - variable.EnvironmentVariableProvider{}, - )) - - injected, err := variableInjector.ReplaceInString(string(df.Contents())) - if err != nil { - return df, fmt.Errorf("cannot inject local environment variables: %w", err) - } - - df = defFile{fileutil.New(df.AbsPath(), []byte(injected))} - - return df, nil -} - -func (a runTestAction) apply(ctx context.Context, df defFile) (defFile, error) { - defType, err := getTypeFromFile(df) - if err != nil { - return df, fmt.Errorf("cannot get type from definition file: %w", err) - } - a.logger.Debug("definition file type", zap.String("type", defType)) - - switch defType { - case "Test": - return a.applyTest(ctx, df) - case "Transaction": - return a.applyTransaction(ctx, df) - default: - return df, fmt.Errorf("unknown type %s", defType) - } -} - -func (a runTestAction) applyTest(ctx context.Context, df defFile) (defFile, error) { - df, err := a.injectLocalEnvVars(ctx, df) - if err != nil { - return df, fmt.Errorf("cannot inject local env vars: %w", err) - } - - a.logger.Debug("applying test", - zap.String("absolutePath", df.AbsPath()), - ) - - updated, err := a.tests.Apply(ctx, df.File, a.yamlFormat) - if err != nil { - return df, fmt.Errorf("could not read test file: %w", err) - } - - df = defFile{fileutil.New(df.AbsPath(), []byte(updated))} - - return df, nil -} - -func (a runTestAction) applyTransaction(ctx context.Context, df defFile) (defFile, error) { - df, err := a.injectLocalEnvVars(ctx, df) - if err != nil { - return df, fmt.Errorf("cannot inject local env vars: %w", err) - } - - a.logger.Debug("applying transaction", - zap.String("absolutePath", df.AbsPath()), - ) - - updated, err := a.transactions.Apply(ctx, df.File, a.yamlFormat) - if err != nil { - return df, fmt.Errorf("could not read transaction file: %w", err) - } - - df = defFile{fileutil.New(df.AbsPath(), []byte(updated))} - return df, nil -} - -func getTypeFromFile(df defFile) (string, error) { - var raw map[string]any - err := yaml.Unmarshal(df.Contents(), &raw) - if err != nil { - return "", fmt.Errorf("cannot unmarshal definition file: %w", err) - } - - if raw["type"] == nil { - return "", fmt.Errorf("missing type in definition file") - } - - defType, ok := raw["type"].(string) - if !ok { - return "", fmt.Errorf("type is not a string") - } - - return defType, nil - -} - -type envVar struct { - Name string - DefaultValue string - UserValue string -} - -func (ev envVar) value() string { - if ev.UserValue != "" { - return ev.UserValue - } - - return ev.DefaultValue -} - -type envVars []envVar - -func (evs envVars) toOpenapi() []openapi.EnvironmentValue { - vars := make([]openapi.EnvironmentValue, len(evs)) - for i, ev := range evs { - vars[i] = openapi.EnvironmentValue{ - Key: openapi.PtrString(ev.Name), - Value: openapi.PtrString(ev.value()), - } - } - - return vars -} - -func (evs envVars) unique() envVars { - seen := make(map[string]bool) - vars := make(envVars, 0, len(evs)) - for _, ev := range evs { - if seen[ev.Name] { - continue - } - - seen[ev.Name] = true - vars = append(vars, ev) - } - - return vars -} - -type missingEnvVarsError envVars - -func (e missingEnvVarsError) Error() string { - return fmt.Sprintf("missing env vars: %v", []envVar(e)) -} - -type runResult struct { - ResourceType string - Resource any - Run any -} - -func (a runTestAction) run(ctx context.Context, df defFile, envID string, ev envVars) (runResult, error) { - res := runResult{} - - defType, err := getTypeFromFile(df) - if err != nil { - return res, fmt.Errorf("cannot get type from file: %w", err) - } - res.ResourceType = defType - - a.logger.Debug("running definition", - zap.String("type", defType), - zap.String("envID", envID), - zap.Any("envVars", ev), - ) - - runInfo := openapi.RunInformation{ - EnvironmentId: openapi.PtrString(envID), - Variables: ev.toOpenapi(), - Metadata: getMetadata(), - } - - switch defType { - case "Test": - var test openapi.TestResource - err = yaml.Unmarshal(df.Contents(), &test) - if err != nil { - a.logger.Error("error parsing test", zap.String("content", string(df.Contents())), zap.Error(err)) - return res, fmt.Errorf("could not unmarshal test yaml: %w", err) - } - - req := a.openapiClient.ApiApi. - RunTest(ctx, test.Spec.GetId()). - RunInformation(runInfo) - - a.logger.Debug("running test", zap.String("id", test.Spec.GetId())) - - run, resp, err := a.openapiClient.ApiApi.RunTestExecute(req) - err = a.handleRunError(resp, err) - if err != nil { - return res, err - } - - full, err := a.tests.Get(ctx, test.Spec.GetId(), a.jsonFormat) - if err != nil { - return res, fmt.Errorf("cannot get full test '%s': %w", test.Spec.GetId(), err) - } - err = json.Unmarshal([]byte(full), &test) - if err != nil { - return res, fmt.Errorf("cannot get full test '%s': %w", test.Spec.GetId(), err) - } - - res.Resource = test - res.Run = *run - - case "Transaction": - var tran openapi.TransactionResource - err = yaml.Unmarshal(df.Contents(), &tran) - if err != nil { - a.logger.Error("error parsing transaction", zap.String("content", string(df.Contents())), zap.Error(err)) - return res, fmt.Errorf("could not unmarshal transaction yaml: %w", err) - } - - req := a.openapiClient.ApiApi. - RunTransaction(ctx, tran.Spec.GetId()). - RunInformation(runInfo) - - a.logger.Debug("running transaction", zap.String("id", tran.Spec.GetId())) - - run, resp, err := a.openapiClient.ApiApi.RunTransactionExecute(req) - err = a.handleRunError(resp, err) - if err != nil { - return res, err - } - - full, err := a.transactions.Get(ctx, tran.Spec.GetId(), a.jsonFormat) - if err != nil { - return res, fmt.Errorf("cannot get full transaction '%s': %w", tran.Spec.GetId(), err) - } - err = json.Unmarshal([]byte(full), &tran) - if err != nil { - return res, fmt.Errorf("cannot get full transaction '%s': %w", tran.Spec.GetId(), err) - } - - res.Resource = tran - res.Run = *run - default: - return res, fmt.Errorf("unknown type: %s", defType) - } - - a.logger.Debug("definition run", - zap.String("type", defType), - zap.String("envID", envID), - zap.Any("envVars", ev), - zap.Any("response", res), - ) - - return res, nil -} - -func (a runTestAction) handleRunError(resp *http.Response, reqErr error) error { - body, err := io.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("could not read response body: %w", err) - } - resp.Body.Close() - - if resp.StatusCode == http.StatusNotFound { - return fmt.Errorf("resource not found in server") - } - - if resp.StatusCode == http.StatusUnprocessableEntity { - return buildMissingEnvVarsError(body) - } - - if reqErr != nil { - a.logger.Error("error running transaction", zap.Error(err), zap.String("body", string(body))) - return fmt.Errorf("could not run transaction: %w", err) - } - - return nil -} - -func buildMissingEnvVarsError(body []byte) error { - var missingVarsErrResp openapi.MissingVariablesError - err := json.Unmarshal(body, &missingVarsErrResp) - if err != nil { - return fmt.Errorf("could not unmarshal response body: %w", err) - } - - missingVars := envVars{} - - for _, missingVarErr := range missingVarsErrResp.MissingVariables { - for _, missingVar := range missingVarErr.Variables { - missingVars = append(missingVars, envVar{ - Name: missingVar.GetKey(), - DefaultValue: missingVar.GetDefaultValue(), - }) - } - } - - return missingEnvVarsError(missingVars.unique()) -} - -func (a runTestAction) askForMissingVars(missingVars []envVar) []envVar { - ui.DefaultUI.Warning("Some variables are required by one or more tests") - ui.DefaultUI.Info("Fill the values for each variable:") - - filledVariables := make([]envVar, 0, len(missingVars)) - - for _, missingVar := range missingVars { - answer := missingVar - answer.UserValue = ui.DefaultUI.TextInput(missingVar.Name, missingVar.DefaultValue) - filledVariables = append(filledVariables, answer) - } - - a.logger.Debug("filled variables", zap.Any("variables", filledVariables)) - - return filledVariables -} - -func (a runTestAction) formatResult(result runResult, hasResults bool) string { - switch result.ResourceType { - case "Test": - return a.formatTestResult(result, hasResults) - case "Transaction": - return a.formatTransactionResult(result, hasResults) - } - return "" -} - -func (a runTestAction) formatTestResult(result runResult, hasResults bool) string { - test := result.Resource.(openapi.TestResource) - run := result.Run.(openapi.TestRun) - - tro := formatters.TestRunOutput{ - HasResults: hasResults, - Test: test.GetSpec(), - Run: run, - } - - formatter := formatters.TestRun(a.config, true) - return formatter.Format(tro) -} -func (a runTestAction) formatTransactionResult(result runResult, hasResults bool) string { - tran := result.Resource.(openapi.TransactionResource) - run := result.Run.(openapi.TransactionRun) - - tro := formatters.TransactionRunOutput{ - HasResults: hasResults, - Transaction: tran.GetSpec(), - Run: run, - } - - return formatters. - TransactionRun(a.config, true). - Format(tro) -} - -func (a runTestAction) waitForResult(ctx context.Context, run runResult) (runResult, error) { - switch run.ResourceType { - case "Test": - tr, err := a.waitForTestResult(ctx, run) - if err != nil { - return run, err - } - - run.Run = tr - return run, nil - - case "Transaction": - tr, err := a.waitForTransactionResult(ctx, run) - if err != nil { - return run, err - } - - run.Run = tr - return run, nil - } - return run, fmt.Errorf("unknown resource type: %s", run.ResourceType) -} - -func (a runTestAction) waitForTestResult(ctx context.Context, result runResult) (openapi.TestRun, error) { - var ( - testRun openapi.TestRun - lastError error - wg sync.WaitGroup - ) - - test := result.Resource.(openapi.TestResource) - run := result.Run.(openapi.TestRun) - - wg.Add(1) - ticker := time.NewTicker(1 * time.Second) // TODO: change to websockets - go func() { - for range ticker.C { - readyTestRun, err := a.isTestReady(ctx, test.Spec.GetId(), run.GetId()) - if err != nil { - lastError = err - wg.Done() - return - } - - if readyTestRun != nil { - testRun = *readyTestRun - wg.Done() - return - } - } - }() - wg.Wait() - - if lastError != nil { - return openapi.TestRun{}, lastError - } - - return testRun, nil -} - -func (a runTestAction) isTestReady(ctx context.Context, testID string, testRunID string) (*openapi.TestRun, error) { - runID, err := strconv.Atoi(testRunID) - if err != nil { - return nil, fmt.Errorf("invalid transaction run id format: %w", err) - } - - req := a.openapiClient.ApiApi.GetTestRun(ctx, testID, int32(runID)) - run, _, err := a.openapiClient.ApiApi.GetTestRunExecute(req) - if err != nil { - return &openapi.TestRun{}, fmt.Errorf("could not execute GetTestRun request: %w", err) - } - - if utils.RunStateIsFinished(run.GetState()) { - return run, nil - } - - return nil, nil -} - -func (a runTestAction) waitForTransactionResult(ctx context.Context, result runResult) (openapi.TransactionRun, error) { - var ( - transactionRun openapi.TransactionRun - lastError error - wg sync.WaitGroup - ) - - tran := result.Resource.(openapi.TransactionResource) - run := result.Run.(openapi.TransactionRun) - - wg.Add(1) - ticker := time.NewTicker(1 * time.Second) // TODO: change to websockets - go func() { - for range ticker.C { - readyTransactionRun, err := a.isTransactionReady(ctx, tran.Spec.GetId(), run.GetId()) - if err != nil { - lastError = err - wg.Done() - return - } - - if readyTransactionRun != nil { - transactionRun = *readyTransactionRun - wg.Done() - return - } - } - }() - wg.Wait() - - if lastError != nil { - return openapi.TransactionRun{}, lastError - } - - return transactionRun, nil -} - -func (a runTestAction) isTransactionReady(ctx context.Context, transactionID, transactionRunID string) (*openapi.TransactionRun, error) { - runID, err := strconv.Atoi(transactionRunID) - if err != nil { - return nil, fmt.Errorf("invalid transaction run id format: %w", err) - } - - req := a.openapiClient.ApiApi.GetTransactionRun(ctx, transactionID, int32(runID)) - run, _, err := a.openapiClient.ApiApi.GetTransactionRunExecute(req) - if err != nil { - return nil, fmt.Errorf("could not execute GetTestRun request: %w", err) - } - - if utils.RunStateIsFinished(run.GetState()) { - return run, nil - } - - return nil, nil -} - -func (a runTestAction) writeJUnitReport(ctx context.Context, result runResult, outputFile string) error { - test := result.Resource.(openapi.TestResource) - run := result.Run.(openapi.TestRun) - runID, err := strconv.Atoi(run.GetId()) - if err != nil { - return fmt.Errorf("invalid run id format: %w", err) - } - - req := a.openapiClient.ApiApi.GetRunResultJUnit(ctx, test.Spec.GetId(), int32(runID)) - junit, _, err := a.openapiClient.ApiApi.GetRunResultJUnitExecute(req) - if err != nil { - return fmt.Errorf("could not execute request: %w", err) - } - - f, err := os.Create(junit) - if err != nil { - return fmt.Errorf("could not create junit output file: %w", err) - } - - _, err = f.WriteString(outputFile) - - return err - -} - -func getMetadata() map[string]string { - ci := cienvironment.DetectCIEnvironment() - if ci == nil { - return map[string]string{} - } - - metadata := map[string]string{ - "name": ci.Name, - "url": ci.URL, - "buildNumber": ci.BuildNumber, - } - - if ci.Git != nil { - metadata["branch"] = ci.Git.Branch - metadata["tag"] = ci.Git.Tag - metadata["revision"] = ci.Git.Revision - } - - return metadata -} - -type defFile struct { - fileutil.File -} diff --git a/cli/cmd/config.go b/cli/cmd/config.go index 40b19ac295..b689714ba2 100644 --- a/cli/cmd/config.go +++ b/cli/cmd/config.go @@ -9,6 +9,7 @@ import ( "github.com/kubeshop/tracetest/cli/analytics" "github.com/kubeshop/tracetest/cli/config" "github.com/kubeshop/tracetest/cli/formatters" + "github.com/kubeshop/tracetest/cli/openapi" "github.com/kubeshop/tracetest/cli/utils" "github.com/spf13/cobra" "go.uber.org/zap" @@ -18,6 +19,7 @@ import ( var ( cliLogger = &zap.Logger{} cliConfig config.Config + openapiClient = &openapi.APIClient{} versionText string isVersionMatch bool ) @@ -57,6 +59,7 @@ func setupCommand(options ...setupOption) func(cmd *cobra.Command, args []string overrideConfig() setupVersion() setupResources() + setupRunners() if config.shouldValidateConfig { validateConfig(cmd, args) @@ -83,17 +86,20 @@ func overrideConfig() { } } -func setupOutputFormat(cmd *cobra.Command) { - if cmd.GroupID != "resources" && output == string(formatters.Empty) { - output = string(formatters.DefaultOutput) - } +func setupRunners() { + c := utils.GetAPIClient(cliConfig) + *openapiClient = *c +} +func setupOutputFormat(cmd *cobra.Command) { o := formatters.Output(output) + if output == "" { + o = formatters.Pretty + } if !formatters.ValidOutput(o) { fmt.Fprintf(os.Stderr, "Invalid output format %s. Available formats are [%s]\n", output, outputFormatsString) ExitCLI(1) } - formatters.SetOutput(o) } func loadConfig(cmd *cobra.Command, args []string) { diff --git a/cli/cmd/legacy_test_cmd.go b/cli/cmd/legacy_test_cmd.go index eb26de5b4a..fca472da6e 100644 --- a/cli/cmd/legacy_test_cmd.go +++ b/cli/cmd/legacy_test_cmd.go @@ -1,15 +1,20 @@ package cmd import ( + "os" + "strings" + + "github.com/kubeshop/tracetest/cli/pkg/resourcemanager" "github.com/spf13/cobra" ) var testCmd = &cobra.Command{ - GroupID: cmdGroupTests.ID, - Use: "test", - Short: "Manage your tracetest tests", - Long: "Manage your tracetest tests", - PreRun: setupCommand(), + GroupID: cmdGroupTests.ID, + Use: "test", + Short: "Manage your tracetest tests", + Long: "Manage your tracetest tests", + Deprecated: "Please use `tracetest (apply|delete|export|get|list) test` commands instead.", + PreRun: setupCommand(), Run: func(cmd *cobra.Command, args []string) { cmd.Help() }, @@ -40,6 +45,54 @@ var testExportCmd = &cobra.Command{ PostRun: teardownCommand, } +var ( + runTestFileDefinition, + runTestEnvID, + runTestJUnit string + runTestWaitForResult bool +) + +var testRunCmd = &cobra.Command{ + Use: "run", + Short: "Run a test on your Tracetest server", + Long: "Run a test on your Tracetest server", + Deprecated: "Please use `tracetest run test` command instead.", + PreRun: setupCommand(), + Run: func(_ *cobra.Command, _ []string) { + // map old flags to new ones + runParams.DefinitionFile = runTestFileDefinition + runParams.EnvID = runTestEnvID + runParams.SkipResultWait = !runTestWaitForResult + runParams.JUnitOuptutFile = runTestJUnit + + fileType := getFileType(runTestFileDefinition) + + runCmd.Run(listCmd, []string{fileType}) + }, + PostRun: teardownCommand, +} + +// getFileType returns the value of the `type` field defined in the definition file. +// if any error happens, it will return `test` as default, and errors will be ignored. +func getFileType(filePath string) string { + const defaultFileType = "test" + + contents, err := os.ReadFile(filePath) + if err != nil { + return defaultFileType + } + + var definition struct { + Type string `json:"type"` + } + yaml := resourcemanager.Formats.Get(resourcemanager.FormatYAML) + if err := yaml.Unmarshal(contents, &definition); err != nil { + return defaultFileType + } + + return strings.ToLower(definition.Type) +} + func init() { rootCmd.AddCommand(testCmd) @@ -50,4 +103,12 @@ func init() { testExportCmd.PersistentFlags().StringVarP(&exportParams.ResourceID, "id", "", "", "id of the test") testExportCmd.PersistentFlags().StringVarP(&exportParams.OutputFile, "output", "o", "", "file to be created with definition") testCmd.AddCommand(testExportCmd) + + // run + testRunCmd.PersistentFlags().StringVarP(&runTestEnvID, "environment", "e", "", "id of the environment to be used") + testRunCmd.PersistentFlags().StringVarP(&runTestFileDefinition, "definition", "d", "", "path to the definition file to be run") + testRunCmd.PersistentFlags().BoolVarP(&runTestWaitForResult, "wait-for-result", "w", false, "wait for the test result to print it's result") + testRunCmd.PersistentFlags().StringVarP(&runTestJUnit, "junit", "j", "", "path to the junit file that will be generated") + + testCmd.AddCommand(testRunCmd) } diff --git a/cli/cmd/resource_delete_cmd.go b/cli/cmd/resource_delete_cmd.go index b73b13ccf6..13fd1e1ee5 100644 --- a/cli/cmd/resource_delete_cmd.go +++ b/cli/cmd/resource_delete_cmd.go @@ -2,6 +2,7 @@ package cmd import ( "context" + "errors" "fmt" "github.com/kubeshop/tracetest/cli/pkg/resourcemanager" @@ -35,6 +36,9 @@ func init() { } result, err := resourceClient.Delete(ctx, deleteParams.ResourceID, resultFormat) + if errors.Is(err, resourcemanager.ErrNotFound) { + return "", errors.New(result) + } if err != nil { return "", err } diff --git a/cli/cmd/resource_export_cmd.go b/cli/cmd/resource_export_cmd.go index 7f04165811..cba48c3d7e 100644 --- a/cli/cmd/resource_export_cmd.go +++ b/cli/cmd/resource_export_cmd.go @@ -31,12 +31,7 @@ func init() { } // export is ALWAYS yaml, so we can hardcode it here - resultFormat, err := resourcemanager.Formats.Get("yaml") - if err != nil { - return "", err - } - - result, err := resourceClient.Get(ctx, exportParams.ResourceID, resultFormat) + result, err := resourceClient.Get(ctx, exportParams.ResourceID, resourcemanager.Formats.Get("yaml")) if err != nil { return "", err } diff --git a/cli/cmd/resource_get_cmd.go b/cli/cmd/resource_get_cmd.go index 130a1bc551..b10dece6d5 100644 --- a/cli/cmd/resource_get_cmd.go +++ b/cli/cmd/resource_get_cmd.go @@ -2,6 +2,7 @@ package cmd import ( "context" + "errors" "github.com/kubeshop/tracetest/cli/pkg/resourcemanager" "github.com/spf13/cobra" @@ -34,6 +35,9 @@ func init() { } result, err := resourceClient.Get(ctx, getParams.ResourceID, resultFormat) + if errors.Is(err, resourcemanager.ErrNotFound) { + return result, nil + } if err != nil { return "", err } diff --git a/cli/cmd/resource_run_cmd.go b/cli/cmd/resource_run_cmd.go new file mode 100644 index 0000000000..759e4ac595 --- /dev/null +++ b/cli/cmd/resource_run_cmd.go @@ -0,0 +1,101 @@ +package cmd + +import ( + "context" + "fmt" + + "github.com/kubeshop/tracetest/cli/runner" + "github.com/kubeshop/tracetest/cli/utils" + "github.com/spf13/cobra" +) + +var ( + runParams = &runParameters{} + runCmd *cobra.Command +) + +func init() { + runCmd = &cobra.Command{ + GroupID: cmdGroupResources.ID, + Use: "run " + runnableResourceList(), + Short: "run resources", + Long: "run resources", + PreRun: setupCommand(), + Run: WithResourceMiddleware(func(_ *cobra.Command, args []string) (string, error) { + resourceType := resourceParams.ResourceName + ctx := context.Background() + + r, err := runnerRegistry.Get(resourceType) + if err != nil { + return "", fmt.Errorf("resource type '%s' cannot be run", resourceType) + } + + orchestrator := runner.Orchestrator( + cliLogger, + utils.GetAPIClient(cliConfig), + environmentClient, + ) + + runParams := runner.RunOptions{ + ID: runParams.ID, + DefinitionFile: runParams.DefinitionFile, + EnvID: runParams.EnvID, + SkipResultWait: runParams.SkipResultWait, + JUnitOuptutFile: runParams.JUnitOuptutFile, + } + + exitCode, err := orchestrator.Run(ctx, r, runParams, output) + if err != nil { + return "", err + } + + ExitCLI(exitCode) + + // ExitCLI will exit the process, so this return is just to satisfy the compiler + return "", nil + + }, runParams), + PostRun: teardownCommand, + } + + runCmd.Flags().StringVarP(&runParams.DefinitionFile, "file", "f", "", "path to the definition file") + runCmd.Flags().StringVar(&runParams.ID, "id", "", "id of the resource to run") + runCmd.PersistentFlags().StringVarP(&runParams.EnvID, "environment", "e", "", "environment file or ID to be used") + runCmd.PersistentFlags().BoolVarP(&runParams.SkipResultWait, "skip-result-wait", "W", false, "do not wait for results. exit immediately after test run started") + runCmd.PersistentFlags().StringVarP(&runParams.JUnitOuptutFile, "junit", "j", "", "file path to save test results in junit format") + rootCmd.AddCommand(runCmd) +} + +type runParameters struct { + ID string + DefinitionFile string + EnvID string + SkipResultWait bool + JUnitOuptutFile string +} + +func (p runParameters) Validate(cmd *cobra.Command, args []string) []error { + errs := []error{} + if p.DefinitionFile == "" && p.ID == "" { + errs = append(errs, paramError{ + Parameter: "resource", + Message: "you must specify a definition file or resource ID", + }) + } + + if p.DefinitionFile != "" && p.ID != "" { + errs = append(errs, paramError{ + Parameter: "resource", + Message: "you cannot specify both a definition file and resource ID", + }) + } + + if p.JUnitOuptutFile != "" && p.SkipResultWait { + errs = append(errs, paramError{ + Parameter: "junit", + Message: "--junit option is incompatible with --skip-result-wait option", + }) + } + + return errs +} diff --git a/cli/cmd/resources.go b/cli/cmd/resources.go index 4af432ad51..fd6a7676e4 100644 --- a/cli/cmd/resources.go +++ b/cli/cmd/resources.go @@ -9,18 +9,47 @@ import ( "github.com/Jeffail/gabs/v2" "github.com/kubeshop/tracetest/cli/analytics" + "github.com/kubeshop/tracetest/cli/formatters" "github.com/kubeshop/tracetest/cli/pkg/fileutil" "github.com/kubeshop/tracetest/cli/pkg/resourcemanager" "github.com/kubeshop/tracetest/cli/preprocessor" + "github.com/kubeshop/tracetest/cli/runner" ) var resourceParams = &resourceParameters{} + +var ( + runnerRegistry = runner.NewRegistry(). + Register(runner.TestRunner( + testClient, + openapiClient, + formatters.TestRun(func() string { return cliConfig.URL() }, true), + )). + Register(runner.TransactionRunner( + transactionClient, + openapiClient, + formatters.TransactionRun(func() string { return cliConfig.URL() }, true), + )) +) + var ( httpClient = &resourcemanager.HTTPClient{} + environmentClient = resourcemanager.NewClient( + httpClient, cliLogger, + "environment", "environments", + resourcemanager.WithTableConfig(resourcemanager.TableConfig{ + Cells: []resourcemanager.TableCellConfig{ + {Header: "ID", Path: "spec.id"}, + {Header: "NAME", Path: "spec.name"}, + {Header: "DESCRIPTION", Path: "spec.description"}, + }, + }), + ) + testPreprocessor = preprocessor.Test(cliLogger) testClient = resourcemanager.NewClient( - httpClient, + httpClient, cliLogger, "test", "tests", resourcemanager.WithTableConfig(resourcemanager.TableConfig{ Cells: []resourcemanager.TableCellConfig{ @@ -55,11 +84,7 @@ var ( ) transactionPreprocessor = preprocessor.Transaction(cliLogger, func(ctx context.Context, input fileutil.File) (fileutil.File, error) { - formatYAML, err := resourcemanager.Formats.Get(resourcemanager.FormatYAML) - if err != nil { - return input, fmt.Errorf("cannot get yaml format: %w", err) - } - updated, err := testClient.Apply(ctx, input, formatYAML) + updated, err := testClient.Apply(ctx, input, resourcemanager.Formats.Get(resourcemanager.FormatYAML)) if err != nil { return input, fmt.Errorf("cannot apply test: %w", err) } @@ -67,10 +92,38 @@ var ( return fileutil.New(input.AbsPath(), []byte(updated)), nil }) + transactionClient = resourcemanager.NewClient( + httpClient, cliLogger, + "transaction", "transactions", + resourcemanager.WithTableConfig(resourcemanager.TableConfig{ + Cells: []resourcemanager.TableCellConfig{ + {Header: "ID", Path: "spec.id"}, + {Header: "NAME", Path: "spec.name"}, + {Header: "VERSION", Path: "spec.version"}, + {Header: "STEPS", Path: "spec.summary.steps"}, + {Header: "RUNS", Path: "spec.summary.runs"}, + {Header: "LAST RUN TIME", Path: "spec.summary.lastRun.time"}, + {Header: "LAST RUN SUCCESSES", Path: "spec.summary.lastRun.passes"}, + {Header: "LAST RUN FAILURES", Path: "spec.summary.lastRun.fails"}, + }, + ItemModifier: func(item *gabs.Container) error { + // set spec.summary.steps to the number of steps in the transaction + item.SetP(len(item.Path("spec.steps").Children()), "spec.summary.steps") + + if err := formatItemDate(item, "spec.summary.lastRun.time"); err != nil { + return err + } + + return nil + }, + }), + resourcemanager.WithApplyPreProcessor(transactionPreprocessor.Preprocess), + ) + resources = resourcemanager.NewRegistry(). Register( resourcemanager.NewClient( - httpClient, + httpClient, cliLogger, "config", "configs", resourcemanager.WithTableConfig(resourcemanager.TableConfig{ Cells: []resourcemanager.TableCellConfig{ @@ -83,7 +136,7 @@ var ( ). Register( resourcemanager.NewClient( - httpClient, + httpClient, cliLogger, "analyzer", "analyzers", resourcemanager.WithTableConfig(resourcemanager.TableConfig{ Cells: []resourcemanager.TableCellConfig{ @@ -103,7 +156,7 @@ var ( ). Register( resourcemanager.NewClient( - httpClient, + httpClient, cliLogger, "pollingprofile", "pollingprofiles", resourcemanager.WithTableConfig(resourcemanager.TableConfig{ Cells: []resourcemanager.TableCellConfig{ @@ -112,11 +165,12 @@ var ( {Header: "STRATEGY", Path: "spec.strategy"}, }, }), + resourcemanager.WithResourceType("PollingProfile"), ), ). Register( resourcemanager.NewClient( - httpClient, + httpClient, cliLogger, "demo", "demos", resourcemanager.WithTableConfig(resourcemanager.TableConfig{ Cells: []resourcemanager.TableCellConfig{ @@ -130,7 +184,7 @@ var ( ). Register( resourcemanager.NewClient( - httpClient, + httpClient, cliLogger, "datastore", "datastores", resourcemanager.WithTableConfig(resourcemanager.TableConfig{ Cells: []resourcemanager.TableCellConfig{ @@ -149,50 +203,11 @@ var ( }, }), resourcemanager.WithDeleteSuccessMessage("DataStore removed. Defaulting back to no-tracing mode"), + resourcemanager.WithResourceType("DataStore"), ), ). - Register( - resourcemanager.NewClient( - httpClient, - "environment", "environments", - resourcemanager.WithTableConfig(resourcemanager.TableConfig{ - Cells: []resourcemanager.TableCellConfig{ - {Header: "ID", Path: "spec.id"}, - {Header: "NAME", Path: "spec.name"}, - {Header: "DESCRIPTION", Path: "spec.description"}, - }, - }), - ), - ). - Register( - resourcemanager.NewClient( - httpClient, - "transaction", "transactions", - resourcemanager.WithTableConfig(resourcemanager.TableConfig{ - Cells: []resourcemanager.TableCellConfig{ - {Header: "ID", Path: "spec.id"}, - {Header: "NAME", Path: "spec.name"}, - {Header: "VERSION", Path: "spec.version"}, - {Header: "STEPS", Path: "spec.summary.steps"}, - {Header: "RUNS", Path: "spec.summary.runs"}, - {Header: "LAST RUN TIME", Path: "spec.summary.lastRun.time"}, - {Header: "LAST RUN SUCCESSES", Path: "spec.summary.lastRun.passes"}, - {Header: "LAST RUN FAILURES", Path: "spec.summary.lastRun.fails"}, - }, - ItemModifier: func(item *gabs.Container) error { - // set spec.summary.steps to the number of steps in the transaction - item.SetP(len(item.Path("spec.steps").Children()), "spec.summary.steps") - - if err := formatItemDate(item, "spec.summary.lastRun.time"); err != nil { - return err - } - - return nil - }, - }), - resourcemanager.WithApplyPreProcessor(transactionPreprocessor.Preprocess), - ), - ). + Register(environmentClient). + Register(transactionClient). Register(testClient) ) @@ -200,6 +215,10 @@ func resourceList() string { return strings.Join(resources.List(), "|") } +func runnableResourceList() string { + return strings.Join(runnerRegistry.List(), "|") +} + func setupResources() { extraHeaders := http.Header{} extraHeaders.Set("x-client-id", analytics.ClientID()) diff --git a/cli/cmd/test_run_cmd.go b/cli/cmd/test_run_cmd.go deleted file mode 100644 index 8b34971cbc..0000000000 --- a/cli/cmd/test_run_cmd.go +++ /dev/null @@ -1,71 +0,0 @@ -package cmd - -import ( - "context" - "fmt" - - "github.com/kubeshop/tracetest/cli/actions" - "github.com/kubeshop/tracetest/cli/utils" - "github.com/spf13/cobra" -) - -var ( - runTestFileDefinition, - runTestEnvID, - runTestJUnit string - runTestWaitForResult bool -) - -var testRunCmd = &cobra.Command{ - Use: "run", - Short: "Run a test on your Tracetest server", - Long: "Run a test on your Tracetest server", - PreRun: setupCommand(), - Run: WithResultHandler(func(_ *cobra.Command, _ []string) (string, error) { - ctx := context.Background() - client := utils.GetAPIClient(cliConfig) - - envClient, err := resources.Get("environment") - if err != nil { - return "", fmt.Errorf("failed to get environment client: %w", err) - } - - testClient, err := resources.Get("test") - if err != nil { - return "", fmt.Errorf("failed to get test client: %w", err) - } - - transactionsClient, err := resources.Get("transaction") - if err != nil { - return "", fmt.Errorf("failed to get transaction client: %w", err) - } - - runTestAction := actions.NewRunTestAction( - cliConfig, - cliLogger, - client, - testClient, - transactionsClient, - envClient, - ExitCLI, - ) - actionArgs := actions.RunResourceArgs{ - DefinitionFile: runTestFileDefinition, - EnvID: runTestEnvID, - WaitForResult: runTestWaitForResult, - JUnit: runTestJUnit, - } - - err = runTestAction.Run(ctx, actionArgs) - return "", err - }), - PostRun: teardownCommand, -} - -func init() { - testRunCmd.PersistentFlags().StringVarP(&runTestEnvID, "environment", "e", "", "id of the environment to be used") - testRunCmd.PersistentFlags().StringVarP(&runTestFileDefinition, "definition", "d", "", "path to the definition file to be run") - testRunCmd.PersistentFlags().BoolVarP(&runTestWaitForResult, "wait-for-result", "w", false, "wait for the test result to print it's result") - testRunCmd.PersistentFlags().StringVarP(&runTestJUnit, "junit", "j", "", "path to the junit file that will be generated") - testCmd.AddCommand(testRunCmd) -} diff --git a/cli/formatters/outputs.go b/cli/formatters/outputs.go index a98c29deca..7e3d97b042 100644 --- a/cli/formatters/outputs.go +++ b/cli/formatters/outputs.go @@ -5,27 +5,16 @@ import "golang.org/x/exp/slices" type Output string var ( - CurrentOutput = DefaultOutput - Outputs = []Output{ Pretty, JSON, YAML, - Empty, } - - DefaultOutput = Pretty - - Empty Output = "" Pretty Output = "pretty" JSON Output = "json" YAML Output = "yaml" ) -func SetOutput(o Output) { - CurrentOutput = o -} - func OuputsStr() []string { out := make([]string, len(Outputs)) for i, o := range Outputs { diff --git a/cli/formatters/test_run.go b/cli/formatters/test_run.go index 190dac0cc2..146077c2d7 100644 --- a/cli/formatters/test_run.go +++ b/cli/formatters/test_run.go @@ -5,9 +5,7 @@ import ( "encoding/json" "fmt" - "github.com/kubeshop/tracetest/cli/config" "github.com/kubeshop/tracetest/cli/openapi" - "github.com/kubeshop/tracetest/cli/utils" "github.com/pterm/pterm" ) @@ -17,7 +15,7 @@ const ( ) type testRun struct { - config config.Config + baseURLFn func() string colorsEnabled bool padding int } @@ -30,9 +28,9 @@ func WithPadding(padding int) testRunFormatterOption { } } -func TestRun(config config.Config, colorsEnabled bool, options ...testRunFormatterOption) testRun { +func TestRun(baseURLFn func() string, colorsEnabled bool, options ...testRunFormatterOption) testRun { testRun := testRun{ - config: config, + baseURLFn: baseURLFn, colorsEnabled: colorsEnabled, } @@ -45,17 +43,18 @@ func TestRun(config config.Config, colorsEnabled bool, options ...testRunFormatt type TestRunOutput struct { HasResults bool `json:"-"` + IsFailed bool `json:"-"` Test openapi.Test `json:"test"` Run openapi.TestRun `json:"testRun"` RunWebURL string `json:"testRunWebUrl"` } -func (f testRun) Format(output TestRunOutput) string { - switch CurrentOutput { - case Pretty: - return f.pretty(output) +func (f testRun) Format(output TestRunOutput, format Output) string { + switch format { case JSON: return f.json(output) + case Pretty, "": + return f.pretty(output) } return "" @@ -78,7 +77,7 @@ func (f testRun) json(output TestRunOutput) string { } func (f testRun) pretty(output TestRunOutput) string { - if utils.RunStateIsFailed(output.Run.GetState()) { + if output.IsFailed { return f.getColoredText(false, fmt.Sprintf("%s\n%s", f.formatMessage("%s %s (%s)", FAILED_TEST_ICON, @@ -95,7 +94,7 @@ func (f testRun) pretty(output TestRunOutput) string { return f.formatSuccessfulTest(output.Test, output.Run) } - if output.Run.Result.AllPassed == nil || !*output.Run.Result.AllPassed { + if !output.Run.Result.GetAllPassed() { return f.formatFailedTest(output.Test, output.Run) } @@ -276,7 +275,7 @@ func (f testRun) getColoredText(passed bool, text string) string { } func (f testRun) GetRunLink(testID, runID string) string { - return fmt.Sprintf("%s://%s/test/%s/run/%s/test", f.config.Scheme, f.config.Endpoint, testID, runID) + return fmt.Sprintf("%s/test/%s/run/%s/test", f.baseURLFn(), testID, runID) } func (f testRun) getDeepLink(baseLink string, index int, spanID string) string { diff --git a/cli/formatters/test_run_test.go b/cli/formatters/test_run_test.go index 6c54f0fc3e..17290813e9 100644 --- a/cli/formatters/test_run_test.go +++ b/cli/formatters/test_run_test.go @@ -3,12 +3,15 @@ package formatters_test import ( "testing" - "github.com/kubeshop/tracetest/cli/config" "github.com/kubeshop/tracetest/cli/formatters" "github.com/kubeshop/tracetest/cli/openapi" "github.com/stretchr/testify/assert" ) +func baseURL() string { + return "http://localhost:11633" +} + func TestJSON(t *testing.T) { in := formatters.TestRunOutput{ Test: openapi.Test{ @@ -24,18 +27,13 @@ func TestJSON(t *testing.T) { }, } - formatter := formatters.TestRun(config.Config{ - Scheme: "http", - Endpoint: "localhost:11633", - }, false) + formatter := formatters.TestRun(baseURL, false) - formatters.SetOutput(formatters.JSON) - actual := formatter.Format(in) + actual := formatter.Format(in, formatters.JSON) expected := `{"results":{"allPassed":true},"testRunWebUrl":"http://localhost:11633/test/9876543/run/1/test"}` assert.JSONEq(t, expected, actual) - formatters.SetOutput(formatters.DefaultOutput) } func TestSuccessfulTestRunOutput(t *testing.T) { @@ -52,11 +50,8 @@ func TestSuccessfulTestRunOutput(t *testing.T) { }, }, } - formatter := formatters.TestRun(config.Config{ - Scheme: "http", - Endpoint: "localhost:11633", - }, false) - output := formatter.Format(in) + formatter := formatters.TestRun(baseURL, false) + output := formatter.Format(in, formatters.Pretty) assert.Equal(t, "✔ Testcase 1 (http://localhost:11633/test/9876543/run/1/test)\n", output) } @@ -104,11 +99,8 @@ func TestSuccessfulTestRunOutputWithResult(t *testing.T) { }, }, } - formatter := formatters.TestRun(config.Config{ - Scheme: "http", - Endpoint: "localhost:11633", - }, false) - output := formatter.Format(in) + formatter := formatters.TestRun(baseURL, false) + output := formatter.Format(in, formatters.Pretty) expectedOutput := `✔ Testcase 1 (http://localhost:11633/test/9876543/run/1/test) ✔ Validate span duration ` @@ -193,11 +185,8 @@ func TestFailingTestOutput(t *testing.T) { }, } - formatter := formatters.TestRun(config.Config{ - Scheme: "http", - Endpoint: "localhost:11633", - }, false) - output := formatter.Format(in) + formatter := formatters.TestRun(baseURL, false) + output := formatter.Format(in, formatters.Pretty) expectedOutput := `✘ Testcase 2 (http://localhost:11633/test/9876543/run/1/test) ✔ Validate span duration ✔ #123456 @@ -287,11 +276,8 @@ func TestFailingTestOutputWithPadding(t *testing.T) { }, } - formatter := formatters.TestRun(config.Config{ - Scheme: "http", - Endpoint: "localhost:11633", - }, false, formatters.WithPadding(1)) - output := formatter.Format(in) + formatter := formatters.TestRun(baseURL, false, formatters.WithPadding(1)) + output := formatter.Format(in, formatters.Pretty) expectedOutput := ` ✘ Testcase 2 (http://localhost:11633/test/9876543/run/1/test) ✔ Validate span duration ✔ #123456 diff --git a/cli/formatters/transaction_run.go b/cli/formatters/transaction_run.go index 70218af88c..2752e8d254 100644 --- a/cli/formatters/transaction_run.go +++ b/cli/formatters/transaction_run.go @@ -5,22 +5,21 @@ import ( "fmt" "strings" - "github.com/kubeshop/tracetest/cli/config" "github.com/kubeshop/tracetest/cli/openapi" "github.com/pterm/pterm" ) type transactionRun struct { - config config.Config + baseURLFn func() string colorsEnabled bool testRunFormatter testRun } -func TransactionRun(config config.Config, colorsEnabled bool) transactionRun { +func TransactionRun(baseURLFn func() string, colorsEnabled bool) transactionRun { return transactionRun{ - config: config, + baseURLFn: baseURLFn, colorsEnabled: colorsEnabled, - testRunFormatter: TestRun(config, colorsEnabled, WithPadding(1)), + testRunFormatter: TestRun(baseURLFn, colorsEnabled, WithPadding(1)), } } @@ -31,13 +30,14 @@ type TransactionRunOutput struct { RunWebURL string `json:"transactionRunWebUrl"` } -func (f transactionRun) Format(output TransactionRunOutput) string { +func (f transactionRun) Format(output TransactionRunOutput, format Output) string { output.RunWebURL = f.getRunLink(output.Transaction.GetId(), output.Run.GetId()) - switch CurrentOutput { - case Pretty: - return f.pretty(output) + + switch format { case JSON: return f.json(output) + case Pretty, "": + return f.pretty(output) } return "" @@ -78,16 +78,14 @@ func (f transactionRun) json(output TransactionRunOutput) string { } func (f transactionRun) pretty(output TransactionRunOutput) string { + message := fmt.Sprintf("%s %s (%s)\n", PASSED_TEST_ICON, output.Transaction.GetName(), output.RunWebURL) if !output.HasResults { - return "" + return f.getColoredText(true, message) } - link := output.RunWebURL allStepsPassed := f.allTransactionStepsPassed(output) - message := fmt.Sprintf("%s %s (%s)\n", PASSED_TEST_ICON, output.Transaction.GetName(), link) - if !allStepsPassed { - message = fmt.Sprintf("%s %s (%s)\n", FAILED_TEST_ICON, output.Transaction.GetName(), link) + message = fmt.Sprintf("%s %s (%s)\n", FAILED_TEST_ICON, output.Transaction.GetName(), output.RunWebURL) } // the transaction name + all steps @@ -131,5 +129,5 @@ func (f transactionRun) getColoredText(passed bool, text string) string { } func (f transactionRun) getRunLink(transactionID, runID string) string { - return fmt.Sprintf("%s://%s/transaction/%s/run/%s", f.config.Scheme, f.config.Endpoint, transactionID, runID) + return fmt.Sprintf("%s/transaction/%s/run/%s", f.baseURLFn(), transactionID, runID) } diff --git a/cli/go.mod b/cli/go.mod index f198b1f555..2b379c615d 100644 --- a/cli/go.mod +++ b/cli/go.mod @@ -7,6 +7,7 @@ require ( github.com/alexeyco/simpletable v1.0.0 github.com/compose-spec/compose-go v1.5.1 github.com/cucumber/ci-environment/go v0.0.0-20230703185945-ddbd134c44fd + github.com/davecgh/go-spew v1.1.1 github.com/denisbrodbeck/machineid v1.0.1 github.com/goccy/go-yaml v1.11.0 github.com/kubeshop/tracetest/server v0.0.0-20230512142545-cb5e526e06f9 @@ -17,6 +18,8 @@ require ( github.com/stretchr/testify v1.8.4 go.uber.org/zap v1.23.0 golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15 + golang.org/x/text v0.8.0 + gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -26,7 +29,6 @@ require ( github.com/benbjohnson/clock v1.3.0 // indirect github.com/containerd/console v1.0.3 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect github.com/distribution/distribution/v3 v3.0.0-20220907155224-78b9c98c5c31 // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.5.0 // indirect @@ -79,10 +81,8 @@ require ( golang.org/x/sync v0.1.0 // indirect golang.org/x/sys v0.6.0 // indirect golang.org/x/term v0.5.0 // indirect - golang.org/x/text v0.8.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/grpc v1.53.0 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/cli/pkg/resourcemanager/apply.go b/cli/pkg/resourcemanager/apply.go index 59fbf76646..837d9dd8c5 100644 --- a/cli/pkg/resourcemanager/apply.go +++ b/cli/pkg/resourcemanager/apply.go @@ -2,21 +2,67 @@ package resourcemanager import ( "context" + "errors" "fmt" "io" "net/http" + "net/http/httputil" "github.com/Jeffail/gabs/v2" "github.com/kubeshop/tracetest/cli/pkg/fileutil" + "go.uber.org/zap" ) const VerbApply Verb = "apply" type applyPreProcessorFn func(context.Context, fileutil.File) (fileutil.File, error) +func (c Client) validType(inputFile fileutil.File) error { + c.logger.Debug("Validating resource type", zap.String("inputFile", inputFile.AbsPath())) + + var raw any + err := (yamlFormat{}).Unmarshal(inputFile.Contents(), &raw) + if err != nil { + return fmt.Errorf("cannot unmarshal yaml: %w", err) + } + c.logger.Debug("Unmarshaled yaml", zap.Any("raw", raw)) + + parsed := gabs.Wrap(raw) + rawType := parsed.Path("type").Data() + if rawType == nil { + return errors.New("cannot find type in yaml") + } + c.logger.Debug("Found type", zap.String("type", fmt.Sprintf("%T", rawType))) + t, ok := rawType.(string) + if !ok { + return fmt.Errorf("cannot parse type from yaml: %w", err) + } + c.logger.Debug("Parsed type", zap.String("type", t)) + + if t != c.resourceType() { + return fmt.Errorf("cannot apply %s to %s resource", t, c.resourceType()) + } + + c.logger.Debug("resource type is valid") + + return nil + +} + func (c Client) Apply(ctx context.Context, inputFile fileutil.File, requestedFormat Format) (string, error) { originalInputFile := inputFile + if err := c.validType(inputFile); err != nil { + return "", err + } + + c.logger.Debug("Applying resource", + zap.String("format", requestedFormat.String()), + zap.String("resource", c.resourceName), + zap.String("inputFile", inputFile.AbsPath()), + zap.String("contents", string(inputFile.Contents())), + ) + if c.options.applyPreProcessor != nil { var err error inputFile, err = c.options.applyPreProcessor(ctx, inputFile) @@ -25,6 +71,11 @@ func (c Client) Apply(ctx context.Context, inputFile fileutil.File, requestedFor } } + c.logger.Debug("preprocessed", + zap.String("inputFile", inputFile.AbsPath()), + zap.String("contents", string(inputFile.Contents())), + ) + url := c.client.url(c.resourceNamePlural) req, err := http.NewRequestWithContext(ctx, http.MethodPut, url.String(), inputFile.Reader()) if err != nil { @@ -39,11 +90,7 @@ func (c Client) Apply(ctx context.Context, inputFile fileutil.File, requestedFor // the files must be in yaml format, so we can safely force the content type, // even if it doesn't matcht he user's requested format - yamlFormat, err := Formats.Get(FormatYAML) - if err != nil { - return "", fmt.Errorf("cannot get json format: %w", err) - } - req.Header.Set("Content-Type", yamlFormat.ContentType()) + req.Header.Set("Content-Type", (yamlFormat{}).ContentType()) // final request looks like this: // PUT {server}/{resourceNamePlural} @@ -54,12 +101,22 @@ func (c Client) Apply(ctx context.Context, inputFile fileutil.File, requestedFor // // This means that we'll send the request body as YAML (read from the user provided file) // and we'll get the reponse in the users's requrested format. + d, _ := httputil.DumpRequestOut(req, true) + c.logger.Debug("apply request", + zap.String("request", string(d)), + ) + resp, err := c.client.do(req) if err != nil { return "", fmt.Errorf("cannot execute Apply request: %w", err) } defer resp.Body.Close() + d, _ = httputil.DumpResponse(resp, true) + c.logger.Debug("apply response", + zap.String("response", string(d)), + ) + if !isSuccessResponse(resp) { err := parseRequestError(resp, requestedFormat) @@ -71,10 +128,10 @@ func (c Client) Apply(ctx context.Context, inputFile fileutil.File, requestedFor return "", fmt.Errorf("cannot read Apply response: %w", err) } + c.logger.Debug("file has id?", zap.Bool("hasID", originalInputFile.HasID())) // if the original file doesn't have an ID, we need to get the server generated ID from the response // and write it to the original file if !originalInputFile.HasID() { - jsonBody, err := requestedFormat.ToJSON(body) if err != nil { return "", fmt.Errorf("cannot convert response body to JSON format: %w", err) @@ -90,6 +147,8 @@ func (c Client) Apply(ctx context.Context, inputFile fileutil.File, requestedFor return "", fmt.Errorf("cannot get ID from Apply response") } + c.logger.Debug("New ID", zap.String("id", id)) + originalInputFile, err = originalInputFile.SetID(id) if err != nil { return "", fmt.Errorf("cannot set ID on input file: %w", err) @@ -99,7 +158,6 @@ func (c Client) Apply(ctx context.Context, inputFile fileutil.File, requestedFor if err != nil { return "", fmt.Errorf("cannot write updated input file: %w", err) } - } return requestedFormat.Format(string(body), c.options.tableConfig) diff --git a/cli/pkg/resourcemanager/client.go b/cli/pkg/resourcemanager/client.go index 9f18cd8226..31b60390f2 100644 --- a/cli/pkg/resourcemanager/client.go +++ b/cli/pkg/resourcemanager/client.go @@ -7,6 +7,10 @@ import ( "net/url" "path" "strings" + + "go.uber.org/zap" + "golang.org/x/text/cases" + "golang.org/x/text/language" ) type Verb string @@ -15,6 +19,7 @@ type Client struct { client *HTTPClient resourceName string resourceNamePlural string + logger *zap.Logger options options } @@ -60,12 +65,14 @@ func (c HTTPClient) do(req *http.Request) (*http.Response, error) { // This configuration work both for a single resource from a Get, or a ResourceList from a List func NewClient( httpClient *HTTPClient, + logger *zap.Logger, resourceName, resourceNamePlural string, opts ...option) Client { c := Client{ client: httpClient, resourceName: resourceName, resourceNamePlural: resourceNamePlural, + logger: logger, } for _, opt := range opts { @@ -75,6 +82,20 @@ func NewClient( return c } +func (c Client) resourceType() string { + if c.options.resourceType != "" { + return c.options.resourceType + } + // language.Und means Undefined + caser := cases.Title(language.Und, cases.NoLower) + return caser.String(c.resourceName) +} + +var ErrNotFound = requestError{ + Code: http.StatusNotFound, + Message: "Resource not found", +} + type requestError struct { Code int `json:"code"` Message string `json:"error"` @@ -84,6 +105,11 @@ func (e requestError) Error() string { return e.Message } +func (e requestError) Is(target error) bool { + t, ok := target.(requestError) + return ok && t.Code == e.Code +} + func isSuccessResponse(resp *http.Response) bool { // successfull http status codes are 2xx return resp.StatusCode >= 200 && resp.StatusCode < 300 diff --git a/cli/pkg/resourcemanager/delete.go b/cli/pkg/resourcemanager/delete.go index cf4302378a..3ecaa8e58e 100644 --- a/cli/pkg/resourcemanager/delete.go +++ b/cli/pkg/resourcemanager/delete.go @@ -2,6 +2,7 @@ package resourcemanager import ( "context" + "errors" "fmt" "net/http" "strings" @@ -29,9 +30,8 @@ func (c Client) Delete(ctx context.Context, id string, format Format) (string, e if !isSuccessResponse(resp) { err := parseRequestError(resp, format) - reqErr, ok := err.(requestError) - if ok && reqErr.Code == http.StatusNotFound { - return "", fmt.Errorf("Resource %s with ID %s not found", c.resourceName, id) + if errors.Is(err, ErrNotFound) { + return fmt.Sprintf("Resource %s with ID %s not found", c.resourceName, id), ErrNotFound } return "", fmt.Errorf("could not Delete resource: %w", err) diff --git a/cli/pkg/resourcemanager/format.go b/cli/pkg/resourcemanager/format.go index 88206d8da9..e2db674306 100644 --- a/cli/pkg/resourcemanager/format.go +++ b/cli/pkg/resourcemanager/format.go @@ -28,20 +28,25 @@ func (f formatRegistry) GetWithFallback(format, fallback string) (Format, error) } if format == "" { - return f.Get(fallback) + format = fallback } - return f.Get(format) + actualFormat := f.Get(format) + if actualFormat == nil { + return nil, fmt.Errorf("unknown format '%s'", format) + } + + return actualFormat, nil } -func (f formatRegistry) Get(format string) (Format, error) { +func (f formatRegistry) Get(format string) Format { for _, fr := range f { if fr.String() == format { - return fr, nil + return fr } } - return nil, fmt.Errorf("format '%s' not supported", format) + return nil } var Formats = formatRegistry{ diff --git a/cli/pkg/resourcemanager/get.go b/cli/pkg/resourcemanager/get.go index 89db55160b..f266b7183f 100644 --- a/cli/pkg/resourcemanager/get.go +++ b/cli/pkg/resourcemanager/get.go @@ -2,9 +2,13 @@ package resourcemanager import ( "context" + "errors" "fmt" "io" "net/http" + "net/http/httputil" + + "go.uber.org/zap" ) const VerbGet Verb = "get" @@ -20,6 +24,10 @@ func (c Client) Get(ctx context.Context, id string, format Format) (string, erro if err != nil { return "", fmt.Errorf("cannot build Get request: %w", err) } + d, _ := httputil.DumpRequestOut(req, true) + c.logger.Debug("get request", + zap.String("request", string(d)), + ) resp, err := c.client.do(req) if err != nil { @@ -27,14 +35,19 @@ func (c Client) Get(ctx context.Context, id string, format Format) (string, erro } defer resp.Body.Close() + d, _ = httputil.DumpResponse(resp, true) + c.logger.Debug("apply response", + zap.String("response", string(d)), + ) + if !isSuccessResponse(resp) { err := parseRequestError(resp, format) - reqErr, ok := err.(requestError) - if ok && reqErr.Code == http.StatusNotFound { - return fmt.Sprintf("Resource %s with ID %s not found", c.resourceName, id), nil + if errors.Is(err, ErrNotFound) { + return fmt.Sprintf("Resource %s with ID %s not found", c.resourceName, id), ErrNotFound } return "", fmt.Errorf("could not Get resource: %w", err) + } body, err := io.ReadAll(resp.Body) diff --git a/cli/pkg/resourcemanager/options.go b/cli/pkg/resourcemanager/options.go index c3200b64eb..6984621992 100644 --- a/cli/pkg/resourcemanager/options.go +++ b/cli/pkg/resourcemanager/options.go @@ -4,6 +4,7 @@ type options struct { applyPreProcessor applyPreProcessorFn tableConfig TableConfig deleteSuccessMsg string + resourceType string } type option func(*options) @@ -25,3 +26,9 @@ func WithTableConfig(tableConfig TableConfig) option { o.tableConfig = tableConfig } } + +func WithResourceType(resourceType string) option { + return func(o *options) { + o.resourceType = resourceType + } +} diff --git a/cli/runner/env_vars.go b/cli/runner/env_vars.go new file mode 100644 index 0000000000..f75c15d789 --- /dev/null +++ b/cli/runner/env_vars.go @@ -0,0 +1,98 @@ +package runner + +import ( + "fmt" + + "github.com/kubeshop/tracetest/cli/openapi" + "github.com/kubeshop/tracetest/cli/ui" +) + +func askForMissingVars(missingVars []envVar) []envVar { + ui.DefaultUI.Warning("Some variables are required by one or more tests") + ui.DefaultUI.Info("Fill the values for each variable:") + + filledVariables := make([]envVar, 0, len(missingVars)) + + for _, missingVar := range missingVars { + answer := missingVar + answer.UserValue = ui.DefaultUI.TextInput(missingVar.Name, missingVar.DefaultValue) + filledVariables = append(filledVariables, answer) + } + + return filledVariables +} + +type envVar struct { + Name string + DefaultValue string + UserValue string +} + +func (ev envVar) value() string { + if ev.UserValue != "" { + return ev.UserValue + } + + return ev.DefaultValue +} + +type envVars []envVar + +func (evs envVars) toOpenapi() []openapi.EnvironmentValue { + vars := make([]openapi.EnvironmentValue, len(evs)) + for i, ev := range evs { + vars[i] = openapi.EnvironmentValue{ + Key: openapi.PtrString(ev.Name), + Value: openapi.PtrString(ev.value()), + } + } + + return vars +} + +func (evs envVars) unique() envVars { + seen := make(map[string]bool) + vars := make(envVars, 0, len(evs)) + for _, ev := range evs { + if seen[ev.Name] { + continue + } + + seen[ev.Name] = true + vars = append(vars, ev) + } + + return vars +} + +type missingEnvVarsError envVars + +func (e missingEnvVarsError) Error() string { + return fmt.Sprintf("missing env vars: %v", []envVar(e)) +} + +func (e missingEnvVarsError) Is(target error) bool { + _, ok := target.(missingEnvVarsError) + return ok +} + +func buildMissingEnvVarsError(body []byte) error { + var missingVarsErrResp openapi.MissingVariablesError + err := jsonFormat.Unmarshal(body, &missingVarsErrResp) + if err != nil { + return fmt.Errorf("could not unmarshal response body: %w", err) + } + + missingVars := envVars{} + + for _, missingVarErr := range missingVarsErrResp.MissingVariables { + for _, missingVar := range missingVarErr.Variables { + missingVars = append(missingVars, envVar{ + Name: missingVar.GetKey(), + DefaultValue: missingVar.GetDefaultValue(), + }) + } + } + + return missingEnvVarsError(missingVars.unique()) +} diff --git a/cli/runner/orchestrator.go b/cli/runner/orchestrator.go new file mode 100644 index 0000000000..b620ae68b2 --- /dev/null +++ b/cli/runner/orchestrator.go @@ -0,0 +1,375 @@ +package runner + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "os" + "sync" + "time" + + cienvironment "github.com/cucumber/ci-environment/go" + "github.com/davecgh/go-spew/spew" + "github.com/kubeshop/tracetest/cli/openapi" + "github.com/kubeshop/tracetest/cli/pkg/fileutil" + "github.com/kubeshop/tracetest/cli/pkg/resourcemanager" + "github.com/kubeshop/tracetest/cli/variable" + "go.uber.org/zap" + "gopkg.in/yaml.v2" +) + +// RunOptions defines options for running a resource +// ID and DefinitionFile are mutually exclusive and the only required options +type RunOptions struct { + // ID of the resource to run + ID string + + // path to the file with resource definition + // the file will be applied before running + DefinitionFile string + + // environmentID or path to the file with environment definition + EnvID string + + // By default the runner will wait for the result of the run + // if this option is true, the wait will be skipped + SkipResultWait bool + + // Optional path to the file where the result of the run will be saved + // in JUnit xml format + JUnitOuptutFile string +} + +// RunResult holds the result of the run +// Resources +type RunResult struct { + // The resource being run. If has been preprocessed, this needs to be the updated version + Resource any + + // The result of the run. It can be anything the resource needs for validating and formatting the result + Run any + + // If true, it means that the current run is ready to be presented to the user + Finished bool + + // Whether the run has passed or not. Used to determine exit code + Passed bool +} + +// Runner defines interface for running a resource +type Runner interface { + // Name of the runner. must match the resource name it handles + Name() string + + // Apply the given file and return a resource. The resource can be of any type. + // It will then be used by Run method + Apply(context.Context, fileutil.File) (resource any, _ error) + + // GetByID gets the resource by ID. This method is used to get the resource when running from id + GetByID(_ context.Context, id string) (resource any, _ error) + + // StartRun starts running the resource and return the result. This method should not wait for the test to finish + StartRun(_ context.Context, resource any, _ openapi.RunInformation) (RunResult, error) + + // UpdateResult is regularly called by the orchestrator to check the status of the run + UpdateResult(context.Context, RunResult) (RunResult, error) + + // JUnitResult returns the result of the run in JUnit format + JUnitResult(context.Context, RunResult) (string, error) + + // Format the result of the run into a string + FormatResult(_ RunResult, format string) string +} + +func Orchestrator( + logger *zap.Logger, + openapiClient *openapi.APIClient, + environments resourcemanager.Client, +) orchestrator { + return orchestrator{ + logger: logger, + openapiClient: openapiClient, + environments: environments, + } +} + +type orchestrator struct { + logger *zap.Logger + + openapiClient *openapi.APIClient + environments resourcemanager.Client +} + +var ( + yamlFormat = resourcemanager.Formats.Get(resourcemanager.FormatYAML) + jsonFormat = resourcemanager.Formats.Get(resourcemanager.FormatJSON) +) + +const ( + ExitCodeSuccess = 0 + ExitCodeGeneralError = 1 + ExitCodeTestNotPassed = 2 +) + +func (o orchestrator) Run(ctx context.Context, r Runner, opts RunOptions, outputFormat string) (exitCode int, _ error) { + + o.logger.Debug( + "Running test from definition", + zap.String("definitionFile", opts.DefinitionFile), + zap.String("ID", opts.ID), + zap.String("envID", opts.EnvID), + zap.Bool("skipResultsWait", opts.SkipResultWait), + zap.String("junitOutputFile", opts.JUnitOuptutFile), + ) + + envID, err := o.resolveEnvID(ctx, opts.EnvID) + if err != nil { + return ExitCodeGeneralError, fmt.Errorf("cannot resolve environment id: %w", err) + } + o.logger.Debug("env resolved", zap.String("ID", envID)) + + var resource any + if opts.DefinitionFile != "" { + f, err := fileutil.Read(opts.DefinitionFile) + if err != nil { + return ExitCodeGeneralError, fmt.Errorf("cannot read definition file %s: %w", opts.DefinitionFile, err) + } + df := f + o.logger.Debug("Definition file read", zap.String("absolutePath", df.AbsPath())) + + df, err = o.injectLocalEnvVars(ctx, df) + if err != nil { + return ExitCodeGeneralError, fmt.Errorf("cannot inject local env vars: %w", err) + } + + resource, err = r.Apply(ctx, df) + if err != nil { + return ExitCodeGeneralError, fmt.Errorf("cannot apply definition file: %w", err) + } + o.logger.Debug("Definition file applied", zap.String("updated", string(df.Contents()))) + } else { + o.logger.Debug("Definition file not provided, fetching resource by ID", zap.String("ID", opts.ID)) + resource, err = r.GetByID(ctx, opts.ID) + if err != nil { + return ExitCodeGeneralError, fmt.Errorf("cannot get resource by ID: %w", err) + } + o.logger.Debug("Resource fetched by ID", zap.String("ID", opts.ID), zap.Any("resource", resource)) + } + + var result RunResult + var ev envVars + + // iterate until we have all env vars, + // or the server returns an actual error + for { + runInfo := openapi.RunInformation{ + EnvironmentId: &envID, + Variables: ev.toOpenapi(), + Metadata: getMetadata(), + } + + result, err = r.StartRun(ctx, resource, runInfo) + if err == nil { + break + } + if !errors.Is(err, missingEnvVarsError{}) { + // actual error, return + return ExitCodeGeneralError, fmt.Errorf("cannot run test: %w", err) + } + + // missing vars error + ev = askForMissingVars([]envVar(err.(missingEnvVarsError))) + o.logger.Debug("filled variables", zap.Any("variables", ev)) + } + + if opts.SkipResultWait { + fmt.Println(r.FormatResult(result, outputFormat)) + return ExitCodeSuccess, nil + } + + result, err = o.waitForResult(ctx, r, result) + if err != nil { + return ExitCodeGeneralError, fmt.Errorf("cannot wait for test result: %w", err) + } + + fmt.Println(r.FormatResult(result, outputFormat)) + + err = o.writeJUnitReport(ctx, r, result, opts.JUnitOuptutFile) + if err != nil { + return ExitCodeGeneralError, fmt.Errorf("cannot write junit report: %w", err) + } + + exitCode = ExitCodeSuccess + if !result.Passed { + exitCode = ExitCodeTestNotPassed + } + + return exitCode, nil +} + +func (o orchestrator) resolveEnvID(ctx context.Context, envID string) (string, error) { + if !fileutil.IsFilePath(envID) { + o.logger.Debug("envID is not a file path", zap.String("envID", envID)) + + // validate that env exists + _, err := o.environments.Get(ctx, envID, resourcemanager.Formats.Get(resourcemanager.FormatYAML)) + if errors.Is(err, resourcemanager.ErrNotFound) { + return "", fmt.Errorf("environment '%s' not found", envID) + } + if err != nil { + return "", fmt.Errorf("cannot get environment '%s': %w", envID, err) + } + + o.logger.Debug("envID is valid") + + return envID, nil + } + + f, err := fileutil.Read(envID) + if err != nil { + return "", fmt.Errorf("cannot read environment file %s: %w", envID, err) + } + + o.logger.Debug("envID is a file path", zap.String("filePath", envID), zap.Any("file", f)) + updatedEnv, err := o.environments.Apply(ctx, f, yamlFormat) + if err != nil { + return "", fmt.Errorf("could not read environment file: %w", err) + } + + var env openapi.EnvironmentResource + err = yaml.Unmarshal([]byte(updatedEnv), &env) + if err != nil { + o.logger.Error("error parsing json", zap.String("content", updatedEnv), zap.Error(err)) + return "", fmt.Errorf("could not unmarshal environment json: %w", err) + } + + return env.Spec.GetId(), nil +} + +func (o orchestrator) injectLocalEnvVars(ctx context.Context, df fileutil.File) (fileutil.File, error) { + variableInjector := variable.NewInjector(variable.WithVariableProvider( + variable.EnvironmentVariableProvider{}, + )) + + injected, err := variableInjector.ReplaceInString(string(df.Contents())) + if err != nil { + return df, fmt.Errorf("cannot inject local environment variables: %w", err) + } + + df = fileutil.New(df.AbsPath(), []byte(injected)) + + return df, nil +} + +func (o orchestrator) waitForResult(ctx context.Context, r Runner, result RunResult) (RunResult, error) { + var ( + updatedResult RunResult + lastError error + wg sync.WaitGroup + ) + + wg.Add(1) + ticker := time.NewTicker(1 * time.Second) // TODO: change to websockets + go func() { + for range ticker.C { + updated, err := r.UpdateResult(ctx, result) + o.logger.Debug("updated result", zap.String("result", spew.Sdump(updated))) + if err != nil { + o.logger.Debug("UpdateResult failed", zap.Error(err)) + lastError = err + wg.Done() + return + } + + if updated.Finished { + o.logger.Debug("result is finished") + updatedResult = updated + wg.Done() + return + } + o.logger.Debug("still waiting") + } + }() + wg.Wait() + + if lastError != nil { + return RunResult{}, lastError + } + + return updatedResult, nil +} + +var ErrJUnitNotSupported = errors.New("junit report is not supported for this resource type") + +func (a orchestrator) writeJUnitReport(ctx context.Context, r Runner, result RunResult, outputFile string) error { + if outputFile == "" { + a.logger.Debug("no junit output file specified") + return nil + } + + a.logger.Debug("saving junit report", zap.String("outputFile", outputFile)) + + report, err := r.JUnitResult(ctx, result) + if err != nil { + return err + } + + a.logger.Debug("junit report", zap.String("report", report)) + f, err := os.Create(outputFile) + if err != nil { + return fmt.Errorf("could not create junit output file: %w", err) + } + + _, err = f.WriteString(report) + + return err +} + +func getMetadata() map[string]string { + ci := cienvironment.DetectCIEnvironment() + if ci == nil { + return map[string]string{} + } + + metadata := map[string]string{ + "name": ci.Name, + "url": ci.URL, + "buildNumber": ci.BuildNumber, + } + + if ci.Git != nil { + metadata["branch"] = ci.Git.Branch + metadata["tag"] = ci.Git.Tag + metadata["revision"] = ci.Git.Revision + } + + return metadata +} + +// HandleRunError handles errors returned by the server when running a test. +// It normalizes the handling of general errors, like 404, +// but more importantly, it processes the missing environment variables error +// so the orchestrator can request them from the user. +func HandleRunError(resp *http.Response, reqErr error) error { + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("could not read response body: %w", err) + } + resp.Body.Close() + + if resp.StatusCode == http.StatusNotFound { + return fmt.Errorf("resource not found in server") + } + + if resp.StatusCode == http.StatusUnprocessableEntity { + return buildMissingEnvVarsError(body) + } + + if reqErr != nil { + return fmt.Errorf("could not run transaction: %w", err) + } + + return nil +} diff --git a/cli/runner/registry.go b/cli/runner/registry.go new file mode 100644 index 0000000000..ab80a0a76e --- /dev/null +++ b/cli/runner/registry.go @@ -0,0 +1,35 @@ +package runner + +import ( + "fmt" +) + +type Registry map[string]Runner + +func NewRegistry() Registry { + return Registry{} +} + +func (r Registry) Register(runner Runner) Registry { + r[runner.Name()] = runner + return r +} + +var ErrNotFound = fmt.Errorf("runner not found") + +func (r Registry) Get(name string) (Runner, error) { + if runner, ok := r[name]; ok { + return runner, nil + } + + return nil, ErrNotFound +} + +func (r Registry) List() []string { + var list []string + for name := range r { + list = append(list, name) + } + + return list +} diff --git a/cli/runner/test_runner.go b/cli/runner/test_runner.go new file mode 100644 index 0000000000..d6c0f1d151 --- /dev/null +++ b/cli/runner/test_runner.go @@ -0,0 +1,169 @@ +package runner + +import ( + "context" + "fmt" + "strconv" + + "github.com/kubeshop/tracetest/cli/formatters" + "github.com/kubeshop/tracetest/cli/openapi" + "github.com/kubeshop/tracetest/cli/pkg/fileutil" + "github.com/kubeshop/tracetest/cli/pkg/resourcemanager" +) + +type testFormatter interface { + Format(output formatters.TestRunOutput, format formatters.Output) string +} + +type testRunner struct { + client resourcemanager.Client + openapiClient *openapi.APIClient + formatter testFormatter +} + +func TestRunner( + client resourcemanager.Client, + openapiClient *openapi.APIClient, + formatter testFormatter, +) Runner { + return testRunner{ + client: client, + openapiClient: openapiClient, + formatter: formatter, + } +} + +func (r testRunner) Name() string { + return "test" +} + +func (r testRunner) GetByID(ctx context.Context, id string) (resource any, _ error) { + jsonTest, err := r.client.Get(ctx, id, jsonFormat) + if err != nil { + return nil, fmt.Errorf("cannot get test '%s': %w", id, err) + } + + var test openapi.TestResource + err = jsonFormat.Unmarshal([]byte(jsonTest), &test) + if err != nil { + return nil, fmt.Errorf("cannot unmarshal test definition file: %w", err) + } + + return test, nil +} + +func (r testRunner) Apply(ctx context.Context, df fileutil.File) (resource any, _ error) { + updated, err := r.client.Apply(ctx, df, yamlFormat) + if err != nil { + return nil, fmt.Errorf("could not read test file: %w", err) + } + + var parsed openapi.TestResource + err = yamlFormat.Unmarshal([]byte(updated), &parsed) + if err != nil { + return nil, fmt.Errorf("cannot unmarshal test definition file: %w", err) + } + + return parsed, nil +} + +func (r testRunner) StartRun(ctx context.Context, resource any, runInfo openapi.RunInformation) (RunResult, error) { + test := resource.(openapi.TestResource) + run, resp, err := r.openapiClient.ApiApi. + RunTest(ctx, test.Spec.GetId()). + RunInformation(runInfo). + Execute() + + err = HandleRunError(resp, err) + if err != nil { + return RunResult{}, err + } + + full, err := r.client.Get(ctx, test.Spec.GetId(), jsonFormat) + if err != nil { + return RunResult{}, fmt.Errorf("cannot get full test '%s': %w", test.Spec.GetId(), err) + } + err = jsonFormat.Unmarshal([]byte(full), &test) + if err != nil { + return RunResult{}, fmt.Errorf("cannot get full test '%s': %w", test.Spec.GetId(), err) + } + + return RunResult{ + Resource: test, + Run: *run, + }, nil +} + +func (r testRunner) UpdateResult(ctx context.Context, result RunResult) (RunResult, error) { + test := result.Resource.(openapi.TestResource) + run := result.Run.(openapi.TestRun) + runID, err := strconv.Atoi(run.GetId()) + if err != nil { + return RunResult{}, fmt.Errorf("invalid test run id format: %w", err) + } + + updated, _, err := r.openapiClient.ApiApi. + GetTestRun(ctx, test.Spec.GetId(), int32(runID)). + Execute() + + if err != nil { + return RunResult{}, err + } + + passed := !isStateFailed(updated.GetState()) && updated.Result.GetAllPassed() + + return RunResult{ + Resource: test, + Run: *updated, + Finished: isStateFinished(updated.GetState()), + Passed: passed, + }, nil +} + +func (r testRunner) JUnitResult(ctx context.Context, result RunResult) (string, error) { + test := result.Resource.(openapi.TestResource) + run := result.Run.(openapi.TestRun) + runID, err := strconv.Atoi(run.GetId()) + if err != nil { + return "", fmt.Errorf("invalid run id format: %w", err) + } + + junit, _, err := r.openapiClient.ApiApi. + GetRunResultJUnit( + ctx, + test.Spec.GetId(), + int32(runID), + ). + Execute() + if err != nil { + return "", fmt.Errorf("could not execute request: %w", err) + } + + return junit, nil +} + +func (r testRunner) FormatResult(result RunResult, format string) string { + test := result.Resource.(openapi.TestResource) + run := result.Run.(openapi.TestRun) + + tro := formatters.TestRunOutput{ + HasResults: result.Finished, + IsFailed: isStateFailed(run.GetState()), + Test: test.GetSpec(), + Run: run, + } + + return r.formatter.Format(tro, formatters.Output(format)) +} + +func isStateFinished(state string) bool { + return isStateFailed(state) || state == "FINISHED" +} + +func isStateFailed(state string) bool { + return state == "TRIGGER_FAILED" || + state == "TRACE_FAILED" || + state == "ASSERTION_FAILED" || + state == "ANALYZING_ERROR" || + state == "FAILED" // this one is for backwards compatibility +} diff --git a/cli/runner/transaction_runner.go b/cli/runner/transaction_runner.go new file mode 100644 index 0000000000..f86d6a3c36 --- /dev/null +++ b/cli/runner/transaction_runner.go @@ -0,0 +1,146 @@ +package runner + +import ( + "context" + "fmt" + "strconv" + + "github.com/kubeshop/tracetest/cli/formatters" + "github.com/kubeshop/tracetest/cli/openapi" + "github.com/kubeshop/tracetest/cli/pkg/fileutil" + "github.com/kubeshop/tracetest/cli/pkg/resourcemanager" +) + +type transactionFormatter interface { + Format(output formatters.TransactionRunOutput, format formatters.Output) string +} + +type transactionRunner struct { + client resourcemanager.Client + openapiClient *openapi.APIClient + formatter transactionFormatter +} + +func TransactionRunner( + client resourcemanager.Client, + openapiClient *openapi.APIClient, + formatter transactionFormatter, +) Runner { + return transactionRunner{ + client: client, + openapiClient: openapiClient, + formatter: formatter, + } +} + +func (r transactionRunner) Name() string { + return "transaction" +} + +func (r transactionRunner) GetByID(_ context.Context, id string) (resource any, _ error) { + jsonTransaction, err := r.client.Get(context.Background(), id, jsonFormat) + if err != nil { + return nil, fmt.Errorf("cannot get transaction '%s': %w", id, err) + } + + var transaction openapi.TransactionResource + err = jsonFormat.Unmarshal([]byte(jsonTransaction), &transaction) + if err != nil { + return nil, fmt.Errorf("cannot unmarshal transaction definition file: %w", err) + } + + return transaction, nil +} + +func (r transactionRunner) Apply(ctx context.Context, df fileutil.File) (resource any, _ error) { + updated, err := r.client.Apply(ctx, df, yamlFormat) + if err != nil { + return nil, fmt.Errorf("could not read transaction file: %w", err) + } + + var parsed openapi.TransactionResource + err = yamlFormat.Unmarshal([]byte(updated), &parsed) + if err != nil { + return nil, fmt.Errorf("cannot unmarshal transaction definition file: %w", err) + } + + return parsed, nil +} + +func (r transactionRunner) StartRun(ctx context.Context, resource any, runInfo openapi.RunInformation) (RunResult, error) { + tran := resource.(openapi.TransactionResource) + run, resp, err := r.openapiClient.ApiApi. + RunTransaction(ctx, tran.Spec.GetId()). + RunInformation(runInfo). + Execute() + + err = HandleRunError(resp, err) + if err != nil { + return RunResult{}, err + } + + full, err := r.client.Get(ctx, tran.Spec.GetId(), jsonFormat) + if err != nil { + return RunResult{}, fmt.Errorf("cannot get full transaction '%s': %w", tran.Spec.GetId(), err) + } + err = jsonFormat.Unmarshal([]byte(full), &tran) + if err != nil { + return RunResult{}, fmt.Errorf("cannot get full transaction '%s': %w", tran.Spec.GetId(), err) + } + + return RunResult{ + Resource: tran, + Run: *run, + }, nil +} + +func (r transactionRunner) UpdateResult(ctx context.Context, result RunResult) (RunResult, error) { + transaction := result.Resource.(openapi.TransactionResource) + run := result.Run.(openapi.TransactionRun) + runID, err := strconv.Atoi(run.GetId()) + if err != nil { + return RunResult{}, fmt.Errorf("invalid transaction run id format: %w", err) + } + + updated, _, err := r.openapiClient.ApiApi. + GetTransactionRun(ctx, transaction.Spec.GetId(), int32(runID)). + Execute() + + if err != nil { + return RunResult{}, err + } + + allPassed := true + for _, s := range updated.GetSteps() { + if !s.Result.GetAllPassed() { + allPassed = false + break + } + } + + passed := !isStateFailed(updated.GetState()) && allPassed + + return RunResult{ + Resource: transaction, + Run: *updated, + Finished: isStateFinished(updated.GetState()), + Passed: passed, + }, nil +} + +func (r transactionRunner) JUnitResult(ctx context.Context, result RunResult) (string, error) { + return "", ErrJUnitNotSupported +} + +func (r transactionRunner) FormatResult(result RunResult, format string) string { + transaction := result.Resource.(openapi.TransactionResource) + run := result.Run.(openapi.TransactionRun) + + tro := formatters.TransactionRunOutput{ + HasResults: result.Finished, + Transaction: transaction.GetSpec(), + Run: run, + } + + return r.formatter.Format(tro, formatters.Output(format)) +} diff --git a/examples/keptn-integration/job-config.yaml b/examples/keptn-integration/job-config.yaml index ae97c356eb..c5ad3a8141 100644 --- a/examples/keptn-integration/job-config.yaml +++ b/examples/keptn-integration/job-config.yaml @@ -14,8 +14,8 @@ actions: args: - --config - /keptn/data/tracetest-cli-config.yaml - - test - run - - --definition + - test + - --file - /keptn/data/test-definition.yaml - - --wait-for-result + diff --git a/examples/quick-start-github-actions/.github/workflows/start-and-test-on-main.yaml b/examples/quick-start-github-actions/.github/workflows/start-and-test-on-main.yaml index ae154cd988..7acdd03f1e 100644 --- a/examples/quick-start-github-actions/.github/workflows/start-and-test-on-main.yaml +++ b/examples/quick-start-github-actions/.github/workflows/start-and-test-on-main.yaml @@ -26,9 +26,9 @@ jobs: - name: Run tests via the Tracetest CLI run: | - tracetest test run -d ./tracetest/tests/test-api.yaml -w - tracetest test run -d ./tracetest/tests/test-api-and-av.yaml -w - tracetest test run -d ./tracetest/tests/transaction-api.yaml -w + tracetest run test -f ./tracetest/tests/test-api.yaml + tracetest run test -f ./tracetest/tests/test-api-and-av.yaml + tracetest run test -f ./tracetest/tests/transaction-api.yaml - name: Stop containers if: always() diff --git a/examples/quick-start-github-actions/.github/workflows/start-and-test-on-schedule.yaml b/examples/quick-start-github-actions/.github/workflows/start-and-test-on-schedule.yaml index 8d3913085c..fbdd065ab6 100644 --- a/examples/quick-start-github-actions/.github/workflows/start-and-test-on-schedule.yaml +++ b/examples/quick-start-github-actions/.github/workflows/start-and-test-on-schedule.yaml @@ -24,9 +24,9 @@ jobs: - name: Run tests via the Tracetest CLI run: | - tracetest test run -d ./tracetest/tests/test-api.yaml -w - tracetest test run -d ./tracetest/tests/test-api-and-av.yaml -w - tracetest test run -d ./tracetest/tests/transaction-api.yaml -w + tracetest run test -f ./tracetest/tests/test-api.yaml + tracetest run test -f ./tracetest/tests/test-api-and-av.yaml + tracetest run test -f ./tracetest/tests/transaction-api.yaml - name: Stop containers if: always() diff --git a/examples/quick-start-tekton/install-and-run-tracetest.yaml b/examples/quick-start-tekton/install-and-run-tracetest.yaml index ff25780e4e..5996676bb8 100644 --- a/examples/quick-start-tekton/install-and-run-tracetest.yaml +++ b/examples/quick-start-tekton/install-and-run-tracetest.yaml @@ -36,10 +36,10 @@ spec: script: | # Configure and Run Tracetest CLI tracetest configure -g --endpoint http://tracetest.tracetest.svc.cluster.local:11633/ - tracetest test run -d /workspace/test-api.yaml -w + tracetest run test -f /workspace/test-api.yaml volumeMounts: - name: custom mountPath: /workspace volumes: - name: custom - emptyDir: {} \ No newline at end of file + emptyDir: {} diff --git a/examples/table-driven-test/table-test.bash b/examples/table-driven-test/table-test.bash index 7fcb5f9255..df99e645c7 100755 --- a/examples/table-driven-test/table-test.bash +++ b/examples/table-driven-test/table-test.bash @@ -44,7 +44,7 @@ EOF echo "Running test" $TRACETEST apply environment --file "${envFile}" > /dev/null - $TRACETEST test run --definition "${TEST_DEFINITION}" --environment "${envFile}" --wait-for-result + $TRACETEST run test --file "${TEST_DEFINITION}" --environment "${envFile}" ((line=line+1)) diff --git a/examples/tracetest-synthetic-monitoring/.github/workflows/synthetic-monitoring.yaml b/examples/tracetest-synthetic-monitoring/.github/workflows/synthetic-monitoring.yaml index 02fd3feecd..5c4d4562f5 100644 --- a/examples/tracetest-synthetic-monitoring/.github/workflows/synthetic-monitoring.yaml +++ b/examples/tracetest-synthetic-monitoring/.github/workflows/synthetic-monitoring.yaml @@ -43,7 +43,7 @@ jobs: - name: Run syntethic monitoring tests id: monitoring run: | - tracetest test run -d test-api.yaml + tracetest test run -f test-api.yaml - name: Send custom JSON data to Slack workflow if: ${{ failure() }} diff --git a/examples/tracetesting-kubernetes/test-apiserver.bash b/examples/tracetesting-kubernetes/test-apiserver.bash index a5eeff617a..bf3507a980 100755 --- a/examples/tracetesting-kubernetes/test-apiserver.bash +++ b/examples/tracetesting-kubernetes/test-apiserver.bash @@ -74,7 +74,7 @@ TRACE_ID=$(cat $traces | jq -r '.data | first' | jq -r '.traceID') echo "-> TraceID": $TRACE_ID testFile=$(mktemp) -tracetestCommand="tracetest test run --definition $testFile --wait-for-result" +tracetestCommand="tracetest run test --file $testFile" cat $TEST_FILE | sed "8s/.*/ id: $TRACE_ID/" > $testFile if [ "$DEBUG" == "yes" ]; then diff --git a/examples/tracetesting-kubernetes/test-kubelet.bash b/examples/tracetesting-kubernetes/test-kubelet.bash index 0c27a49518..d4bbccbc40 100755 --- a/examples/tracetesting-kubernetes/test-kubelet.bash +++ b/examples/tracetesting-kubernetes/test-kubelet.bash @@ -84,7 +84,7 @@ TRACE_ID=$(cat $traces | jq -r '.data | first' | jq -r '.traceID') echo "-> TraceID": $TRACE_ID testFile=$(mktemp) -tracetestCommand="tracetest test run --definition $testFile --wait-for-result" +tracetestCommand="tracetest run test --file $testFile" cat $TEST_FILE | sed "8s/.*/ id: $TRACE_ID/" > $testFile DEBUG=yes diff --git a/go.work.sum b/go.work.sum index f86509c926..5e885c52f9 100644 --- a/go.work.sum +++ b/go.work.sum @@ -400,9 +400,6 @@ google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c/go.mod h1:rZS5c/ZV google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd/go.mod h1:cTsE614GARnxrLsqKREzmNYJACSWWpAWdNMwnD7c2BE= google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= -google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= diff --git a/testing/cli-e2etest/Makefile b/testing/cli-e2etest/Makefile index 83d7c66b04..7e4d6a1717 100644 --- a/testing/cli-e2etest/Makefile +++ b/testing/cli-e2etest/Makefile @@ -15,7 +15,7 @@ test: # run tests for this application export TAG=$(TAG); \ export ENABLE_CLI_DEBUG=$(ENABLE_CLI_DEBUG); \ go clean -testcache; \ - go test -timeout 300s -p 1 ./...; + go test -v -timeout 300s -p 1 ./...; test/debug: # run tests for this application with debug mode enabled export ENABLE_CLI_DEBUG="true"; \ diff --git a/testing/cli-e2etest/environment/jaeger/server-setup/tracetest-provision.yaml b/testing/cli-e2etest/environment/jaeger/server-setup/tracetest-provision.yaml index a47e7af05e..2e91879e12 100644 --- a/testing/cli-e2etest/environment/jaeger/server-setup/tracetest-provision.yaml +++ b/testing/cli-e2etest/environment/jaeger/server-setup/tracetest-provision.yaml @@ -5,5 +5,5 @@ spec: strategy: periodic default: true periodic: - retryDelay: 5s - timeout: 10m + retryDelay: 500ms + timeout: 1m diff --git a/testing/cli-e2etest/testscenarios/pollingprofile/get_pollingprofile_test.go b/testing/cli-e2etest/testscenarios/pollingprofile/get_pollingprofile_test.go index fb8ceddb4f..3eb1a442ea 100644 --- a/testing/cli-e2etest/testscenarios/pollingprofile/get_pollingprofile_test.go +++ b/testing/cli-e2etest/testscenarios/pollingprofile/get_pollingprofile_test.go @@ -47,8 +47,8 @@ func TestGetPollingProfile(t *testing.T) { require.Equal("Default", pollingProfile.Spec.Name) require.True(pollingProfile.Spec.Default) require.Equal("periodic", pollingProfile.Spec.Strategy) - require.Equal("5s", pollingProfile.Spec.Periodic.RetryDelay) - require.Equal("10m", pollingProfile.Spec.Periodic.Timeout) + require.Equal("500ms", pollingProfile.Spec.Periodic.RetryDelay) + require.Equal("1m", pollingProfile.Spec.Periodic.Timeout) }) addGetPollingProfilePreReqs(t, env) diff --git a/testing/cli-e2etest/testscenarios/test/run_test_with_http_trigger_and_environment_file_test.go b/testing/cli-e2etest/testscenarios/test/run_test_test.go similarity index 60% rename from testing/cli-e2etest/testscenarios/test/run_test_with_http_trigger_and_environment_file_test.go rename to testing/cli-e2etest/testscenarios/test/run_test_test.go index 9e41dc9426..7e59e0a04d 100644 --- a/testing/cli-e2etest/testscenarios/test/run_test_with_http_trigger_and_environment_file_test.go +++ b/testing/cli-e2etest/testscenarios/test/run_test_test.go @@ -11,6 +11,32 @@ import ( "github.com/stretchr/testify/require" ) +func TestRunTransactionInsteadOfTest(t *testing.T) { + t.Run("should fail if transaction resource is selected", func(t *testing.T) { + // setup isolated e2e environment + env := environment.CreateAndStart(t) + defer env.Close(t) + + cliConfig := env.GetCLIConfigPath(t) + + // instantiate require with testing helper + require := require.New(t) + + // Given I am a Tracetest CLI user + // And I have my server recently created + // And the datasource is already set + + // When I try to run a transaction + // Then it should pass + testFil := env.GetTestResourcePath(t, "import") + + command := fmt.Sprintf("run transaction -f %s", testFil) + result := tracetestcli.Exec(t, command, tracetestcli.WithCLIConfig(cliConfig)) + helpers.RequireExitCodeEqual(t, result, 1) + require.Contains(result.StdErr, "cannot apply Test to Transaction resource") + }) +} + func TestRunTestWithHttpTriggerAndEnvironmentFile(t *testing.T) { // setup isolated e2e environment env := environment.CreateAndStart(t, environment.WithDataStoreEnabled(), environment.WithPokeshop()) @@ -37,7 +63,7 @@ func TestRunTestWithHttpTriggerAndEnvironmentFile(t *testing.T) { environmentFile := env.GetTestResourcePath(t, "environment-file") testFile := env.GetTestResourcePath(t, "http-trigger-with-environment-file") - command := fmt.Sprintf("test run -w -d %s --environment %s", testFile, environmentFile) + command := fmt.Sprintf("run test -f %s --environment %s", testFile, environmentFile) result = tracetestcli.Exec(t, command, tracetestcli.WithCLIConfig(cliConfig)) helpers.RequireExitCodeEqual(t, result, 0) require.Contains(result.StdOut, "✔ It should add a Pokemon correctly") @@ -86,10 +112,54 @@ func TestRunTestWithHttpTriggerAndEnvironmentFile(t *testing.T) { testFile := env.GetTestResourcePath(t, "http-trigger-with-environment-file") - command := fmt.Sprintf("test run -w -d %s --environment pokeapi-env", testFile) + command := fmt.Sprintf("run test -f %s --environment pokeapi-env", testFile) result = tracetestcli.Exec(t, command, tracetestcli.WithCLIConfig(cliConfig)) helpers.RequireExitCodeEqual(t, result, 0) require.Contains(result.StdOut, "✔ It should add a Pokemon correctly") require.Contains(result.StdOut, "✔ It should save the correct data") }) } + +func TestRunTestWithGrpcTrigger(t *testing.T) { + // setup isolated e2e environment + env := environment.CreateAndStart(t, environment.WithDataStoreEnabled(), environment.WithPokeshop()) + defer env.Close(t) + + cliConfig := env.GetCLIConfigPath(t) + + t.Run("should pass when using an embedded protobuf string in the test", func(t *testing.T) { + // instantiate require with testing helper + require := require.New(t) + + // Given I am a Tracetest CLI user + // And I have my server recently created + // And the datasource is already set + + // When I try to run a test with a gRPC trigger with embedded protobuf + // Then it should pass + testFile := env.GetTestResourcePath(t, "grpc-trigger-embedded-protobuf") + + command := fmt.Sprintf("run test -f %s", testFile) + result := tracetestcli.Exec(t, command, tracetestcli.WithCLIConfig(cliConfig)) + helpers.RequireExitCodeEqual(t, result, 0) + require.Contains(result.StdOut, "✔ It calls Pokeshop correctly") // checks if the assertion was succeeded + }) + + t.Run("should pass when referencing a protobuf file in the test", func(t *testing.T) { + // instantiate require with testing helper + require := require.New(t) + + // Given I am a Tracetest CLI user + // And I have my server recently created + // And the datasource is already set + + // When I try to run a test with a gRPC trigger with a reference to a protobuf file + // Then it should pass + testFile := env.GetTestResourcePath(t, "grpc-trigger-reference-protobuf") + + command := fmt.Sprintf("run test -f %s", testFile) + result := tracetestcli.Exec(t, command, tracetestcli.WithCLIConfig(cliConfig)) + helpers.RequireExitCodeEqual(t, result, 0) + require.Contains(result.StdOut, "✔ It calls Pokeshop correctly") // checks if the assertion was succeeded + }) +} diff --git a/testing/cli-e2etest/testscenarios/test/run_test_with_grpc_trigger_test.go b/testing/cli-e2etest/testscenarios/test/run_test_with_grpc_trigger_test.go deleted file mode 100644 index e626b32154..0000000000 --- a/testing/cli-e2etest/testscenarios/test/run_test_with_grpc_trigger_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package test - -import ( - "fmt" - "testing" - - "github.com/kubeshop/tracetest/cli-e2etest/environment" - "github.com/kubeshop/tracetest/cli-e2etest/helpers" - "github.com/kubeshop/tracetest/cli-e2etest/tracetestcli" - "github.com/stretchr/testify/require" -) - -func TestRunTestWithGrpcTrigger(t *testing.T) { - // setup isolated e2e environment - env := environment.CreateAndStart(t, environment.WithDataStoreEnabled(), environment.WithPokeshop()) - defer env.Close(t) - - cliConfig := env.GetCLIConfigPath(t) - - t.Run("should pass when using an embedded protobuf string in the test", func(t *testing.T) { - // instantiate require with testing helper - require := require.New(t) - - // Given I am a Tracetest CLI user - // And I have my server recently created - // And the datasource is already set - - // When I try to run a test with a gRPC trigger with embedded protobuf - // Then it should pass - testFile := env.GetTestResourcePath(t, "grpc-trigger-embedded-protobuf") - - command := fmt.Sprintf("test run -w -d %s", testFile) - result := tracetestcli.Exec(t, command, tracetestcli.WithCLIConfig(cliConfig)) - helpers.RequireExitCodeEqual(t, result, 0) - require.Contains(result.StdOut, "✔ It calls Pokeshop correctly") // checks if the assertion was succeeded - }) - - t.Run("should pass when referencing a protobuf file in the test", func(t *testing.T) { - // instantiate require with testing helper - require := require.New(t) - - // Given I am a Tracetest CLI user - // And I have my server recently created - // And the datasource is already set - - // When I try to run a test with a gRPC trigger with a reference to a protobuf file - // Then it should pass - testFile := env.GetTestResourcePath(t, "grpc-trigger-reference-protobuf") - - command := fmt.Sprintf("test run -w -d %s", testFile) - result := tracetestcli.Exec(t, command, tracetestcli.WithCLIConfig(cliConfig)) - helpers.RequireExitCodeEqual(t, result, 0) - require.Contains(result.StdOut, "✔ It calls Pokeshop correctly") // checks if the assertion was succeeded - }) -} diff --git a/testing/cli-e2etest/testscenarios/transaction/apply_transaction_test.go b/testing/cli-e2etest/testscenarios/transaction/apply_transaction_test.go index 24812acd51..7cee97e738 100644 --- a/testing/cli-e2etest/testscenarios/transaction/apply_transaction_test.go +++ b/testing/cli-e2etest/testscenarios/transaction/apply_transaction_test.go @@ -91,6 +91,7 @@ func TestApplyTransaction(t *testing.T) { // Then it should be applied with success and it should not update // the steps with its ids. transactionWithoutIDPath := env.GetTestResourcePath(t, "new-transaction-without-id") + helpers.Copy(transactionWithoutIDPath+".tpl", transactionWithoutIDPath) helpers.RemoveIDFromTransactionFile(t, transactionWithoutIDPath) diff --git a/testing/cli-e2etest/testscenarios/transaction/resources/.gitignore b/testing/cli-e2etest/testscenarios/transaction/resources/.gitignore new file mode 100644 index 0000000000..57c1775944 --- /dev/null +++ b/testing/cli-e2etest/testscenarios/transaction/resources/.gitignore @@ -0,0 +1 @@ +new-transaction-without-id.yaml diff --git a/testing/cli-e2etest/testscenarios/transaction/resources/new-transaction-without-id.yaml b/testing/cli-e2etest/testscenarios/transaction/resources/new-transaction-without-id.yaml.tpl similarity index 100% rename from testing/cli-e2etest/testscenarios/transaction/resources/new-transaction-without-id.yaml rename to testing/cli-e2etest/testscenarios/transaction/resources/new-transaction-without-id.yaml.tpl diff --git a/testing/cli-e2etest/testscenarios/transaction/run_transaction_test.go b/testing/cli-e2etest/testscenarios/transaction/run_transaction_test.go index 5900f802cc..35d9faa907 100644 --- a/testing/cli-e2etest/testscenarios/transaction/run_transaction_test.go +++ b/testing/cli-e2etest/testscenarios/transaction/run_transaction_test.go @@ -11,13 +11,37 @@ import ( ) func TestRunTransaction(t *testing.T) { - // setup isolated e2e environment - env := environment.CreateAndStart(t, environment.WithDataStoreEnabled(), environment.WithPokeshop()) - defer env.Close(t) + t.Run("should fail if test resource is selected", func(t *testing.T) { + // setup isolated e2e environment + env := environment.CreateAndStart(t) + defer env.Close(t) - cliConfig := env.GetCLIConfigPath(t) + cliConfig := env.GetCLIConfigPath(t) + + // instantiate require with testing helper + require := require.New(t) + + // Given I am a Tracetest CLI user + // And I have my server recently created + // And the datasource is already set + + // When I try to run a transaction + // Then it should pass + transactionFile := env.GetTestResourcePath(t, "transaction-to-run") + + command := fmt.Sprintf("run test -f %s", transactionFile) + result := tracetestcli.Exec(t, command, tracetestcli.WithCLIConfig(cliConfig)) + helpers.RequireExitCodeEqual(t, result, 1) + require.Contains(result.StdErr, "cannot apply Transaction to Test resource") + }) t.Run("should pass", func(t *testing.T) { + // setup isolated e2e environment + env := environment.CreateAndStart(t, environment.WithDataStoreEnabled(), environment.WithPokeshop()) + defer env.Close(t) + + cliConfig := env.GetCLIConfigPath(t) + // instantiate require with testing helper require := require.New(t) @@ -29,7 +53,7 @@ func TestRunTransaction(t *testing.T) { // Then it should pass transactionFile := env.GetTestResourcePath(t, "transaction-to-run") - command := fmt.Sprintf("test run -w -d %s", transactionFile) + command := fmt.Sprintf("run transaction -f %s", transactionFile) result := tracetestcli.Exec(t, command, tracetestcli.WithCLIConfig(cliConfig)) helpers.RequireExitCodeEqual(t, result, 0) require.Contains(result.StdOut, "Transaction To Run") // transaction name diff --git a/testing/cli-smoketest/run.bash b/testing/cli-smoketest/run.bash index 639e5b4ef8..94342ec0a6 100755 --- a/testing/cli-smoketest/run.bash +++ b/testing/cli-smoketest/run.bash @@ -50,7 +50,7 @@ EXIT_STATUS=0 run_cli_command '--help' || EXIT_STATUS=$? run_cli_command 'version' || EXIT_STATUS=$? -run_cli_command 'test run --definition ./tests/simple-test.yaml --wait-for-result' || EXIT_STATUS=$? +run_cli_command 'run test --file ./tests/simple-test.yaml' || EXIT_STATUS=$? echo "" echo "Tests done! Exit code: $EXIT_STATUS" diff --git a/testing/cli-smoketest/tests/simple-test.yaml b/testing/cli-smoketest/tests/simple-test.yaml index 4f49c4519b..926d6e4b8f 100644 --- a/testing/cli-smoketest/tests/simple-test.yaml +++ b/testing/cli-smoketest/tests/simple-test.yaml @@ -1,5 +1,4 @@ type: Test - spec: id: e9c6cff9-974d-4263-8a23-22f1e9f975aa name: List all tracetest tests diff --git a/testing/server-tracetesting/run.bash b/testing/server-tracetesting/run.bash index 3ae6b43981..e8f87bff3d 100755 --- a/testing/server-tracetesting/run.bash +++ b/testing/server-tracetesting/run.bash @@ -65,10 +65,10 @@ mkdir -p results/responses run_test_suite_for_feature() { feature=$1 - junit_output='results/'$feature'_test_suite.xml' + # junit_output='results/'$feature'_test_suite.xml' definition='./features/'$feature'/_test_suite.yml' - testCMD="$TRACETEST_CLI --config ./config.yml test run --definition $definition --environment ./tracetesting-env.yaml --wait-for-result --junit $junit_output" + testCMD="$TRACETEST_CLI --config ./config.yml run transaction --file $definition --environment ./tracetesting-env.yaml" echo $testCMD $testCMD return $? diff --git a/web/src/services/CliCommand.service.ts b/web/src/services/CliCommand.service.ts index 1db419b7a9..71d491f62f 100644 --- a/web/src/services/CliCommand.service.ts +++ b/web/src/services/CliCommand.service.ts @@ -55,7 +55,7 @@ const CliCommandService = () => ({ const command = Object.entries(options).reduce( (acc, [option, enabled]) => this.applyOptions[option as CliCommandOption]({command: acc, enabled, test, environmentId}), - `test run -d ${fileName}` + `run test -f ${fileName}` ); return `${command} -o ${format}`;