Update CLI e2e test engine to support CLI debug and add error handlin…

…g for 404 errors on Get (#2573)

cli/cmd/config.go

@@ -114,7 +114,7 @@ func overrideConfig() {
 		if err != nil {
 			msg := fmt.Sprintf("cannot parse endpoint %s", overrideEndpoint)
 			cliLogger.Error(msg, zap.Error(err))
-			os.Exit(1)
+			ExitCLI(1)
 		}
 		cliConfig.Scheme = scheme
 		cliConfig.Endpoint = endpoint
@@ -129,7 +129,7 @@ func setupOutputFormat(cmd *cobra.Command) {
 	o := formatters.Output(output)
 	if !formatters.ValidOutput(o) {
 		fmt.Fprintf(os.Stderr, "Invalid output format %s. Available formats are [%s]\n", output, outputFormatsString)
-		os.Exit(1)
+		ExitCLI(1)
 	}
 	formatters.SetOutput(o)
 }

cli/cmd/docgen_cmd.go

@@ -26,22 +26,22 @@ var docGenCmd = &cobra.Command{
 		err := os.MkdirAll(docsOutputDir, os.ModePerm)
 		if err != nil {
 			fmt.Println(fmt.Errorf("could not create output dir: %w", err).Error())
-			os.Exit(1)
+			ExitCLI(1)
 		}
 
 		err = doc.GenMarkdownTreeCustom(rootCmd, docsOutputDir, func(s string) string {
 			return "# CLI Reference\n"
 		}, func(s string) string { return s })
 		if err != nil {
 			fmt.Println(fmt.Errorf("could not generate documentation: %w", err).Error())
-			os.Exit(1)
+			ExitCLI(1)
 		}
 
 		if docusaurusFolder != "" {
 			err = generateDocusaurusSidebar(docsOutputDir, docusaurusFolder)
 			if err != nil {
 				fmt.Println(fmt.Errorf("could not create docusaurus sidebar file: %w", err).Error())
-				os.Exit(1)
+				ExitCLI(1)
 			}
 		}
 	},

cli/cmd/get_cmd.go

@@ -2,10 +2,12 @@ package cmd
 
 import (
 	"context"
+	"errors"
 	"fmt"
 
 	"github.com/kubeshop/tracetest/cli/analytics"
 	"github.com/kubeshop/tracetest/cli/formatters"
+	"github.com/kubeshop/tracetest/cli/utils"
 	"github.com/spf13/cobra"
 )
 
@@ -40,7 +42,9 @@ var getCmd = &cobra.Command{
 		}
 
 		resource, err := resourceActions.Get(ctx, resourceID)
-		if err != nil {
+		if err != nil && errors.Is(err, utils.ResourceNotFound) {
+			return fmt.Sprintf("Resource %s with ID %s not found", resourceType, resourceID), nil
+		} else if err != nil {
 			return "", err
 		}
 

cli/cmd/middleware.go

@@ -2,7 +2,6 @@ package cmd
 
 import (
 	"fmt"
-	"os"
 
 	"github.com/spf13/cobra"
 	"go.uber.org/zap"
@@ -20,7 +19,7 @@ Version
 %s
 
 An error ocurred when executing the command`, versionText), zap.Error(err))
-			os.Exit(1)
+			ExitCLI(1)
 			return
 		}
 

cli/cmd/root.go

@@ -18,7 +18,8 @@ var (
 	outputFormatsString = strings.Join(outputFormats, "|")
 
 	// overrides
-	overrideEndpoint string
+	overrideEndpoint   string
+	cliExitInterceptor func(code int)
 )
 
 var rootCmd = &cobra.Command{
@@ -32,10 +33,23 @@ var rootCmd = &cobra.Command{
 func Execute() {
 	if err := rootCmd.Execute(); err != nil {
 		fmt.Fprintln(os.Stderr, err)
-		os.Exit(1)
+		ExitCLI(1)
 	}
 }
 
+func ExitCLI(errorCode int) {
+	if cliExitInterceptor != nil {
+		cliExitInterceptor(errorCode)
+		return
+	}
+
+	os.Exit(errorCode)
+}
+
+func RegisterCLIExitInterceptor(interceptor func(int)) {
+	cliExitInterceptor = interceptor
+}
+
 var (
 	cmdGroupConfig = &cobra.Group{
 		ID:    "configuration",

cli/utils/api.go

@@ -159,6 +159,10 @@ func (resourceClient ResourceClient) Get(ctx context.Context, id string) (*file.
 	}
 
 	defer resp.Body.Close()
+	if resp.StatusCode == http.StatusNotFound {
+		return nil, ResourceNotFound
+	}
+
 	if resp.StatusCode != http.StatusOK {
 		body, err := ioutil.ReadAll(resp.Body)
 		if err != nil {

cli/utils/errors.go

@@ -0,0 +1,7 @@
+package utils
+
+import "errors"
+
+var (
+	ResourceNotFound = errors.New("resource not found in API")
+)

docs/docs/deployment/kubernetes.mdx

@@ -14,7 +14,7 @@ This is an example of a production-ready deployment, but real-world deployments
 This setup is ideal for CI/CD environments and QA teams working in shared environments. You can use a remote or local ([minikube](https://minikube.sigs.k8s.io/docs/start/), [kind](https://kind.sigs.k8s.io/), [k3d](https://k3d.io/), etc) cluster.
 :::
 
-You have two options to install Tracetest on Kubernetes: 
+You have two options to install Tracetest on Kubernetes:
 
 - Using the [Tracetest CLI](../getting-started/installation) to guide your installation
 - Using the official [Helm chart](https://github.com/kubeshop/helm-charts/tree/main/charts/tracetest)

testing/cli-e2etest/README.md

@@ -0,0 +1,138 @@
+# Tracetest CLI e2e tests
+
+In this folder we have the End-to-end tests done on the CLI to guarantee that the CLI is working fine. 
+The main idea is to test every CLI command against Tracetest server with different data stores and different Operating systems.
+
+
+## Implementation status
+
+| Linux              | Windows | MacOS  |
+| ------------------ | ------- | ------ |
+| :white_check_mark: | :soon:  | :soon: |
+
+## Tracetest Data Store
+
+| Jaeger             | Tempo  | OpenSearch | SignalFx | OTLP   | ElasticAPM | New Relic | Lightstep | Datadog | AWS X-Ray | Honeycomb |
+| ------------------ | ------ | ---------- | -------- | ------ | ---------- | --------- | --------- | ------- | --------- | --------- |
+| :white_check_mark: | :soon: | :soon:     | :soon:   | :soon: | :soon:     | :soon:    | :soon:    | :soon:  | :soon:    | :soon:    |
+
+## CLI Commands to Test
+
+### Misc and Flags
+
+| CLI Command    | Tested             | Test scenarios                                    |
+| -------------- | ------------------ | ------------------------------------------------- |
+| `version`      | :white_check_mark: | [VersionCommand](./testscenarios/version_test.go) |
+| `help`         | :white_check_mark: | [HelpCommand](./testscenarios/help_test.go)       |
+| `--help`, `-h` | :white_check_mark: | [HelpCommand](./testscenarios/help_test.go)       |
+| `--config`     | :white_check_mark: | All scenarios                                     |
+
+### Tests and Test Runner
+
+| CLI Command                                                        | Tested             | Test scenarios |
+| ------------------------------------------------------------------ | ------------------ | -------------- |
+| `test list`                                                        | :yellow_circle:    | |
+| `test run -d [test-definition]`                                    | :yellow_circle:    | |
+| `test run -d [test-definition] -e [environment-id]`                | :yellow_circle:    | |
+| `test run -d [test-definition] -e [environment-definition]`        | :yellow_circle:    | |
+| `test run -d [transaction-definition]`                             | :yellow_circle:    | |
+| `test run -d [transaction-definition] -e [environment-id]`         | :yellow_circle:    | |
+| `test run -d [transaction-definition] -e [environment-definition]` | :yellow_circle:    | |
+
+### Resources: Config
+
+| CLI Command                                           | Tested          | Test scenarios |
+| ----------------------------------------------------- | ----------------| -------------- |
+| `apply config -f [config-file]`                       | :yellow_circle: | |
+| `delete config --id current`                          | :yellow_circle: | |
+| `export config --id current --file [config-file]`     | :yellow_circle: | |
+| `get config --id current --output pretty`             | :yellow_circle: | |
+| `get config --id current --output json`               | :yellow_circle: | |
+| `get config --id current --output yaml`               | :yellow_circle: | |
+| `list config --output pretty`                         | :yellow_circle: | |
+| `list config --output json`                           | :yellow_circle: | |
+| `list config --output yaml`                           | :yellow_circle: | |
+### Resources: Data Store
+
+| CLI Command                                              | Tested             | Test scenarios |
+| -------------------------------------------------------- | ------------------ | -------------- |
+| `apply datastore -f [data-store-file]`                   | :white_check_mark: | [ApplyNewDatastore](./testscenarios/datastore/apply_new_datastore_test.go) |
+| `delete datastore --id current`                          | :white_check_mark: | [DeleteDatastore](./testscenarios/datastore/delete_datastore_test.go) |
+| `export datastore --id current --file [data-store-file]` | :yellow_circle:    | |
+| `get datastore --id current --output pretty`             | :white_check_mark: | [ApplyNewDatastore](./testscenarios/datastore/apply_new_datastore_test.go), [DeleteDatastore](./testscenarios/datastore/delete_datastore_test.go) |
+| `get datastore --id current --output json`               | :yellow_circle:    | |
+| `get datastore --id current --output yaml`               | :yellow_circle:    | |
+| `list datastore --output pretty`                         | :white_check_mark: | [ListDatastore](./testscenarios/datastore/list_datastore_test.go) |
+| `list datastore --output json`                           | :white_check_mark: | [ListDatastore](./testscenarios/datastore/list_datastore_test.go) |
+| `list datastore --output yaml`                           | :white_check_mark: | [ListDatastore](./testscenarios/datastore/list_datastore_test.go) |
+
+### Resources: Demo
+
+| CLI Command                                          | Tested             | Test scenarios |
+| ---------------------------------------------------- | ------------------ | -------------- |
+| `apply demo -f [new-demo-file]`                      | :yellow_circle:    | |
+| `apply demo -f [existing-demo-file]`                 | :yellow_circle:    | |
+| `delete demo --id [existing-id]`                     | :yellow_circle:    | |
+| `delete demo --id [non-existing-id]`                 | :yellow_circle:    | |
+| `export demo --id current --file [demo-file]`        | :yellow_circle:    | |
+| `get demo --id [non-existing-id]`                    | :yellow_circle:    | |
+| `get demo --id [existing-id] --output pretty`        | :yellow_circle:    | |
+| `get demo --id [existing-id] --output json`          | :yellow_circle:    | |
+| `get demo --id [existing-id] --output yaml`          | :yellow_circle:    | |
+| `list demo --output pretty`                          | :yellow_circle:    | |
+| `list demo --output json`                            | :yellow_circle:    | |
+| `list demo --output yaml`                            | :yellow_circle:    | |
+| `list demo --skip 1 --take 2`                        | :yellow_circle:    | |
+| `list demo --sortBy name --sortDirection desc`       | :yellow_circle:    | |
+
+### Resources: Environment
+
+| CLI Command                                                 | Tested             | Test scenarios |
+| ----------------------------------------------------------- | ------------------ | -------------- |
+| `apply environment -f [new-environment-file]`               | :yellow_circle:    | |
+| `apply environment -f [existing-environment-file]`          | :yellow_circle:    | |
+| `delete environment --id [existing-id]`                     | :yellow_circle:    | |
+| `delete environment --id [non-existing-id]`                 | :yellow_circle:    | |
+| `export environment --id current --file [environment-file]` | :yellow_circle:    | |
+| `get environment --id [non-existing-id]`                    | :yellow_circle:    | |
+| `get environment --id [existing-id] --output pretty`        | :yellow_circle:    | |
+| `get environment --id [existing-id] --output json`          | :yellow_circle:    | |
+| `get environment --id [existing-id] --output yaml`          | :yellow_circle:    | |
+| `list environment --output pretty`                          | :yellow_circle:    | |
+| `list environment --output json`                            | :yellow_circle:    | |
+| `list environment --output yaml`                            | :yellow_circle:    | |
+| `list environment --skip 1 --take 2`                        | :yellow_circle:    | |
+| `list environment --sortBy name --sortDirection desc`       | :yellow_circle:    | |
+
+### Resources: PollingProfile
+
+| CLI Command                                                           | Tested          | Test scenarios |
+| --------------------------------------------------------------------- | --------------- | -------------- |
+| `apply pollingprofile -f [pollingprofile-file]`                       | :yellow_circle: | |
+| `delete pollingprofile --id current`                                  | :yellow_circle: | |
+| `export pollingprofile --id current --file [pollingprofile-file]`     | :yellow_circle: | |
+| `get pollingprofile --id current --output pretty`                     | :yellow_circle: | |
+| `get pollingprofile --id current --output json`                       | :yellow_circle: | |
+| `get pollingprofile --id current --output yaml`                       | :yellow_circle: | |
+| `list pollingprofile --output pretty`                                 | :yellow_circle: | |
+| `list pollingprofile --output json`                                   | :yellow_circle: | |
+| `list pollingprofile --output yaml`                                   | :yellow_circle: | |
+
+### Resources: Transactions
+
+| CLI Command                                                 | Tested             | Test scenarios |
+| ----------------------------------------------------------- | ------------------ | -------------- |
+| `apply transaction -f [new-transaction-file]`               | :yellow_circle:    | |
+| `apply transaction -f [existing-transaction-file]`          | :yellow_circle:    | |
+| `delete transaction --id [existing-id]`                     | :yellow_circle:    | |
+| `delete transaction --id [non-existing-id]`                 | :yellow_circle:    | |
+| `export transaction --id current --file [transaction-file]` | :yellow_circle:    | |
+| `get transaction --id [non-existing-id]`                    | :yellow_circle:    | |
+| `get transaction --id [existing-id] --output pretty`        | :yellow_circle:    | |
+| `get transaction --id [existing-id] --output json`          | :yellow_circle:    | |
+| `get transaction --id [existing-id] --output yaml`          | :yellow_circle:    | |
+| `list transaction --output pretty`                          | :yellow_circle:    | |
+| `list transaction --output json`                            | :yellow_circle:    | |
+| `list transaction --output yaml`                            | :yellow_circle:    | |
+| `list transaction --skip 1 --take 2`                        | :yellow_circle:    | |
+| `list transaction --sortBy name --sortDirection desc`       | :yellow_circle:    | |

testing/cli-e2etest/config/config.go

@@ -0,0 +1,35 @@
+package config
+
+import "os"
+
+type EnvironmentVars struct {
+	EnableCLIDebug   bool
+	TracetestCommand string
+	TestEnvironment  string
+}
+
+var instance *EnvironmentVars
+
+func GetConfigAsEnvVars() *EnvironmentVars {
+	if instance != nil {
+		return instance
+	}
+
+	enableCLIDebug := (os.Getenv("ENABLE_CLI_DEBUG") == "true")
+
+	tracetestCommand := os.Getenv("TRACETEST_COMMAND")
+	if tracetestCommand == "" {
+		tracetestCommand = "tracetest"
+	}
+
+	testEnvironment := os.Getenv("TEST_ENVIRONMENT")
+	if testEnvironment == "" {
+		testEnvironment = "jaeger"
+	}
+
+	return &EnvironmentVars{
+		EnableCLIDebug:   enableCLIDebug,
+		TracetestCommand: tracetestCommand,
+		TestEnvironment:  testEnvironment,
+	}
+}

testing/cli-e2etest/environment/manager.go

@@ -2,7 +2,6 @@ package environment
 
 import (
 	"fmt"
-	"os"
 	"path"
 	"runtime"
 	"sync"
@@ -11,14 +10,14 @@ import (
 	"time"
 
 	"github.com/kubeshop/tracetest/cli-e2etest/command"
+	"github.com/kubeshop/tracetest/cli-e2etest/config"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/exp/slices"
 )
 
 var (
 	mutex               = sync.Mutex{}
 	envCounter    int64 = 0
-	defaultEnv          = "jaeger"
 	supportedEnvs       = []string{"jaeger"}
 )
 
@@ -27,7 +26,8 @@ type Manager interface {
 	Start(t *testing.T)
 	Close(t *testing.T)
 	GetCLIConfigPath(t *testing.T) string
-	GetManisfestResourcePath(t *testing.T, manifestName string) string
+	GetEnvironmentResourcePath(t *testing.T, resourceName string) string
+	GetTestResourcePath(t *testing.T, resourceName string) string
 }
 
 type internalManager struct {
@@ -42,11 +42,7 @@ func CreateAndStart(t *testing.T) Manager {
 	mutex.Lock()
 	defer mutex.Unlock()
 
-	environmentName := os.Getenv("TEST_ENVIRONMENT")
-
-	if environmentName == "" {
-		environmentName = defaultEnv
-	}
+	environmentName := config.GetConfigAsEnvVars().TestEnvironment
 
 	if !slices.Contains(supportedEnvs, environmentName) {
 		t.Fatalf("environment %s not registered", environmentName)
@@ -59,7 +55,7 @@ func CreateAndStart(t *testing.T) Manager {
 }
 
 func getExecutingDir() string {
-	_, filename, _, _ := runtime.Caller(0)
+	_, filename, _, _ := runtime.Caller(0) // get file of the getExecutingDir caller
 	return path.Dir(filename)
 }
 
@@ -125,7 +121,14 @@ func (m *internalManager) GetCLIConfigPath(t *testing.T) string {
 	return fmt.Sprintf("%s/%s/cli-config.yaml", currentDir, m.environmentType)
 }
 
-func (m *internalManager) GetManisfestResourcePath(t *testing.T, manifestName string) string {
+func (m *internalManager) GetEnvironmentResourcePath(t *testing.T, resourceName string) string {
 	currentDir := getExecutingDir()
-	return fmt.Sprintf("%s/%s/resources/%s.yaml", currentDir, m.environmentType, manifestName)
+	return fmt.Sprintf("%s/%s/resources/%s.yaml", currentDir, m.environmentType, resourceName)
+}
+
+func (m *internalManager) GetTestResourcePath(t *testing.T, resourceName string) string {
+	_, filename, _, _ := runtime.Caller(1) // get file of the GetTestResourcePath caller
+	testDir := path.Dir(filename)
+
+	return fmt.Sprintf("%s/resources/%s.yaml", testDir, resourceName)
 }