diff --git a/api/v1/testkube.yaml b/api/v1/testkube.yaml index 02378741628..2877d0e5e2d 100644 --- a/api/v1/testkube.yaml +++ b/api/v1/testkube.yaml @@ -3608,12 +3608,24 @@ components: example: record: "true" prefix: "some-" + command: + type: array + description: "executor image command" + example: ["curl"] + items: + type: string args: type: array description: "additional arguments/flags passed to executor binary" example: ["--concurrency", "2", "--remote", "--some", "blabla"] items: type: string + args_mode: + type: string + description: usage mode for arguments + enum: + - append + - override variables: $ref: "#/components/schemas/Variables" variablesFile: @@ -3671,7 +3683,7 @@ components: example: execution-c01d7cf6-ec3f-47f0-9556-a5d6e9009a43 artifactRequest: $ref: "#/components/schemas/ArtifactRequest" - description: configuration parameters for storing container executor test artifacts + description: configuration parameters for storing test artifacts preRunScript: type: string description: script to run before test execution @@ -3989,24 +4001,21 @@ components: - $ref: "#/components/schemas/RepositoryParameters" ArtifactRequest: - description: artifact request body for container executors with test artifacts + description: artifact request body with test artifacts type: object - required: - - storageClassName - - volumeMountPath properties: storageClassName: type: string - description: artifact storage class name + description: artifact storage class name for container executor example: artifact-volume-local volumeMountPath: type: string - description: artifact volume mount path + description: artifact volume mount path for container executor dirs: type: array items: type: string - description: artifact directories + description: artifact directories for scraping ArtifactUpdateRequest: description: artifact request update body @@ -4059,7 +4068,7 @@ components: example: "7934600f-b367-48dd-b981-4353304362fb" command: type: array - description: "container executor image command" + description: "executor image command" items: type: string example: @@ -4073,6 +4082,12 @@ components: - "--repeats" - "5" - "--insecure" + args_mode: + type: string + description: usage mode for arguments + enum: + - append + - override image: type: string description: container image, executor will run inside this image @@ -4137,7 +4152,7 @@ components: example: execution-c01d7cf6-ec3f-47f0-9556-a5d6e9009a43 artifactRequest: $ref: "#/components/schemas/ArtifactRequest" - description: configuration parameters for storing container executor test artifacts + description: configuration parameters for storing test artifacts jobTemplate: type: string description: job template extensions @@ -4339,14 +4354,14 @@ components: $ref: "#/components/schemas/LocalObjectReference" command: type: array - description: "container executor image command" + description: "executor image command" items: type: string example: - "curl" args: type: array - description: "additional executor binary arguments" + description: "additional executor binary argument" items: type: string example: diff --git a/cmd/kubectl-testkube/commands/crds/tests_crds.go b/cmd/kubectl-testkube/commands/crds/tests_crds.go index 86444bd0fa8..df5de85f1b2 100644 --- a/cmd/kubectl-testkube/commands/crds/tests_crds.go +++ b/cmd/kubectl-testkube/commands/crds/tests_crds.go @@ -166,7 +166,14 @@ func processPostmanFiles(cmd *cobra.Command, args []string) error { } } - test.ExecutionRequest = &testkube.ExecutionRequest{Args: flags.ExecutorArgs, Envs: flags.Envs, Variables: vars, PreRunScript: scriptBody} + test.ExecutionRequest = &testkube.ExecutionRequest{ + Command: flags.Command, + Args: flags.ExecutorArgs, + ArgsMode: flags.ArgsMode, + Envs: flags.Envs, + Variables: vars, + PreRunScript: scriptBody, + } detectedTests[testName] = test return nil }) diff --git a/cmd/kubectl-testkube/commands/executors/create.go b/cmd/kubectl-testkube/commands/executors/create.go index 080d0c06ca9..5ebc6b2e77a 100644 --- a/cmd/kubectl-testkube/commands/executors/create.go +++ b/cmd/kubectl-testkube/commands/executors/create.go @@ -68,8 +68,8 @@ func NewCreateExecutorCmd() *cobra.Command { cmd.Flags().StringVarP(&uri, "uri", "u", "", "if resource need to be loaded from URI") cmd.Flags().StringVar(&image, "image", "", "image used for executor") cmd.Flags().StringArrayVar(&imagePullSecretNames, "image-pull-secrets", []string{}, "secret name used to pull the image in executor") - cmd.Flags().StringArrayVar(&command, "command", []string{}, "command passed to image in container executor") - cmd.Flags().StringArrayVar(&executorArgs, "args", []string{}, "args passed to image in container executor") + cmd.Flags().StringArrayVar(&command, "command", []string{}, "command passed to image in executor") + cmd.Flags().StringArrayVar(&executorArgs, "args", []string{}, "args passed to image in executor") cmd.Flags().StringVarP(&jobTemplate, "job-template", "j", "", "if executor needs to be launched using custom job specification, then a path to template file should be provided") cmd.Flags().StringToStringVarP(&labels, "label", "l", nil, "label key value pair: --label key1=value1") cmd.Flags().StringArrayVar(&features, "feature", []string{}, "feature provided by executor") diff --git a/cmd/kubectl-testkube/commands/executors/update.go b/cmd/kubectl-testkube/commands/executors/update.go index 09fd379aeed..16a1655da27 100644 --- a/cmd/kubectl-testkube/commands/executors/update.go +++ b/cmd/kubectl-testkube/commands/executors/update.go @@ -47,8 +47,8 @@ func UpdateExecutorCmd() *cobra.Command { cmd.Flags().StringVarP(&uri, "uri", "u", "", "if resource need to be loaded from URI") cmd.Flags().StringVar(&image, "image", "", "image used for executor") cmd.Flags().StringArrayVar(&imagePullSecretNames, "image-pull-secrets", []string{}, "secret name used to pull the image in executor") - cmd.Flags().StringArrayVar(&command, "command", []string{}, "command passed to image in container executor") - cmd.Flags().StringArrayVar(&executorArgs, "args", []string{}, "args passed to image in container executor") + cmd.Flags().StringArrayVar(&command, "command", []string{}, "command passed to image in executor") + cmd.Flags().StringArrayVar(&executorArgs, "args", []string{}, "args passed to image in executor") cmd.Flags().StringVarP(&jobTemplate, "job-template", "j", "", "if executor needs to be launched using custom job specification, then a path to template file should be provided") cmd.Flags().StringToStringVarP(&labels, "label", "l", nil, "label key value pair: --label key1=value1") cmd.Flags().StringArrayVar(&features, "feature", []string{}, "feature provided by executor") diff --git a/cmd/kubectl-testkube/commands/tests/common.go b/cmd/kubectl-testkube/commands/tests/common.go index 78dc98c734b..d6a3e820d20 100644 --- a/cmd/kubectl-testkube/commands/tests/common.go +++ b/cmd/kubectl-testkube/commands/tests/common.go @@ -226,7 +226,7 @@ func newArtifactRequestFromFlags(cmd *cobra.Command) (request *testkube.Artifact return nil, err } - if artifactStorageClassName != "" && artifactVolumeMountPath != "" { + if artifactStorageClassName != "" || artifactVolumeMountPath != "" || len(dirs) != 0 { request = &testkube.ArtifactRequest{ StorageClassName: artifactStorageClassName, VolumeMountPath: artifactVolumeMountPath, @@ -335,6 +335,7 @@ func newExecutionRequestFromFlags(cmd *cobra.Command) (request *testkube.Executi return nil, err } + argsMode := cmd.Flag("args-mode").Value.String() executionName := cmd.Flag("execution-name").Value.String() envs, err := cmd.Flags().GetStringToString("env") if err != nil { @@ -441,6 +442,7 @@ func newExecutionRequestFromFlags(cmd *cobra.Command) (request *testkube.Executi Image: image, Command: command, Args: executorArgs, + ArgsMode: argsMode, ImagePullSecrets: imageSecrets, Envs: envs, SecretEnvs: secretEnvs, @@ -772,6 +774,10 @@ func newExecutionUpdateRequestFromFlags(cmd *cobra.Command) (request *testkube.E "https-proxy", &request.HttpsProxy, }, + { + "args-mode", + &request.ArgsMode, + }, } var nonEmpty bool diff --git a/cmd/kubectl-testkube/commands/tests/create.go b/cmd/kubectl-testkube/commands/tests/create.go index 669cbfd8215..163c73a711f 100644 --- a/cmd/kubectl-testkube/commands/tests/create.go +++ b/cmd/kubectl-testkube/commands/tests/create.go @@ -23,6 +23,7 @@ type CreateCommonFlags struct { SecretVariables map[string]string Schedule string ExecutorArgs []string + ArgsMode string ExecutionName string VariablesFile string Envs map[string]string @@ -101,9 +102,6 @@ func NewCreateTestsCmd() *cobra.Command { ui.ExitOnError("validating passed flags", err) } - err = validateArtifactRequest(flags.ArtifactStorageClassName, flags.ArtifactVolumeMountPath, flags.ArtifactDirs) - ui.ExitOnError("validating artifact flags", err) - options, err := NewUpsertTestOptionsFromFlags(cmd) ui.ExitOnError("getting test options", err) @@ -180,7 +178,9 @@ func AddCreateFlags(cmd *cobra.Command, flags *CreateCommonFlags) { cmd.Flags().StringToStringVarP(&flags.Variables, "variable", "v", nil, "variable key value pair: --variable key1=value1") cmd.Flags().StringToStringVarP(&flags.SecretVariables, "secret-variable", "s", nil, "secret variable key value pair: --secret-variable key1=value1") cmd.Flags().StringVarP(&flags.Schedule, "schedule", "", "", "test schedule in a cron job form: * * * * *") + cmd.Flags().StringArrayVar(&flags.Command, "command", []string{}, "command passed to image in executor") cmd.Flags().StringArrayVarP(&flags.ExecutorArgs, "executor-args", "", []string{}, "executor binary additional arguments") + cmd.Flags().StringVarP(&flags.ArgsMode, "args-mode", "", "append", "usage mode for arguments. one of append|override") cmd.Flags().StringVarP(&flags.ExecutionName, "execution-name", "", "", "execution name, if empty will be autogenerated") cmd.Flags().StringVarP(&flags.VariablesFile, "variables-file", "", "", "variables file path, e.g. postman env file - will be passed to executor if supported") cmd.Flags().StringToStringVarP(&flags.Envs, "env", "", map[string]string{}, "envs in a form of name1=val1 passed to executor") @@ -191,11 +191,10 @@ func AddCreateFlags(cmd *cobra.Command, flags *CreateCommonFlags) { cmd.Flags().StringArrayVarP(&flags.CopyFiles, "copy-files", "", []string{}, "file path mappings from host to pod of form source:destination") cmd.Flags().StringVar(&flags.Image, "image", "", "image for container executor") cmd.Flags().StringArrayVar(&flags.ImagePullSecretNames, "image-pull-secrets", []string{}, "secret name used to pull the image in container executor") - cmd.Flags().StringArrayVar(&flags.Command, "command", []string{}, "command passed to image in container executor") cmd.Flags().Int64Var(&flags.Timeout, "timeout", 0, "duration in seconds for test to timeout. 0 disables timeout.") cmd.Flags().StringVar(&flags.ArtifactStorageClassName, "artifact-storage-class-name", "", "artifact storage class name for container executor") cmd.Flags().StringVar(&flags.ArtifactVolumeMountPath, "artifact-volume-mount-path", "", "artifact volume mount path for container executor") - cmd.Flags().StringArrayVarP(&flags.ArtifactDirs, "artifact-dir", "", []string{}, "artifact dirs for container executor") + cmd.Flags().StringArrayVarP(&flags.ArtifactDirs, "artifact-dir", "", []string{}, "artifact dirs for scraping") cmd.Flags().StringVar(&flags.JobTemplate, "job-template", "", "job template file path for extensions to job template") cmd.Flags().StringVar(&flags.CronJobTemplate, "cronjob-template", "", "cron job template file path for extensions to cron job template") cmd.Flags().StringVarP(&flags.PreRunScript, "prerun-script", "", "", "path to script to be run before test execution") @@ -244,13 +243,3 @@ func validateExecutorTypeAndContent(executorType, contentType string, executors return nil } - -func validateArtifactRequest(artifactStorageClassName, artifactVolumeMountPath string, artifactDirs []string) error { - if artifactStorageClassName != "" || artifactVolumeMountPath != "" || len(artifactDirs) != 0 { - if artifactStorageClassName == "" || artifactVolumeMountPath == "" { - return fmt.Errorf("both artifact storage class name and mount path should be provided") - } - } - - return nil -} diff --git a/cmd/kubectl-testkube/commands/tests/renderer/execution_obj.go b/cmd/kubectl-testkube/commands/tests/renderer/execution_obj.go index 590280cd354..be3320eae4a 100644 --- a/cmd/kubectl-testkube/commands/tests/renderer/execution_obj.go +++ b/cmd/kubectl-testkube/commands/tests/renderer/execution_obj.go @@ -38,8 +38,12 @@ func ExecutionRenderer(ui *ui.UI, obj interface{}) error { renderer.RenderVariables(execution.Variables) + if len(execution.Command) > 0 { + ui.Warn("Command: ", execution.Command...) + } + if len(execution.Args) > 0 { - ui.Warn("Args: ", execution.Args...) + ui.Warn("Args: ", execution.Args...) } if execution.Content != nil && execution.Content.Repository != nil { diff --git a/cmd/kubectl-testkube/commands/tests/renderer/test_obj.go b/cmd/kubectl-testkube/commands/tests/renderer/test_obj.go index c6c7b7a3dc9..e41db81aeb2 100644 --- a/cmd/kubectl-testkube/commands/tests/renderer/test_obj.go +++ b/cmd/kubectl-testkube/commands/tests/renderer/test_obj.go @@ -84,9 +84,14 @@ func TestRenderer(ui *ui.UI, obj interface{}) error { renderer.RenderVariables(test.ExecutionRequest.Variables) } + if len(test.ExecutionRequest.Command) > 0 { + ui.Warn(" Command: ", test.ExecutionRequest.Command...) + } + if len(test.ExecutionRequest.Args) > 0 { ui.Warn(" Args: ", test.ExecutionRequest.Args...) } + ui.Warn(" Args mode: ", test.ExecutionRequest.ArgsMode) if len(test.ExecutionRequest.Envs) > 0 { ui.NL() diff --git a/cmd/kubectl-testkube/commands/tests/run.go b/cmd/kubectl-testkube/commands/tests/run.go index 2dd78f5c9c3..a159d307564 100644 --- a/cmd/kubectl-testkube/commands/tests/run.go +++ b/cmd/kubectl-testkube/commands/tests/run.go @@ -57,6 +57,8 @@ func NewRunTestCmd() *cobra.Command { format string masks []string runningContext string + command []string + argsMode string ) cmd := &cobra.Command{ @@ -81,9 +83,6 @@ func NewRunTestCmd() *cobra.Command { executorArgs, err := testkube.PrepareExecutorArgs(binaryArgs) ui.ExitOnError("getting args", err) - err = validateArtifactRequest(artifactStorageClassName, artifactVolumeMountPath, artifactDirs) - ui.ExitOnError("validating artifact flags", err) - envConfigMaps, envSecrets, err := newEnvReferencesFromFlags(cmd) ui.WarnOnError("getting env config maps and secrets", err) @@ -108,13 +107,20 @@ func NewRunTestCmd() *cobra.Command { scraperTemplateContent = string(b) } + mode := "" + if cmd.Flag("args-mode").Changed { + mode = argsMode + } + var executions []testkube.Execution client, namespace := common.GetClient(cmd) options := apiv1.ExecuteTestOptions{ ExecutionVariables: variables, ExecutionVariablesFileContent: paramsFileContent, ExecutionLabels: executionLabels, + Command: command, Args: executorArgs, + ArgsMode: mode, SecretEnvs: secretEnvs, HTTPProxy: httpProxy, HTTPSProxy: httpsProxy, @@ -132,7 +138,7 @@ func NewRunTestCmd() *cobra.Command { }, } - if artifactStorageClassName != "" && artifactVolumeMountPath != "" { + if artifactStorageClassName != "" || artifactVolumeMountPath != "" || len(artifactDirs) != 0 { options.ArtifactRequest = &testkube.ArtifactRequest{ StorageClassName: artifactStorageClassName, VolumeMountPath: artifactVolumeMountPath, @@ -243,7 +249,9 @@ func NewRunTestCmd() *cobra.Command { cmd.Flags().StringVarP(&variablesFile, "variables-file", "", "", "variables file path, e.g. postman env file - will be passed to executor if supported") cmd.Flags().StringToStringVarP(&variables, "variable", "v", map[string]string{}, "execution variable passed to executor") cmd.Flags().StringToStringVarP(&secretVariables, "secret-variable", "s", map[string]string{}, "execution secret variable passed to executor") + cmd.Flags().StringArrayVar(&command, "command", []string{}, "command passed to image in executor") cmd.Flags().StringArrayVarP(&binaryArgs, "args", "", []string{}, "executor binary additional arguments") + cmd.Flags().StringVarP(&argsMode, "args-mode", "", "append", "usage mode for argumnets. one of append|override") cmd.Flags().BoolVarP(&watchEnabled, "watch", "f", false, "watch for changes after start") cmd.Flags().StringVar(&downloadDir, "download-dir", "artifacts", "download dir") cmd.Flags().BoolVarP(&downloadArtifactsEnabled, "download-artifacts", "d", false, "downlaod artifacts automatically") @@ -259,7 +267,7 @@ func NewRunTestCmd() *cobra.Command { cmd.Flags().StringArrayVarP(©Files, "copy-files", "", []string{}, "file path mappings from host to pod of form source:destination") cmd.Flags().StringVar(&artifactStorageClassName, "artifact-storage-class-name", "", "artifact storage class name for container executor") cmd.Flags().StringVar(&artifactVolumeMountPath, "artifact-volume-mount-path", "", "artifact volume mount path for container executor") - cmd.Flags().StringArrayVarP(&artifactDirs, "artifact-dir", "", []string{}, "artifact dirs for container executor") + cmd.Flags().StringArrayVarP(&artifactDirs, "artifact-dir", "", []string{}, "artifact dirs for scraping") cmd.Flags().StringVar(&jobTemplate, "job-template", "", "job template file path for extensions to job template") cmd.Flags().StringVarP(&gitBranch, "git-branch", "", "", "if uri is git repository we can set additional branch parameter") cmd.Flags().StringVarP(&gitCommit, "git-commit", "", "", "if uri is git repository we can use commit id (sha) parameter") diff --git a/cmd/kubectl-testkube/commands/tests/update.go b/cmd/kubectl-testkube/commands/tests/update.go index 8497049473f..0a9d53c4804 100644 --- a/cmd/kubectl-testkube/commands/tests/update.go +++ b/cmd/kubectl-testkube/commands/tests/update.go @@ -27,6 +27,7 @@ func NewUpdateTestsCmd() *cobra.Command { secretVariables map[string]string schedule string executorArgs []string + argsMode string executionName string variablesFile string envs map[string]string @@ -102,7 +103,9 @@ func NewUpdateTestsCmd() *cobra.Command { cmd.Flags().StringToStringVarP(&variables, "variable", "v", nil, "variable key value pair: -v key1=value1") cmd.Flags().StringToStringVarP(&secretVariables, "secret-variable", "s", nil, "secret variable key value pair: -s key1=value1") cmd.Flags().StringVarP(&schedule, "schedule", "", "", "test schedule in a cron job form: * * * * *") + cmd.Flags().StringArrayVarP(&command, "command", "", []string{}, "command passed to image in executor") cmd.Flags().StringArrayVarP(&executorArgs, "executor-args", "", []string{}, "executor binary additional arguments") + cmd.Flags().StringVarP(&argsMode, "args-mode", "", "append", "usage mode for arguments. one of append|override") cmd.Flags().StringVarP(&executionName, "execution-name", "", "", "execution name, if empty will be autogenerated") cmd.Flags().StringVarP(&variablesFile, "variables-file", "", "", "variables file path, e.g. postman env file - will be passed to executor if supported") cmd.Flags().StringToStringVarP(&envs, "env", "", map[string]string{}, "envs in a form of name1=val1 passed to executor") @@ -115,14 +118,13 @@ func NewUpdateTestsCmd() *cobra.Command { cmd.Flags().StringArrayVarP(©Files, "copy-files", "", []string{}, "file path mappings from host to pod of form source:destination") cmd.Flags().StringVarP(&image, "image", "i", "", "image for container executor") cmd.Flags().StringArrayVar(&imagePullSecretNames, "image-pull-secrets", []string{}, "secret name used to pull the image in container executor") - cmd.Flags().StringArrayVarP(&command, "command", "", []string{}, "command passed to image in container executor") cmd.Flags().Int64Var(&timeout, "timeout", 0, "duration in seconds for test to timeout. 0 disables timeout.") cmd.Flags().StringVarP(&gitWorkingDir, "git-working-dir", "", "", "if repository contains multiple directories with tests (like monorepo) and one starting directory we can set working directory parameter") cmd.Flags().StringVarP(&gitCertificateSecret, "git-certificate-secret", "", "", "if git repository is private we can use certificate as an auth parameter stored in a kubernetes secret name") cmd.Flags().StringVarP(&gitAuthType, "git-auth-type", "", "basic", "auth type for git requests one of basic|header") cmd.Flags().StringVar(&artifactStorageClassName, "artifact-storage-class-name", "", "artifact storage class name for container executor") cmd.Flags().StringVar(&artifactVolumeMountPath, "artifact-volume-mount-path", "", "artifact volume mount path for container executor") - cmd.Flags().StringArrayVarP(&artifactDirs, "artifact-dir", "", []string{}, "artifact dirs for container executor") + cmd.Flags().StringArrayVarP(&artifactDirs, "artifact-dir", "", []string{}, "artifact dirs for scraping") cmd.Flags().StringVar(&jobTemplate, "job-template", "", "job template file path for extensions to job template") cmd.Flags().StringVar(&cronJobTemplate, "cronjob-template", "", "cron job template file path for extensions to cron job template") cmd.Flags().StringVarP(&preRunScript, "prerun-script", "", "", "path to script to be run before test execution") diff --git a/config/job-container-template.yml b/config/job-container-template.yml index af170ea5a33..8e565aa9e87 100644 --- a/config/job-container-template.yml +++ b/config/job-container-template.yml @@ -29,8 +29,10 @@ spec: mountPath: /etc/certs {{- end }} {{- if .ArtifactRequest }} + {{- if .ArtifactRequest.VolumeMountPath }} - name: artifact-volume mountPath: {{ .ArtifactRequest.VolumeMountPath }} + {{- end }} {{- end }} {{- range $configmap := .EnvConfigMaps }} {{- if and $configmap.Mount $configmap.Reference }} @@ -76,8 +78,10 @@ spec: mountPath: /etc/certs {{- end }} {{- if .ArtifactRequest }} + {{- if .ArtifactRequest.VolumeMountPath }} - name: artifact-volume mountPath: {{ .ArtifactRequest.VolumeMountPath }} + {{- end }} {{- end }} {{- range $configmap := .EnvConfigMaps }} {{- if and $configmap.Mount $configmap.Reference }} @@ -100,9 +104,11 @@ spec: secretName: {{ .CertificateSecret }} {{- end }} {{- if .ArtifactRequest }} + {{- if and .ArtifactRequest.VolumeMountPath .ArtifactRequest.StorageClassName }} - name: artifact-volume persistentVolumeClaim: claimName: {{ .Name }}-pvc + {{- end }} {{- end }} {{- range $configmap := .EnvConfigMaps }} {{- if and $configmap.Mount $configmap.Reference }} diff --git a/config/job-scraper-template.yml b/config/job-scraper-template.yml index abaf3204d32..89c84c7a620 100644 --- a/config/job-scraper-template.yml +++ b/config/job-scraper-template.yml @@ -21,15 +21,19 @@ spec: - "/bin/runner" - '{{ .Jsn }}' {{- if .ArtifactRequest }} + {{- if .ArtifactRequest.VolumeMountPath }} volumeMounts: - name: artifact-volume mountPath: {{ .ArtifactRequest.VolumeMountPath }} + {{- end }} {{- end }} {{- if .ArtifactRequest }} + {{- if and .ArtifactRequest.VolumeMountPath .ArtifactRequest.StorageClassName }} volumes: - name: artifact-volume persistentVolumeClaim: claimName: {{ .Name }}-pvc + {{- end }} {{- end }} restartPolicy: Never {{- if .ServiceAccountName }} diff --git a/contrib/executor/artillery/cmd/agent/main.go b/contrib/executor/artillery/cmd/agent/main.go index 5cddd777513..36d7006a249 100644 --- a/contrib/executor/artillery/cmd/agent/main.go +++ b/contrib/executor/artillery/cmd/agent/main.go @@ -7,11 +7,10 @@ import ( "github.com/pkg/errors" - "github.com/kubeshop/testkube/pkg/envs" - "github.com/kubeshop/testkube/pkg/executor/output" - "github.com/kubeshop/testkube/contrib/executor/artillery/pkg/runner" + "github.com/kubeshop/testkube/pkg/envs" "github.com/kubeshop/testkube/pkg/executor/agent" + "github.com/kubeshop/testkube/pkg/executor/output" "github.com/kubeshop/testkube/pkg/ui" ) diff --git a/contrib/executor/artillery/pkg/runner/artillery.go b/contrib/executor/artillery/pkg/runner/artillery.go index 82b11b5bf8e..f80488538d3 100644 --- a/contrib/executor/artillery/pkg/runner/artillery.go +++ b/contrib/executor/artillery/pkg/runner/artillery.go @@ -5,11 +5,10 @@ import ( "fmt" "os" "path/filepath" + "strings" "github.com/pkg/errors" - "github.com/kubeshop/testkube/pkg/executor/scraper/factory" - "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/envs" "github.com/kubeshop/testkube/pkg/executor" @@ -18,6 +17,7 @@ import ( "github.com/kubeshop/testkube/pkg/executor/output" "github.com/kubeshop/testkube/pkg/executor/runner" "github.com/kubeshop/testkube/pkg/executor/scraper" + "github.com/kubeshop/testkube/pkg/executor/scraper/factory" "github.com/kubeshop/testkube/pkg/ui" ) @@ -72,28 +72,42 @@ func (r *ArtilleryRunner) Run(ctx context.Context, execution testkube.Execution) } testDir, _ := filepath.Split(path) - args := []string{"run", path} envManager := env.NewManagerWithVars(execution.Variables) envManager.GetReferenceVars(envManager.Variables) + var envFile string if len(envManager.Variables) != 0 { - envFile, err := CreateEnvFile(envManager.Variables) + envFile, err = CreateEnvFile(envManager.Variables) if err != nil { return result, err } defer os.Remove(envFile) - args = append(args, "--dotenv", envFile) output.PrintEvent("created dotenv file", envFile) } // artillery test result output file testReportFile := filepath.Join(testDir, "test-report.json") - // append args from execution - args = append(args, "-o", testReportFile) + args := execution.Args + for i := len(args) - 1; i >= 0; i-- { + if envFile == "" && (args[i] == "--dotenv" || args[i] == "") { + args = append(args[:i], args[i+1:]...) + continue + } + + if args[i] == "" { + args[i] = envFile + } + + if args[i] == "" { + args[i] = testReportFile + } - args = append(args, execution.Args...) + if args[i] == "" { + args[i] = path + } + } runPath := testDir if execution.Content.Repository != nil && execution.Content.Repository.WorkingDir != "" { @@ -101,7 +115,9 @@ func (r *ArtilleryRunner) Run(ctx context.Context, execution testkube.Execution) } // run executor - out, rerr := executor.Run(runPath, "artillery", envManager, args...) + command := strings.Join(execution.Command, " ") + output.PrintLogf("%s Test run command %s %s", ui.IconRocket, command, strings.Join(args, " ")) + out, rerr := executor.Run(runPath, command, envManager, args...) out = envManager.ObfuscateSecrets(out) @@ -118,6 +134,10 @@ func (r *ArtilleryRunner) Run(ctx context.Context, execution testkube.Execution) directories := []string{ testReportFile, } + if execution.ArtifactRequest != nil && len(execution.ArtifactRequest.Dirs) != 0 { + directories = append(directories, execution.ArtifactRequest.Dirs...) + } + err = r.Scraper.Scrape(ctx, directories, execution) if err != nil { return *result.Err(err), errors.Wrap(err, "error scraping artifacts for Artillery executor") diff --git a/contrib/executor/artillery/pkg/runner/artillery_test.go b/contrib/executor/artillery/pkg/runner/artillery_test.go index 9bc99ad14bd..4572841a86b 100644 --- a/contrib/executor/artillery/pkg/runner/artillery_test.go +++ b/contrib/executor/artillery/pkg/runner/artillery_test.go @@ -5,13 +5,11 @@ import ( "os" "testing" - "github.com/kubeshop/testkube/pkg/utils/test" - - "github.com/kubeshop/testkube/pkg/envs" - "github.com/stretchr/testify/assert" "github.com/kubeshop/testkube/pkg/api/v1/testkube" + "github.com/kubeshop/testkube/pkg/envs" + "github.com/kubeshop/testkube/pkg/utils/test" ) func TestRun(t *testing.T) { diff --git a/contrib/executor/curl/cmd/agent/main.go b/contrib/executor/curl/cmd/agent/main.go index cfa0adaf91a..f08c25b53ab 100644 --- a/contrib/executor/curl/cmd/agent/main.go +++ b/contrib/executor/curl/cmd/agent/main.go @@ -7,21 +7,21 @@ import ( "github.com/pkg/errors" - "github.com/kubeshop/testkube/pkg/envs" - "github.com/kubeshop/testkube/pkg/executor/output" - "github.com/kubeshop/testkube/contrib/executor/curl/pkg/runner" + "github.com/kubeshop/testkube/pkg/envs" "github.com/kubeshop/testkube/pkg/executor/agent" + "github.com/kubeshop/testkube/pkg/executor/output" "github.com/kubeshop/testkube/pkg/ui" ) func main() { + ctx := context.Background() params, err := envs.LoadTestkubeVariables() if err != nil { output.PrintError(os.Stderr, errors.Errorf("could not initialize cURL Executor environment variables: %v", err)) os.Exit(1) } - r, err := runner.NewCurlRunner(params) + r, err := runner.NewCurlRunner(ctx, params) if err != nil { log.Fatalf("%s Could not run cURL tests: %s", ui.IconCross, err.Error()) } diff --git a/contrib/executor/curl/pkg/runner/runner.go b/contrib/executor/curl/pkg/runner/runner.go index 33bde8486ad..1939289bcf1 100644 --- a/contrib/executor/curl/pkg/runner/runner.go +++ b/contrib/executor/curl/pkg/runner/runner.go @@ -10,7 +10,6 @@ import ( "strings" "github.com/pkg/errors" - "go.uber.org/zap" "github.com/kubeshop/testkube/pkg/api/v1/testkube" @@ -20,32 +19,45 @@ import ( "github.com/kubeshop/testkube/pkg/executor/env" outputPkg "github.com/kubeshop/testkube/pkg/executor/output" "github.com/kubeshop/testkube/pkg/executor/runner" + "github.com/kubeshop/testkube/pkg/executor/scraper" + "github.com/kubeshop/testkube/pkg/executor/scraper/factory" "github.com/kubeshop/testkube/pkg/log" "github.com/kubeshop/testkube/pkg/ui" ) -const CurlAdditionalFlags = "-is" - // CurlRunner is used to run curl commands. type CurlRunner struct { Params envs.Params Fetcher contentPkg.ContentFetcher Log *zap.SugaredLogger + Scraper scraper.Scraper } var _ runner.Runner = &CurlRunner{} -func NewCurlRunner(params envs.Params) (*CurlRunner, error) { +func NewCurlRunner(ctx context.Context, params envs.Params) (*CurlRunner, error) { outputPkg.PrintLogf("%s Preparing test runner", ui.IconTruck) - return &CurlRunner{ + var err error + r := &CurlRunner{ Log: log.DefaultLogger, Params: params, Fetcher: contentPkg.NewFetcher(""), - }, nil + } + + r.Scraper, err = factory.TryGetScrapper(ctx, params) + if err != nil { + return nil, err + } + + return r, nil } func (r *CurlRunner) Run(ctx context.Context, execution testkube.Execution) (result testkube.ExecutionResult, err error) { + if r.Scraper != nil { + defer r.Scraper.Close() + } + outputPkg.PrintLogf("%s Preparing for test run", ui.IconTruck) var runnerInput CurlRunnerInput if r.Params.GitUsername != "" || r.Params.GitToken != "" { @@ -92,15 +104,23 @@ func (r *CurlRunner) Run(ctx context.Context, execution testkube.Execution) (res } outputPkg.PrintLogf("%s Successfully filled the input templates", ui.IconCheckMark) - command := runnerInput.Command[0] + command := "" + var args []string + if len(execution.Command) != 0 { + command = execution.Command[0] + args = execution.Command[1:] + } + + if len(runnerInput.Command) != 0 { + command = runnerInput.Command[0] + args = runnerInput.Command[1:] + } + if command != "curl" { outputPkg.PrintLogf("%s you can run only `curl` commands with this executor but passed: `%s`", ui.IconCross, command) return result, errors.Errorf("you can run only `curl` commands with this executor but passed: `%s`", command) } - runnerInput.Command[0] = CurlAdditionalFlags - - args := runnerInput.Command args = append(args, execution.Args...) runPath := "" @@ -108,6 +128,7 @@ func (r *CurlRunner) Run(ctx context.Context, execution testkube.Execution) (res runPath = filepath.Join(r.Params.DataDir, "repo", execution.Content.Repository.WorkingDir) } + outputPkg.PrintLogf("%s Test run command %s %s", ui.IconRocket, command, strings.Join(args, " ")) output, err := executor.Run(runPath, command, envManager, args...) output = envManager.ObfuscateSecrets(output) @@ -116,6 +137,15 @@ func (r *CurlRunner) Run(ctx context.Context, execution testkube.Execution) (res return *result.Err(err), nil } + // scrape artifacts first even if there are errors above + if r.Params.ScrapperEnabled && execution.ArtifactRequest != nil && len(execution.ArtifactRequest.Dirs) != 0 { + outputPkg.PrintLogf("Scraping directories: %v", execution.ArtifactRequest.Dirs) + + if err := r.Scraper.Scrape(ctx, execution.ArtifactRequest.Dirs, execution); err != nil { + return *result.WithErrors(err), nil + } + } + outputString := string(output) result.Output = outputString responseStatus, err := getResponseCode(outputString) diff --git a/contrib/executor/cypress/cmd/agent/main.go b/contrib/executor/cypress/cmd/agent/main.go index 6917c5f84f3..ae20861e626 100644 --- a/contrib/executor/cypress/cmd/agent/main.go +++ b/contrib/executor/cypress/cmd/agent/main.go @@ -4,11 +4,10 @@ import ( "context" "os" - "github.com/kubeshop/testkube/pkg/envs" - "github.com/pkg/errors" "github.com/kubeshop/testkube/contrib/executor/cypress/pkg/runner" + "github.com/kubeshop/testkube/pkg/envs" "github.com/kubeshop/testkube/pkg/executor/agent" "github.com/kubeshop/testkube/pkg/executor/output" ) diff --git a/contrib/executor/cypress/pkg/runner/cypress.go b/contrib/executor/cypress/pkg/runner/cypress.go index 3519cd74ae9..16093b5ebbd 100644 --- a/contrib/executor/cypress/pkg/runner/cypress.go +++ b/contrib/executor/cypress/pkg/runner/cypress.go @@ -7,8 +7,6 @@ import ( "path/filepath" "strings" - "github.com/kubeshop/testkube/pkg/executor/scraper" - "github.com/joshdk/go-junit" "github.com/pkg/errors" @@ -18,6 +16,7 @@ import ( "github.com/kubeshop/testkube/pkg/executor/env" "github.com/kubeshop/testkube/pkg/executor/output" "github.com/kubeshop/testkube/pkg/executor/runner" + "github.com/kubeshop/testkube/pkg/executor/scraper" "github.com/kubeshop/testkube/pkg/executor/scraper/factory" "github.com/kubeshop/testkube/pkg/ui" ) @@ -94,7 +93,8 @@ func (r *CypressRunner) Run(ctx context.Context, execution testkube.Execution) ( } // handle project local Cypress version install (`Cypress` app) - out, err = executor.Run(runPath, "./node_modules/cypress/bin/cypress", nil, "install") + command := strings.Join(execution.Command, " ") + out, err = executor.Run(runPath, command, nil, "install") if err != nil { return result, errors.Errorf("cypress binary install error: %v\n\n%s", err, out) } @@ -110,18 +110,36 @@ func (r *CypressRunner) Run(ctx context.Context, execution testkube.Execution) ( } junitReportPath := filepath.Join(projectPath, "results/junit.xml") - args := []string{"run", "--reporter", "junit", "--reporter-options", fmt.Sprintf("mochaFile=%s,toConsole=false", junitReportPath), - "--env", strings.Join(envVars, ",")} + var project string if execution.Content.Repository.WorkingDir != "" { - args = append(args, "--project", projectPath) + project = projectPath } // append args from execution - args = append(args, execution.Args...) + args := execution.Args + for i := len(args) - 1; i >= 0; i-- { + if project == "" && (args[i] == "--project" || args[i] == "") { + args = append(args[:i], args[i+1:]...) + continue + } + + if args[i] == "" { + args[i] = project + } + + if strings.Contains(args[i], "") { + args[i] = strings.ReplaceAll(args[i], "", junitReportPath) + } + + if args[i] == "" { + args[i] = strings.Join(envVars, ",") + } + } // run cypress inside repo directory ignore execution error in case of failed test - out, err = executor.Run(runPath, "./node_modules/cypress/bin/cypress", envManager, args...) + output.PrintLogf("%s Test run command %s %s", ui.IconRocket, command, strings.Join(args, " ")) + out, err = executor.Run(runPath, command, envManager, args...) out = envManager.ObfuscateSecrets(out) suites, serr := junit.IngestFile(junitReportPath) result = MapJunitToExecutionResults(out, suites) @@ -134,6 +152,10 @@ func (r *CypressRunner) Run(ctx context.Context, execution testkube.Execution) ( filepath.Join(projectPath, "cypress/screenshots"), } + if execution.ArtifactRequest != nil && len(execution.ArtifactRequest.Dirs) != 0 { + directories = append(directories, execution.ArtifactRequest.Dirs...) + } + output.PrintLogf("Scraping directories: %v", directories) if err := r.Scraper.Scrape(ctx, directories, execution); err != nil { diff --git a/contrib/executor/cypress/pkg/runner/cypress_test.go b/contrib/executor/cypress/pkg/runner/cypress_test.go index 28717db086f..3389a7710d2 100644 --- a/contrib/executor/cypress/pkg/runner/cypress_test.go +++ b/contrib/executor/cypress/pkg/runner/cypress_test.go @@ -7,14 +7,12 @@ import ( "path/filepath" "testing" - "github.com/kubeshop/testkube/pkg/utils/test" - - "github.com/kubeshop/testkube/pkg/envs" - cp "github.com/otiai10/copy" "github.com/stretchr/testify/assert" "github.com/kubeshop/testkube/pkg/api/v1/testkube" + "github.com/kubeshop/testkube/pkg/envs" + "github.com/kubeshop/testkube/pkg/utils/test" ) func TestRun_Integration(t *testing.T) { @@ -50,6 +48,20 @@ func TestRun_Integration(t *testing.T) { Path: "", }, }, + Command: []string{ + "./node_modules/cypress/bin/cypress", + }, + Args: []string{ + "run", + "--reporter", + "junit", + "--reporter-options", + "mochaFile=,toConsole=false", + "--project", + "", + "--env", + "", + }, }) assert.NoErrorf(t, err, "Cypress Test Failed: ResultErr: %v, Err: %v ", result.ErrorMessage, err) @@ -71,6 +83,21 @@ func TestRunErrors(t *testing.T) { } execution := testkube.NewQueuedExecution() + execution.Command = []string{ + "./node_modules/cypress/bin/cypress", + } + + execution.Args = []string{ + "run", + "--reporter", + "junit", + "--reporter-options", + "mochaFile=,toConsole=false", + "--project", + "", + "--env", + "", + } // when _, err = runner.Run(ctx, *execution) diff --git a/contrib/executor/ginkgo/cmd/agent/main.go b/contrib/executor/ginkgo/cmd/agent/main.go index f9fe2476f9e..73520dd5e3c 100644 --- a/contrib/executor/ginkgo/cmd/agent/main.go +++ b/contrib/executor/ginkgo/cmd/agent/main.go @@ -4,11 +4,10 @@ import ( "context" "os" - "github.com/kubeshop/testkube/pkg/envs" - "github.com/pkg/errors" "github.com/kubeshop/testkube/contrib/executor/ginkgo/pkg/runner" + "github.com/kubeshop/testkube/pkg/envs" "github.com/kubeshop/testkube/pkg/executor/agent" "github.com/kubeshop/testkube/pkg/executor/output" ) diff --git a/contrib/executor/ginkgo/pkg/runner/runner.go b/contrib/executor/ginkgo/pkg/runner/runner.go index c0ecd942ca1..f84844c572f 100644 --- a/contrib/executor/ginkgo/pkg/runner/runner.go +++ b/contrib/executor/ginkgo/pkg/runner/runner.go @@ -7,11 +7,8 @@ import ( "path/filepath" "strings" - "github.com/pkg/errors" - - "github.com/kubeshop/testkube/pkg/executor/scraper/factory" - "github.com/joshdk/go-junit" + "github.com/pkg/errors" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/envs" @@ -21,11 +18,11 @@ import ( "github.com/kubeshop/testkube/pkg/executor/output" "github.com/kubeshop/testkube/pkg/executor/runner" "github.com/kubeshop/testkube/pkg/executor/scraper" + "github.com/kubeshop/testkube/pkg/executor/scraper/factory" "github.com/kubeshop/testkube/pkg/ui" ) var ginkgoDefaultParams = InitializeGinkgoParams() -var ginkgoBin = "ginkgo" func NewGinkgoRunner(ctx context.Context, params envs.Params) (*GinkgoRunner, error) { output.PrintLogf("%s Preparing test runner", ui.IconTruck) @@ -98,13 +95,19 @@ func (r *GinkgoRunner) Run(ctx context.Context, execution testkube.Execution) (r path = filepath.Join(r.Params.DataDir, "repo", execution.Content.Repository.Path) } + reportFile := "report.xml" + if ginkgoParams["GinkgoJunitReport"] != "" { + values := strings.Split(ginkgoParams["GinkgoJunitReport"], " ") + if len(values) > 1 { + reportFile = values[1] + } + } + // Set up ginkgo potential args - ginkgoArgs, err := BuildGinkgoArgs(ginkgoParams, path, runPath) + args, err := BuildGinkgoArgs(ginkgoParams, path, runPath, reportFile, execution) if err != nil { return result, err } - ginkgoPassThroughFlags := BuildGinkgoPassThroughFlags(execution) - ginkgoArgsAndFlags := append(ginkgoArgs, ginkgoPassThroughFlags...) // set up reports directory reportsPath := filepath.Join(path, "reports") @@ -118,14 +121,16 @@ func (r *GinkgoRunner) Run(ctx context.Context, execution testkube.Execution) (r // check Ginkgo version output.PrintLogf("%s Checking Ginkgo CLI version", ui.IconTruck) - _, err = executor.Run(runPath, ginkgoBin, envManager, "version") + command := strings.Join(execution.Command, " ") + _, err = executor.Run(runPath, command, envManager, "version") if err != nil { output.PrintLogf("%s error checking Ginkgo CLI version: %s", ui.IconCross, err.Error()) return result, err } // run executor here - out, err := executor.Run(runPath, ginkgoBin, envManager, ginkgoArgsAndFlags...) + output.PrintLogf("%s Test run command %s %s", ui.IconRocket, command, strings.Join(args, " ")) + out, err := executor.Run(runPath, command, envManager, args...) out = envManager.ObfuscateSecrets(out) // generate report/result @@ -136,13 +141,12 @@ func (r *GinkgoRunner) Run(ctx context.Context, execution testkube.Execution) (r return result, moveErr } } - if ginkgoParams["GinkgoJunitReport"] != "" { - moveErr := MoveReport(runPath, reportsPath, strings.Split(ginkgoParams["GinkgoJunitReport"], " ")[1]) - if moveErr != nil { - output.PrintLogf("%s could not move Junit report: %s", ui.IconCross, moveErr.Error()) - return result, moveErr - } + + moveErr := MoveReport(runPath, reportsPath, reportFile) + if moveErr != nil { + output.PrintLogf("%s could not move Junit report: %s", ui.IconCross, moveErr.Error()) } + if ginkgoParams["GinkgoTeamCityReport"] != "" { moveErr := MoveReport(runPath, reportsPath, strings.Split(ginkgoParams["GinkgoTeamCityReport"], " ")[1]) if moveErr != nil { @@ -150,9 +154,12 @@ func (r *GinkgoRunner) Run(ctx context.Context, execution testkube.Execution) (r return result, moveErr } } - suites, serr := junit.IngestFile(filepath.Join(reportsPath, strings.Split(ginkgoParams["GinkgoJunitReport"], " ")[1])) - result = MapJunitToExecutionResults(out, suites) - output.PrintLogf("%s Mapped Junit to Execution Results...", ui.IconCheckMark) + + suites, serr := junit.IngestFile(filepath.Join(reportsPath, reportFile)) + if serr == nil { + result = MapJunitToExecutionResults(out, suites) + output.PrintLogf("%s Mapped Junit to Execution Results...", ui.IconCheckMark) + } // scrape artifacts first even if there are errors above @@ -161,6 +168,10 @@ func (r *GinkgoRunner) Run(ctx context.Context, execution testkube.Execution) (r reportsPath, } + if execution.ArtifactRequest != nil && len(execution.ArtifactRequest.Dirs) != 0 { + directories = append(directories, execution.ArtifactRequest.Dirs...) + } + if err := r.Scraper.Scrape(ctx, directories, execution); err != nil { return *result.Err(err), errors.Wrap(err, "error scraping artifacts for Ginkgo executor") } @@ -184,30 +195,30 @@ func InitializeGinkgoParams() map[string]string { ginkgoParams := make(map[string]string) ginkgoParams["GinkgoTestPackage"] = "" - ginkgoParams["GinkgoRecursive"] = "-r" // -r - ginkgoParams["GinkgoParallel"] = "-p" // -p - ginkgoParams["GinkgoParallelProcs"] = "" // --procs N - ginkgoParams["GinkgoCompilers"] = "" // --compilers N - ginkgoParams["GinkgoRandomize"] = "--randomize-all" // --randomize-all - ginkgoParams["GinkgoRandomizeSuites"] = "--randomize-suites" // --randomize-suites - ginkgoParams["GinkgoLabelFilter"] = "" // --label-filter QUERY - ginkgoParams["GinkgoFocusFilter"] = "" // --focus REGEXP - ginkgoParams["GinkgoSkipFilter"] = "" // --skip REGEXP - ginkgoParams["GinkgoUntilItFails"] = "" // --until-it-fails - ginkgoParams["GinkgoRepeat"] = "" // --repeat N - ginkgoParams["GinkgoFlakeAttempts"] = "" // --flake-attempts N - ginkgoParams["GinkgoTimeout"] = "" // --timeout=duration - ginkgoParams["GinkgoSkipPackage"] = "" // --skip-package list,of,packages - ginkgoParams["GinkgoFailFast"] = "" // --fail-fast - ginkgoParams["GinkgoKeepGoing"] = "--keep-going" // --keep-going - ginkgoParams["GinkgoFailOnPending"] = "" // --fail-on-pending - ginkgoParams["GinkgoCover"] = "" // --cover - ginkgoParams["GinkgoCoverProfile"] = "" // --coverprofile cover.profile - ginkgoParams["GinkgoRace"] = "" // --race - ginkgoParams["GinkgoTrace"] = "--trace" // --trace - ginkgoParams["GinkgoJsonReport"] = "" // --json-report report.json [will be stored in reports/filename] - ginkgoParams["GinkgoJunitReport"] = "--junit-report report.xml" // --junit-report report.xml [will be stored in reports/filename] - ginkgoParams["GinkgoTeamCityReport"] = "" // --teamcity-report report.teamcity [will be stored in reports/filename] + ginkgoParams["GinkgoRecursive"] = "" // -r + ginkgoParams["GinkgoParallel"] = "" // -p + ginkgoParams["GinkgoParallelProcs"] = "" // --procs N + ginkgoParams["GinkgoCompilers"] = "" // --compilers N + ginkgoParams["GinkgoRandomize"] = "" // --randomize-all + ginkgoParams["GinkgoRandomizeSuites"] = "" // --randomize-suites + ginkgoParams["GinkgoLabelFilter"] = "" // --label-filter QUERY + ginkgoParams["GinkgoFocusFilter"] = "" // --focus REGEXP + ginkgoParams["GinkgoSkipFilter"] = "" // --skip REGEXP + ginkgoParams["GinkgoUntilItFails"] = "" // --until-it-fails + ginkgoParams["GinkgoRepeat"] = "" // --repeat N + ginkgoParams["GinkgoFlakeAttempts"] = "" // --flake-attempts N + ginkgoParams["GinkgoTimeout"] = "" // --timeout=duration + ginkgoParams["GinkgoSkipPackage"] = "" // --skip-package list,of,packages + ginkgoParams["GinkgoFailFast"] = "" // --fail-fast + ginkgoParams["GinkgoKeepGoing"] = "" // --keep-going + ginkgoParams["GinkgoFailOnPending"] = "" // --fail-on-pending + ginkgoParams["GinkgoCover"] = "" // --cover + ginkgoParams["GinkgoCoverProfile"] = "" // --coverprofile cover.profile + ginkgoParams["GinkgoRace"] = "" // --race + ginkgoParams["GinkgoTrace"] = "" // --trace + ginkgoParams["GinkgoJsonReport"] = "" // --json-report report.json [will be stored in reports/filename] + ginkgoParams["GinkgoJunitReport"] = "" // --junit-report report.xml [will be stored in reports/filename] + ginkgoParams["GinkgoTeamCityReport"] = "" // --teamcity-report report.teamcity [will be stored in reports/filename] output.PrintLogf("%s Initial Ginkgo parameters prepared: %s", ui.IconCheckMark, ginkgoParams) return ginkgoParams @@ -234,58 +245,61 @@ func FindGinkgoParams(execution *testkube.Execution, defaultParams map[string]st return retVal } -func BuildGinkgoArgs(params map[string]string, path, runPath string) ([]string, error) { +func BuildGinkgoArgs(params map[string]string, path, runPath, reportFile string, execution testkube.Execution) ([]string, error) { output.PrintLogf("%s Building Ginkgo arguments from params", ui.IconWorld) - var args []string - for k, p := range params { - if p == "" { - continue - } - if k != "GinkgoTestPackage" { - args = append(args, strings.Split(p, " ")...) + args := execution.Args + for i := range args { + if args[i] == "" { + var envVars []string + for k, p := range params { + if p == "" { + continue + } + if k != "GinkgoTestPackage" { + envVars = append(envVars, strings.Split(p, " ")...) + } + } + + newArgs := make([]string, len(args)+len(envVars)-1) + copy(newArgs, args[:i]) + copy(newArgs[i:], envVars) + copy(newArgs[i+len(envVars):], args[i+1:]) + args = newArgs + break } } + var rp string if params["GinkgoTestPackage"] != "" { if path != runPath { - args = append(args, filepath.Join(path, params["GinkgoTestPackage"])) + rp = filepath.Join(path, params["GinkgoTestPackage"]) } else { - args = append(args, params["GinkgoTestPackage"]) + rp = params["GinkgoTestPackage"] } } else { if path != runPath { - args = append(args, path) + rp = path } } - output.PrintLogf("%s Ginkgo arguments from params built: %s", ui.IconCheckMark, args) - return args, nil -} - -// BuildGinkgoPassThroughFlags should always be called after FindGinkgoParams so that it only -// acts on the "left over" Variables that are to be treated as pass through -// flags to GInkgo -func BuildGinkgoPassThroughFlags(execution testkube.Execution) []string { - output.PrintLogf("%s Building Ginkgo flags", ui.IconWorld) - - vars := execution.Variables - args := execution.Args - var flags []string - for _, v := range vars { - os.Setenv(v.Name, v.Value) - } + for i := len(args) - 1; i >= 0; i-- { + if rp == "" && args[i] == "" { + args = append(args[:i], args[i+1:]...) + continue + } - if len(args) > 0 { - flags = append(flags, args...) - } + if args[i] == "" { + args[i] = rp + } - if len(flags) > 0 { - flags = append([]string{"--"}, flags...) + if args[i] == "" { + args[i] = reportFile + } } - output.PrintLogf("%s Ginkgo flags built: %s", ui.IconCheckMark, flags) - return flags + output.PrintLogf("%s Ginkgo arguments from params built: %s", ui.IconCheckMark, args) + return args, nil } // Validate checks if Execution has valid data in context of Ginkgo executor diff --git a/contrib/executor/ginkgo/pkg/runner/runner_integration_test.go b/contrib/executor/ginkgo/pkg/runner/runner_integration_test.go index 718e8d873dd..62c7754db4e 100644 --- a/contrib/executor/ginkgo/pkg/runner/runner_integration_test.go +++ b/contrib/executor/ginkgo/pkg/runner/runner_integration_test.go @@ -5,13 +5,11 @@ import ( "os" "testing" - "github.com/kubeshop/testkube/pkg/utils/test" - - "github.com/kubeshop/testkube/pkg/envs" - "github.com/stretchr/testify/assert" "github.com/kubeshop/testkube/pkg/api/v1/testkube" + "github.com/kubeshop/testkube/pkg/envs" + "github.com/kubeshop/testkube/pkg/utils/test" ) const repoURI = "https://github.com/kubeshop/testkube-executor-ginkgo.git" @@ -53,6 +51,21 @@ func TestRun_Integration(t *testing.T) { }, }, Variables: vars, + Command: []string{ + "ginkgo", + }, + Args: []string{ + "-r", + "-p", + "--randomize-all", + "--randomize-suites", + "--keep-going", + "--trace", + "--junit-report", + "", + "", + "", + }, }) assert.Equal(t, testkube.ExecutionStatusPassed, result.Status) @@ -87,6 +100,21 @@ func TestRun_Integration(t *testing.T) { }, }, Variables: vars, + Command: []string{ + "ginkgo", + }, + Args: []string{ + "-r", + "-p", + "--randomize-all", + "--randomize-suites", + "--keep-going", + "--trace", + "--junit-report", + "", + "", + "", + }, }) assert.Equal(t, testkube.ExecutionStatusFailed, result.Status) diff --git a/contrib/executor/ginkgo/pkg/runner/runner_test.go b/contrib/executor/ginkgo/pkg/runner/runner_test.go index 81d9b9d79eb..e99df7d70b4 100644 --- a/contrib/executor/ginkgo/pkg/runner/runner_test.go +++ b/contrib/executor/ginkgo/pkg/runner/runner_test.go @@ -1,7 +1,6 @@ package runner import ( - "os" "testing" "github.com/stretchr/testify/assert" @@ -12,20 +11,6 @@ import ( func TestRun(t *testing.T) { t.Parallel() - t.Run("InitializeGinkgoParams should should set up some default parameters for ginkgo", func(t *testing.T) { - t.Parallel() - - defaultParams := InitializeGinkgoParams() - assert.Equal(t, "", defaultParams["GinkgoTestPackage"]) - assert.Equal(t, "-r", defaultParams["GinkgoRecursive"]) - assert.Equal(t, "-p", defaultParams["GinkgoParallel"]) - assert.Equal(t, "--randomize-all", defaultParams["GinkgoRandomize"]) - assert.Equal(t, "--randomize-suites", defaultParams["GinkgoRandomizeSuites"]) - assert.Equal(t, "--trace", defaultParams["GinkgoTrace"]) - assert.Equal(t, "--junit-report report.xml", defaultParams["GinkgoJunitReport"]) - - }) - t.Run("FindGoinkgoParams should override default params when provided with new value", func(t *testing.T) { t.Parallel() @@ -50,53 +35,4 @@ func TestRun(t *testing.T) { assert.Equal(t, "e2e", mappedParams["GinkgoTestPackage"]) assert.Equal(t, "", mappedParams["GinkgoRecursive"]) }) - - t.Run("BuildGinkgoArgs should build ginkgo args slice", func(t *testing.T) { - t.Parallel() - - defaultParams := InitializeGinkgoParams() - argSlice, err := BuildGinkgoArgs(defaultParams, "", "") - assert.Nil(t, err) - assert.Contains(t, argSlice, "-r") - assert.Contains(t, argSlice, "-p") - assert.Contains(t, argSlice, "--randomize-all") - assert.Contains(t, argSlice, "--randomize-suites") - assert.Contains(t, argSlice, "--trace") - assert.Contains(t, argSlice, "--junit-report") - assert.Contains(t, argSlice, "report.xml") - }) - - t.Run("BuildGinkgoPassThroughFlags should build pass through flags slice from leftover Variables and from Args", func(t *testing.T) { - t.Parallel() - - variables := make(map[string]testkube.Variable) - variableOne := testkube.Variable{ - Name: "one", - Value: "one", - Type_: testkube.VariableTypeBasic, - } - variableTwo := testkube.Variable{ - Name: "two", - Value: "two", - Type_: testkube.VariableTypeBasic, - } - variables["GinkgoPassThroughOne"] = variableOne - variables["GinkgoPassThroughTwo"] = variableTwo - - args := []string{ - "--three", - "--four=four", - } - - execution := testkube.Execution{ - Variables: variables, - Args: args, - } - passThroughs := BuildGinkgoPassThroughFlags(execution) - assert.Contains(t, passThroughs, "--") - assert.Equal(t, os.Getenv("one"), "one") - assert.Equal(t, os.Getenv("two"), "two") - assert.Contains(t, passThroughs, "--three") - assert.Contains(t, passThroughs, "--four=four") - }) } diff --git a/contrib/executor/gradle/cmd/agent/main.go b/contrib/executor/gradle/cmd/agent/main.go index 572388ade8f..a6a05fb3feb 100644 --- a/contrib/executor/gradle/cmd/agent/main.go +++ b/contrib/executor/gradle/cmd/agent/main.go @@ -6,18 +6,23 @@ import ( "github.com/pkg/errors" - "github.com/kubeshop/testkube/pkg/envs" - "github.com/kubeshop/testkube/pkg/executor/output" - "github.com/kubeshop/testkube/contrib/executor/gradle/pkg/runner" + "github.com/kubeshop/testkube/pkg/envs" "github.com/kubeshop/testkube/pkg/executor/agent" + "github.com/kubeshop/testkube/pkg/executor/output" ) func main() { + ctx := context.Background() params, err := envs.LoadTestkubeVariables() if err != nil { output.PrintError(os.Stderr, errors.Errorf("could not initialize Gradle Executor environment variables: %v", err)) os.Exit(1) } - agent.Run(context.Background(), runner.NewRunner(params), os.Args) + r, err := runner.NewRunner(ctx, params) + if err != nil { + output.PrintError(os.Stderr, errors.Errorf("could not initialize runner: %v", err)) + os.Exit(1) + } + agent.Run(ctx, r, os.Args) } diff --git a/contrib/executor/gradle/pkg/runner/runner.go b/contrib/executor/gradle/pkg/runner/runner.go index e7c67d87ea9..cb7f5048df3 100644 --- a/contrib/executor/gradle/pkg/runner/runner.go +++ b/contrib/executor/gradle/pkg/runner/runner.go @@ -8,33 +8,46 @@ import ( "path/filepath" "strings" - "github.com/kubeshop/testkube/pkg/envs" - - "github.com/pkg/errors" - "github.com/joshdk/go-junit" + "github.com/pkg/errors" "github.com/kubeshop/testkube/pkg/api/v1/testkube" + "github.com/kubeshop/testkube/pkg/envs" "github.com/kubeshop/testkube/pkg/executor" "github.com/kubeshop/testkube/pkg/executor/env" "github.com/kubeshop/testkube/pkg/executor/output" "github.com/kubeshop/testkube/pkg/executor/runner" + "github.com/kubeshop/testkube/pkg/executor/scraper" + "github.com/kubeshop/testkube/pkg/executor/scraper/factory" "github.com/kubeshop/testkube/pkg/ui" ) -func NewRunner(params envs.Params) *GradleRunner { +func NewRunner(ctx context.Context, params envs.Params) (*GradleRunner, error) { output.PrintLogf("%s Preparing test runner", ui.IconTruck) - return &GradleRunner{ + var err error + r := &GradleRunner{ params: params, } + + r.Scraper, err = factory.TryGetScrapper(ctx, params) + if err != nil { + return nil, err + } + + return r, nil } type GradleRunner struct { - params envs.Params + params envs.Params + Scraper scraper.Scraper } func (r *GradleRunner) Run(ctx context.Context, execution testkube.Execution) (result testkube.ExecutionResult, err error) { + if r.Scraper != nil { + defer r.Scraper.Close() + } + output.PrintLogf("%s Preparing for test run", ui.IconTruck) err = r.Validate(execution) if err != nil { @@ -76,28 +89,48 @@ func (r *GradleRunner) Run(ctx context.Context, execution testkube.Execution) (r } // determine the Gradle command to use - gradleCommand := "gradle" + gradleCommand := strings.Join(execution.Command, " ") gradleWrapper := filepath.Join(directory, "gradlew") _, err = os.Stat(gradleWrapper) - if err == nil { + if gradleCommand == "gradle" && err == nil { // then we use the wrapper instead gradleCommand = "./gradlew" } // pass additional executor arguments/flags to Gradle - args := []string{"--no-daemon"} - args = append(args, execution.Args...) - + args := execution.Args + var taskName string task := strings.Split(execution.TestType, "/")[1] if !strings.EqualFold(task, "project") { // then use the test subtype as task name - args = append(args, task) + taskName = task } runPath := directory + var project string if execution.Content.Repository != nil && execution.Content.Repository.WorkingDir != "" { runPath = filepath.Join(r.params.DataDir, "repo", execution.Content.Repository.WorkingDir) - args = append(args, "-p", directory) + project = directory + } + + for i := len(args) - 1; i >= 0; i-- { + if taskName == "" && args[i] == "" { + args = append(args[:i], args[i+1:]...) + continue + } + + if project == "" && (args[i] == "-p" || args[i] == "") { + args = append(args[:i], args[i+1:]...) + continue + } + + if args[i] == "" { + args[i] = taskName + } + + if args[i] == "" { + args[i] = project + } } output.PrintEvent("Running task: "+task, directory, gradleCommand, args) @@ -127,6 +160,15 @@ func (r *GradleRunner) Run(ctx context.Context, execution testkube.Execution) (r } } + // scrape artifacts first even if there are errors above + if r.params.ScrapperEnabled && execution.ArtifactRequest != nil && len(execution.ArtifactRequest.Dirs) != 0 { + output.PrintLogf("Scraping directories: %v", execution.ArtifactRequest.Dirs) + + if err := r.Scraper.Scrape(ctx, execution.ArtifactRequest.Dirs, execution); err != nil { + return *result.WithErrors(err), nil + } + } + result.Output = string(out) result.OutputType = "text/plain" diff --git a/contrib/executor/gradle/pkg/runner/runner_integration_test.go b/contrib/executor/gradle/pkg/runner/runner_integration_test.go index 51f68136bb8..48f81ea9377 100644 --- a/contrib/executor/gradle/pkg/runner/runner_integration_test.go +++ b/contrib/executor/gradle/pkg/runner/runner_integration_test.go @@ -8,15 +8,12 @@ import ( "path/filepath" "testing" - "github.com/kubeshop/testkube/pkg/utils/test" - - "github.com/kubeshop/testkube/pkg/envs" - cp "github.com/otiai10/copy" - "github.com/stretchr/testify/assert" "github.com/kubeshop/testkube/pkg/api/v1/testkube" + "github.com/kubeshop/testkube/pkg/envs" + "github.com/kubeshop/testkube/pkg/utils/test" ) func TestRunGradle_Integration(t *testing.T) { @@ -37,7 +34,9 @@ func TestRunGradle_Integration(t *testing.T) { // given params := envs.Params{DataDir: tempDir} - runner := NewRunner(params) + runner, err := NewRunner(context.Background(), params) + assert.NoError(t, err) + execution := testkube.NewQueuedExecution() execution.TestType = "gradle/project" execution.Content = &testkube.TestContent{ @@ -47,7 +46,8 @@ func TestRunGradle_Integration(t *testing.T) { Branch: "main", }, } - execution.Args = []string{"test"} + execution.Command = []string{"gradle"} + execution.Args = []string{"test", "--no-daemon", "", "-p", ""} // when result, err := runner.Run(ctx, *execution) @@ -69,7 +69,9 @@ func TestRunGradle_Integration(t *testing.T) { // given params := envs.Params{DataDir: tempDir} - runner := NewRunner(params) + runner, err := NewRunner(context.Background(), params) + assert.NoError(t, err) + execution := testkube.NewQueuedExecution() execution.TestType = "gradle/test" execution.Content = &testkube.TestContent{ @@ -79,6 +81,8 @@ func TestRunGradle_Integration(t *testing.T) { Branch: "main", }, } + execution.Command = []string{"gradle"} + execution.Args = []string{"--no-daemon", "", "-p", ""} assert.NoError(t, os.Setenv("TESTKUBE_GRADLE", "true")) // when @@ -103,11 +107,15 @@ func TestRunErrors_Integration(t *testing.T) { t.Parallel() // given params := envs.Params{DataDir: "/unknown"} - runner := NewRunner(params) + runner, err := NewRunner(context.Background(), params) + assert.NoError(t, err) + execution := testkube.NewQueuedExecution() + execution.Command = []string{"gradle"} + execution.Args = []string{"--no-daemon", "", "-p", ""} // when - _, err := runner.Run(ctx, *execution) + _, err = runner.Run(ctx, *execution) // then assert.Error(t, err) @@ -120,13 +128,17 @@ func TestRunErrors_Integration(t *testing.T) { // given params := envs.Params{DataDir: tempDir} - runner := NewRunner(params) + runner, err := NewRunner(context.Background(), params) + assert.NoError(t, err) + execution := testkube.NewQueuedExecution() execution.TestType = "gradle/project" execution.Content = testkube.NewStringTestContent("") + execution.Command = []string{"gradle"} + execution.Args = []string{"--no-daemon", "", "-p", ""} // when - _, err := runner.Run(ctx, *execution) + _, err = runner.Run(ctx, *execution) // then assert.EqualError(t, err, "gradle executor handles only repository based tests, but repository is nil") @@ -144,7 +156,9 @@ func TestRunErrors_Integration(t *testing.T) { // given params := envs.Params{DataDir: tempDir} - runner := NewRunner(params) + runner, err := NewRunner(context.Background(), params) + assert.NoError(t, err) + execution := testkube.NewQueuedExecution() execution.TestType = "gradle/project" execution.Content = &testkube.TestContent{ @@ -154,7 +168,8 @@ func TestRunErrors_Integration(t *testing.T) { Branch: "main", }, } - + execution.Command = []string{"gradle"} + execution.Args = []string{"--no-daemon", "", "-p", ""} // when result, err := runner.Run(ctx, *execution) diff --git a/contrib/executor/init/cmd/agent/main.go b/contrib/executor/init/cmd/agent/main.go index 504760a05a2..5805fa7f05b 100644 --- a/contrib/executor/init/cmd/agent/main.go +++ b/contrib/executor/init/cmd/agent/main.go @@ -6,11 +6,10 @@ import ( "github.com/pkg/errors" - "github.com/kubeshop/testkube/pkg/envs" - "github.com/kubeshop/testkube/pkg/executor/output" - "github.com/kubeshop/testkube/contrib/executor/init/pkg/runner" + "github.com/kubeshop/testkube/pkg/envs" "github.com/kubeshop/testkube/pkg/executor/agent" + "github.com/kubeshop/testkube/pkg/executor/output" ) func main() { diff --git a/contrib/executor/init/pkg/runner/runner.go b/contrib/executor/init/pkg/runner/runner.go index fdf72ef2c78..23b90f6e39d 100755 --- a/contrib/executor/init/pkg/runner/runner.go +++ b/contrib/executor/init/pkg/runner/runner.go @@ -5,8 +5,6 @@ import ( "os" "path/filepath" - "github.com/kubeshop/testkube/pkg/storage/minio" - "github.com/pkg/errors" "github.com/kubeshop/testkube/pkg/api/v1/testkube" @@ -15,6 +13,7 @@ import ( "github.com/kubeshop/testkube/pkg/executor/content" "github.com/kubeshop/testkube/pkg/executor/output" "github.com/kubeshop/testkube/pkg/executor/runner" + "github.com/kubeshop/testkube/pkg/storage/minio" "github.com/kubeshop/testkube/pkg/ui" ) @@ -84,7 +83,7 @@ func (r *InitRunner) Run(ctx context.Context, execution testkube.Execution) (res output.PrintLogf("%s Could not chmod for data dir: %s", ui.IconCross, err.Error()) } - if execution.ArtifactRequest != nil { + if execution.ArtifactRequest != nil && execution.ArtifactRequest.VolumeMountPath != "" { _, err = executor.Run(execution.ArtifactRequest.VolumeMountPath, "chmod", nil, []string{"-R", "777", "."}...) if err != nil { output.PrintLogf("%s Could not chmod for artifacts dir: %s", ui.IconCross, err.Error()) diff --git a/contrib/executor/init/pkg/runner/runner_test.go b/contrib/executor/init/pkg/runner/runner_test.go index 289ec255b75..977c20213f7 100755 --- a/contrib/executor/init/pkg/runner/runner_test.go +++ b/contrib/executor/init/pkg/runner/runner_test.go @@ -4,11 +4,10 @@ import ( "context" "testing" - "github.com/kubeshop/testkube/pkg/envs" - "github.com/stretchr/testify/assert" "github.com/kubeshop/testkube/pkg/api/v1/testkube" + "github.com/kubeshop/testkube/pkg/envs" ) func TestRun(t *testing.T) { diff --git a/contrib/executor/jmeter/cmd/agent/main.go b/contrib/executor/jmeter/cmd/agent/main.go index 20caece58f0..eead0f714bf 100644 --- a/contrib/executor/jmeter/cmd/agent/main.go +++ b/contrib/executor/jmeter/cmd/agent/main.go @@ -6,11 +6,10 @@ import ( "github.com/pkg/errors" - "github.com/kubeshop/testkube/pkg/envs" - "github.com/kubeshop/testkube/pkg/executor/output" - "github.com/kubeshop/testkube/contrib/executor/jmeter/pkg/runner" + "github.com/kubeshop/testkube/pkg/envs" "github.com/kubeshop/testkube/pkg/executor/agent" + "github.com/kubeshop/testkube/pkg/executor/output" ) func main() { diff --git a/contrib/executor/jmeter/pkg/runner/runner.go b/contrib/executor/jmeter/pkg/runner/runner.go index ac874d936b0..fef01f68fb7 100644 --- a/contrib/executor/jmeter/pkg/runner/runner.go +++ b/contrib/executor/jmeter/pkg/runner/runner.go @@ -5,8 +5,7 @@ import ( "fmt" "os" "path/filepath" - - "github.com/kubeshop/testkube/pkg/executor/scraper" + "strings" "github.com/pkg/errors" @@ -17,6 +16,7 @@ import ( "github.com/kubeshop/testkube/pkg/executor/env" "github.com/kubeshop/testkube/pkg/executor/output" "github.com/kubeshop/testkube/pkg/executor/runner" + "github.com/kubeshop/testkube/pkg/executor/scraper" "github.com/kubeshop/testkube/pkg/executor/scraper/factory" "github.com/kubeshop/testkube/pkg/ui" ) @@ -149,16 +149,49 @@ func (r *JMeterRunner) Run(ctx context.Context, execution testkube.Execution) (r jtlPath := filepath.Join(outputDir, "report.jtl") reportPath := filepath.Join(outputDir, "report") jmeterLogPath := filepath.Join(outputDir, "jmeter.log") - args := []string{"-n", "-j", jmeterLogPath, "-t", path, "-l", jtlPath, "-e", "-o", reportPath} - args = append(args, params...) + args := execution.Args + for i := range args { + if args[i] == "" { + args[i] = path + } + + if args[i] == "" { + args[i] = jtlPath + } + + if args[i] == "" { + args[i] = reportPath + } + + if args[i] == "" { + args[i] = jmeterLogPath + } + } + + for i := range args { + if args[i] == "" { + newArgs := make([]string, len(args)+len(params)-1) + copy(newArgs, args[:i]) + copy(newArgs[i:], params) + copy(newArgs[i+len(params):], args[i+1:]) + args = newArgs + break + } + } - // append args from execution - args = append(args, execution.Args...) output.PrintLogf("%s Using arguments: %v", ui.IconWorld, args) - mainCmd := getEntrypoint() + entryPoint := getEntryPoint() + for i := range execution.Command { + if execution.Command[i] == "" { + execution.Command[i] = entryPoint + } + } + + command := strings.Join(execution.Command, " ") // run JMeter inside repo directory ignore execution error in case of failed test - out, err := executor.Run(runPath, mainCmd, envManager, args...) + output.PrintLogf("%s Test run command %s %s", ui.IconRocket, command, strings.Join(args, " ")) + out, err := executor.Run(runPath, command, envManager, args...) if err != nil { return *result.WithErrors(errors.Errorf("jmeter run error: %v", err)), nil } @@ -179,9 +212,11 @@ func (r *JMeterRunner) Run(ctx context.Context, execution testkube.Execution) (r directories := []string{ outputDir, } + if execution.ArtifactRequest != nil && len(execution.ArtifactRequest.Dirs) != 0 { + directories = append(directories, execution.ArtifactRequest.Dirs...) + } output.PrintLogf("Scraping directories: %v", directories) - if err := r.Scraper.Scrape(ctx, directories, execution); err != nil { return *executionResult.Err(err), errors.Wrap(err, "error scraping artifacts for JMeter executor") } @@ -190,7 +225,7 @@ func (r *JMeterRunner) Run(ctx context.Context, execution testkube.Execution) (r return executionResult, nil } -func getEntrypoint() (entrypoint string) { +func getEntryPoint() (entrypoint string) { if entrypoint = os.Getenv("ENTRYPOINT_CMD"); entrypoint != "" { return entrypoint } diff --git a/contrib/executor/jmeter/pkg/runner/runner_integration_test.go b/contrib/executor/jmeter/pkg/runner/runner_integration_test.go index 2f4c0aa47ee..7a16a935d1f 100644 --- a/contrib/executor/jmeter/pkg/runner/runner_integration_test.go +++ b/contrib/executor/jmeter/pkg/runner/runner_integration_test.go @@ -6,13 +6,11 @@ import ( "path/filepath" "testing" - "github.com/kubeshop/testkube/pkg/utils/test" - "github.com/kubeshop/testkube/pkg/envs" - "github.com/stretchr/testify/assert" "github.com/kubeshop/testkube/pkg/api/v1/testkube" + "github.com/kubeshop/testkube/pkg/utils/test" ) func TestRun_Integration(t *testing.T) { @@ -37,6 +35,22 @@ func TestRun_Integration(t *testing.T) { execution := testkube.NewQueuedExecution() execution.TestType = "jmeter/test" execution.Content = testkube.NewStringTestContent("") + execution.Command = []string{ + "", + } + execution.Args = []string{ + "-n", + "-j", + "", + "-t", + "", + "-l", + "", + "-e", + "-o", + "", + "", + } writeTestContent(t, tempDir, "../../examples/kubeshop.jmx") execution.Variables = map[string]testkube.Variable{} @@ -66,6 +80,22 @@ func TestRun_Integration(t *testing.T) { execution := testkube.NewQueuedExecution() execution.TestType = "jmeter/test" execution.Content = testkube.NewStringTestContent("") + execution.Command = []string{ + "", + } + execution.Args = []string{ + "-n", + "-j", + "", + "-t", + "", + "-l", + "", + "-e", + "-o", + "", + "", + } writeTestContent(t, tempDir, "../../examples/kubeshop_failed.jmx") execution.Variables = map[string]testkube.Variable{} @@ -95,10 +125,34 @@ func TestRun_Integration(t *testing.T) { execution := testkube.NewQueuedExecution() execution.TestType = "jmeter/test" execution.Content = testkube.NewStringTestContent("") + execution.Command = []string{ + "", + } + execution.Args = []string{ + "-n", + "-j", + "", + "-t", + "", + "-l", + "", + "-e", + "-o", + "", + "", + "-Jthreads", + "10", + "-Jrampup", + "0", + "-Jloopcount", + "1", + "-Jip", + "sampleip", + "-Jport", + "1234", + } writeTestContent(t, tempDir, "../../examples/kubeshop.jmx") - execution.Args = []string{"-Jthreads", "10", "-Jrampup", "0", "-Jloopcount", "1", "-Jip", "sampleip", "-Jport", "1234"} - result, err := runner.Run(ctx, *execution) assert.NoError(t, err) diff --git a/contrib/executor/k6/cmd/agent/main.go b/contrib/executor/k6/cmd/agent/main.go index a11e83634f0..4ccea704a62 100644 --- a/contrib/executor/k6/cmd/agent/main.go +++ b/contrib/executor/k6/cmd/agent/main.go @@ -2,22 +2,29 @@ package main import ( "context" + "log" "os" "github.com/pkg/errors" - "github.com/kubeshop/testkube/pkg/envs" - "github.com/kubeshop/testkube/pkg/executor/output" - "github.com/kubeshop/testkube/contrib/executor/k6/pkg/runner" + "github.com/kubeshop/testkube/pkg/envs" "github.com/kubeshop/testkube/pkg/executor/agent" + "github.com/kubeshop/testkube/pkg/executor/output" + "github.com/kubeshop/testkube/pkg/ui" ) func main() { + ctx := context.Background() params, err := envs.LoadTestkubeVariables() if err != nil { output.PrintError(os.Stderr, errors.Errorf("could not initialize K6 Executor environment variables: %v", err)) os.Exit(1) } - agent.Run(context.Background(), runner.NewRunner(params), os.Args) + r, err := runner.NewRunner(ctx, params) + if err != nil { + log.Fatalf("%s Could not run cURL tests: %s", ui.IconCross, err.Error()) + } + + agent.Run(ctx, r, os.Args) } diff --git a/contrib/executor/k6/pkg/runner/runner.go b/contrib/executor/k6/pkg/runner/runner.go index 3a0d4b3c7f3..a8d6a2035ba 100644 --- a/contrib/executor/k6/pkg/runner/runner.go +++ b/contrib/executor/k6/pkg/runner/runner.go @@ -7,28 +7,38 @@ import ( "path/filepath" "strings" - "github.com/kubeshop/testkube/pkg/envs" - "github.com/pkg/errors" "github.com/kubeshop/testkube/pkg/api/v1/testkube" + "github.com/kubeshop/testkube/pkg/envs" "github.com/kubeshop/testkube/pkg/executor" "github.com/kubeshop/testkube/pkg/executor/env" outputPkg "github.com/kubeshop/testkube/pkg/executor/output" "github.com/kubeshop/testkube/pkg/executor/runner" + "github.com/kubeshop/testkube/pkg/executor/scraper" + "github.com/kubeshop/testkube/pkg/executor/scraper/factory" "github.com/kubeshop/testkube/pkg/ui" ) -func NewRunner(params envs.Params) *K6Runner { +func NewRunner(ctx context.Context, params envs.Params) (*K6Runner, error) { outputPkg.PrintLogf("%s Preparing test runner", ui.IconTruck) - return &K6Runner{ + var err error + r := &K6Runner{ Params: params, } + + r.Scraper, err = factory.TryGetScrapper(ctx, params) + if err != nil { + return nil, err + } + + return r, nil } type K6Runner struct { - Params envs.Params + Params envs.Params + Scraper scraper.Scraper } var _ runner.Runner = &K6Runner{} @@ -37,6 +47,10 @@ const K6Cloud = "cloud" const K6Run = "run" func (r *K6Runner) Run(ctx context.Context, execution testkube.Execution) (result testkube.ExecutionResult, err error) { + if r.Scraper != nil { + defer r.Scraper.Close() + } + outputPkg.PrintLogf("%s Preparing for test run", ui.IconTruck) // check that the datadir exists @@ -46,8 +60,7 @@ func (r *K6Runner) Run(ctx context.Context, execution testkube.Execution) (resul return result, err } - var args []string - + var k6Command string k6TestType := strings.Split(execution.TestType, "/") if len(k6TestType) != 2 { outputPkg.PrintLogf("%s Invalid test type %s", ui.IconCross, execution.TestType) @@ -56,18 +69,19 @@ func (r *K6Runner) Run(ctx context.Context, execution testkube.Execution) (resul k6Subtype := k6TestType[1] if k6Subtype == K6Cloud { - args = append(args, K6Cloud) + k6Command = K6Cloud } else { - args = append(args, K6Run) + k6Command = K6Run } + var envVars []string envManager := env.NewManagerWithVars(execution.Variables) envManager.GetReferenceVars(envManager.Variables) for _, variable := range envManager.Variables { if variable.Name != "K6_CLOUD_TOKEN" { // pass to k6 using -e option envvar := fmt.Sprintf("%s=%s", variable.Name, variable.Value) - args = append(args, "-e", envvar) + envVars = append(envVars, "-e", envvar) } } @@ -77,21 +91,19 @@ func (r *K6Runner) Run(ctx context.Context, execution testkube.Execution) (resul if key != "K6_CLOUD_TOKEN" { // pass to k6 using -e option envvar := fmt.Sprintf("%s=%s", key, value) - args = append(args, "-e", envvar) + envVars = append(envVars, "-e", envvar) } } - // pass additional executor arguments/flags to k6 - args = append(args, execution.Args...) - var directory string - + var testPath string + args := execution.Args // in case of a test file execution we will pass the // file path as final parameter to k6 if execution.Content.Type_ == string(testkube.TestContentTypeString) || execution.Content.Type_ == string(testkube.TestContentTypeFileURI) { directory = r.Params.DataDir - args = append(args, "test-content") + testPath = "test-content" } // in case of Git directory we will run k6 here and @@ -114,13 +126,15 @@ func (r *K6Runner) Run(ctx context.Context, execution testkube.Execution) (resul } if fileInfo.IsDir() { - args[len(args)-1] = filepath.Join(path, args[len(args)-1]) + testPath = filepath.Join(path, args[len(args)-1]) + args = append(args[:len(args)-1], args[len(args):]...) + } else { - args = append(args, path) + testPath = path } // sanity checking for test script - scriptFile := filepath.Join(directory, workingDir, args[len(args)-1]) + scriptFile := filepath.Join(directory, workingDir, testPath) fileInfo, err = os.Stat(scriptFile) if errors.Is(err, os.ErrNotExist) || fileInfo.IsDir() { outputPkg.PrintLogf("%s k6 test script %s not found", ui.IconCross, scriptFile) @@ -128,14 +142,46 @@ func (r *K6Runner) Run(ctx context.Context, execution testkube.Execution) (resul } } - outputPkg.PrintEvent("Running", directory, "k6", args) + for i := range args { + if args[i] == "" { + args[i] = k6Command + } + + if args[i] == "" { + args[i] = testPath + } + } + + for i := range args { + if args[i] == "" { + newArgs := make([]string, len(args)+len(envVars)-1) + copy(newArgs, args[:i]) + copy(newArgs[i:], envVars) + copy(newArgs[i+len(envVars):], args[i+1:]) + args = newArgs + break + } + } + + command := strings.Join(execution.Command, " ") + outputPkg.PrintEvent("Running", directory, command, args) runPath := directory if execution.Content.Repository != nil && execution.Content.Repository.WorkingDir != "" { runPath = filepath.Join(directory, execution.Content.Repository.WorkingDir) } - output, err := executor.Run(runPath, "k6", envManager, args...) + output, err := executor.Run(runPath, command, envManager, args...) output = envManager.ObfuscateSecrets(output) + + // scrape artifacts first even if there are errors above + if r.Params.ScrapperEnabled && execution.ArtifactRequest != nil && len(execution.ArtifactRequest.Dirs) != 0 { + outputPkg.PrintLogf("Scraping directories: %v", execution.ArtifactRequest.Dirs) + + if err := r.Scraper.Scrape(ctx, execution.ArtifactRequest.Dirs, execution); err != nil { + return *result.WithErrors(err), nil + } + } + return finalExecutionResult(string(output), err), nil } diff --git a/contrib/executor/k6/pkg/runner/runner_integration_test.go b/contrib/executor/k6/pkg/runner/runner_integration_test.go index 010f147beb1..4691fe82fd9 100644 --- a/contrib/executor/k6/pkg/runner/runner_integration_test.go +++ b/contrib/executor/k6/pkg/runner/runner_integration_test.go @@ -6,13 +6,11 @@ import ( "path/filepath" "testing" - "github.com/kubeshop/testkube/pkg/utils/test" - - "github.com/kubeshop/testkube/pkg/envs" - "github.com/stretchr/testify/assert" "github.com/kubeshop/testkube/pkg/api/v1/testkube" + "github.com/kubeshop/testkube/pkg/envs" + "github.com/kubeshop/testkube/pkg/utils/test" ) func TestRunFiles_Integration(t *testing.T) { @@ -29,10 +27,20 @@ func TestRunFiles_Integration(t *testing.T) { defer os.RemoveAll(tempDir) params := envs.Params{DataDir: tempDir} - runner := NewRunner(params) + runner, err := NewRunner(context.Background(), params) + assert.NoError(t, err) + execution := testkube.NewQueuedExecution() execution.Content = testkube.NewStringTestContent("") execution.TestType = "k6/script" + execution.Command = []string{ + "k6", + } + execution.Args = []string{ + "", + "", + "", + } writeTestContent(t, tempDir, "../../examples/k6-test-script.js") // when @@ -52,10 +60,20 @@ func TestRunFiles_Integration(t *testing.T) { defer os.RemoveAll(tempDir) params := envs.Params{DataDir: tempDir} - runner := NewRunner(params) + runner, err := NewRunner(context.Background(), params) + assert.NoError(t, err) + execution := testkube.NewQueuedExecution() execution.Content = testkube.NewStringTestContent("") execution.TestType = "k6/script" + execution.Command = []string{ + "k6", + } + execution.Args = []string{ + "", + "", + "", + } writeTestContent(t, tempDir, "../../examples/k6-test-failing-script.js") // when @@ -75,11 +93,24 @@ func TestRunFiles_Integration(t *testing.T) { defer os.RemoveAll(tempDir) params := envs.Params{DataDir: tempDir} - runner := NewRunner(params) + runner, err := NewRunner(context.Background(), params) + assert.NoError(t, err) + execution := testkube.NewQueuedExecution() execution.Content = testkube.NewStringTestContent("") execution.TestType = "k6/run" - execution.Args = []string{"--vus", "2", "--duration", "1s"} + execution.Command = []string{ + "k6", + } + execution.Args = []string{ + "", + "", + "", + "--vus", + "2", + "--duration", + "1s", + } writeTestContent(t, tempDir, "../../examples/k6-test-script.js") // when @@ -99,10 +130,20 @@ func TestRunFiles_Integration(t *testing.T) { defer os.RemoveAll(tempDir) params := envs.Params{DataDir: tempDir} - runner := NewRunner(params) + runner, err := NewRunner(context.Background(), params) + assert.NoError(t, err) + execution := testkube.NewQueuedExecution() execution.Content = testkube.NewStringTestContent("") execution.TestType = "k6/script" + execution.Command = []string{ + "k6", + } + execution.Args = []string{ + "", + "", + "", + } execution.Envs = map[string]string{"TARGET_HOSTNAME": "kubeshop.github.io"} writeTestContent(t, tempDir, "../../examples/k6-test-environment.js") @@ -130,10 +171,20 @@ func TestRunAdvanced_Integration(t *testing.T) { defer os.RemoveAll(tempDir) params := envs.Params{DataDir: tempDir} - runner := NewRunner(params) + runner, err := NewRunner(context.Background(), params) + assert.NoError(t, err) + execution := testkube.NewQueuedExecution() execution.Content = testkube.NewStringTestContent("") execution.TestType = "k6/run" + execution.Command = []string{ + "k6", + } + execution.Args = []string{ + "", + "", + "", + } writeTestContent(t, tempDir, "../../examples/k6-test-scenarios.js") // when @@ -153,10 +204,20 @@ func TestRunAdvanced_Integration(t *testing.T) { defer os.RemoveAll(tempDir) params := envs.Params{DataDir: tempDir} - runner := NewRunner(params) + runner, err := NewRunner(context.Background(), params) + assert.NoError(t, err) + execution := testkube.NewQueuedExecution() execution.Content = testkube.NewStringTestContent("") execution.TestType = "k6/script" + execution.Command = []string{ + "k6", + } + execution.Args = []string{ + "", + "", + "", + } writeTestContent(t, tempDir, "../../examples/k6-test-thresholds.js") // when @@ -195,7 +256,9 @@ func TestRunDirs_Integtaion(t *testing.T) { t.Run("Run k6 from directory with script argument", func(t *testing.T) { params := envs.Params{DataDir: tempDir} - runner := NewRunner(params) + runner, err := NewRunner(context.Background(), params) + assert.NoError(t, err) + execution := testkube.NewQueuedExecution() execution.Content = &testkube.TestContent{ Type_: string(testkube.TestContentTypeGitDir), @@ -205,7 +268,17 @@ func TestRunDirs_Integtaion(t *testing.T) { }, } execution.TestType = "k6/script" - execution.Args = []string{"--duration", "1s", "k6-test-script.js"} + execution.Command = []string{ + "k6", + } + execution.Args = []string{ + "", + "", + "", + "--duration", + "1s", + "k6-test-script.js", + } // when result, err := runner.Run(ctx, *execution) @@ -231,10 +304,20 @@ func TestRunErrors_Integration(t *testing.T) { defer os.RemoveAll(tempDir) params := envs.Params{DataDir: tempDir} - runner := NewRunner(params) + runner, err := NewRunner(context.Background(), params) + assert.NoError(t, err) + execution := testkube.NewQueuedExecution() execution.Content = testkube.NewStringTestContent("") execution.TestType = "k6/script" + execution.Command = []string{ + "k6", + } + execution.Args = []string{ + "", + "", + "", + } // when result, err := runner.Run(ctx, *execution) @@ -253,11 +336,24 @@ func TestRunErrors_Integration(t *testing.T) { defer os.RemoveAll(tempDir) params := envs.Params{DataDir: tempDir} - runner := NewRunner(params) + runner, err := NewRunner(context.Background(), params) + assert.NoError(t, err) + execution := testkube.NewQueuedExecution() execution.Content = testkube.NewStringTestContent("") execution.TestType = "k6/script" - execution.Args = []string{"--vues", "2", "--duration", "5"} + execution.Command = []string{ + "k6", + } + execution.Args = []string{ + "", + "", + "", + "--vues", + "2", + "--duration", + "5", + } // when result, err := runner.Run(ctx, *execution) @@ -276,7 +372,9 @@ func TestRunErrors_Integration(t *testing.T) { defer os.RemoveAll(tempDir) params := envs.Params{DataDir: tempDir} - runner := NewRunner(params) + runner, err := NewRunner(context.Background(), params) + assert.NoError(t, err) + execution := testkube.NewQueuedExecution() execution.Content = &testkube.TestContent{ Type_: string(testkube.TestContentTypeGitDir), @@ -287,7 +385,14 @@ func TestRunErrors_Integration(t *testing.T) { }, } execution.TestType = "k6/script" - execution.Args = []string{} + execution.Command = []string{ + "k6", + } + execution.Args = []string{ + "", + "", + "", + } // when result, err := runner.Run(ctx, *execution) diff --git a/contrib/executor/kubepug/cmd/agent/main.go b/contrib/executor/kubepug/cmd/agent/main.go index 4c6b61f7cab..98c4e9011b6 100644 --- a/contrib/executor/kubepug/cmd/agent/main.go +++ b/contrib/executor/kubepug/cmd/agent/main.go @@ -2,22 +2,28 @@ package main import ( "context" + "log" "os" "github.com/pkg/errors" - "github.com/kubeshop/testkube/pkg/envs" - "github.com/kubeshop/testkube/pkg/executor/output" - "github.com/kubeshop/testkube/contrib/executor/kubepug/pkg/runner" + "github.com/kubeshop/testkube/pkg/envs" "github.com/kubeshop/testkube/pkg/executor/agent" + "github.com/kubeshop/testkube/pkg/executor/output" + "github.com/kubeshop/testkube/pkg/ui" ) func main() { + ctx := context.Background() params, err := envs.LoadTestkubeVariables() if err != nil { output.PrintError(os.Stderr, errors.Errorf("could not initialize Kubepug Executor environment variables: %v", err)) os.Exit(1) } - agent.Run(context.Background(), runner.NewRunner(params), os.Args) + r, err := runner.NewRunner(ctx, params) + if err != nil { + log.Fatalf("%s could not run artillery tests: %s", ui.IconCross, err.Error()) + } + agent.Run(ctx, r, os.Args) } diff --git a/contrib/executor/kubepug/pkg/runner/result.go b/contrib/executor/kubepug/pkg/runner/result.go index 2d4d8f2cb3e..4ff13f69c0f 100644 --- a/contrib/executor/kubepug/pkg/runner/result.go +++ b/contrib/executor/kubepug/pkg/runner/result.go @@ -4,7 +4,6 @@ import ( "encoding/json" "github.com/pkg/errors" - kubepug "github.com/rikatz/kubepug/pkg/results" ) diff --git a/contrib/executor/kubepug/pkg/runner/result_test.go b/contrib/executor/kubepug/pkg/runner/result_test.go index b9d86d43020..64427905230 100644 --- a/contrib/executor/kubepug/pkg/runner/result_test.go +++ b/contrib/executor/kubepug/pkg/runner/result_test.go @@ -4,9 +4,8 @@ import ( "fmt" "testing" - "github.com/stretchr/testify/assert" - kubepug "github.com/rikatz/kubepug/pkg/results" + "github.com/stretchr/testify/assert" ) func TestResultParser(t *testing.T) { diff --git a/contrib/executor/kubepug/pkg/runner/runner.go b/contrib/executor/kubepug/pkg/runner/runner.go index 00258e7dbaf..5140d6af716 100644 --- a/contrib/executor/kubepug/pkg/runner/runner.go +++ b/contrib/executor/kubepug/pkg/runner/runner.go @@ -8,40 +8,51 @@ import ( "path/filepath" "strings" - "github.com/kubeshop/testkube/pkg/envs" - - "github.com/pkg/errors" - kubepug "github.com/rikatz/kubepug/pkg/results" "github.com/kubeshop/testkube/pkg/api/v1/testkube" + "github.com/kubeshop/testkube/pkg/envs" "github.com/kubeshop/testkube/pkg/executor" "github.com/kubeshop/testkube/pkg/executor/content" "github.com/kubeshop/testkube/pkg/executor/env" "github.com/kubeshop/testkube/pkg/executor/output" "github.com/kubeshop/testkube/pkg/executor/runner" + "github.com/kubeshop/testkube/pkg/executor/scraper" + "github.com/kubeshop/testkube/pkg/executor/scraper/factory" "github.com/kubeshop/testkube/pkg/ui" ) -func NewRunner(params envs.Params) *KubepugRunner { +func NewRunner(ctx context.Context, params envs.Params) (*KubepugRunner, error) { output.PrintLogf("%s Preparing test runner", ui.IconTruck) - return &KubepugRunner{ + var err error + r := &KubepugRunner{ Fetcher: content.NewFetcher(""), params: params, } + + r.Scraper, err = factory.TryGetScrapper(ctx, params) + if err != nil { + return nil, err + } + + return r, nil } // KubepugRunner runs kubepug against cluster type KubepugRunner struct { Fetcher content.ContentFetcher params envs.Params + Scraper scraper.Scraper } var _ runner.Runner = &KubepugRunner{} // Run runs the kubepug executable and parses it's output to be Testkube-compatible func (r *KubepugRunner) Run(ctx context.Context, execution testkube.Execution) (testkube.ExecutionResult, error) { + if r.Scraper != nil { + defer r.Scraper.Close() + } output.PrintLogf("%s Preparing for test run", ui.IconTruck) path, err := r.Fetcher.Fetch(execution.Content) @@ -77,13 +88,24 @@ func (r *KubepugRunner) Run(ctx context.Context, execution testkube.Execution) ( runPath = filepath.Join(r.params.DataDir, "repo", execution.Content.Repository.WorkingDir) } - out, err := executor.Run(runPath, "kubepug", envManager, args...) + command := strings.Join(execution.Command, " ") + output.PrintLogf("%s Test run command %s %s", ui.IconRocket, command, strings.Join(args, " ")) + out, err := executor.Run(runPath, command, envManager, args...) out = envManager.ObfuscateSecrets(out) if err != nil { output.PrintLogf("%s Could not execute kubepug: %s", ui.IconCross, err.Error()) return testkube.ExecutionResult{}, fmt.Errorf("could not execute kubepug: %w", err) } + // scrape artifacts first even if there are errors above + if r.params.ScrapperEnabled && execution.ArtifactRequest != nil && len(execution.ArtifactRequest.Dirs) != 0 { + output.PrintLogf("Scraping directories: %v", execution.ArtifactRequest.Dirs) + + if err := r.Scraper.Scrape(ctx, execution.ArtifactRequest.Dirs, execution); err != nil { + return testkube.ExecutionResult{}, fmt.Errorf("could not scrape kubepug directories: %w", err) + } + } + var kubepugResult kubepug.Result err = json.Unmarshal(out, &kubepugResult) if err != nil { @@ -163,15 +185,12 @@ func getResultStatus(r kubepug.Result) *testkube.ExecutionStatus { // buildArgs builds up the arguments for func buildArgs(args []string, inputPath string) ([]string, error) { - for _, a := range args { - if strings.Contains(a, "--format") { - return []string{}, fmt.Errorf("the Testkube Kubepug executor does not accept the \"--format\" parameter: %s", a) - } - if strings.Contains(a, "--input-file") { - return []string{}, errors.Errorf("the Testkube Kubepug executor does not accept the \"--input-file\" parameter: %s", a) + for i := range args { + if args[i] == "" { + args[i] = inputPath } } - return append(args, "--format=json", "--input-file", inputPath), nil + return args, nil } // GetType returns runner type diff --git a/contrib/executor/kubepug/pkg/runner/runner_test.go b/contrib/executor/kubepug/pkg/runner/runner_test.go index 6d2856f87cd..a8ded43c8de 100644 --- a/contrib/executor/kubepug/pkg/runner/runner_test.go +++ b/contrib/executor/kubepug/pkg/runner/runner_test.go @@ -4,13 +4,11 @@ import ( "context" "testing" - "github.com/kubeshop/testkube/pkg/utils/test" - - "github.com/kubeshop/testkube/pkg/envs" - "github.com/stretchr/testify/assert" "github.com/kubeshop/testkube/pkg/api/v1/testkube" + "github.com/kubeshop/testkube/pkg/envs" + "github.com/kubeshop/testkube/pkg/utils/test" ) func TestRunString_Integration(t *testing.T) { @@ -22,9 +20,17 @@ func TestRunString_Integration(t *testing.T) { t.Run("runner should return success and empty result on empty string", func(t *testing.T) { t.Parallel() - runner := NewRunner(envs.Params{}) + runner, err := NewRunner(context.Background(), envs.Params{}) + assert.NoError(t, err) + execution := testkube.NewQueuedExecution() execution.Content = testkube.NewStringTestContent("") + execution.Command = []string{"kubepug"} + execution.Args = []string{ + "--format=json", + "--input-file", + "", + } result, err := runner.Run(ctx, *execution) @@ -38,8 +44,16 @@ func TestRunString_Integration(t *testing.T) { t.Run("runner should return success and empty result on passing yaml", func(t *testing.T) { t.Parallel() - runner := NewRunner(envs.Params{}) + runner, err := NewRunner(context.Background(), envs.Params{}) + assert.NoError(t, err) + execution := testkube.NewQueuedExecution() + execution.Command = []string{"kubepug"} + execution.Args = []string{ + "--format=json", + "--input-file", + "", + } execution.Content = testkube.NewStringTestContent(` apiVersion: v1 kind: ConfigMap @@ -65,8 +79,16 @@ metadata: t.Run("runner should return failure and list of deprecated APIs result on yaml containing deprecated API", func(t *testing.T) { t.Parallel() - runner := NewRunner(envs.Params{}) + runner, err := NewRunner(context.Background(), envs.Params{}) + assert.NoError(t, err) + execution := testkube.NewQueuedExecution() + execution.Command = []string{"kubepug"} + execution.Args = []string{ + "--format=json", + "--input-file", + "", + } execution.Content = testkube.NewStringTestContent(` apiVersion: v1 conditions: @@ -98,8 +120,16 @@ func TestRunFileURI_Integration(t *testing.T) { t.Run("runner should return success on valid yaml gist file URI", func(t *testing.T) { t.Parallel() - runner := NewRunner(envs.Params{}) + runner, err := NewRunner(context.Background(), envs.Params{}) + assert.NoError(t, err) + execution := testkube.NewQueuedExecution() + execution.Command = []string{"kubepug"} + execution.Args = []string{ + "--format=json", + "--input-file", + "", + } execution.Content = &testkube.TestContent{ Type_: string(testkube.TestContentTypeFileURI), Uri: "https://gist.githubusercontent.com/vLia/" + @@ -118,8 +148,16 @@ func TestRunFileURI_Integration(t *testing.T) { t.Run("runner should return failure on yaml gist file URI with deprecated/deleted APIs", func(t *testing.T) { t.Parallel() - runner := NewRunner(envs.Params{}) + runner, err := NewRunner(context.Background(), envs.Params{}) + assert.NoError(t, err) + execution := testkube.NewQueuedExecution() + execution.Command = []string{"kubepug"} + execution.Args = []string{ + "--format=json", + "--input-file", + "", + } execution.Content = &testkube.TestContent{ Type_: string(testkube.TestContentTypeFileURI), Uri: "https://gist.githubusercontent.com/vLia/" + @@ -145,8 +183,16 @@ func TestRunGitFile_Integration(t *testing.T) { t.Run("runner should return error on non-existent Git path", func(t *testing.T) { t.Parallel() - runner := NewRunner(envs.Params{}) + runner, err := NewRunner(context.Background(), envs.Params{}) + assert.NoError(t, err) + execution := testkube.NewQueuedExecution() + execution.Command = []string{"kubepug"} + execution.Args = []string{ + "--format=json", + "--input-file", + "", + } execution.Content = &testkube.TestContent{ Type_: string(testkube.TestContentTypeGitFile), Repository: &testkube.Repository{ @@ -156,7 +202,7 @@ func TestRunGitFile_Integration(t *testing.T) { }, } - _, err := runner.Run(ctx, *execution) + _, err = runner.Run(ctx, *execution) assert.Error(t, err) }) @@ -164,8 +210,16 @@ func TestRunGitFile_Integration(t *testing.T) { t.Run("runner should return deprecated and deleted APIs on Git file containing deprecated and delete API definitions", func(t *testing.T) { t.Parallel() - runner := NewRunner(envs.Params{}) + runner, err := NewRunner(context.Background(), envs.Params{}) + assert.NoError(t, err) + execution := testkube.NewQueuedExecution() + execution.Command = []string{"kubepug"} + execution.Args = []string{ + "--format=json", + "--input-file", + "", + } execution.Content = &testkube.TestContent{ Type_: string(testkube.TestContentTypeGitFile), Repository: &testkube.Repository{ @@ -194,8 +248,16 @@ func TestRunGitDirectory_Integration(t *testing.T) { t.Run("runner should return success on manifests from Git directory", func(t *testing.T) { t.Parallel() - runner := NewRunner(envs.Params{}) + runner, err := NewRunner(context.Background(), envs.Params{}) + assert.NoError(t, err) + execution := testkube.NewQueuedExecution() + execution.Command = []string{"kubepug"} + execution.Args = []string{ + "--format=json", + "--input-file", + "", + } execution.Content = &testkube.TestContent{ Type_: string(testkube.TestContentTypeGitDir), Repository: &testkube.Repository{ @@ -225,8 +287,16 @@ func TestRunWithSpecificK8sVersion_Integration(t *testing.T) { "on yaml containing deprecated API with current K8s version", func(t *testing.T) { t.Parallel() - runner := NewRunner(envs.Params{}) + runner, err := NewRunner(context.Background(), envs.Params{}) + assert.NoError(t, err) + execution := testkube.NewQueuedExecution() + execution.Command = []string{"kubepug"} + execution.Args = []string{ + "--format=json", + "--input-file", + "", + } execution.Content = testkube.NewStringTestContent(` apiVersion: v1 conditions: @@ -251,9 +321,15 @@ metadata: t.Run("runner should return success on yaml containing deprecated API with old K8s version", func(t *testing.T) { t.Parallel() - runner := NewRunner(envs.Params{}) + runner, err := NewRunner(context.Background(), envs.Params{}) + assert.NoError(t, err) + execution := testkube.NewQueuedExecution() + execution.Command = []string{"kubepug"} execution.Args = []string{ + "--format=json", + "--input-file", + "", "--k8s-version=v1.18.0", // last version v1/ComponentStatus was valid } execution.Content = testkube.NewStringTestContent(` diff --git a/contrib/executor/maven/cmd/agent/main.go b/contrib/executor/maven/cmd/agent/main.go index 3ac71eb0744..0ee0dcee12f 100644 --- a/contrib/executor/maven/cmd/agent/main.go +++ b/contrib/executor/maven/cmd/agent/main.go @@ -6,18 +6,23 @@ import ( "github.com/pkg/errors" - "github.com/kubeshop/testkube/pkg/envs" - "github.com/kubeshop/testkube/pkg/executor/output" - "github.com/kubeshop/testkube/contrib/executor/maven/pkg/runner" + "github.com/kubeshop/testkube/pkg/envs" "github.com/kubeshop/testkube/pkg/executor/agent" + "github.com/kubeshop/testkube/pkg/executor/output" ) func main() { + ctx := context.Background() params, err := envs.LoadTestkubeVariables() if err != nil { output.PrintError(os.Stderr, errors.Errorf("could not initialize Maven Executor environment variables: %v", err)) os.Exit(1) } - agent.Run(context.Background(), runner.NewRunner(params), os.Args) + r, err := runner.NewRunner(ctx, params) + if err != nil { + output.PrintError(os.Stderr, errors.Errorf("could not initialize runner: %v", err)) + os.Exit(1) + } + agent.Run(ctx, r, os.Args) } diff --git a/contrib/executor/maven/pkg/runner/runner.go b/contrib/executor/maven/pkg/runner/runner.go index 606594a2869..54631095aee 100644 --- a/contrib/executor/maven/pkg/runner/runner.go +++ b/contrib/executor/maven/pkg/runner/runner.go @@ -8,35 +8,48 @@ import ( "path/filepath" "strings" - "github.com/kubeshop/testkube/pkg/envs" - - "github.com/pkg/errors" - "github.com/joshdk/go-junit" + "github.com/pkg/errors" "github.com/kubeshop/testkube/pkg/api/v1/testkube" + "github.com/kubeshop/testkube/pkg/envs" "github.com/kubeshop/testkube/pkg/executor" "github.com/kubeshop/testkube/pkg/executor/env" outputPkg "github.com/kubeshop/testkube/pkg/executor/output" "github.com/kubeshop/testkube/pkg/executor/runner" + "github.com/kubeshop/testkube/pkg/executor/scraper" + "github.com/kubeshop/testkube/pkg/executor/scraper/factory" "github.com/kubeshop/testkube/pkg/ui" ) -func NewRunner(params envs.Params) *MavenRunner { +func NewRunner(ctx context.Context, params envs.Params) (*MavenRunner, error) { outputPkg.PrintLogf("%s Preparing test runner", ui.IconTruck) - return &MavenRunner{ + var err error + r := &MavenRunner{ params: params, } + + r.Scraper, err = factory.TryGetScrapper(ctx, params) + if err != nil { + return nil, err + } + + return r, nil } type MavenRunner struct { - params envs.Params + params envs.Params + Scraper scraper.Scraper } var _ runner.Runner = &MavenRunner{} func (r *MavenRunner) Run(ctx context.Context, execution testkube.Execution) (result testkube.ExecutionResult, err error) { + if r.Scraper != nil { + defer r.Scraper.Close() + } + outputPkg.PrintLogf("%s Preparing for test run", ui.IconTruck) err = r.Validate(execution) if err != nil { @@ -72,10 +85,10 @@ func (r *MavenRunner) Run(ctx context.Context, execution testkube.Execution) (re } // determine the Maven command to use - mavenCommand := "mvn" + mavenCommand := strings.Join(execution.Command, " ") mavenWrapper := filepath.Join(directory, "mvnw") _, err = os.Stat(mavenWrapper) - if err == nil { + if mavenCommand == "mvn" && err == nil { // then we use the wrapper instead mavenCommand = "./mvnw" } @@ -84,33 +97,33 @@ func (r *MavenRunner) Run(ctx context.Context, execution testkube.Execution) (re envManager.GetReferenceVars(envManager.Variables) // pass additional executor arguments/flags to Gradle - var args []string - args = append(args, execution.Args...) - + args := execution.Args + var settingsXML string if execution.VariablesFile != "" { outputPkg.PrintLogf("%s Creating settings.xml file", ui.IconWorld) - settingsXML, err := createSettingsXML(directory, execution.VariablesFile) + settingsXML, err = createSettingsXML(directory, execution.VariablesFile) if err != nil { outputPkg.PrintLogf("%s Could not create settings.xml", ui.IconCross) return *result.Err(errors.New("could not create settings.xml")), nil } outputPkg.PrintLogf("%s Successfully created settings.xml", ui.IconCheckMark) - args = append(args, "--settings", settingsXML) } goal := strings.Split(execution.TestType, "/")[1] + var goalName string if !strings.EqualFold(goal, "project") { // use the test subtype as goal or phase when != project // in case of project there is need to pass additional args - args = append(args, goal) + goalName = goal } // workaround for https://github.com/eclipse/che/issues/13926 os.Unsetenv("MAVEN_CONFIG") currentUser, err := user.Current() + var mavenHome string if err == nil && currentUser.Name == "maven" { - args = append(args, "-Duser.home=/home/maven") + mavenHome = "/home/maven" } runPath := directory @@ -118,6 +131,36 @@ func (r *MavenRunner) Run(ctx context.Context, execution testkube.Execution) (re runPath = filepath.Join(r.params.DataDir, "repo", execution.Content.Repository.WorkingDir) } + for i := len(args) - 1; i >= 0; i-- { + if goalName == "" && args[i] == "" { + args = append(args[:i], args[i+1:]...) + continue + } + + if settingsXML == "" && (args[i] == "--settings" || args[i] == "") { + args = append(args[:i], args[i+1:]...) + continue + } + + if mavenHome == "" && (args[i] == "--Duser.home" || args[i] == "") { + args = append(args[:i], args[i+1:]...) + continue + } + + if args[i] == "" { + args[i] = goalName + } + + if args[i] == "" { + args[i] = settingsXML + } + + if args[i] == "" { + args[i] = mavenHome + } + } + + outputPkg.PrintEvent("Running goal: "+goal, mavenHome, mavenCommand, args) output, err := executor.Run(runPath, mavenCommand, envManager, args...) output = envManager.ObfuscateSecrets(output) @@ -137,6 +180,15 @@ func (r *MavenRunner) Run(ctx context.Context, execution testkube.Execution) (re } } + // scrape artifacts first even if there are errors above + if r.params.ScrapperEnabled && execution.ArtifactRequest != nil && len(execution.ArtifactRequest.Dirs) != 0 { + outputPkg.PrintLogf("Scraping directories: %v", execution.ArtifactRequest.Dirs) + + if err := r.Scraper.Scrape(ctx, execution.ArtifactRequest.Dirs, execution); err != nil { + return *result.WithErrors(err), nil + } + } + result.Output = string(output) result.OutputType = "text/plain" diff --git a/contrib/executor/maven/pkg/runner/runner_integration_test.go b/contrib/executor/maven/pkg/runner/runner_integration_test.go index 32b69725e56..ed825638980 100644 --- a/contrib/executor/maven/pkg/runner/runner_integration_test.go +++ b/contrib/executor/maven/pkg/runner/runner_integration_test.go @@ -7,15 +7,12 @@ import ( "path/filepath" "testing" - "github.com/kubeshop/testkube/pkg/utils/test" - - "github.com/kubeshop/testkube/pkg/envs" - cp "github.com/otiai10/copy" - "github.com/stretchr/testify/assert" "github.com/kubeshop/testkube/pkg/api/v1/testkube" + "github.com/kubeshop/testkube/pkg/envs" + "github.com/kubeshop/testkube/pkg/utils/test" ) func TestRun_Integration(t *testing.T) { @@ -36,7 +33,9 @@ func TestRun_Integration(t *testing.T) { // given params := envs.Params{DataDir: tempDir} - runner := NewRunner(params) + runner, err := NewRunner(context.Background(), params) + assert.NoError(t, err) + execution := testkube.NewQueuedExecution() execution.TestType = "maven/test" execution.Content = &testkube.TestContent{ @@ -46,6 +45,8 @@ func TestRun_Integration(t *testing.T) { Branch: "main", }, } + execution.Command = []string{"mvn"} + execution.Args = []string{"--settings", "", "", "-Duser.home", ""} execution.Variables = map[string]testkube.Variable{ "wrapper": {Name: "TESTKUBE_MAVEN_WRAPPER", Value: "true", Type_: testkube.VariableTypeBasic}, @@ -73,7 +74,9 @@ func TestRun_Integration(t *testing.T) { // given params := envs.Params{DataDir: tempDir} - runner := NewRunner(params) + runner, err := NewRunner(context.Background(), params) + assert.NoError(t, err) + execution := testkube.NewQueuedExecution() execution.TestType = "maven/project" execution.Content = &testkube.TestContent{ @@ -83,7 +86,8 @@ func TestRun_Integration(t *testing.T) { Branch: "main", }, } - execution.Args = []string{"test"} + execution.Command = []string{"mvn"} + execution.Args = []string{"test", "--settings", "", "", "-Duser.home", ""} execution.Variables = map[string]testkube.Variable{ "wrapper": {Name: "TESTKUBE_MAVEN", Value: "true", Type_: testkube.VariableTypeBasic}, } @@ -110,7 +114,9 @@ func TestRun_Integration(t *testing.T) { // given params := envs.Params{DataDir: tempDir} - runner := NewRunner(params) + runner, err := NewRunner(context.Background(), params) + assert.NoError(t, err) + execution := testkube.NewQueuedExecution() execution.TestType = "maven/test" execution.Content = &testkube.TestContent{ @@ -120,6 +126,8 @@ func TestRun_Integration(t *testing.T) { Branch: "main", }, } + execution.Command = []string{"mvn"} + execution.Args = []string{"--settings", "", "", "-Duser.home", ""} execution.Envs = map[string]string{"TESTKUBE_MAVEN_WRAPPER": "true"} assert.NoError(t, os.Setenv("TESTKUBE_MAVEN_WRAPPER", "true")) @@ -144,7 +152,9 @@ func TestRun_Integration(t *testing.T) { // given params := envs.Params{DataDir: tempDir} - runner := NewRunner(params) + runner, err := NewRunner(context.Background(), params) + assert.NoError(t, err) + execution := testkube.NewQueuedExecution() execution.TestType = "maven/test" execution.Content = &testkube.TestContent{ @@ -154,6 +164,8 @@ func TestRun_Integration(t *testing.T) { Branch: "main", }, } + execution.Command = []string{"mvn"} + execution.Args = []string{"--settings", "", "", "-Duser.home", ""} execution.Variables = map[string]testkube.Variable{ "wrapper": {Name: "TESTKUBE_MAVEN", Value: "true", Type_: testkube.VariableTypeBasic}, } @@ -182,11 +194,15 @@ func TestRunErrors_Integration(t *testing.T) { t.Parallel() // given params := envs.Params{DataDir: "/unknown"} - runner := NewRunner(params) + runner, err := NewRunner(context.Background(), params) + assert.NoError(t, err) + execution := testkube.NewQueuedExecution() + execution.Command = []string{"mvn"} + execution.Args = []string{"--settings", "", "", "-Duser.home", ""} // when - _, err := runner.Run(ctx, *execution) + _, err = runner.Run(ctx, *execution) // then assert.Error(t, err) @@ -201,10 +217,14 @@ func TestRunErrors_Integration(t *testing.T) { // given params := envs.Params{DataDir: tempDir} - runner := NewRunner(params) + runner, err := NewRunner(context.Background(), params) + assert.NoError(t, err) + execution := testkube.NewQueuedExecution() execution.TestType = "maven/test" execution.Content = testkube.NewStringTestContent("") + execution.Command = []string{"mvn"} + execution.Args = []string{"--settings", "", "", "-Duser.home", ""} // when _, err = runner.Run(ctx, *execution) @@ -225,7 +245,9 @@ func TestRunErrors_Integration(t *testing.T) { // given params := envs.Params{DataDir: tempDir} - runner := NewRunner(params) + runner, err := NewRunner(context.Background(), params) + assert.NoError(t, err) + execution := testkube.NewQueuedExecution() execution.TestType = "maven/test" execution.Content = &testkube.TestContent{ @@ -235,6 +257,8 @@ func TestRunErrors_Integration(t *testing.T) { Branch: "main", }, } + execution.Command = []string{"mvn"} + execution.Args = []string{"--settings", "", "", "-Duser.home", ""} // when result, err := runner.Run(ctx, *execution) @@ -264,7 +288,9 @@ func TestRunMavenProject_Integration(t *testing.T) { // given params := envs.Params{DataDir: tempDir} - runner := NewRunner(params) + runner, err := NewRunner(context.Background(), params) + assert.NoError(t, err) + execution := testkube.NewQueuedExecution() execution.TestType = "maven/project" execution.Content = &testkube.TestContent{ @@ -274,7 +300,8 @@ func TestRunMavenProject_Integration(t *testing.T) { Branch: "main", }, } - execution.Args = []string{"test"} + execution.Command = []string{"mvn"} + execution.Args = []string{"test", "--settings", "", "", "-Duser.home", ""} execution.Variables = map[string]testkube.Variable{ "wrapper": {Name: "TESTKUBE_MAVEN", Value: "true", Type_: testkube.VariableTypeBasic}, } diff --git a/contrib/executor/playwright/cmd/agent/main.go b/contrib/executor/playwright/cmd/agent/main.go index 2f55964284d..0758f5e27a3 100644 --- a/contrib/executor/playwright/cmd/agent/main.go +++ b/contrib/executor/playwright/cmd/agent/main.go @@ -4,11 +4,10 @@ import ( "context" "os" - "github.com/kubeshop/testkube/pkg/envs" - "github.com/pkg/errors" "github.com/kubeshop/testkube/contrib/executor/playwright/pkg/runner" + "github.com/kubeshop/testkube/pkg/envs" "github.com/kubeshop/testkube/pkg/executor/agent" "github.com/kubeshop/testkube/pkg/executor/output" ) diff --git a/contrib/executor/playwright/pkg/runner/playwright.go b/contrib/executor/playwright/pkg/runner/playwright.go index f26fb440625..b2c2076c15e 100644 --- a/contrib/executor/playwright/pkg/runner/playwright.go +++ b/contrib/executor/playwright/pkg/runner/playwright.go @@ -5,19 +5,18 @@ import ( "fmt" "os" "path/filepath" - - "github.com/kubeshop/testkube/pkg/executor/scraper" + "strings" "github.com/pkg/errors" - "github.com/kubeshop/testkube/pkg/executor/scraper/factory" - "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/envs" "github.com/kubeshop/testkube/pkg/executor" "github.com/kubeshop/testkube/pkg/executor/env" "github.com/kubeshop/testkube/pkg/executor/output" "github.com/kubeshop/testkube/pkg/executor/runner" + "github.com/kubeshop/testkube/pkg/executor/scraper" + "github.com/kubeshop/testkube/pkg/executor/scraper/factory" "github.com/kubeshop/testkube/pkg/ui" ) @@ -74,25 +73,40 @@ func (r *PlaywrightRunner) Run(ctx context.Context, execution testkube.Execution output.PrintLogf("%s Dependencies successfully installed", ui.IconBox) } - var runner string - var args []string - + var depManager, depCommand string if r.dependency == "pnpm" { - runner = "pnpm" - args = []string{"dlx", "playwright", "test"} + depManager = "pnpm" + depCommand = "dlx" } else { - runner = "npx" - args = []string{"playwright", "test"} + depManager = "npx" } - args = append(args, execution.Args...) + args := execution.Args + for i := range execution.Command { + if execution.Command[i] == "" { + execution.Command[i] = depManager + } + } + + for i := len(args) - 1; i >= 0; i-- { + if depCommand == "" && args[i] == "" { + args = append(args[:i], args[i+1:]...) + continue + } + + if args[i] == "" { + args[i] = depCommand + } + } envManager := env.NewManagerWithVars(execution.Variables) envManager.GetReferenceVars(envManager.Variables) - output.PrintEvent("Running", runPath, "playwright", args) - out, runErr := executor.Run(runPath, runner, envManager, args...) + command := strings.Join(execution.Command, " ") + output.PrintEvent("Running", runPath, command, args) + out, runErr := executor.Run(runPath, command, envManager, args...) out = envManager.ObfuscateSecrets(out) + if runErr != nil { output.PrintLogf("%s Test run failed", ui.IconCross) result = testkube.ExecutionResult{ @@ -109,7 +123,8 @@ func (r *PlaywrightRunner) Run(ctx context.Context, execution testkube.Execution } if r.Params.ScrapperEnabled { - if err = scrapeArtifacts(ctx, r, execution); err != nil { + reportFile := "playwright-report" + if err = scrapeArtifacts(ctx, r, execution, reportFile); err != nil { return result, err } } @@ -125,25 +140,28 @@ func (r *PlaywrightRunner) GetType() runner.Type { return runner.TypeMain } -func scrapeArtifacts(ctx context.Context, r *PlaywrightRunner, execution testkube.Execution) (err error) { +func scrapeArtifacts(ctx context.Context, r *PlaywrightRunner, execution testkube.Execution, reportName string) (err error) { projectPath := filepath.Join(r.Params.DataDir, "repo", execution.Content.Repository.Path) - originalName := "playwright-report" - compressedName := originalName + "-zip" - + compressedName := reportName + "-zip" if _, err := executor.Run(projectPath, "mkdir", nil, compressedName); err != nil { output.PrintLogf("%s Artifact scraping failed: making dir %s", ui.IconCross, compressedName) return errors.Errorf("mkdir error: %v", err) } - if _, err := executor.Run(projectPath, "zip", nil, compressedName+"/"+originalName+".zip", "-r", originalName); err != nil { - output.PrintLogf("%s Artifact scraping failed: zipping reports %s", ui.IconCross, originalName) + if _, err := executor.Run(projectPath, "zip", nil, compressedName+"/"+reportName+".zip", "-r", reportName); err != nil { + output.PrintLogf("%s Artifact scraping failed: zipping reports %s", ui.IconCross, reportName) return errors.Errorf("zip error: %v", err) } directories := []string{ filepath.Join(projectPath, compressedName), } + + if execution.ArtifactRequest != nil && len(execution.ArtifactRequest.Dirs) != 0 { + directories = append(directories, execution.ArtifactRequest.Dirs...) + } + output.PrintLogf("Scraping directories: %v", directories) if err := r.Scraper.Scrape(ctx, directories, execution); err != nil { diff --git a/contrib/executor/playwright/pkg/runner/playwright_test.go b/contrib/executor/playwright/pkg/runner/playwright_test.go index ed0efd84c54..3b809f47c0d 100644 --- a/contrib/executor/playwright/pkg/runner/playwright_test.go +++ b/contrib/executor/playwright/pkg/runner/playwright_test.go @@ -6,15 +6,12 @@ import ( "path/filepath" "testing" - "github.com/kubeshop/testkube/pkg/utils/test" - - "github.com/kubeshop/testkube/pkg/envs" - + cp "github.com/otiai10/copy" "github.com/stretchr/testify/assert" "github.com/kubeshop/testkube/pkg/api/v1/testkube" - - cp "github.com/otiai10/copy" + "github.com/kubeshop/testkube/pkg/envs" + "github.com/kubeshop/testkube/pkg/utils/test" ) func TestRun_Integration(t *testing.T) { @@ -49,6 +46,14 @@ func TestRun_Integration(t *testing.T) { Path: "", }, }, + Command: []string{ + "", + }, + Args: []string{ + "", + "playwright", + "test", + }, }) assert.Equal(t, result.Status, testkube.ExecutionStatusPassed) diff --git a/contrib/executor/postman/cmd/agent/main.go b/contrib/executor/postman/cmd/agent/main.go index efa56293d99..83be2eb1c28 100644 --- a/contrib/executor/postman/cmd/agent/main.go +++ b/contrib/executor/postman/cmd/agent/main.go @@ -16,12 +16,13 @@ import ( ) func main() { + ctx := context.Background() params, err := envs.LoadTestkubeVariables() if err != nil { output.PrintError(os.Stderr, errors.Errorf("could not initialize Postman Executor environment variables: %v", err)) os.Exit(1) } - r, err := newman.NewNewmanRunner(params) + r, err := newman.NewNewmanRunner(ctx, params) if err != nil { log.Fatalf("%s could not run Postman tests: %s", ui.IconCross, err.Error()) } diff --git a/contrib/executor/postman/pkg/runner/newman/newman.go b/contrib/executor/postman/pkg/runner/newman/newman.go index 17025434395..7a97d666516 100644 --- a/contrib/executor/postman/pkg/runner/newman/newman.go +++ b/contrib/executor/postman/pkg/runner/newman/newman.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/envs" @@ -14,29 +15,44 @@ import ( "github.com/kubeshop/testkube/pkg/executor/env" "github.com/kubeshop/testkube/pkg/executor/output" "github.com/kubeshop/testkube/pkg/executor/runner" + "github.com/kubeshop/testkube/pkg/executor/scraper" + "github.com/kubeshop/testkube/pkg/executor/scraper/factory" "github.com/kubeshop/testkube/pkg/tmp" "github.com/kubeshop/testkube/pkg/ui" ) -func NewNewmanRunner(params envs.Params) (*NewmanRunner, error) { +func NewNewmanRunner(ctx context.Context, params envs.Params) (*NewmanRunner, error) { output.PrintLog(fmt.Sprintf("%s Preparing test runner", ui.IconTruck)) - return &NewmanRunner{ + var err error + r := &NewmanRunner{ Params: params, Fetcher: content.NewFetcher(""), - }, nil + } + + r.Scraper, err = factory.TryGetScrapper(ctx, params) + if err != nil { + return nil, err + } + + return r, nil } // NewmanRunner struct for newman based runner type NewmanRunner struct { Params envs.Params Fetcher content.ContentFetcher + Scraper scraper.Scraper } var _ runner.Runner = &NewmanRunner{} // Run runs particular test content on top of newman binary func (r *NewmanRunner) Run(ctx context.Context, execution testkube.Execution) (result testkube.ExecutionResult, err error) { + if r.Scraper != nil { + defer r.Scraper.Close() + } + output.PrintLog(fmt.Sprintf("%s Preparing for test run", ui.IconTruck)) if r.Params.GitUsername != "" || r.Params.GitToken != "" { @@ -73,11 +89,20 @@ func (r *NewmanRunner) Run(ctx context.Context, execution testkube.Execution) (r } tmpName := tmp.Name() + ".json" + args := execution.Args + for i := range args { + if args[i] == "" { + args[i] = envpath + } - args := []string{ - "run", path, "-e", envpath, "--reporters", "cli,json", "--reporter-json-export", tmpName, + if args[i] == "" { + args[i] = tmpName + } + + if args[i] == "" { + args[i] = path + } } - args = append(args, execution.Args...) runPath := "" if execution.Content.Repository != nil && execution.Content.Repository.WorkingDir != "" { @@ -86,7 +111,9 @@ func (r *NewmanRunner) Run(ctx context.Context, execution testkube.Execution) (r // we'll get error here in case of failed test too so we treat this as // starter test execution with failed status - out, err := executor.Run(runPath, "newman", envManager, args...) + command := strings.Join(execution.Command, " ") + output.PrintLogf("%s Test run command %s %s", ui.IconRocket, command, strings.Join(args, " ")) + out, err := executor.Run(runPath, command, envManager, args...) out = envManager.ObfuscateSecrets(out) @@ -101,6 +128,15 @@ func (r *NewmanRunner) Run(ctx context.Context, execution testkube.Execution) (r result = MapMetadataToResult(newmanResult) output.PrintLog(fmt.Sprintf("%s Mapped Newman result successfully", ui.IconCheckMark)) + // scrape artifacts first even if there are errors above + if r.Params.ScrapperEnabled && execution.ArtifactRequest != nil && len(execution.ArtifactRequest.Dirs) != 0 { + output.PrintLogf("Scraping directories: %v", execution.ArtifactRequest.Dirs) + + if err := r.Scraper.Scrape(ctx, execution.ArtifactRequest.Dirs, execution); err != nil { + return *result.WithErrors(err), nil + } + } + // catch errors if any if err != nil { return *result.Err(err), nil diff --git a/contrib/executor/postman/pkg/runner/newman/newman_test.go b/contrib/executor/postman/pkg/runner/newman/newman_test.go index 38b18a66d85..f866b6ff26f 100644 --- a/contrib/executor/postman/pkg/runner/newman/newman_test.go +++ b/contrib/executor/postman/pkg/runner/newman/newman_test.go @@ -23,7 +23,7 @@ func TestRun_Integration(t *testing.T) { test.IntegrationTest(t) t.Parallel() // given - runner, err := NewNewmanRunner(envs.Params{}) + runner, err := NewNewmanRunner(context.Background(), envs.Params{}) assert.NoError(t, err) // and test server for getting newman responses @@ -39,6 +39,17 @@ func TestRun_Integration(t *testing.T) { execution := testkube.Execution{ Content: testkube.NewStringTestContent(fmt.Sprintf(exampleCollection, port, port)), + Command: []string{"newman"}, + Args: []string{ + "run", + "", + "-e", + "", + "--reporters", + "cli,json", + "--reporter-json-export", + "", + }, } ctx := context.Background() diff --git a/contrib/executor/scraper/cmd/agent/main.go b/contrib/executor/scraper/cmd/agent/main.go index 30e0cd44ca9..eff510336c2 100644 --- a/contrib/executor/scraper/cmd/agent/main.go +++ b/contrib/executor/scraper/cmd/agent/main.go @@ -4,11 +4,10 @@ import ( "context" "os" - "github.com/kubeshop/testkube/pkg/envs" - "github.com/pkg/errors" "github.com/kubeshop/testkube/contrib/executor/scraper/pkg/runner" + "github.com/kubeshop/testkube/pkg/envs" "github.com/kubeshop/testkube/pkg/executor/agent" "github.com/kubeshop/testkube/pkg/executor/output" ) diff --git a/contrib/executor/scraper/pkg/runner/runner.go b/contrib/executor/scraper/pkg/runner/runner.go index 3ae39d71b5d..31ac1736e91 100644 --- a/contrib/executor/scraper/pkg/runner/runner.go +++ b/contrib/executor/scraper/pkg/runner/runner.go @@ -6,16 +6,14 @@ import ( "os" "path/filepath" - "github.com/kubeshop/testkube/pkg/executor/scraper" - "github.com/pkg/errors" - "github.com/kubeshop/testkube/pkg/executor/scraper/factory" - "github.com/kubeshop/testkube/pkg/api/v1/testkube" "github.com/kubeshop/testkube/pkg/envs" "github.com/kubeshop/testkube/pkg/executor/output" "github.com/kubeshop/testkube/pkg/executor/runner" + "github.com/kubeshop/testkube/pkg/executor/scraper" + "github.com/kubeshop/testkube/pkg/executor/scraper/factory" ) // NewRunner creates scraper runner @@ -51,6 +49,10 @@ func (r *ScraperRunner) Run(ctx context.Context, execution testkube.Execution) ( return *result.Err(errors.Errorf("executor only support artifact based tests")), nil } + if execution.ArtifactRequest.VolumeMountPath == "" || execution.ArtifactRequest.StorageClassName == "" { + return *result.Err(errors.Errorf("artifact request should have not empty volume mount path and storage class name")), nil + } + _, err = os.Stat(execution.ArtifactRequest.VolumeMountPath) if errors.Is(err, os.ErrNotExist) { return result, err diff --git a/contrib/executor/scraper/pkg/runner/runner_test.go b/contrib/executor/scraper/pkg/runner/runner_test.go index a4e21f745a6..6f1205e2c4c 100644 --- a/contrib/executor/scraper/pkg/runner/runner_test.go +++ b/contrib/executor/scraper/pkg/runner/runner_test.go @@ -7,14 +7,11 @@ import ( "testing" "github.com/golang/mock/gomock" - - "github.com/kubeshop/testkube/pkg/executor/scraper" - - "github.com/kubeshop/testkube/pkg/envs" - "github.com/stretchr/testify/assert" "github.com/kubeshop/testkube/pkg/api/v1/testkube" + "github.com/kubeshop/testkube/pkg/envs" + "github.com/kubeshop/testkube/pkg/executor/scraper" ) func TestRun(t *testing.T) { @@ -23,7 +20,7 @@ func TestRun(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() - e := testkube.Execution{ArtifactRequest: &testkube.ArtifactRequest{VolumeMountPath: "."}} + e := testkube.Execution{ArtifactRequest: &testkube.ArtifactRequest{VolumeMountPath: ".", StorageClassName: "standard"}} tests := []struct { name string scraper func(id string, directories []string) error @@ -48,7 +45,7 @@ func TestRun(t *testing.T) { { name: "failing scraper", scraper: func(id string, directories []string) error { return errors.New("Scraping failed") }, - execution: testkube.Execution{ArtifactRequest: &testkube.ArtifactRequest{VolumeMountPath: "."}}, + execution: testkube.Execution{ArtifactRequest: &testkube.ArtifactRequest{VolumeMountPath: ".", StorageClassName: "standard"}}, expectedError: "error scraping artifacts from container executor: Scraping failed", expectedStatus: testkube.ExecutionStatusFailed, scraperBuilder: func() scraper.Scraper { diff --git a/contrib/executor/soapui/cmd/agent/main.go b/contrib/executor/soapui/cmd/agent/main.go index 9e795d9187e..1c9eb51866b 100644 --- a/contrib/executor/soapui/cmd/agent/main.go +++ b/contrib/executor/soapui/cmd/agent/main.go @@ -4,11 +4,10 @@ import ( "context" "os" - "github.com/kubeshop/testkube/pkg/envs" - "github.com/pkg/errors" "github.com/kubeshop/testkube/contrib/executor/soapui/pkg/runner" + "github.com/kubeshop/testkube/pkg/envs" "github.com/kubeshop/testkube/pkg/executor/agent" "github.com/kubeshop/testkube/pkg/executor/output" ) diff --git a/contrib/executor/soapui/pkg/runner/runner.go b/contrib/executor/soapui/pkg/runner/runner.go index 76b15ac28ae..bd8f6b2d973 100644 --- a/contrib/executor/soapui/pkg/runner/runner.go +++ b/contrib/executor/soapui/pkg/runner/runner.go @@ -28,7 +28,6 @@ func NewRunner(ctx context.Context, params envs.Params) (*SoapUIRunner, error) { var err error r := &SoapUIRunner{ - SoapUIExecPath: "/usr/local/SmartBear/EntryPoint.sh", SoapUILogsPath: "/home/soapui/.soapuios/logs", Params: params, } @@ -43,7 +42,6 @@ func NewRunner(ctx context.Context, params envs.Params) (*SoapUIRunner, error) { // SoapUIRunner runs SoapUI tests type SoapUIRunner struct { - SoapUIExecPath string SoapUILogsPath string Scraper scraper.Scraper Params envs.Params @@ -91,6 +89,10 @@ func (r *SoapUIRunner) Run(ctx context.Context, execution testkube.Execution) (r if r.Params.ScrapperEnabled { directories := []string{r.SoapUILogsPath} + if execution.ArtifactRequest != nil && len(execution.ArtifactRequest.Dirs) != 0 { + directories = append(directories, execution.ArtifactRequest.Dirs...) + } + output.PrintLogf("Scraping directories: %v", directories) if err := r.Scraper.Scrape(ctx, directories, execution); err != nil { @@ -104,7 +106,11 @@ func (r *SoapUIRunner) Run(ctx context.Context, execution testkube.Execution) (r // setUpEnvironment sets up the COMMAND_LINE environment variable to // contain the incoming arguments and to point to the test file path func setUpEnvironment(args []string, testFilePath string) { - args = append(args, testFilePath) + for i := range args { + if args[i] == "" { + args[i] = testFilePath + } + } os.Setenv("COMMAND_LINE", strings.Join(args, " ")) } @@ -119,7 +125,9 @@ func (r *SoapUIRunner) runSoapUI(execution *testkube.Execution) testkube.Executi runPath = filepath.Join(r.Params.DataDir, "repo", execution.Content.Repository.WorkingDir) } - output, err := executor.Run(runPath, "/bin/sh", envManager, r.SoapUIExecPath) + command := strings.Join(execution.Command, " ") + output.PrintLogf("%s Test run command %s %s", ui.IconRocket, command, strings.Join(execution.Args, " ")) + output, err := executor.Run(runPath, command, envManager, execution.Args...) output = envManager.ObfuscateSecrets(output) if err != nil { return testkube.ExecutionResult{ diff --git a/contrib/executor/soapui/pkg/runner/runner_test.go b/contrib/executor/soapui/pkg/runner/runner_test.go index ff901c1f95d..4d6d44c3148 100644 --- a/contrib/executor/soapui/pkg/runner/runner_test.go +++ b/contrib/executor/soapui/pkg/runner/runner_test.go @@ -7,16 +7,13 @@ import ( "path/filepath" "testing" - "github.com/kubeshop/testkube/pkg/utils/test" - "github.com/golang/mock/gomock" - - "github.com/kubeshop/testkube/pkg/envs" - "github.com/kubeshop/testkube/pkg/executor/scraper" - "github.com/stretchr/testify/assert" "github.com/kubeshop/testkube/pkg/api/v1/testkube" + "github.com/kubeshop/testkube/pkg/envs" + "github.com/kubeshop/testkube/pkg/executor/scraper" + "github.com/kubeshop/testkube/pkg/utils/test" ) func TestRun_Integration(t *testing.T) { @@ -109,11 +106,17 @@ func TestRun_Integration(t *testing.T) { params := envs.Params{DataDir: tempDir, ScrapperEnabled: true} runner := SoapUIRunner{ SoapUILogsPath: "/logs", - SoapUIExecPath: file.Name(), Params: params, Scraper: test.scraperBuilder(), } + test.execution.Command = []string{ + "/bin/sh", + file.Name(), + } + test.execution.Args = []string{ + "", + } res, err := runner.Run(context.Background(), test.execution) if test.expectedError == "" { assert.NoError(t, err) diff --git a/docs/docs/concepts/tests/tests-creating.md b/docs/docs/concepts/tests/tests-creating.md index a258b4b6fe5..31fb54f70ca 100644 --- a/docs/docs/concepts/tests/tests-creating.md +++ b/docs/docs/concepts/tests/tests-creating.md @@ -344,6 +344,18 @@ testkube run test maven-example-file-test --args "--settings" --args "/data/uplo By default, there is a 10 second timeout limit on all requests on the client side, and a 1 GB body size limit on the server side. To update the timeout, use `--upload-timeout` with [Go-compatible duration formats](https://pkg.go.dev/time#ParseDuration). +### Redefining the Prebuilt Executor command and arguments + +Each of Testkube Prebuilt executors has a default command and arguments it uses to execute the test. They are provided as a part of Executor CRD and can be either ovveriden or appended during test creation or execution, for example: + +```sh +testkube create test --name maven-example-test --git-uri https://github.com/kubeshop/testkube-executor-maven.git --git-path examples/hello-maven --type maven/test --git-branch main --command "mvn" --args-mode "override" --executor-args="--settings -Duser.home " +``` + +```sh title="Expected output:" +Test created maven-example-test 🥇 +``` + ### Changing the Default Job Template Used for Test Execution You can always create your own custom executor with its own job template definition used for test execution. But sometimes you just need to adjust an existing job template of a standard Testkube executor with a few parameters. In this case you can use additional parameter `--job-template` when you create or run the test: @@ -448,9 +460,11 @@ spec: - "/bin/runner" - '{{ .Jsn }}' {{- if .ArtifactRequest }} + {{- if .ArtifactRequest.VolumeMountPath }} volumeMounts: - name: artifact-volume mountPath: {{ .ArtifactRequest.VolumeMountPath }} + {{- end }} {{- end }} resources: limits: diff --git a/docs/docs/reference/cli/testkube_create_executor.md b/docs/docs/reference/cli/testkube_create_executor.md index 6e7113b14d7..59785b34c1f 100644 --- a/docs/docs/reference/cli/testkube_create_executor.md +++ b/docs/docs/reference/cli/testkube_create_executor.md @@ -13,8 +13,8 @@ testkube create executor [flags] ### Options ``` - --args stringArray args passed to image in container executor - --command stringArray command passed to image in container executor + --args stringArray args passed to image in executor + --command stringArray command passed to image in executor --content-type stringArray list of supported content types for executor --docs-uri string URI to executor docs --executor-type string executor type, container or job (defaults to job) (default "job") diff --git a/docs/docs/reference/cli/testkube_create_test.md b/docs/docs/reference/cli/testkube_create_test.md index 2278200eece..9557f16d23e 100644 --- a/docs/docs/reference/cli/testkube_create_test.md +++ b/docs/docs/reference/cli/testkube_create_test.md @@ -1,4 +1,4 @@ -# testkube create test +## testkube create test Create new Test @@ -13,10 +13,11 @@ testkube create test [flags] ### Options ``` - --artifact-dir stringArray artifact dirs for container executor + --args-mode string usage mode for arguments. one of append|override (default "append") + --artifact-dir stringArray artifact dirs for scraping --artifact-storage-class-name string artifact storage class name for container executor --artifact-volume-mount-path string artifact volume mount path for container executor - --command stringArray command passed to image in container executor + --command stringArray command passed to image in executor --copy-files stringArray file path mappings from host to pod of form source:destination --cronjob-template string cron job template file path for extensions to cron job template --env stringToString envs in a form of name1=val1 passed to executor (default []) diff --git a/docs/docs/reference/cli/testkube_generate_tests-crds.md b/docs/docs/reference/cli/testkube_generate_tests-crds.md index d8a086d1b5f..92cf1366ccd 100644 --- a/docs/docs/reference/cli/testkube_generate_tests-crds.md +++ b/docs/docs/reference/cli/testkube_generate_tests-crds.md @@ -25,7 +25,7 @@ testkube generate tests-crds [flags] --artifact-dir stringArray artifact dirs for container executor --artifact-storage-class-name string artifact storage class name for container executor --artifact-volume-mount-path string artifact volume mount path for container executor - --command stringArray command passed to image in container executor + --command stringArray command passed to image in executor --copy-files stringArray file path mappings from host to pod of form source:destination --cronjob-template string cron job template file path for extensions to cron job template --env stringToString envs in a form of name1=val1 passed to executor (default []) diff --git a/docs/docs/reference/cli/testkube_run_test.md b/docs/docs/reference/cli/testkube_run_test.md index 8aadad82ae5..5881bbc3ce2 100644 --- a/docs/docs/reference/cli/testkube_run_test.md +++ b/docs/docs/reference/cli/testkube_run_test.md @@ -1,4 +1,4 @@ -# testkube run test +## testkube run test Starts new test @@ -14,9 +14,11 @@ testkube run test [flags] ``` --args stringArray executor binary additional arguments - --artifact-dir stringArray artifact dirs for container executor + --args-mode string usage mode for argumnets. one of append|override (default "append") + --artifact-dir stringArray artifact dirs for scraping --artifact-storage-class-name string artifact storage class name for container executor --artifact-volume-mount-path string artifact volume mount path for container executor + --command stringArray command passed to image in executor --concurrency int concurrency level for multiple test execution (default 10) --context string running context description for test execution --copy-files stringArray file path mappings from host to pod of form source:destination diff --git a/docs/docs/reference/cli/testkube_update_executor.md b/docs/docs/reference/cli/testkube_update_executor.md index 0e76d49eb82..128ce0d91d0 100644 --- a/docs/docs/reference/cli/testkube_update_executor.md +++ b/docs/docs/reference/cli/testkube_update_executor.md @@ -13,8 +13,8 @@ testkube update executor [flags] ### Options ``` - --args stringArray args passed to image in container executor - --command stringArray command passed to image in container executor + --args stringArray args passed to image in executor + --command stringArray command passed to image in executor --content-type stringArray list of supported content types for executor --docs-uri string URI to executor docs --executor-type string executor type, container or job (defaults to job) (default "job") diff --git a/docs/docs/reference/cli/testkube_update_test.md b/docs/docs/reference/cli/testkube_update_test.md index 3f9be6c35e3..ea3116856ff 100644 --- a/docs/docs/reference/cli/testkube_update_test.md +++ b/docs/docs/reference/cli/testkube_update_test.md @@ -1,4 +1,4 @@ -# testkube update test +## testkube update test Update test @@ -13,10 +13,11 @@ testkube update test [flags] ### Options ``` - --artifact-dir stringArray artifact dirs for container executor + --args-mode string usage mode for arguments. one of append|override (default "append") + --artifact-dir stringArray artifact dirs for scraping --artifact-storage-class-name string artifact storage class name for container executor --artifact-volume-mount-path string artifact volume mount path for container executor - --command stringArray command passed to image in container executor + --command stringArray command passed to image in executor --copy-files stringArray file path mappings from host to pod of form source:destination --cronjob-template string cron job template file path for extensions to cron job template --execution-name string execution name, if empty will be autogenerated diff --git a/docs/docs/test-types/container-executor.mdx b/docs/docs/test-types/container-executor.mdx index 304a7189f28..811f2c2c16f 100644 --- a/docs/docs/test-types/container-executor.mdx +++ b/docs/docs/test-types/container-executor.mdx @@ -59,8 +59,8 @@ Aliases: executor, exec, ex Flags: - --args stringArray args passed to image in container executor - --command stringArray command passed to image in container executor + --args stringArray args passed to image in executor + --command stringArray command passed to image in executor --content-type stringArray list of supported content types for executor --docs-uri string URI to executor docs --executor-type string executor type, container or job (defaults to job) (default "job") diff --git a/docs/docs/test-types/executor-artillery.mdx b/docs/docs/test-types/executor-artillery.mdx index b52ebc46c81..daf7801b06a 100644 --- a/docs/docs/test-types/executor-artillery.mdx +++ b/docs/docs/test-types/executor-artillery.mdx @@ -5,6 +5,10 @@ import Admonition from "@theme/Admonition"; # Artillery.io The Artillery executor allows you to run Artillery tests with Testkube. +Default command for this executor: artillery +Default arguments for this executor command: run <runPath> --dotenv <envFile> -o <reportFile> +(parameters in <> are calculated at test execution) + export const ExecutorInfo = () => { return (
diff --git a/docs/docs/test-types/executor-curl.mdx b/docs/docs/test-types/executor-curl.mdx index d05d9355f62..3f79ba311a9 100644 --- a/docs/docs/test-types/executor-curl.mdx +++ b/docs/docs/test-types/executor-curl.mdx @@ -4,6 +4,9 @@ import TabItem from "@theme/TabItem"; # cURL Testkube is able to run cURL commands as tests. +Default command for this executor: curl +Default arguments for this executor command: -is + ## Abstraction over cURL Testkube executor provides an abstraction over cURL that allow you to create JSON-based cURL test files. They allow you to combine a cURL command with expected results: diff --git a/docs/docs/test-types/executor-cypress.mdx b/docs/docs/test-types/executor-cypress.mdx index 45eaf5743ba..632c6bd5d88 100644 --- a/docs/docs/test-types/executor-cypress.mdx +++ b/docs/docs/test-types/executor-cypress.mdx @@ -6,6 +6,10 @@ import Admonition from "@theme/Admonition"; Our dedicated Cypress executor allows running Cypress tests with Testkube - directly from your Git repository. +Default command for this executor: ./node_modules/cypress/bin/cypress +Default arguments for this executor command: run --reporter junit --reporter-options mochaFile=<reportFile>,toConsole=false --project <projectPath> --env <envVars> +(parameters in <> are calculated at test execution) + export const ExecutorInfo = () => { return (
diff --git a/docs/docs/test-types/executor-ginkgo.md b/docs/docs/test-types/executor-ginkgo.md index f1b6e2ec6d8..53ad6be2a31 100644 --- a/docs/docs/test-types/executor-ginkgo.md +++ b/docs/docs/test-types/executor-ginkgo.md @@ -4,6 +4,10 @@ import Admonition from "@theme/Admonition"; Our dedicated Ginkgo executor allows running Ginkgo tests with Testkube - directly from your Git repository. +Default command for this executor: ginkgo +Default arguments for this executor command: -r -p --randomize-all --randomize-suites --keep-going --trace --junit-report <reportFile> <envVars> <runPath> +(parameters in <> are calculated at test execution) + export const ExecutorInfo = () => { return (
diff --git a/docs/docs/test-types/executor-gradle.md b/docs/docs/test-types/executor-gradle.md index 65f24966426..e25d7223ee3 100644 --- a/docs/docs/test-types/executor-gradle.md +++ b/docs/docs/test-types/executor-gradle.md @@ -2,6 +2,9 @@ Testkube allows running Gradle based tasks that could also be tests. For example, we can easily run JUnit tests in Testkube. +Default command for this executor: gradle +Default arguments for this executor command: --no-daemon <taskName> -p <projectDir> +(parameters in <> are calculated at test execution) ## **Test Environment** diff --git a/docs/docs/test-types/executor-jmeter.md b/docs/docs/test-types/executor-jmeter.md index 15a3eb50e4f..6559b42266d 100644 --- a/docs/docs/test-types/executor-jmeter.md +++ b/docs/docs/test-types/executor-jmeter.md @@ -4,6 +4,10 @@ import Admonition from "@theme/Admonition"; [JMeter](https://jmeter.apache.org/) is an integral part of Testkube. The Testkube JMeter executor is installed by default during the Testkube installation. +Default command for this executor: <entryPoint> +Default arguments for this executor command: -n -j <logFile> -t <runPath> -l <jtlFile> -e -o <reportFile> <envVars> +(parameters in <> are calculated at test execution) + diff --git a/docs/docs/test-types/executor-k6.mdx b/docs/docs/test-types/executor-k6.mdx index 4950ed50062..9fd4b364c0c 100644 --- a/docs/docs/test-types/executor-k6.mdx +++ b/docs/docs/test-types/executor-k6.mdx @@ -6,6 +6,10 @@ import Admonition from "@theme/Admonition"; Testkube's k6 executor provides a convenient way of running k6 tests. +Default command for this executor: k6 +Default arguments for this executor command: <k6Command> <envVars> <runPath> +(parameters in <> are calculated at test execution) + diff --git a/docs/docs/test-types/executor-kubepug.md b/docs/docs/test-types/executor-kubepug.md index 45dcfb4476d..35c3ed81158 100644 --- a/docs/docs/test-types/executor-kubepug.md +++ b/docs/docs/test-types/executor-kubepug.md @@ -3,6 +3,10 @@ [KubePug](https://github.com/rikatz/kubepug) is a kubectl plugin checking for deprecated Kubernetes clusters or deprecated versions of Kubernetes manifests. It can connect to both your cluster directly and it can run on input files. For security, Testkube only supports scanning input files via the KubePug executor. +Default command for this executor: kubepug +Default arguments for this executor command: --format=json --input-file <runPath> +(parameters in <> are calculated at test execution) + Running the KubePug Testkube executor does not require any special installation; Testkube comes with the ability to run Kubepug immediately after installation. ## Testing Manifests diff --git a/docs/docs/test-types/executor-maven.md b/docs/docs/test-types/executor-maven.md index f0fcf8c5e0c..b7cb396f23d 100644 --- a/docs/docs/test-types/executor-maven.md +++ b/docs/docs/test-types/executor-maven.md @@ -4,6 +4,10 @@ import Admonition from "@theme/Admonition"; Testkube allows you to run Maven-based tasks which could be also tests. For example, we can easily run JUnit tests in Testkube now. +Default command for this executor: mvn +Default arguments for this executor command: --settings <settingsFile> <goalName> -Duser.home <mavenHome> +(parameters in <> are calculated at test execution) + export const ExecutorInfo = () => { return (
diff --git a/docs/docs/test-types/executor-playwright.md b/docs/docs/test-types/executor-playwright.md index 77e1f353597..c360e46fe0a 100644 --- a/docs/docs/test-types/executor-playwright.md +++ b/docs/docs/test-types/executor-playwright.md @@ -5,6 +5,10 @@ import Admonition from "@theme/Admonition"; Starting from the Testkube Helm chart version 1.9.5, it is possible to use Testkube to manage your Playwright tests inside your Kubernetes cluster. +Default command for this executor: <depManager> +Default arguments for this executor command: <depCommand> playwright test +(parameters in <> are calculated at test execution) + export const ExecutorInfo = () => { return (
diff --git a/docs/docs/test-types/executor-postman.mdx b/docs/docs/test-types/executor-postman.mdx index 7df50540e67..71f13ba53dc 100644 --- a/docs/docs/test-types/executor-postman.mdx +++ b/docs/docs/test-types/executor-postman.mdx @@ -6,6 +6,10 @@ import Admonition from "@theme/Admonition"; Testkube is able to run Postman collections inside your Kubernetes cluster so it can be used to test internal or external services. +Default command for this executor: newman +Default arguments for this executor command: run <runPath> -e <envFile> --reporters cli,json --reporter-json-export <reportFile> +(parameters in <> are calculated at test execution) + export const ExecutorInfo = () => { return (
diff --git a/docs/docs/test-types/executor-soapui.md b/docs/docs/test-types/executor-soapui.md index 36487ac7565..20062af9695 100644 --- a/docs/docs/test-types/executor-soapui.md +++ b/docs/docs/test-types/executor-soapui.md @@ -4,6 +4,10 @@ import Admonition from "@theme/Admonition"; [SoapUI](https://www.soapui.org) is an open-source tool used for the end-to-end testing of REST, SOAP and GraphQL APIs, as well as JMS, JDBC and other web services. Testkube supports the SoapUI executor implementation. +Default command for this executor: /bin/sh /usr/local/SmartBear/EntryPoint.sh +Default arguments for this executor command: <runPath> +(parameters in <> are calculated at test execution) + **Check out our [blog post](https://kubeshop.io/blog/run-kubernetes-tests-with-soapui-and-testkube) to follow tutorial steps to Learn how to run functional tests in Kubernetes with SoapUI and Testkube.** Testkube supports the [SoapUI](https://www.soapui.org) executor implementation. diff --git a/go.mod b/go.mod index 8a57636f831..25bd8ff5358 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/gorilla/websocket v1.5.0 github.com/joshdk/go-junit v1.0.0 github.com/kelseyhightower/envconfig v1.4.0 - github.com/kubeshop/testkube-operator v1.10.8-0.20230503165649-71e7f7ff4822 + github.com/kubeshop/testkube-operator v1.10.8-0.20230503173604-b1213c70c7b7 github.com/minio/minio-go/v7 v7.0.47 github.com/montanaflynn/stats v0.6.6 github.com/moogar0880/problems v0.1.1 diff --git a/go.sum b/go.sum index 75c9b8b9dd2..385c628b1e1 100644 --- a/go.sum +++ b/go.sum @@ -350,8 +350,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kubeshop/testkube-operator v1.10.8-0.20230503165649-71e7f7ff4822 h1:34/aSA0XqLW9SLd1McqmVyCbruyjAnYxBC5dvC7w02o= -github.com/kubeshop/testkube-operator v1.10.8-0.20230503165649-71e7f7ff4822/go.mod h1:6Rs8MugOzaMcthGzobf6GBlRzbOFiK/GJiuYN6MCfEw= +github.com/kubeshop/testkube-operator v1.10.8-0.20230503173604-b1213c70c7b7 h1:tHIi/wawsIrAcAZo5BwDwcvpuVGonTrfrYq2cHlkuiU= +github.com/kubeshop/testkube-operator v1.10.8-0.20230503173604-b1213c70c7b7/go.mod h1:6Rs8MugOzaMcthGzobf6GBlRzbOFiK/GJiuYN6MCfEw= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= diff --git a/pkg/api/v1/client/interface.go b/pkg/api/v1/client/interface.go index 292aeaf9016..945bec6fcd3 100644 --- a/pkg/api/v1/client/interface.go +++ b/pkg/api/v1/client/interface.go @@ -156,7 +156,9 @@ type ExecuteTestOptions struct { ExecutionVariables map[string]testkube.Variable ExecutionVariablesFileContent string ExecutionLabels map[string]string + Command []string Args []string + ArgsMode string Envs map[string]string SecretEnvs map[string]string HTTPProxy string diff --git a/pkg/api/v1/client/test.go b/pkg/api/v1/client/test.go index 9d240905154..2e1a666425d 100644 --- a/pkg/api/v1/client/test.go +++ b/pkg/api/v1/client/test.go @@ -135,7 +135,9 @@ func (c TestClient) ExecuteTest(id, executionName string, options ExecuteTestOpt VariablesFile: options.ExecutionVariablesFileContent, Variables: options.ExecutionVariables, Envs: options.Envs, + Command: options.Command, Args: options.Args, + ArgsMode: options.ArgsMode, SecretEnvs: options.SecretEnvs, HttpProxy: options.HTTPProxy, HttpsProxy: options.HTTPSProxy, @@ -171,7 +173,9 @@ func (c TestClient) ExecuteTests(selector string, concurrencyLevel int, options VariablesFile: options.ExecutionVariablesFileContent, Variables: options.ExecutionVariables, Envs: options.Envs, + Command: options.Command, Args: options.Args, + ArgsMode: options.ArgsMode, SecretEnvs: options.SecretEnvs, HttpProxy: options.HTTPProxy, HttpsProxy: options.HTTPSProxy, diff --git a/pkg/api/v1/testkube/model_artifact_request.go b/pkg/api/v1/testkube/model_artifact_request.go index f0ad9a3335e..d3b724afcd9 100644 --- a/pkg/api/v1/testkube/model_artifact_request.go +++ b/pkg/api/v1/testkube/model_artifact_request.go @@ -9,12 +9,12 @@ */ package testkube -// artifact request body for container executors with test artifacts +// artifact request body with test artifacts type ArtifactRequest struct { - // artifact storage class name + // artifact storage class name for container executor StorageClassName string `json:"storageClassName"` - // artifact volume mount path + // artifact volume mount path for container executor VolumeMountPath string `json:"volumeMountPath"` - // artifact directories + // artifact directories for scraping Dirs []string `json:"dirs,omitempty"` } diff --git a/pkg/api/v1/testkube/model_execution.go b/pkg/api/v1/testkube/model_execution.go index d473160ad8c..432b4a549a6 100644 --- a/pkg/api/v1/testkube/model_execution.go +++ b/pkg/api/v1/testkube/model_execution.go @@ -32,8 +32,12 @@ type Execution struct { // Environment variables passed to executor. // Deprecated: use Basic Variables instead Envs map[string]string `json:"envs,omitempty"` + // executor image command + Command []string `json:"command,omitempty"` // additional arguments/flags passed to executor binary - Args []string `json:"args,omitempty"` + Args []string `json:"args,omitempty"` + // usage mode for arguments + ArgsMode string `json:"args_mode,omitempty"` Variables map[string]Variable `json:"variables,omitempty"` // variables file content - need to be in format for particular executor (e.g. postman envs file) VariablesFile string `json:"variablesFile,omitempty"` diff --git a/pkg/api/v1/testkube/model_execution_request.go b/pkg/api/v1/testkube/model_execution_request.go index d0a1a162ce7..4bcc10048ce 100644 --- a/pkg/api/v1/testkube/model_execution_request.go +++ b/pkg/api/v1/testkube/model_execution_request.go @@ -28,10 +28,12 @@ type ExecutionRequest struct { TestSecretUUID string `json:"testSecretUUID,omitempty"` // test suite secret uuid, if it's run as a part of test suite TestSuiteSecretUUID string `json:"testSuiteSecretUUID,omitempty"` - // container executor image command + // executor image command Command []string `json:"command,omitempty"` // additional executor binary arguments Args []string `json:"args,omitempty"` + // usage mode for arguments + ArgsMode string `json:"args_mode,omitempty"` // container image, executor will run inside this image Image string `json:"image,omitempty"` // container image pull secrets diff --git a/pkg/api/v1/testkube/model_execution_update_request.go b/pkg/api/v1/testkube/model_execution_update_request.go index 268c39b4598..f21f4fc443d 100644 --- a/pkg/api/v1/testkube/model_execution_update_request.go +++ b/pkg/api/v1/testkube/model_execution_update_request.go @@ -28,10 +28,12 @@ type ExecutionUpdateRequest struct { TestSecretUUID *string `json:"testSecretUUID,omitempty"` // test suite secret uuid, if it's run as a part of test suite TestSuiteSecretUUID *string `json:"testSuiteSecretUUID,omitempty"` - // container executor image command + // executor image command Command *[]string `json:"command,omitempty"` // additional executor binary arguments Args *[]string `json:"args,omitempty"` + // usage mode for arguments + ArgsMode *string `json:"args_mode,omitempty"` // container image, executor will run inside this image Image *string `json:"image,omitempty"` // container image pull secrets diff --git a/pkg/api/v1/testkube/model_executor.go b/pkg/api/v1/testkube/model_executor.go index de83775f42c..a71b3c00c7f 100644 --- a/pkg/api/v1/testkube/model_executor.go +++ b/pkg/api/v1/testkube/model_executor.go @@ -17,9 +17,9 @@ type Executor struct { Image string `json:"image,omitempty"` // container image pull secrets ImagePullSecrets []LocalObjectReference `json:"imagePullSecrets,omitempty"` - // container executor image command + // executor image command Command []string `json:"command,omitempty"` - // additional executor binary arguments + // additional executor binary argument Args []string `json:"args,omitempty"` // Types defines what types can be handled by executor e.g. \"postman/collection\", \":curl/command\" etc Types []string `json:"types,omitempty"` diff --git a/pkg/api/v1/testkube/model_executor_update_request.go b/pkg/api/v1/testkube/model_executor_update_request.go index fc65f501fb7..589078bcc08 100644 --- a/pkg/api/v1/testkube/model_executor_update_request.go +++ b/pkg/api/v1/testkube/model_executor_update_request.go @@ -21,9 +21,9 @@ type ExecutorUpdateRequest struct { Image *string `json:"image,omitempty"` // container image pull secrets ImagePullSecrets *[]LocalObjectReference `json:"imagePullSecrets,omitempty"` - // container executor image command + // executor image command Command *[]string `json:"command,omitempty"` - // additional executor binary arguments + // additional executor binary argument Args *[]string `json:"args,omitempty"` // Types defines what types can be handled by executor e.g. \"postman/collection\", \":curl/command\" etc Types *[]string `json:"types,omitempty"` diff --git a/pkg/api/v1/testkube/model_executor_upsert_request.go b/pkg/api/v1/testkube/model_executor_upsert_request.go index 78daa6d7c95..8b8fdd6b764 100644 --- a/pkg/api/v1/testkube/model_executor_upsert_request.go +++ b/pkg/api/v1/testkube/model_executor_upsert_request.go @@ -21,9 +21,9 @@ type ExecutorUpsertRequest struct { Image string `json:"image,omitempty"` // container image pull secrets ImagePullSecrets []LocalObjectReference `json:"imagePullSecrets,omitempty"` - // container executor image command + // executor image command Command []string `json:"command,omitempty"` - // additional executor binary arguments + // additional executor binary argument Args []string `json:"args,omitempty"` // Types defines what types can be handled by executor e.g. \"postman/collection\", \":curl/command\" etc Types []string `json:"types"` diff --git a/pkg/api/v1/testkube/model_test_content_extended.go b/pkg/api/v1/testkube/model_test_content_extended.go index b58a92e98ee..b37ec7f0807 100644 --- a/pkg/api/v1/testkube/model_test_content_extended.go +++ b/pkg/api/v1/testkube/model_test_content_extended.go @@ -14,6 +14,13 @@ const ( TestContentTypeEmpty TestContentType = "" ) +type ArgsModeType string + +const ( + ArgsModeTypeAppend ArgsModeType = "append" + ArgsModeTypeOverride ArgsModeType = "override" +) + var ErrTestContentTypeNotFile = fmt.Errorf("unsupported content type use one of: file-uri, git-file, string") var ErrTestContentTypeNotDir = fmt.Errorf("unsupported content type use one of: git-dir") diff --git a/pkg/crd/templates/test.tmpl b/pkg/crd/templates/test.tmpl index 2ae4fb88872..3e3b0b6d52e 100644 --- a/pkg/crd/templates/test.tmpl +++ b/pkg/crd/templates/test.tmpl @@ -79,7 +79,7 @@ spec: schedule: {{ .Schedule }} {{- end }} {{- if .ExecutionRequest }} - {{- if or (.ExecutionRequest.Name) (.ExecutionRequest.NegativeTest) (.ExecutionRequest.VariablesFile) (.ExecutionRequest.HttpProxy) (.ExecutionRequest.HttpsProxy) (ne (len .ExecutionRequest.Variables) 0) (ne (len .ExecutionRequest.Args) 0) (ne (len .ExecutionRequest.Envs) 0) (ne (len .ExecutionRequest.SecretEnvs) 0) (.ExecutionRequest.Image) (ne (len .ExecutionRequest.Command) 0) (ne (len .ExecutionRequest.ImagePullSecrets) 0) (ne .ExecutionRequest.ActiveDeadlineSeconds 0) (.ExecutionRequest.ArtifactRequest) (.ExecutionRequest.JobTemplate) (.ExecutionRequest.CronJobTemplate) (.ExecutionRequest.PreRunScript) (.ExecutionRequest.ScraperTemplate) (ne (len .ExecutionRequest.EnvConfigMaps) 0) (ne (len .ExecutionRequest.EnvSecrets) 0) }} + {{- if or (.ExecutionRequest.Name) (.ExecutionRequest.NegativeTest) (.ExecutionRequest.VariablesFile) (.ExecutionRequest.HttpProxy) (.ExecutionRequest.HttpsProxy) (ne (len .ExecutionRequest.Variables) 0) (ne (len .ExecutionRequest.Args) 0) (ne (len .ExecutionRequest.Envs) 0) (ne (len .ExecutionRequest.SecretEnvs) 0) (.ExecutionRequest.Image) (ne (len .ExecutionRequest.Command) 0) (.ExecutionRequest.ArgsMode) (ne (len .ExecutionRequest.ImagePullSecrets) 0) (ne .ExecutionRequest.ActiveDeadlineSeconds 0) (.ExecutionRequest.ArtifactRequest) (.ExecutionRequest.JobTemplate) (.ExecutionRequest.CronJobTemplate) (.ExecutionRequest.PreRunScript) (.ExecutionRequest.ScraperTemplate) (ne (len .ExecutionRequest.EnvConfigMaps) 0) (ne (len .ExecutionRequest.EnvSecrets) 0) }} executionRequest: {{- if .ExecutionRequest.Name }} name: {{ .ExecutionRequest.Name }} @@ -126,6 +126,9 @@ spec: - {{ . }} {{- end }} {{- end }} + {{- if .ExecutionRequest.ArgsMode }} + argsMode: {{ .ExecutionRequest.ArgsMode }} + {{- end }} {{- if ne (len .ExecutionRequest.Envs) 0 }} envs: {{- range $key, $value := .ExecutionRequest.Envs }} diff --git a/pkg/executor/common.go b/pkg/executor/common.go index 5c4782a6cf5..0fcd8a76b2a 100644 --- a/pkg/executor/common.go +++ b/pkg/executor/common.go @@ -339,6 +339,8 @@ func SyncDefaultExecutors( Types: executor.Executor.Types, ExecutorType: executorv1.ExecutorType(executor.Executor.ExecutorType), Image: executor.Executor.Image, + Command: executor.Executor.Command, + Args: executor.Executor.Args, Features: executorsmapper.MapFeaturesToCRD(executor.Executor.Features), ContentTypes: executorsmapper.MapContentTypesToCRD(executor.Executor.ContentTypes), Meta: executorsmapper.MapMetaToCRD(executor.Executor.Meta), diff --git a/pkg/executor/containerexecutor/containerexecutor.go b/pkg/executor/containerexecutor/containerexecutor.go index b6c787718d6..17bc4b3f763 100644 --- a/pkg/executor/containerexecutor/containerexecutor.go +++ b/pkg/executor/containerexecutor/containerexecutor.go @@ -173,7 +173,8 @@ func (c *ContainerExecutor) Logs(ctx context.Context, id string) (out chan outpu } ids := []string{id} - if supportArtifacts && execution.ArtifactRequest != nil { + if supportArtifacts && execution.ArtifactRequest != nil && + execution.ArtifactRequest.VolumeMountPath != "" && execution.ArtifactRequest.StorageClassName != "" { ids = append(ids, id+"-scraper") } @@ -279,7 +280,8 @@ func (c *ContainerExecutor) createJob(ctx context.Context, execution testkube.Ex return nil, err } - if jobOptions.ArtifactRequest != nil { + if jobOptions.ArtifactRequest != nil && + jobOptions.ArtifactRequest.VolumeMountPath != "" && jobOptions.ArtifactRequest.StorageClassName != "" { c.log.Debug("creating persistent volume claim with options", "options", jobOptions) pvcsClient := c.clientSet.CoreV1().PersistentVolumeClaims(c.namespace) pvcSpec, err := NewPersistentVolumeClaimSpec(c.log, jobOptions) @@ -337,7 +339,8 @@ func (c *ContainerExecutor) updateResultsFromPod( } var scraperLogs []byte - if jobOptions.ArtifactRequest != nil { + if jobOptions.ArtifactRequest != nil && + jobOptions.ArtifactRequest.VolumeMountPath != "" && jobOptions.ArtifactRequest.StorageClassName != "" { c.log.Debug("creating scraper job with options", "options", jobOptions) jobsClient := c.clientSet.BatchV1().Jobs(c.namespace) scraperSpec, err := NewScraperJobSpec(c.log, jobOptions) @@ -516,42 +519,51 @@ func (c *ContainerExecutor) stopExecution(ctx context.Context, execution *testku func NewJobOptionsFromExecutionOptions(options client.ExecuteOptions) *JobOptions { // for args, command and image, HTTP request takes priority, then test spec, then executor var args []string - switch { - case len(options.Request.Args) != 0: + argsMode := options.Request.ArgsMode + if options.TestSpec.ExecutionRequest != nil && argsMode == "" { + argsMode = string(options.TestSpec.ExecutionRequest.ArgsMode) + } + + if argsMode == string(testkube.ArgsModeTypeAppend) || argsMode == "" { args = options.Request.Args + if options.TestSpec.ExecutionRequest != nil && len(args) == 0 { + args = options.TestSpec.ExecutionRequest.Args + } - case options.TestSpec.ExecutionRequest != nil && - len(options.TestSpec.ExecutionRequest.Args) != 0: - args = options.TestSpec.ExecutionRequest.Args + args = append(options.ExecutorSpec.Args, args...) + } - case len(options.ExecutorSpec.Command) != 0: - args = options.ExecutorSpec.Args + if argsMode == string(testkube.ArgsModeTypeOverride) { + args = options.Request.Args + if options.TestSpec.ExecutionRequest != nil && len(args) == 0 { + args = options.TestSpec.ExecutionRequest.Args + } } var command []string switch { - case len(options.Request.Command) != 0: - command = options.Request.Command + case len(options.ExecutorSpec.Command) != 0: + command = options.ExecutorSpec.Command case options.TestSpec.ExecutionRequest != nil && len(options.TestSpec.ExecutionRequest.Command) != 0: command = options.TestSpec.ExecutionRequest.Command - case len(options.ExecutorSpec.Command) != 0: - command = options.ExecutorSpec.Command + case len(options.Request.Command) != 0: + command = options.Request.Command } var image string switch { - case options.Request.Image != "": - image = options.Request.Image + case options.ExecutorSpec.Image != "": + image = options.ExecutorSpec.Image case options.TestSpec.ExecutionRequest != nil && options.TestSpec.ExecutionRequest.Image != "": image = options.TestSpec.ExecutionRequest.Image - case options.ExecutorSpec.Image != "": - image = options.ExecutorSpec.Image + case options.Request.Image != "": + image = options.Request.Image } var workingDir string diff --git a/pkg/executor/containerexecutor/containerexecutor_test.go b/pkg/executor/containerexecutor/containerexecutor_test.go index a046d9c05f8..6295963eb7e 100644 --- a/pkg/executor/containerexecutor/containerexecutor_test.go +++ b/pkg/executor/containerexecutor/containerexecutor_test.go @@ -81,6 +81,7 @@ func TestNewExecutorJobSpecEmptyArgs(t *testing.T) { InitImage: "kubeshop/testkube-init-executor:0.7.10", Image: "ubuntu", JobTemplate: defaultJobTemplate, + Command: []string{}, Args: []string{}, } spec, err := NewExecutorJobSpec(logger(), jobOptions) @@ -139,6 +140,7 @@ func TestNewExecutorJobSpecWithoutInitImage(t *testing.T) { InitImage: "", Image: "ubuntu", JobTemplate: defaultJobTemplate, + Command: []string{}, Args: []string{}, } spec, err := NewExecutorJobSpec(logger(), jobOptions) diff --git a/pkg/executor/containerexecutor/templates/job.tmpl b/pkg/executor/containerexecutor/templates/job.tmpl index 51c6a136a49..928dc72fb52 100644 --- a/pkg/executor/containerexecutor/templates/job.tmpl +++ b/pkg/executor/containerexecutor/templates/job.tmpl @@ -29,8 +29,10 @@ spec: mountPath: /etc/certs {{- end }} {{- if .ArtifactRequest }} + {{- if .ArtifactRequest.VolumeMountPath }} - name: artifact-volume mountPath: {{ .ArtifactRequest.VolumeMountPath }} + {{- end }} {{- end }} {{- range $configmap := .EnvConfigMaps }} {{- if and $configmap.Mount $configmap.Reference }} @@ -76,8 +78,10 @@ spec: mountPath: /etc/certs {{- end }} {{- if .ArtifactRequest }} + {{- if .ArtifactRequest.VolumeMountPath }} - name: artifact-volume mountPath: {{ .ArtifactRequest.VolumeMountPath }} + {{- end }} {{- end }} {{- range $configmap := .EnvConfigMaps }} {{- if and $configmap.Mount $configmap.Reference }} @@ -100,9 +104,11 @@ spec: secretName: {{ .CertificateSecret }} {{- end }} {{- if .ArtifactRequest }} + {{- if and .ArtifactRequest.VolumeMountPath .ArtifactRequest.StorageClassName }} - name: artifact-volume persistentVolumeClaim: claimName: {{ .Name }}-pvc + {{- end }} {{- end }} {{- range $configmap := .EnvConfigMaps }} {{- if and $configmap.Mount $configmap.Reference }} diff --git a/pkg/git/checkout.go b/pkg/git/checkout.go index c7f3e9fc5dc..39104ace847 100644 --- a/pkg/git/checkout.go +++ b/pkg/git/checkout.go @@ -52,7 +52,7 @@ func CheckoutCommit(uri, authHeader, path, commit, dir string) (err error) { "git", args..., ) - output.PrintLogf("Git parameters: %v", obfuscateArgs(args, uri, authHeader)) + output.PrintLogf("Git parameters: %s", strings.Join(obfuscateArgs(args, uri, authHeader), " ")) if err != nil { return err } @@ -110,7 +110,7 @@ func Checkout(uri, authHeader, branch, commit, dir string) (outputDir string, er "git", args..., ) - output.PrintLogf("Git parameters: %v", obfuscateArgs(args, uri, authHeader)) + output.PrintLogf("Git parameters: %s", strings.Join(obfuscateArgs(args, uri, authHeader), " ")) if err != nil { return "", err } @@ -152,7 +152,7 @@ func PartialCheckout(uri, authHeader, path, branch, commit, dir string) (outputD "git", args..., ) - output.PrintLogf("Git parameters: %v", obfuscateArgs(args, uri, authHeader)) + output.PrintLogf("Git parameters: %s", strings.Join(obfuscateArgs(args, uri, authHeader), " ")) if err != nil { return "", err } diff --git a/pkg/mapper/tests/kube_openapi.go b/pkg/mapper/tests/kube_openapi.go index 7ac5e5aef73..26107118e6f 100644 --- a/pkg/mapper/tests/kube_openapi.go +++ b/pkg/mapper/tests/kube_openapi.go @@ -136,6 +136,7 @@ func MapExecutionRequestFromSpec(specExecutionRequest *testsv3.ExecutionRequest) TestSuiteSecretUUID: specExecutionRequest.TestSuiteSecretUUID, Command: specExecutionRequest.Command, Args: specExecutionRequest.Args, + ArgsMode: string(specExecutionRequest.ArgsMode), Image: specExecutionRequest.Image, ImagePullSecrets: MapImagePullSecrets(specExecutionRequest.ImagePullSecrets), Envs: specExecutionRequest.Envs, diff --git a/pkg/mapper/tests/kube_openapi_test.go b/pkg/mapper/tests/kube_openapi_test.go index 1f88574a7ca..d26f17ade64 100644 --- a/pkg/mapper/tests/kube_openapi_test.go +++ b/pkg/mapper/tests/kube_openapi_test.go @@ -90,6 +90,7 @@ func TestMapTestCRToAPI(t *testing.T) { Variables: map[string]testkube.Variable{}, VariablesFile: "", Args: []string{"-v", "-X", "GET", "https://testkube.kubeshop.io"}, + ArgsMode: "", Command: []string{"curl"}, Image: "curlimages/curl:7.85.0", ImagePullSecrets: []testkube.LocalObjectReference{ diff --git a/pkg/mapper/tests/openapi_kube.go b/pkg/mapper/tests/openapi_kube.go index 66ad71f24a1..d10a0631d7b 100644 --- a/pkg/mapper/tests/openapi_kube.go +++ b/pkg/mapper/tests/openapi_kube.go @@ -147,6 +147,7 @@ func MapExecutionRequestToSpecExecutionRequest(executionRequest *testkube.Execut TestSecretUUID: executionRequest.TestSecretUUID, TestSuiteSecretUUID: executionRequest.TestSuiteSecretUUID, Args: executionRequest.Args, + ArgsMode: testsv3.ArgsModeType(executionRequest.ArgsMode), Envs: executionRequest.Envs, SecretEnvs: executionRequest.SecretEnvs, Sync: executionRequest.Sync, @@ -466,6 +467,11 @@ func MapExecutionUpdateRequestToSpecExecutionRequest(executionRequest *testkube. } } + if executionRequest.ArgsMode != nil { + request.ArgsMode = testsv3.ArgsModeType(*executionRequest.ArgsMode) + emptyExecution = false + } + var slices = []struct { source *map[string]string destination *map[string]string diff --git a/pkg/scheduler/test_scheduler.go b/pkg/scheduler/test_scheduler.go index e902c98dbd8..4b479467346 100644 --- a/pkg/scheduler/test_scheduler.go +++ b/pkg/scheduler/test_scheduler.go @@ -224,6 +224,7 @@ func newExecutionFromExecutionOptions(options client.ExecuteOptions) testkube.Ex ) execution.Envs = options.Request.Envs + execution.Command = options.Request.Command execution.Args = options.Request.Args execution.VariablesFile = options.Request.VariablesFile execution.Uploads = options.Request.Uploads @@ -264,8 +265,7 @@ func (s *Scheduler) getExecuteOptions(namespace, id string, request testkube.Exe if test.ExecutionRequest != nil { // Test variables lowest priority, then test suite, then test suite execution / test execution request.Variables = mergeVariables(test.ExecutionRequest.Variables, request.Variables) - // Combine test executor args with execution args - request.Args = append(request.Args, test.ExecutionRequest.Args...) + request.Envs = mergeEnvs(request.Envs, test.ExecutionRequest.Envs) request.SecretEnvs = mergeEnvs(request.SecretEnvs, test.ExecutionRequest.SecretEnvs) request.EnvConfigMaps = mergeEnvReferences(request.EnvConfigMaps, test.ExecutionRequest.EnvConfigMaps) @@ -299,6 +299,10 @@ func (s *Scheduler) getExecuteOptions(namespace, id string, request testkube.Exe test.ExecutionRequest.ScraperTemplate, &request.ScraperTemplate, }, + { + test.ExecutionRequest.ArgsMode, + &request.ArgsMode, + }, } for _, field := range fields { @@ -307,6 +311,15 @@ func (s *Scheduler) getExecuteOptions(namespace, id string, request testkube.Exe } } + // Combine test executor args with execution args + if len(request.Command) == 0 { + request.Command = test.ExecutionRequest.Command + } + + if len(request.Args) == 0 { + request.Args = test.ExecutionRequest.Args + } + if request.ActiveDeadlineSeconds == 0 && test.ExecutionRequest.ActiveDeadlineSeconds != 0 { request.ActiveDeadlineSeconds = test.ExecutionRequest.ActiveDeadlineSeconds } @@ -336,16 +349,15 @@ func (s *Scheduler) getExecuteOptions(namespace, id string, request testkube.Exe var imagePullSecrets []string switch { - case len(request.ImagePullSecrets) != 0: + case len(executorCR.Spec.ImagePullSecrets) != 0: + imagePullSecrets = mapK8sImagePullSecrets(executorCR.Spec.ImagePullSecrets) - imagePullSecrets = mapImagePullSecrets(request.ImagePullSecrets) case testCR.Spec.ExecutionRequest != nil && len(testCR.Spec.ExecutionRequest.ImagePullSecrets) != 0: - imagePullSecrets = mapK8sImagePullSecrets(testCR.Spec.ExecutionRequest.ImagePullSecrets) - case len(executorCR.Spec.ImagePullSecrets) != 0: - imagePullSecrets = mapK8sImagePullSecrets(executorCR.Spec.ImagePullSecrets) + case len(request.ImagePullSecrets) != 0: + imagePullSecrets = mapImagePullSecrets(request.ImagePullSecrets) } configMapVars := make(map[string]testkube.Variable, 0) @@ -388,6 +400,14 @@ func (s *Scheduler) getExecuteOptions(namespace, id string, request testkube.Exe request.Variables = mergeVariables(secretVars, request.Variables) } + if len(request.Command) == 0 { + request.Command = executorCR.Spec.Command + } + + if request.ArgsMode == string(testkube.ArgsModeTypeAppend) || request.ArgsMode == "" { + request.Args = append(executorCR.Spec.Args, request.Args...) + } + return client.ExecuteOptions{ TestName: id, Namespace: namespace, diff --git a/pkg/scheduler/test_scheduler_test.go b/pkg/scheduler/test_scheduler_test.go index 047a3b66d02..a40f93bff2b 100644 --- a/pkg/scheduler/test_scheduler_test.go +++ b/pkg/scheduler/test_scheduler_test.go @@ -92,7 +92,7 @@ func TestGetExecuteOptions(t *testing.T) { ExecutorType: "job", URI: "", Image: "cypress", - Args: nil, + Args: []string{}, Command: []string{"run"}, ImagePullSecrets: []k8sv1.LocalObjectReference{{Name: "secret-name1"}, {Name: "secret-name2"}}, Features: nil, @@ -113,8 +113,9 @@ func TestGetExecuteOptions(t *testing.T) { Namespace: "namespace", VariablesFile: "", Variables: map[string]testkube.Variable{"var": testkube.Variable{Name: "one"}}, - Command: []string{}, + Command: []string{"run"}, Args: []string{}, + ArgsMode: "", Image: "executor-image", ImagePullSecrets: []testkube.LocalObjectReference{}, Envs: map[string]string{