From bf31505b49277f4c27ea52ce2b688b9dcab5903f Mon Sep 17 00:00:00 2001 From: Leandro Poroli Date: Mon, 23 Oct 2023 18:44:21 -0500 Subject: [PATCH 1/3] getting the args file from a URL --- .circleci/config.yml | 7 +++++ cli/cli/commands/run/run.go | 52 ++++++++++++++++++++++++++++++++----- 2 files changed, 52 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7b65289a3f..8afa149a3f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -198,6 +198,9 @@ parameters: starlark-test-args-file-test-args-json-relative-path: type: string default: "internal_testsuites/starlark/ci_tests/simple_arg_test/args.json" + starlark-test-args-file-url: + type: string + default: "https://raw.githubusercontent.com/kurtosis-tech/sample-startosis-load/main/args.json" rendertemplate-cli-test-template-relative-path: type: string default: "internal_testsuites/resources/render_template_cli_test/template.txt" @@ -323,6 +326,7 @@ jobs: - "<< pipeline.parameters.file-artifacts-expander-image-filename >>" - "<< pipeline.parameters.starlark-test-args-file-test-main-star-relative-path >>" - "<< pipeline.parameters.starlark-test-args-file-test-args-json-relative-path >>" + - "<< pipeline.parameters.starlark-test-args-file-url >>" build_core_launcher: docker: @@ -768,6 +772,9 @@ jobs: # Execute Simple Starlark Script to test --args-file - run: "${KURTOSIS_BINPATH} run --enclave args-file-test << pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>/<< pipeline.parameters.starlark-test-args-file-test-main-star-relative-path >> --args-file << pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>/<< pipeline.parameters.starlark-test-args-file-test-args-json-relative-path >>" + # Execute Simple Starlark Script to test --args-file from URL + - run: "${KURTOSIS_BINPATH} run --enclave args-file-test-from-url << pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>/<< pipeline.parameters.starlark-test-args-file-test-main-star-relative-path >> --args-file << pipeline.parameters.starlark-test-args-file-url >>" + # Execute github starlark module - run: "${KURTOSIS_BINPATH} run --enclave test-datastore github.com/kurtosis-tech/datastore-army-package '{\"num_datastores\": 2}'" diff --git a/cli/cli/commands/run/run.go b/cli/cli/commands/run/run.go index 3af1903912..64e2a9d429 100644 --- a/cli/cli/commands/run/run.go +++ b/cli/cli/commands/run/run.go @@ -6,6 +6,9 @@ import ( "fmt" "github.com/kurtosis-tech/kurtosis/api/golang/core/lib/starlark_run_config" "gopkg.in/yaml.v2" + "io" + "net/http" + "net/url" "os" "os/signal" "path" @@ -292,15 +295,10 @@ func run( if packageArgs == inputArgsAreEmptyBracesByDefault && packageArgsFile != packageArgsFileDefaultValue { logrus.Debugf("'%v' is empty but '%v' is provided so we will go with the '%v' value", inputArgsArgKey, packageArgsFileFlagKey, packageArgsFileFlagKey) - packageArgsFileBytes, err := os.ReadFile(packageArgsFile) + packageArgs, err = getArgsFromFilepathOrURL(packageArgsFile) if err != nil { - return stacktrace.Propagate(err, "attempted to read file provided by flag '%v' with path '%v' but failed", packageArgsFileFlagKey, packageArgsFile) + return stacktrace.Propagate(err, "An error occurred while getting the package args from filepath or URL '%s'", packageArgsFile) } - packageArgsFileStr := string(packageArgsFileBytes) - if packageArgParsingErr := validateSerializedArgs(packageArgsFileStr); packageArgParsingErr != nil { - return stacktrace.Propagate(err, "attempted to validate '%v' but failed", packageArgsFileFlagKey) - } - packageArgs = packageArgsFileStr } else if packageArgs != inputArgsAreEmptyBracesByDefault && packageArgsFile != packageArgsFileDefaultValue { logrus.Debugf("'%v' arg is not empty; ignoring value of '%v' flag as '%v' arg takes precedence", inputArgsArgKey, packageArgsFileFlagKey, inputArgsArgKey) } @@ -665,3 +663,43 @@ func validateSerializedArgs(serializedArgs string) error { fmt.Errorf("JSON parsing error '%v', YAML parsing error '%v'", jsonError, yamlError), "Error validating args, because it is not a valid JSON or YAML.") } + +func getArgsFromFilepathOrURL(packageArgsFile string) (string, error) { + var packageArgsFileBytes []byte + isFileURL := true + _, err := os.Stat(packageArgsFile) + if err == nil { + isFileURL = false + packageArgsFileBytes, err = os.ReadFile(packageArgsFile) + if err != nil { + return "", stacktrace.Propagate(err, "attempted to read file provided by flag '%v' with path '%v' but failed", packageArgsFileFlagKey, packageArgsFile) + } + } + if err != nil && !os.IsNotExist(err) { + return "", stacktrace.Propagate(err, "An error occurred checking for argument's file existence on '%s'", packageArgsFile) + } + + if isFileURL { + argsFileURL, parseErr := url.Parse(packageArgsFile) + if parseErr != nil { + return "", stacktrace.Propagate(parseErr, "An error occurred while parsing file args URL '%s'", argsFileURL) + } + response, getErr := http.Get(argsFileURL.String()) + if getErr != nil { + return "", stacktrace.Propagate(getErr, "An error occurred getting the args file content from URL '%s'", argsFileURL.String()) + } + defer response.Body.Close() + responseBodyBytes, readAllErr := io.ReadAll(response.Body) + if readAllErr != nil { + return "", stacktrace.Propagate(readAllErr, "An error occurred reading the args file content") + } + packageArgsFileBytes = responseBodyBytes + } + + packageArgsFileStr := string(packageArgsFileBytes) + if packageArgParsingErr := validateSerializedArgs(packageArgsFileStr); packageArgParsingErr != nil { + return "", stacktrace.Propagate(err, "attempted to validate '%v' but failed", packageArgsFileFlagKey) + } + + return packageArgsFileStr, nil +} From 5f978736b2d986daa9c9185ee3043c72665e66ec Mon Sep 17 00:00:00 2001 From: Leandro Poroli Date: Mon, 23 Oct 2023 18:51:54 -0500 Subject: [PATCH 2/3] comment updated --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8afa149a3f..9c765be5f8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -773,7 +773,7 @@ jobs: - run: "${KURTOSIS_BINPATH} run --enclave args-file-test << pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>/<< pipeline.parameters.starlark-test-args-file-test-main-star-relative-path >> --args-file << pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>/<< pipeline.parameters.starlark-test-args-file-test-args-json-relative-path >>" # Execute Simple Starlark Script to test --args-file from URL - - run: "${KURTOSIS_BINPATH} run --enclave args-file-test-from-url << pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>/<< pipeline.parameters.starlark-test-args-file-test-main-star-relative-path >> --args-file << pipeline.parameters.starlark-test-args-file-url >>" + - run: "${KURTOSIS_BINPATH} run --enclave args-file-from-url-test << pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>/<< pipeline.parameters.starlark-test-args-file-test-main-star-relative-path >> --args-file << pipeline.parameters.starlark-test-args-file-url >>" # Execute github starlark module - run: "${KURTOSIS_BINPATH} run --enclave test-datastore github.com/kurtosis-tech/datastore-army-package '{\"num_datastores\": 2}'" From 6f2228ba4b34d8c4140e54516a100dc0244faa38 Mon Sep 17 00:00:00 2001 From: Leandro Poroli Date: Tue, 24 Oct 2023 11:54:44 -0500 Subject: [PATCH 3/3] docs added --- docs/docs/cli-reference/run.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/docs/cli-reference/run.md b/docs/docs/cli-reference/run.md index fa4b2b117d..1f81e36ac0 100644 --- a/docs/docs/cli-reference/run.md +++ b/docs/docs/cli-reference/run.md @@ -77,11 +77,16 @@ run(plan, some_parameter = struct(some_property = "Property value")) `kurtosis run` has additional flags that can further modify its behaviour: -1. The `--args-file` flag can be used to send in a JSON/YAML file as an argument to the Kurtosis Package. Note that if you pass in package arguments as CLI arguments and via the flag, the CLI arguments will be the one used. +1. The `--args-file` flag can be used to send in a JSON/YAML file, from a local file through the filepath or from remote using the URL, as an argument to the Kurtosis Package. Note that if you pass in package arguments as CLI arguments and via the flag, the CLI arguments will be the one used. For example: ```bash kurtosis run github.com/kurtosis-tech/ethereum-package --args-file "devnet-5.json" ``` + or + ```bash + kurtosis run github.com/kurtosis-tech/ethereum-package --args-file "https://www.myhost.com/devnet-5.json" + ``` + 1. The `--dry-run` flag can be used to print the changes proposed by the script without executing them 1. The `--parallelism` flag can be used to specify to what degree of parallelism certain commands can be run. For example: if the script contains an [`add_services`][add-services-reference] instruction and is run with `--parallelism 100`, up to 100 services will be run at one time. 1. The `--enclave` flag can be used to instruct Kurtosis to run the script inside the specified enclave or create a new enclave (with the given enclave [identifier](../concepts-reference/resource-identifier.md)) if one does not exist. If this flag is not used, Kurtosis will create a new enclave with an auto-generated name, and run the script or package inside it.