feat: added --args-file to Kurtosis run (#1451)

## Description:
To make this change work with previous versions of Kurtosis the argument
takes precedence over value provided by the flag; so if a user provides
a non `{}` for input-args; and a value for flag as well, we will go with
the value for the arg

## Is this change user facing?
YES

## References (if applicable):
Closes #1112

.circleci/config.yml

@@ -191,7 +191,13 @@ parameters:
     default: "engine-server-image.tgz"
   startosis-test-script-file-relative-path:
     type: string
-    default: "internal_testsuites/starlark/ci_tests/startosis_simple_script.star"
+    default: "internal_testsuites/starlark/ci_tests/simple_no_arg_test/startosis_simple_script.star"
+  starlark-test-args-file-test-main-star-relative-path:
+    type: string
+    default: "internal_testsuites/starlark/ci_tests/simple_arg_test/main.star"
+  starlark-test-args-file-test-args-json-relative-path:
+    type: string
+    default: "internal_testsuites/starlark/ci_tests/simple_arg_test/args.json"
   rendertemplate-cli-test-template-relative-path:
     type: string
     default: "internal_testsuites/resources/render_template_cli_test/template.txt"
@@ -315,6 +321,8 @@ jobs:
           root: .
           paths:
             - "<< pipeline.parameters.file-artifacts-expander-image-filename >>"
+            - "<< pipeline.parameters.starlark-test-args-file-test-main-star-relative-path >>"
+            - "<< pipeline.parameters.starlark-test-args-file-test-args-json-relative-path >>"
 
   build_core_launcher:
     docker:
@@ -757,6 +765,9 @@ jobs:
       # Execute simple startosis script in new enclave
       - run: "${KURTOSIS_BINPATH} run --enclave test-enclave << pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>/<< pipeline.parameters.startosis-test-script-file-relative-path >>"
 
+      # Execute Simple Starlark Script to test --args-file
+      - run: "${KURTOSIS_BINPATH} run --enclave args-file-test << pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>/<< pipeline.parameters.starlark-test-args-file-test-main-star-relative-path >> --args-file << pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>/<< pipeline.parameters.starlark-test-args-file-test-args-json-relative-path >>"
+
       # Execute github starlark module
       - run: "${KURTOSIS_BINPATH} run --enclave test-datastore github.com/kurtosis-tech/datastore-army-package '{\"num_datastores\": 2}'"
 

cli/cli/commands/run/run.go

@@ -92,6 +92,9 @@ const (
 
 	noConnectFlagKey = "no-connect"
 	noConnectDefault = "false"
+
+	packageArgsFileFlagKey      = "args-file"
+	packageArgsFileDefaultValue = ""
 )
 
 var StarlarkRunCmd = &engine_consuming_kurtosis_command.EngineConsumingKurtosisCommand{
@@ -180,6 +183,12 @@ var StarlarkRunCmd = &engine_consuming_kurtosis_command.EngineConsumingKurtosisC
 			Type:    flags.FlagType_Bool,
 			Default: noConnectDefault,
 		},
+		{
+			Key:     packageArgsFileFlagKey,
+			Usage:   "The file (JSON/YAML) will be used as arguments passed to the Kurtosis Package",
+			Type:    flags.FlagType_String,
+			Default: packageArgsFileDefaultValue,
+		},
 	},
 	Args: []*args.ArgConfig{
 		// TODO add a `Usage` description here when ArgConfig supports it
@@ -209,7 +218,7 @@ func run(
 	args *args.ParsedArgs,
 ) error {
 	// Args parsing and validation
-	serializedJsonArgs, err := args.GetNonGreedyArg(inputArgsArgKey)
+	packageArgs, err := args.GetNonGreedyArg(inputArgsArgKey)
 	if err != nil {
 		return stacktrace.Propagate(err, "An error occurred getting the script/package arguments using flag key '%v'", inputArgsArgKey)
 	}
@@ -275,6 +284,26 @@ func run(
 		return stacktrace.Propagate(err, "Expected a value for the '%v' flag but failed to get it", mainFunctionNameFlagKey)
 	}
 
+	packageArgsFile, err := flags.GetString(packageArgsFileFlagKey)
+	if err != nil {
+		return stacktrace.Propagate(err, "Expected a value for the '%v' flag but failed to get it", packageArgsFileFlagKey)
+	}
+
+	if packageArgs == inputArgsAreEmptyBracesByDefault && packageArgsFile != packageArgsFileDefaultValue {
+		logrus.Debugf("'%v' is empty but '%v' is provided so we will go with the '%v' value", inputArgsArgKey, packageArgsFileFlagKey, packageArgsFileFlagKey)
+		packageArgsFileBytes, err := os.ReadFile(packageArgsFile)
+		if err != nil {
+			return stacktrace.Propagate(err, "attempted to read file provided by flag '%v' with path '%v' but failed", packageArgsFileFlagKey, packageArgsFile)
+		}
+		packageArgsFileStr := string(packageArgsFileBytes)
+		if packageArgParsingErr := validateSerializedArgs(packageArgsFileStr); packageArgParsingErr != nil {
+			return stacktrace.Propagate(err, "attempted to validate '%v' but failed", packageArgsFileFlagKey)
+		}
+		packageArgs = packageArgsFileStr
+	} else if packageArgs != inputArgsAreEmptyBracesByDefault && packageArgsFile != packageArgsFileDefaultValue {
+		logrus.Debugf("'%v' arg is not empty; ignoring value of '%v' flag as '%v' arg takes precedence", inputArgsArgKey, packageArgsFileFlagKey, inputArgsArgKey)
+	}
+
 	kurtosisCtx, err := kurtosis_context.NewKurtosisContextFromLocalEngine()
 	if err != nil {
 		return stacktrace.Propagate(err, "An error occurred connecting to the local Kurtosis engine")
@@ -310,7 +339,7 @@ func run(
 	isStandAloneScript := false
 	packageOrScriptName := starlarkScriptOrPackagePath
 	if isRemotePackage {
-		responseLineChan, cancelFunc, errRunningKurtosis = executeRemotePackage(ctx, enclaveCtx, starlarkScriptOrPackagePath, relativePathToTheMainFile, mainFunctionName, serializedJsonArgs, dryRun, castedParallelism, experimentalFlags)
+		responseLineChan, cancelFunc, errRunningKurtosis = executeRemotePackage(ctx, enclaveCtx, starlarkScriptOrPackagePath, relativePathToTheMainFile, mainFunctionName, packageArgs, dryRun, castedParallelism, experimentalFlags)
 	} else {
 		fileOrDir, err := os.Stat(starlarkScriptOrPackagePath)
 		if err != nil {
@@ -322,7 +351,7 @@ func run(
 			if !strings.HasSuffix(starlarkScriptOrPackagePath, starlarkExtension) {
 				return stacktrace.NewError("Expected a script with a '%s' extension but got file '%v' with a different extension", starlarkExtension, starlarkScriptOrPackagePath)
 			}
-			responseLineChan, cancelFunc, errRunningKurtosis = executeScript(ctx, enclaveCtx, starlarkScriptOrPackagePath, mainFunctionName, serializedJsonArgs, dryRun, castedParallelism, experimentalFlags)
+			responseLineChan, cancelFunc, errRunningKurtosis = executeScript(ctx, enclaveCtx, starlarkScriptOrPackagePath, mainFunctionName, packageArgs, dryRun, castedParallelism, experimentalFlags)
 		} else {
 			// if the path is a file with `kurtosis.yml` at the end it's a module dir
 			// we remove the `kurtosis.yml` to get just the Dir containing the module
@@ -334,7 +363,7 @@ func run(
 			if err != nil {
 				return stacktrace.Propagate(err, "Tried parsing Kurtosis YML at '%v' to get package name but failed", starlarkScriptOrPackagePath)
 			}
-			responseLineChan, cancelFunc, errRunningKurtosis = executePackage(ctx, enclaveCtx, starlarkScriptOrPackagePath, relativePathToTheMainFile, mainFunctionName, serializedJsonArgs, dryRun, castedParallelism, experimentalFlags)
+			responseLineChan, cancelFunc, errRunningKurtosis = executePackage(ctx, enclaveCtx, starlarkScriptOrPackagePath, relativePathToTheMainFile, mainFunctionName, packageArgs, dryRun, castedParallelism, experimentalFlags)
 		}
 	}
 	if errRunningKurtosis != nil {
@@ -558,18 +587,7 @@ func validatePackageArgs(_ context.Context, _ *flags.ParsedFlags, args *args.Par
 	if err != nil {
 		return stacktrace.Propagate(err, "An error occurred getting the script/package arguments using flag key '%v'", inputArgsArgKey)
 	}
-	var result interface{}
-	var jsonError error
-	if jsonError = json.Unmarshal([]byte(serializedArgs), &result); jsonError == nil {
-		return nil
-	}
-	var yamlError error
-	if yamlError = yaml.Unmarshal([]byte(serializedArgs), &result); yamlError == nil {
-		return nil
-	}
-	return stacktrace.Propagate(
-		fmt.Errorf("JSON parsing error '%v', YAML parsing error '%v'", jsonError, yamlError),
-		"Error validating args, because it is not a valid JSON or YAML.")
+	return validateSerializedArgs(serializedArgs)
 }
 
 // parseVerbosityFlag Get the verbosity flag is present, and parse it to a valid Verbosity value
@@ -636,3 +654,18 @@ func scriptPathValidation(scriptPath string) (error, bool) {
 
 	return nil, file_system_path_arg.ContinueWithDefaultValidation
 }
+
+func validateSerializedArgs(serializedArgs string) error {
+	var result interface{}
+	var jsonError error
+	if jsonError = json.Unmarshal([]byte(serializedArgs), &result); jsonError == nil {
+		return nil
+	}
+	var yamlError error
+	if yamlError = yaml.Unmarshal([]byte(serializedArgs), &result); yamlError == nil {
+		return nil
+	}
+	return stacktrace.Propagate(
+		fmt.Errorf("JSON parsing error '%v', YAML parsing error '%v'", jsonError, yamlError),
+		"Error validating args, because it is not a valid JSON or YAML.")
+}

docs/docs/cli-reference/run.md

@@ -33,7 +33,7 @@ If you want to run a non-main branch, tag or commit use the following syntax
 
 ### Arguments
 
-Package behaviour can be customized by passing in JSON-serialized arguments when calling `kurtosis run`.
+Package behaviour can be customized by passing in JSON/YAML-serialized arguments when calling `kurtosis run`.
 
 For example, if your package's `run` function looks like this...
 
@@ -77,6 +77,11 @@ run(plan, some_parameter = struct(some_property = "Property value"))
 
 `kurtosis run` has additional flags that can further modify its behaviour:
 
+1. The `--args-file` flag can be used to send in a JSON/YAML file as an argument to the Kurtosis Package. Note that if you pass in package arguments as CLI arguments and via the flag, the CLI arguments will be the one used.
+   For example:
+   ```bash
+   kurtosis run github.com/kurtosis-tech/ethereum-package --args-file "devnet-5.json"
+   ```
 1. The `--dry-run` flag can be used to print the changes proposed by the script without executing them
 1. The `--parallelism` flag can be used to specify to what degree of parallelism certain commands can be run. For example: if the script contains an [`add_services`][add-services-reference] instruction and is run with `--parallelism 100`, up to 100 services will be run at one time.
 1. The `--enclave` flag can be used to instruct Kurtosis to run the script inside the specified enclave or create a new enclave (with the given enclave [identifier](../concepts-reference/resource-identifier.md)) if one does not exist. If this flag is not used, Kurtosis will create a new enclave with an auto-generated name, and run the script or package inside it.

internal_testsuites/starlark/ci_tests/simple_arg_test/args.json

@@ -0,0 +1,3 @@
+{
+  "name": "John Doe"
+}

internal_testsuites/starlark/ci_tests/simple_arg_test/main.star

@@ -0,0 +1,3 @@
+def run(plan, args):
+    plan.verify(args["name"],  "==", "John Doe")
+    plan.print(args["name"])