Features Test Cmd: "Duplicate" test mode to test Feature Idempotence

docs/features/test.md

@@ -4,9 +4,15 @@ A built-in testing framework for Features is in active development.  This comman
 
 The `test` command utilizes the CLI's `build` and `exec` commands to test Features in your local source tree.  The command will look at the `target` path for mirrored `src` and `test` directories ([example](https://github.com/devcontainers/features).  Without any additional arguments, the `test` command will auto-generate a test for each Feature (pulling the source code directly from `src/<FEATURE>`), and exec `test/<FEATURE>/test.sh` inside of the running container.  
 
-For the test to pass, the container must (1) build and start successfully, and (2) execute the `test.sh` with a success (zero) exit code.  Note that auto-generated tests will execute the given Feature with default options.
+For the test to pass, the container must (1) build the generated dev container and start successfully, and (2) execute the `test.sh` with a success (zero) exit code.  Note that auto-generated tests will execute the given Feature with default options.
 
-Additonally, 'scenarios' can be defined for each Feature to test more complicated cases.  For example, you can use a scenario to test Feature options or >1 Feature in a container.  You can find more information about [writing scenario tests below](#scenarios).
+There are additional 'modes' that run an additional set of tests per Feature if the required assertion script is present.  The table below highlights other modes:
+
+| Mode | Use Case |  Details | CLI Flag |
+| ---- | ----------- | ---------------- | ----- |
+| Auto-generated | Auto-generates a barebones dev container with (1) the target Features, (2) the `--base-image`, and the (3) `--remote-user`) | Quick way to assert the default behavior of a Feature  | `--skip-autogenerated`
+| Scenarios | Define more complicated test scenarios.  Use a scenario to test Feature options or >1 Feature in a container.  | [Details](#scenarios)| `--skip-scenarios`
+| Duplicate Tests | A mode that will generate a dev container installing the same Feature twice with different options | [Details](#duplicate-style-tests)| `--skip-duplicated`
 
 The source code of the sub-command is [here](../../src/spec-node/featuresCLI/test.ts). An example of the command being used in CI can be [found in the `devcontainers/feature-starter` repo](https://github.com/devcontainers/feature-starter/blob/main/.github/workflows/test.yaml) and the [`devcontainers/features` repo](https://github.com/devcontainers/features).
 
@@ -32,6 +38,7 @@ An example project structure can be found below.
 │	│	├── scenarios.json
 │   │   └── some_test_scenario.sh
 │   ├── dotnet
+|   |   ├── duplicate.sh
 │   │   └── test.sh
 │   ├── oryx
 |   |   ├── scenarios.json
@@ -45,7 +52,7 @@ An example project structure can be found below.
 To run all the `dotnet` related tests from a repo structured above, the command would be:
 
 ```bash
-devcontainer features test  -f dotnet -b ubuntu
+devcontainer features test  -f dotnet --base-image ubuntu
 ```
 
 ## Scenarios 
@@ -182,6 +189,70 @@ The `test/_global` directory is a special directory that holds scenario tests no
 
 The `--global-scenarios-only` can be passed to only run the global scenarios.
 
+### Duplicate-style Tests
+
+When executing the command without the `--skip-duplicated` flag, each Feature with a `duplicate.sh` will generate a test installing a given Feature twice (with different options).  This is useful for asserting that a Feature can be installed multiple times without conflict (is idempotent). Additionally, the options used for each distinct Feature are passed into the assertion script, should that be useful to write an assertion.
+
+For example, the `dotnet` Feature above provided a `duplicate.sh`, therefore the test command will generate a dev container test case installing `dotnet` twice.
+
+The generated dev container could look something like this:
+
+#### `devcontainer.json`
+```jsonc
+{
+    "image": "ubuntu",
+    "features": {
+        "./dotnet": {
+            "version": "5",
+            "installUsingApt": "false"
+        },
+        "./dotnet-0": {} // Default
+    }
+}
+```
+
+#### `duplicate.sh`
+```shell
+#!/bin/bash
+
+set -e
+
+# Optional: Import test library
+source dev-container-features-test-lib
+
+# The values of the randomized options will be set as environment variables.
+if [ -z "${VERSION}" ]; then
+	echo "Version of dotnet to install from randomized Feature not set!"
+	exit 1
+fi
+
+if [ -z "${INSTALLUSINGAPT}" ]; then
+	echo "Boolean to install using apt from randomized Feature not set!"
+	exit 1
+fi
+
+# The values of the default options will be set as environment variables.
+if [ -z "${VERSION__DEFAULT}" ]; then
+	echo "Version of dotnet to install default Feature not set!"
+	exit 1
+fi
+
+if [ -z "${INSTALLUSINGAPT__DEFAULT}" ]; then
+	echo "oolean to install using apt from default Feature not set!"
+	exit 1
+fi
+
+check "randomized version of dotnet installed"  bash -c "dotnet --list-sdks | ${VERSION}"
+check "default version of dotnet installed"  bash -c "dotnet --list-sdks | ${VERSION__DEFAULT}"
+
+# ...
+# ...
+
+# Report result
+reportResults
+```
+
+
 ## dev-container-features-test-lib
 
 The `dev-container-features-test-lib` is convenience helper [defined in the CLI](https://github.com/devcontainers/cli/blob/1910ca41015c627b884ddd69ebc52d1e8cdd8cf0/src/spec-node/featuresCLI/utils.ts#L59) that adds several bash functions to organize test asserts. Note that using this libary **is not required**.

src/spec-node/featuresCLI/test.ts

@@ -15,7 +15,9 @@ export function featuresTestOptions(y: Argv) {
 			'filter': { type: 'string', describe: 'Filter current tests to only run scenarios containing this string.  Cannot be combined with \'--skip-scenarios\'.' },
 			'global-scenarios-only': { type: 'boolean', default: false, description: 'Run only scenario tests under \'tests/_global\' .  Cannot be combined with \'-f\'.' },
 			'skip-scenarios': { type: 'boolean', default: false, description: 'Skip all \'scenario\' style tests.  Cannot be combined with \'--global--scenarios-only\'.' },
-			'skip-autogenerated': { type: 'boolean', default: false, description: 'Skip all \'autogenerated\' style tests.' },
+			'skip-autogenerated': { type: 'boolean', default: false, description: 'Skip all \'autogenerated\' style tests (test.sh).' },
+			'skip-duplicated': { type: 'boolean', default: false, description: 'Skip all \'duplicate\' style tests (duplicate.sh).' },
+			'permit-randomization': { type: 'boolean', default: false, description: 'Allow an element of randomness in test cases.' },
 			'base-image': { type: 'string', alias: 'i', default: 'ubuntu:focal', description: 'Base Image. Not used for scenarios.' },  // TODO: Optionally replace 'scenario' configs with this value?
 			'remote-user': { type: 'string', alias: 'u', describe: 'Remote user.  Not used for scenarios.', },  // TODO: Optionally replace 'scenario' configs with this value?
 			'log-level': { choices: ['info' as 'info', 'debug' as 'debug', 'trace' as 'trace'], default: 'info' as 'info', description: 'Log level.' },
@@ -48,17 +50,19 @@ export type FeaturesTestArgs = UnpackArgv<ReturnType<typeof featuresTestOptions>
 export interface FeaturesTestCommandInput {
 	cliHost: CLIHost;
 	pkg: PackageConfiguration;
-	baseImage: string;
 	collectionFolder: string;
 	features?: string[];
 	filter: string | undefined;
 	globalScenariosOnly: boolean;
 	skipScenarios: boolean;
 	skipAutogenerated: boolean;
+	skipDuplicateTest: boolean;
+	permitRandomization: boolean;
+	baseImage: string;
 	remoteUser: string | undefined;
-	quiet: boolean;
-	preserveTestContainers: boolean;
 	logLevel: LogLevel;
+	preserveTestContainers: boolean;
+	quiet: boolean;
 	disposables: (() => Promise<unknown> | undefined)[];
 }
 
@@ -75,10 +79,12 @@ async function featuresTest({
 	'global-scenarios-only': globalScenariosOnly,
 	'skip-scenarios': skipScenarios,
 	'skip-autogenerated': skipAutogenerated,
+	'skip-duplicated': skipDuplicateTest,
+	'permit-randomization': permitRandomization,
 	'remote-user': remoteUser,
-	quiet,
-	'preserve-test-containers': preserveTestContainers,
 	'log-level': inputLogLevel,
+	'preserve-test-containers': preserveTestContainers,
+	quiet,
 }: FeaturesTestArgs) {
 	const disposables: (() => Promise<unknown> | undefined)[] = [];
 	const dispose = async () => {
@@ -106,6 +112,8 @@ async function featuresTest({
 		globalScenariosOnly,
 		skipScenarios,
 		skipAutogenerated,
+		skipDuplicateTest,
+		permitRandomization,
 		remoteUser,
 		preserveTestContainers,
 		disposables