Features Test Cmd: "Duplicate" test mode to test Feature Idempotence

src/spec-node/featuresCLI/test.ts

@@ -15,7 +15,8 @@ export function featuresTestOptions(y: Argv) {
 			'filter': { type: 'string', describe: 'Filter current tests to only run scenarios containing this string.  Cannot be combined with \'--skip-scenarios\'.' },
 			'global-scenarios-only': { type: 'boolean', default: false, description: 'Run only scenario tests under \'tests/_global\' .  Cannot be combined with \'-f\'.' },
 			'skip-scenarios': { type: 'boolean', default: false, description: 'Skip all \'scenario\' style tests.  Cannot be combined with \'--global--scenarios-only\'.' },
-			'skip-autogenerated': { type: 'boolean', default: false, description: 'Skip all \'autogenerated\' style tests.' },
+			'skip-autogenerated': { type: 'boolean', default: false, description: 'Skip all \'autogenerated\' style tests (test.sh).' },
+			'skip-duplicate-test': { type: 'boolean', default: false, description: 'Skip all \'duplicate\' style tests (duplicate.sh).' },
 			'base-image': { type: 'string', alias: 'i', default: 'ubuntu:focal', description: 'Base Image. Not used for scenarios.' },  // TODO: Optionally replace 'scenario' configs with this value?
 			'remote-user': { type: 'string', alias: 'u', describe: 'Remote user.  Not used for scenarios.', },  // TODO: Optionally replace 'scenario' configs with this value?
 			'log-level': { choices: ['info' as 'info', 'debug' as 'debug', 'trace' as 'trace'], default: 'info' as 'info', description: 'Log level.' },
@@ -55,6 +56,7 @@ export interface FeaturesTestCommandInput {
 	globalScenariosOnly: boolean;
 	skipScenarios: boolean;
 	skipAutogenerated: boolean;
+	skipDuplicateTest: boolean;
 	remoteUser: string | undefined;
 	quiet: boolean;
 	preserveTestContainers: boolean;
@@ -75,6 +77,7 @@ async function featuresTest({
 	'global-scenarios-only': globalScenariosOnly,
 	'skip-scenarios': skipScenarios,
 	'skip-autogenerated': skipAutogenerated,
+	'skip-duplicate-test': skipDuplicateTest,
 	'remote-user': remoteUser,
 	quiet,
 	'preserve-test-containers': preserveTestContainers,
@@ -106,6 +109,7 @@ async function featuresTest({
 		globalScenariosOnly,
 		skipScenarios,
 		skipAutogenerated,
+		skipDuplicateTest,
 		remoteUser,
 		preserveTestContainers,
 		disposables

src/spec-node/featuresCLI/testCommandImpl.ts

@@ -12,6 +12,8 @@ import { FeaturesTestCommandInput } from './test';
 import { cpDirectoryLocal, rmLocal } from '../../spec-utils/pfs';
 import { nullLog } from '../../spec-utils/log';
 import { runCommand } from '../../spec-common/commonUtils';
+import { Feature } from '../../spec-configuration/containerFeaturesConfiguration';
+import { getSafeId } from '../containerFeatures';
 
 const TEST_LIBRARY_SCRIPT_NAME = 'dev-container-features-test-lib';
 
@@ -60,7 +62,7 @@ export async function doFeaturesTestCommand(args: FeaturesTestCommandInput): Pro
 	} else {
 		await runFeatureTests(args, testResults);
 
-		// If any features were explictily set to run,
+		// If any features were explicitly set to run,
 		// we know we don't want to run the global tests.
 		if (!features) {
 			await runGlobalFeatureTests(args, testResults);
@@ -103,9 +105,105 @@ async function runGlobalFeatureTests(args: FeaturesTestCommandInput, testResults
 	return testResults;
 }
 
+// Executes the same Feature twice with randomized options to ensure Feature can be installed >1.
+async function runDuplicateTest(args: FeaturesTestCommandInput, feature: string, testResults: TestResult[] = []): Promise<TestResult[]> {
+	const { collectionFolder, cliHost } = args;
+	const scenarioName = `${feature} executed twice with randomized options`;
+
+	const featureTestFolder = path.join(collectionFolder, 'test', feature);
+	const testFileName = 'duplicate.sh';
+	const testFilePath = path.join(featureTestFolder, testFileName);
+	if (!(await cliHost.isFile(testFilePath))) {
+		log(`Skipping duplicate test for ${feature} because '${testFilePath}' does not exist.`, { prefix: '⚠️', });
+		return testResults;
+	}
+
+	//Read Feature's metadata
+	const featureMetadata = await readFeatureMetadata(args, feature);
+	const options = featureMetadata.options || {};
+
+	// For each possible option, generate a random value for each Feature
+	const randomizedOptions: { [key: string]: string | boolean } = {};
+	Object.entries(options).forEach(([key, value]) => {
+		if (value.type === 'boolean') {
+			randomizedOptions[key] = !value.default;
+		}
+		if (value.type === 'string' && 'proposals' in value && value?.proposals?.length) {
+			const randomIndex = Math.floor(Math.random() * value.proposals.length);
+			randomizedOptions[key] = value.proposals[randomIndex];
+			if (randomizedOptions[key] === value.default) {
+				randomizedOptions[key] = value.proposals[(randomIndex + 1) % value.proposals.length];
+			}
+		}
+		if (value.type === 'string' && 'enum' in value && value?.enum?.length) {
+			const randomIndex = Math.floor(Math.random() * value.enum.length);
+			randomizedOptions[key] = value.enum[randomIndex];
+			if (randomizedOptions[key] === value.default) {
+				randomizedOptions[key] = value.enum[(randomIndex + 1) % value.enum.length];
+			}
+		}
+	});
+
+	// Default values
+	const defaultOptions = Object.entries(options).reduce((acc, [key, value]) => {
+		if (value.default === undefined) {
+			return acc;
+		}
+		acc[`${key}__DEFAULT`] = value.default;
+		return acc;
+	}, {} as { [key: string]: string | boolean });
+
+	const config: DevContainerConfig = {
+		image: args.baseImage,
+		features: {
+			[feature]: randomizedOptions, // Randomized option values
+		}
+	};
+
+	// Create Container
+	const workspaceFolder = await generateProjectFromScenario(
+		cliHost,
+		collectionFolder,
+		scenarioName,
+		config,
+		undefined,
+		[{ featureId: feature, featureValue: {} }] // Default option values
+	);
+	const params = await generateDockerParams(workspaceFolder, args);
+	await createContainerFromWorkingDirectory(params, workspaceFolder, args);
+
+	// Move the entire test directory for the given Feature into the workspaceFolder
+	await cpDirectoryLocal(featureTestFolder, workspaceFolder);
+
+	// // Move the test library script into the workspaceFolder
+	await cliHost.writeFile(path.join(workspaceFolder, TEST_LIBRARY_SCRIPT_NAME), Buffer.from(testLibraryScript));
+
+	// Execute Test
+	testResults.push({
+		testName: scenarioName,
+		result: await execTest(testFileName, workspaceFolder, cliHost, { ...randomizedOptions, ...defaultOptions })
+	});
+	return testResults;
+}
+
+async function readFeatureMetadata(args: FeaturesTestCommandInput, feature: string): Promise<Feature> {
+	const { cliHost, collectionFolder } = args;
+	const featureSrcFolder = path.join(collectionFolder, 'src', feature);
+
+	const metadataFile = path.join(featureSrcFolder, 'devcontainer-feature.json');
+	if (!await (cliHost.isFile(metadataFile))) {
+		fail(`Feature '${feature}' does not contain a 'devcontainer-feature.json' file.`);
+	}
+	const buf = await cliHost.readFile(metadataFile);
+	if (!buf || buf.length === 0) {
+		fail(`Failed to read 'devcontainer-feature.json' file for feature '${feature}'`);
+	}
+
+	return jsonc.parse(buf.toString()) as Feature;
+}
 
 async function runFeatureTests(args: FeaturesTestCommandInput, testResults: TestResult[] = []): Promise<TestResult[]> {
-	const { baseImage, collectionFolder, remoteUser, cliHost, skipAutogenerated, skipScenarios } = args;
+	const { baseImage, collectionFolder, remoteUser, cliHost, skipAutogenerated, skipScenarios, skipDuplicateTest } = args;
 	let { features } = args;
 
 	const testsDir = `${collectionFolder}/test`;
@@ -131,7 +229,7 @@ async function runFeatureTests(args: FeaturesTestCommandInput, testResults: Test
 	let workspaceFolder: string | undefined = undefined;
 	let params: DockerResolverParameters | undefined = undefined;
 	if (!skipAutogenerated) {
-		// 1. Generate temporary project with 'baseImage' and all the 'features..'
+		// Generate temporary project with 'baseImage' and all the 'features..'
 		workspaceFolder = await generateDefaultProjectFromFeatures(
 			cliHost,
 			baseImage,
@@ -146,8 +244,8 @@ async function runFeatureTests(args: FeaturesTestCommandInput, testResults: Test
 
 	log('Starting test(s)...\n', { prefix: '\n🏃', info: true });
 
-	// 3. Exec default 'test.sh' script for each feature, in the provided order.
-	//    Also exec a test's test scenarios, if a scenarios.json is present in the feature's test folder.
+	// Exec default 'test.sh' script for each feature, in the provided order.
+	// Also exec a test's test scenarios, if a scenarios.json is present in the feature's test folder.
 	for (const feature of features) {
 		log(`Starting '${feature}' tests...`, { prefix: '🧪' });
 		const featureTestFolder = path.join(collectionFolder, 'test', feature);
@@ -167,6 +265,11 @@ async function runFeatureTests(args: FeaturesTestCommandInput, testResults: Test
 			await doScenario(featureTestFolder, feature, args, testResults);
 		}
 
+		if (!skipDuplicateTest) {
+			log(`Executing duplicate test for feature '${feature}'...`, { prefix: '🧪' });
+			await runDuplicateTest(args, feature, testResults);
+		}
+
 		if (!testResults) {
 			fail(`Failed to run tests`);
 			return []; // We never reach here, we exit via fail().
@@ -357,7 +460,8 @@ async function generateProjectFromScenario(
 	collectionsDirectory: string,
 	scenarioId: string,
 	scenarioObject: DevContainerConfig,
-	targetFeatureOrGlobal: string
+	targetFeatureOrGlobal: string | undefined,
+	additionalFeatures: { featureId: string; featureValue: {} }[] = []
 ): Promise<string> {
 	const tmpFolder = await createTempDevcontainerFolder(cliHost);
 
@@ -385,15 +489,32 @@ async function generateProjectFromScenario(
 		// Reference Feature in the devcontainer.json
 		updatedFeatures[`./${featureId}`] = featureValue;
 	}
+
+	let counter = 0;
+	for (const { featureId, featureValue } of additionalFeatures) {
+		const pathToFeatureSource = `${collectionsDirectory}/src/${featureId}`;
+
+		const orderedFeatureId = `${featureId}-${counter++}`;
+		const destPath = `${tmpFolder}/.devcontainer/${orderedFeatureId}`;
+		await cpDirectoryLocal(pathToFeatureSource, destPath);
+
+		// Reference Feature in the devcontainer.json
+		updatedFeatures[`./${orderedFeatureId}`] = featureValue;
+	}
+
 	scenarioObject.features = updatedFeatures;
 
+	log(`Scenario generated: ${JSON.stringify(scenarioObject, null, 2)}`, { prefix: '\n📝', info: true });
+
 	await cliHost.writeFile(`${tmpFolder}/.devcontainer/devcontainer.json`, Buffer.from(JSON.stringify(scenarioObject)));
 
 	// If the current scenario has a corresponding additional config folder, copy it into the $TMP/.devcontainer directory
 	// This lets the scenario use things like Dockerfiles, shell scripts, etc. in the build.
-	const localPathToAdditionalConfigFolder = `${collectionsDirectory}/test/${targetFeatureOrGlobal}/${scenarioId}`;
-	if (await cliHost.isFolder(localPathToAdditionalConfigFolder)) {
-		await cpDirectoryLocal(localPathToAdditionalConfigFolder, `${tmpFolder}/.devcontainer`);
+	if (targetFeatureOrGlobal) {
+		const localPathToAdditionalConfigFolder = `${collectionsDirectory}/test/${targetFeatureOrGlobal}/${scenarioId}`;
+		if (await cliHost.isFolder(localPathToAdditionalConfigFolder)) {
+			await cpDirectoryLocal(localPathToAdditionalConfigFolder, `${tmpFolder}/.devcontainer`);
+		}
 	}
 
 	// Update permissions on the copied files to make them readable/writable/executable by everyone
@@ -451,19 +572,24 @@ async function launchProject(params: DockerResolverParameters, workspaceFolder:
 	}
 }
 
-async function execTest(testFileName: string, workspaceFolder: string, cliHost: CLIHost) {
+async function execTest(testFileName: string, workspaceFolder: string, cliHost: CLIHost, injectedEnv: { [varName: string]: string | boolean } = {}) {
 	// Ensure all the tests scripts in the workspace folder are executable
 	// Update permissions on the copied files to make them readable/writable/executable by everyone
 	await cliHost.exec({ cmd: 'chmod', args: ['-R', '777', workspaceFolder], output: nullLog });
 
 	const cmd = `./${testFileName}`;
 	const args: string[] = [];
-	return await exec(cmd, args, workspaceFolder);
+	return await exec(cmd, args, workspaceFolder, injectedEnv);
 }
 
-async function exec(cmd: string, args: string[], workspaceFolder: string) {
+async function exec(cmd: string, args: string[], workspaceFolder: string, injectedEnv: { [name: string]: string | boolean } = {}) {
+	const injectedEnvArray = Object.keys(injectedEnv).length > 0
+		? Object.entries(injectedEnv).map(([key, value]) => `${getSafeId(key)}=${value}`)
+		: undefined;
+
 	const execArgs = {
 		...staticExecParams,
+		'remote-env': injectedEnvArray as any,
 		'workspace-folder': workspaceFolder,
 		'skip-feature-auto-mapping': false,
 		cmd,

src/spec-node/featuresCLI/utils.ts

@@ -37,7 +37,6 @@ export const staticExecParams = {
     'override-config': undefined,
     'terminal-rows': undefined,
     'terminal-columns': undefined,
-    'remote-env': undefined,
     'container-id': undefined,
     'mount-workspace-git-root': true,
     'log-level': 'info' as 'info',

src/test/container-features/example-v2-features-sets/simple/src/color/install.sh

@@ -10,4 +10,7 @@ cat > /usr/local/bin/color \
 echo "my favorite color is ${FAVORITE}"
 EOF
 
-chmod +x /usr/local/bin/color
+chmod +x /usr/local/bin/color
+
+cp /usr/local/bin/color /usr/local/bin/color-${FAVORITE}
+chmod +x /usr/local/bin/color-${FAVORITE}

src/test/container-features/example-v2-features-sets/simple/test/color/duplicate.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -e
+
+env
+
+# Optional: Import test library
+source dev-container-features-test-lib
+
+# The values of the randomized options will be set as environment variables.
+if [ -z "${FAVORITE}" ]; then
+	echo "Favorite color from randomized Feature not set!"
+	exit 1
+fi
+
+# The values of the default options will be set as environment variables.
+if [ -z "${FAVORITE__DEFAULT}" ]; then
+	echo "Favorite color from default Feature not set!"
+	exit 1
+fi
+
+# Definition specific tests
+check "runColorCmd" color
+
+# Definition test specific to what option was set.
+check "Feature with randomized options installed correctly" color-"${FAVORITE}"
+
+# Definition test specific to what option was set.
+check "Feature with default options installed correctly" color-"${FAVORITE__DEFAULT}"
+
+# Report result
+reportResults

src/test/container-features/featuresCLICommands.test.ts

@@ -46,6 +46,7 @@ describe('CLI features subcommands', async function () {
 			const expectedTestReport = `  ================== TEST REPORT ==================
 ✅ Passed:      'color'
 ✅ Passed:      'specific_color_scenario'
+✅ Passed:      'color executed twice with randomized options'
 ✅ Passed:      'hello'
 ✅ Passed:      'custom_options'
 ✅ Passed:      'with_external_feature'`;
@@ -152,6 +153,7 @@ describe('CLI features subcommands', async function () {
 			const expectedTestReport = `  ================== TEST REPORT ==================
 ✅ Passed:      'color'
 ✅ Passed:      'specific_color_scenario'
+✅ Passed:      'color executed twice with randomized options'
 ✅ Passed:      'hello'
 ✅ Passed:      'custom_options'
 ✅ Passed:      'with_external_feature'`;
@@ -165,12 +167,12 @@ describe('CLI features subcommands', async function () {
 			assert.isTrue(result.stdout.includes('Ciao, vscode?????'));
 		});
 
-		it('succeeds --skip-autogenerated and subset of features', async function () {
+		it('succeeds --skip-autogenerated and subset of features and --skip-duplicate-test', async function () {
 			const collectionFolder = `${__dirname}/example-v2-features-sets/simple`;
 			let success = false;
 			let result: ExecResult | undefined = undefined;
 			try {
-				result = await shellExec(`${cli} features test -f color --skip-autogenerated --log-level trace ${collectionFolder}`);
+				result = await shellExec(`${cli} features test -f color --skip-autogenerated --skip-duplicate-test --log-level trace ${collectionFolder}`);
 				success = true;
 
 			} catch (error) {