Add CLI DevTunnel sanity test

test/sanity/README.md

@@ -16,7 +16,7 @@ Use -g or -f command-line options to filter tests to match the host platform.
 
 ### Command-Line Options
 
-| Option | Alias | Description |
+|Option|Alias|Description|
 |--------|-------|-------------|
 |`--commit <commit>`|`-c`|The commit to test (required)|
 |`--quality <quality>`|`-q`|The quality to test (required, "stable", "insider" or "exploration")|
@@ -43,7 +43,7 @@ npm run sanity-test -- --commit 19228f26df517fecbfda96c20956f7c521e072be --quali
 
 Platform-specific scripts are provided in the `scripts/` directory to set up the environment and run tests:
 
-| Script | Platform | Description |
+|Script|Platform|Description|
 |--------|----------|-------------|
 |`run-win32.cmd`|Windows|Runs tests using Edge as the Playwright browser|
 |`run-macOS.sh`|macOS|Installs Playwright WebKit and runs tests|
@@ -55,7 +55,7 @@ Platform-specific scripts are provided in the `scripts/` directory to set up the
 
 The `run-docker.sh` script accepts the following options:
 
-| Option | Description |
+|Option|Description|
 |--------|-------------|
 |`--container <name>`|Container dockerfile name (required, e.g., "ubuntu", "alpine")|
 |`--arch <arch>`|Target architecture: amd64, arm64, or arm (default: amd64)|
@@ -67,7 +67,7 @@ All other arguments are passed through to the sanity test runner.
 
 Docker container definitions are provided in the `containers/` directory for testing on various Linux distributions:
 
-| Container | Base Image | Description |
+|Container|Base Image|Description|
 |-----------|------------|-------------|
 |`alpine`|Alpine 3.x|Alpine Linux with musl libc|
 |`centos`|CentOS Stream 9|RHEL-compatible distribution|
@@ -103,7 +103,7 @@ Sanity tests run in Azure Pipelines via the `product-sanity-tests.yml` pipeline.
 
 ### Pipeline Parameters
 
-| Parameter | Description |
+|Parameter|Description|
 |-----------|-------------|
 |`buildQuality`|The quality of the build to test: "exploration", "insider", or "stable"|
 |`buildCommit`|The published build commit SHA|

test/sanity/src/cli.test.ts

@@ -4,8 +4,10 @@
  *--------------------------------------------------------------------------------------------*/
 
 import assert from 'assert';
-import { spawn } from 'child_process';
+import { Browser } from 'playwright';
 import { TestContext } from './context.js';
+import { GitHubAuth } from './githubAuth.js';
+import { UITest } from './uiTest.js';
 
 export function setup(context: TestContext) {
 	context.test('cli-alpine-arm64', ['alpine', 'arm64'], async () => {
@@ -72,45 +74,77 @@ export function setup(context: TestContext) {
 		}
 
 		const result = context.runNoErrors(entryPoint, '--version');
-		const version = result.stdout.trim();
-		assert.ok(version.includes(`(commit ${context.options.commit})`), `Expected CLI version to include commit ${context.options.commit}, got: ${version}`);
-
-		const workspaceDir = context.createTempDir();
-		process.chdir(workspaceDir);
-		context.log(`Changed current directory to: ${workspaceDir}`);
-
-		const args = [
-			'--cli-data-dir', context.createTempDir(),
-			'--user-data-dir', context.createTempDir(),
-			'tunnel',
-			'--accept-server-license-terms',
-			'--server-data-dir', context.createTempDir(),
-			'--extensions-dir', context.createTempDir(),
-		];
-
-		context.log(`Running CLI ${entryPoint} with args ${args.join(' ')}`);
-		const cli = spawn(entryPoint, args, { detached: true });
-
-		cli.stderr.on('data', (data) => {
-			context.error(`[CLI Error] ${data.toString().trim()}`);
-		});
-
-		cli.stdout.on('data', (data) => {
-			const text = data.toString().trim();
-			text.split('\n').forEach((line: string) => {
-				context.log(`[CLI Output] ${line}`);
-			});
-
-			const match = /Using GitHub for authentication/.exec(text);
-			if (match !== null) {
-				context.log(`CLI started successfully and is waiting for authentication`);
-				context.killProcessTree(cli.pid!);
-			}
-		});
+		const version = result.stdout.trim().match(/\(commit ([a-f0-9]+)\)/)?.[1];
+		assert.strictEqual(version, context.options.commit, `Expected commit ${context.options.commit} but got ${version}`);
 
-		await new Promise<void>((resolve, reject) => {
-			cli.on('error', reject);
-			cli.on('exit', resolve);
-		});
+		if (!context.capabilities.has('github-account')) {
+			return;
+		}
+
+		const cliDataDir = context.createTempDir();
+		const test = new UITest(context);
+		const auth = new GitHubAuth(context);
+		let browser: Browser | undefined;
+
+		context.log('Logging out of Dev Tunnel to ensure fresh authentication');
+		context.run(entryPoint, '--cli-data-dir', cliDataDir, 'tunnel', 'user', 'logout');
+
+		context.log('Starting Dev Tunnel to local server using CLI');
+		await context.runCliApp('CLI', entryPoint,
+			[
+				'--cli-data-dir', cliDataDir,
+				'tunnel',
+				'--accept-server-license-terms',
+				'--server-data-dir', context.createTempDir(),
+				'--extensions-dir', test.extensionsDir,
+				'--verbose'
+			],
+			async (line) => {
+				const deviceCode = /To grant access .* use code ([A-Z0-9-]+)/.exec(line)?.[1];
+				if (deviceCode) {
+					context.log(`Device code detected: ${deviceCode}, starting device flow authentication`);
+					browser = await context.launchBrowser();
+					await auth.runDeviceCodeFlow(browser, deviceCode);
+					return;
+				}
+
+				const tunnelUrl = /Open this link in your browser (https?:\/\/[^\s]+)/.exec(line)?.[1];
+				if (tunnelUrl) {
+					const tunnelId = new URL(tunnelUrl).pathname.split('/').pop()!;
+					const url = context.getTunnelUrl(tunnelUrl, test.workspaceDir);
+					context.log(`CLI started successfully with tunnel URL: ${url}`);
+
+					if (!browser) {
+						throw new Error('Browser instance is not available');
+					}
+
+					context.log(`Navigating to ${url}`);
+					const page = await context.getPage(browser.newPage());
+					await page.goto(url);
+
+					context.log('Waiting for the workbench to load');
+					await page.waitForSelector('.monaco-workbench');
+
+					context.log('Selecting GitHub Account');
+					await page.locator('span.monaco-highlighted-label', { hasText: 'GitHub' }).click();
+
+					context.log('Clicking Allow on confirmation dialog');
+					await page.getByRole('button', { name: 'Allow' }).click();
+
+					await auth.runUserWebFlow(page);
+
+					context.log('Waiting for connection to be established');
+					await page.getByRole('button', { name: `remote ${tunnelId}` }).waitFor({ timeout: 5 * 60 * 1000 });
+
+					await test.run(page);
+
+					context.log('Closing browser');
+					await browser.close();
+
+					test.validate();
+					return true;
+				}
+			}
+		);
 	}
 }

test/sanity/src/context.ts

@@ -3,7 +3,7 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { spawnSync, SpawnSyncReturns } from 'child_process';
+import { spawn, spawnSync, SpawnSyncReturns } from 'child_process';
 import { createHash } from 'crypto';
 import fs from 'fs';
 import { test } from 'mocha';
@@ -33,6 +33,7 @@ interface ITargetMetadata {
 export class TestContext {
 	private static readonly authenticodeInclude = /^.+\.(exe|dll|sys|cab|cat|msi|jar|ocx|ps1|psm1|psd1|ps1xml|pssc1)$/i;
 	private static readonly codesignExclude = /node_modules\/(@parcel\/watcher\/build\/Release\/watcher\.node|@vscode\/deviceid\/build\/Release\/windows\.node|@vscode\/ripgrep\/bin\/rg|@vscode\/spdlog\/build\/Release\/spdlog.node|kerberos\/build\/Release\/kerberos.node|@vscode\/native-watchdog\/build\/Release\/watchdog\.node|node-pty\/build\/Release\/(pty\.node|spawn-helper)|vsda\/build\/Release\/vsda\.node|native-watchdog\/build\/Release\/watchdog\.node)$/;
+	private static readonly notarizeExclude = /extensions\/microsoft-authentication\/dist\/libmsalruntime\.dylib$/;
 
 	private readonly tempDirs = new Set<string>();
 	private readonly wslTempDirs = new Set<string>();
@@ -388,14 +389,27 @@ export class TestContext {
 
 		this.log(`Validating codesign signature for ${filePath}`);
 
-		const result = this.run('codesign', '--verify', '--deep', '--strict', '--verbose', filePath);
+		const result = this.run('codesign', '--verify', '--deep', '--strict', '--verbose=2', filePath);
 		if (result.error !== undefined) {
 			this.error(`Failed to run codesign: ${result.error.message}`);
 		}
 
 		if (result.status !== 0) {
 			this.error(`Codesign signature is not valid for ${filePath}: ${result.stderr}`);
 		}
+
+		if (!TestContext.notarizeExclude.test(filePath)) {
+			this.log(`Validating notarization for ${filePath}`);
+
+			const notaryResult = this.run('spctl', '--assess', '--type', 'open', '--context', 'context:primary-signature', '--verbose=2', filePath);
+			if (notaryResult.error !== undefined) {
+				this.error(`Failed to run spctl: ${notaryResult.error.message}`);
+			}
+
+			if (notaryResult.status !== 0) {
+				this.error(`Notarization is not valid for ${filePath}: ${notaryResult.stderr}`);
+			}
+		}
 	}
 
 	/**
@@ -915,17 +929,6 @@ export class TestContext {
 		return dataDir;
 	}
 
-	/**
-	 * Returns the tunnel URL for the VS Code server including vscode-version parameter.
-	 * @param baseUrl The base URL for the VS Code server.
-	 * @returns The tunnel URL with vscode-version parameter.
-	 */
-	public getTunnelUrl(baseUrl: string): string {
-		const url = new URL(baseUrl);
-		url.searchParams.set('vscode-version', this.options.commit);
-		return url.toString();
-	}
-
 	/**
 	 * Launches a web browser for UI testing.
 	 * @returns The launched Browser instance.
@@ -993,6 +996,25 @@ export class TestContext {
 		return url;
 	}
 
+	/**
+	 * Returns the tunnel URL for the VS Code server.
+	 * @param baseUrl The base URL for *vscode.dev/tunnel connection.
+	 * @param workspaceDir Optional folder path to open
+	 * @returns The tunnel URL with folder in pathname.
+	 */
+	public getTunnelUrl(baseUrl: string, workspaceDir?: string): string {
+		const url = new URL(baseUrl);
+		url.searchParams.set('vscode-version', this.options.commit);
+		if (workspaceDir) {
+			let folder = workspaceDir.replaceAll('\\', '/');
+			if (!folder.startsWith('/')) {
+				folder = `/${folder}`;
+			}
+			url.pathname = url.pathname.replace(/\/+$/, '') + folder;
+		}
+		return url.toString();
+	}
+
 	/**
 	 * Returns a random alphanumeric token of length 10.
 	 */
@@ -1026,4 +1048,63 @@ export class TestContext {
 		}
 		return `~/${serverDir}/extensions`;
 	}
+
+	/**
+	 * Runs a VS Code command-line application (such as server or CLI).
+	 * @param name The name of the app as it will appear in logs.
+	 * @param command Command to run.
+	 * @param args Arguments for the command.
+	 * @param onLine Callback to handle output lines.
+	 */
+	public async runCliApp(name: string, command: string, args: string[], onLine: (text: string) => Promise<boolean | void | undefined>) {
+		this.log(`Starting ${name} with command line: ${command} ${args.join(' ')}`);
+
+		const app = spawn(command, args, {
+			shell: /\.(sh|cmd)$/.test(command),
+			detached: !this.capabilities.has('windows'),
+			stdio: ['ignore', 'pipe', 'pipe']
+		});
+
+		try {
+			await new Promise<void>((resolve, reject) => {
+				app.stderr.on('data', (data) => {
+					const text = `[${name}] ${data.toString().trim()}`;
+					if (/ECONNRESET/.test(text)) {
+						this.log(text);
+					} else {
+						reject(new Error(text));
+					}
+				});
+
+				let terminated = false;
+				app.stdout.on('data', (data) => {
+					const text = data.toString().trim();
+					if (/\berror\b/.test(text)) {
+						reject(new Error(`[${name}] ${text}`));
+					}
+
+					for (const line of text.split('\n')) {
+						this.log(`[${name}] ${line}`);
+						onLine(line).then((result) => {
+							if (terminated = !!result) {
+								this.log(`Terminating ${name} process`);
+								resolve();
+							}
+						}).catch(reject);
+					}
+				});
+
+				app.on('error', reject);
+				app.on('exit', (code) => {
+					if (code === 0) {
+						resolve();
+					} else if (!terminated) {
+						reject(new Error(`[${name}] Exited with code ${code}`));
+					}
+				});
+			});
+		} finally {
+			this.killProcessTree(app.pid!);
+		}
+	}
 }

test/sanity/src/desktop.test.ts

@@ -38,34 +38,37 @@ export function setup(context: TestContext) {
 
 	context.test('desktop-darwin-x64-dmg', ['darwin', 'x64', 'desktop'], async () => {
 		const packagePath = await context.downloadTarget('darwin-x64-dmg');
+		context.validateCodesignSignature(packagePath);
 		if (!context.options.downloadOnly) {
-			const mountPoint = context.mountDmg(packagePath);
-			context.validateAllCodesignSignatures(mountPoint);
-			const entryPoint = context.getDesktopEntryPoint(mountPoint);
+			const dir = context.mountDmg(packagePath);
+			context.validateAllCodesignSignatures(dir);
+			const entryPoint = context.getDesktopEntryPoint(dir);
 			await testDesktopApp(entryPoint);
-			context.unmountDmg(mountPoint);
+			context.unmountDmg(dir);
 		}
 	});
 
 	context.test('desktop-darwin-arm64-dmg', ['darwin', 'arm64', 'desktop'], async () => {
 		const packagePath = await context.downloadTarget('darwin-arm64-dmg');
+		context.validateCodesignSignature(packagePath);
 		if (!context.options.downloadOnly) {
-			const mountPoint = context.mountDmg(packagePath);
-			context.validateAllCodesignSignatures(mountPoint);
-			const entryPoint = context.getDesktopEntryPoint(mountPoint);
+			const dir = context.mountDmg(packagePath);
+			context.validateAllCodesignSignatures(dir);
+			const entryPoint = context.getDesktopEntryPoint(dir);
 			await testDesktopApp(entryPoint);
-			context.unmountDmg(mountPoint);
+			context.unmountDmg(dir);
 		}
 	});
 
 	context.test('desktop-darwin-universal-dmg', ['darwin', 'desktop'], async () => {
 		const packagePath = await context.downloadTarget('darwin-universal-dmg');
+		context.validateCodesignSignature(packagePath);
 		if (!context.options.downloadOnly) {
-			const mountPoint = context.mountDmg(packagePath);
-			context.validateAllCodesignSignatures(mountPoint);
-			const entryPoint = context.getDesktopEntryPoint(mountPoint);
+			const dir = context.mountDmg(packagePath);
+			context.validateAllCodesignSignatures(dir);
+			const entryPoint = context.getDesktopEntryPoint(dir);
 			await testDesktopApp(entryPoint);
-			context.unmountDmg(mountPoint);
+			context.unmountDmg(dir);
 		}
 	});
 
@@ -234,7 +237,6 @@ export function setup(context: TestContext) {
 		];
 		args.push(test.workspaceDir);
 
-
 		context.log(`Starting VS Code ${entryPoint} with args ${args.join(' ')}`);
 		const app = await _electron.launch({ executablePath: entryPoint, args });
 		const window = await context.getPage(app.firstWindow());