From 77cf8884d5ad41466dedca53994dc1db324d0e07 Mon Sep 17 00:00:00 2001 From: Dmitriy Vasyura Date: Fri, 27 Mar 2026 13:24:14 -0700 Subject: [PATCH 01/21] Enable CLI DevTunnel sanity tests --- build/azure-pipelines/common/sanity-tests.yml | 19 +++++++++++++++++++ test/sanity/scripts/run-docker.sh | 2 ++ 2 files changed, 21 insertions(+) diff --git a/build/azure-pipelines/common/sanity-tests.yml b/build/azure-pipelines/common/sanity-tests.yml index fdf6b2cd3dd17..7778d30a58c4b 100644 --- a/build/azure-pipelines/common/sanity-tests.yml +++ b/build/azure-pipelines/common/sanity-tests.yml @@ -117,23 +117,39 @@ jobs: workingDirectory: $(TEST_DIR) displayName: Compile Sanity Tests + - task: AzureKeyVault@2 + displayName: "Azure Key Vault: Get Secrets" + inputs: + azureSubscription: vscode + KeyVaultName: vscode-build-secrets + SecretsFilter: "sanity-tests-account,sanity-tests-password" + # Windows - ${{ if eq(parameters.os, 'windows') }}: - script: $(TEST_DIR)/scripts/run-win32.cmd -c $(BUILD_COMMIT) -q $(BUILD_QUALITY) -t $(LOG_FILE) -s $(SCREENSHOTS_DIR) -v ${{ parameters.args }} workingDirectory: $(TEST_DIR) displayName: Run Sanity Tests + env: + GITHUB_ACCOUNT: $(sanity-tests-account) + GITHUB_PASSWORD: $(sanity-tests-password) # macOS - ${{ if eq(parameters.os, 'macOS') }}: - bash: $(TEST_DIR)/scripts/run-macOS.sh -c $(BUILD_COMMIT) -q $(BUILD_QUALITY) -t $(LOG_FILE) -s $(SCREENSHOTS_DIR) -v ${{ parameters.args }} workingDirectory: $(TEST_DIR) displayName: Run Sanity Tests + env: + GITHUB_ACCOUNT: $(sanity-tests-account) + GITHUB_PASSWORD: $(sanity-tests-password) # Native Linux host - ${{ if and(eq(parameters.container, ''), eq(parameters.os, 'linux')) }}: - bash: $(TEST_DIR)/scripts/run-ubuntu.sh -c $(BUILD_COMMIT) -q $(BUILD_QUALITY) -t $(LOG_FILE) -s $(SCREENSHOTS_DIR) -v ${{ parameters.args }} workingDirectory: $(TEST_DIR) displayName: Run Sanity Tests + env: + GITHUB_ACCOUNT: $(sanity-tests-account) + GITHUB_PASSWORD: $(sanity-tests-password) # Linux Docker container - ${{ if ne(parameters.container, '') }}: @@ -164,6 +180,9 @@ jobs: ${{ parameters.args }} workingDirectory: $(TEST_DIR) displayName: Run Sanity Tests + env: + GITHUB_ACCOUNT: $(sanity-tests-account) + GITHUB_PASSWORD: $(sanity-tests-password) - bash: | mkdir -p "$(DOCKER_CACHE_DIR)" diff --git a/test/sanity/scripts/run-docker.sh b/test/sanity/scripts/run-docker.sh index 8b3da44b1f701..51d4f6921323b 100755 --- a/test/sanity/scripts/run-docker.sh +++ b/test/sanity/scripts/run-docker.sh @@ -43,6 +43,8 @@ docker run \ --rm \ --platform "linux/$ARCH" \ --volume "$ROOT_DIR:/root" \ + ${GITHUB_ACCOUNT:+--env GITHUB_ACCOUNT="$GITHUB_ACCOUNT"} \ + ${GITHUB_PASSWORD:+--env GITHUB_PASSWORD="$GITHUB_PASSWORD"} \ --entrypoint sh \ "$CONTAINER" \ /root/containers/entrypoint.sh $ARGS From 29d1729246d13ff47aae0327c1d987917a34fa35 Mon Sep 17 00:00:00 2001 From: Dmitriy Vasyura Date: Fri, 27 Mar 2026 14:26:29 -0700 Subject: [PATCH 02/21] Refactor dev tunnel tests into separate file. --- test/sanity/src/cli.test.ts | 75 --------------- test/sanity/src/devTunnel.test.ts | 146 ++++++++++++++++++++++++++++++ test/sanity/src/main.ts | 2 + 3 files changed, 148 insertions(+), 75 deletions(-) create mode 100644 test/sanity/src/devTunnel.test.ts diff --git a/test/sanity/src/cli.test.ts b/test/sanity/src/cli.test.ts index 8732acde0b7ac..52e1672d54ee4 100644 --- a/test/sanity/src/cli.test.ts +++ b/test/sanity/src/cli.test.ts @@ -4,10 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import assert from 'assert'; -import { Browser, Page } from 'playwright'; import { TestContext } from './context.js'; -import { GitHubAuth } from './githubAuth.js'; -import { UITest } from './uiTest.js'; export function setup(context: TestContext) { context.test('cli-alpine-arm64', ['alpine', 'arm64'], async () => { @@ -78,77 +75,5 @@ export function setup(context: TestContext) { const result = context.runNoErrors(entryPoint, '--version'); const version = result.stdout.trim().match(/\(commit ([a-f0-9]+)\)/)?.[1]; assert.strictEqual(version, context.options.commit, `Expected commit ${context.options.commit} but got ${version}`); - - if (!context.capabilities.has('github-account')) { - return; - } - - const cliDataDir = context.createTempDir(); - const test = new UITest(context); - const auth = new GitHubAuth(context); - let browser: Browser | undefined; - let page: Page | undefined; - - context.log('Logging out of Dev Tunnel to ensure fresh authentication'); - context.run(entryPoint, '--cli-data-dir', cliDataDir, 'tunnel', 'user', 'logout'); - - context.log('Starting Dev Tunnel to local server using CLI'); - await context.runCliApp('CLI', entryPoint, - [ - '--cli-data-dir', cliDataDir, - 'tunnel', - '--accept-server-license-terms', - '--server-data-dir', context.createTempDir(), - '--extensions-dir', test.extensionsDir, - '--verbose' - ], - async (line) => { - const deviceCode = /To grant access .* use code ([A-Z0-9-]+)/.exec(line)?.[1]; - if (deviceCode) { - context.log(`Device code detected: ${deviceCode}, starting device flow authentication`); - browser = await context.launchBrowser(); - page = await context.getPage(browser.newPage()); - await auth.runDeviceCodeFlow(page, deviceCode); - return; - } - - const tunnelUrl = /Open this link in your browser (https?:\/\/[^\s]+)/.exec(line)?.[1]; - if (tunnelUrl) { - const tunnelId = new URL(tunnelUrl).pathname.split('/').pop()!; - const url = context.getTunnelUrl(tunnelUrl, test.workspaceDir); - context.log(`CLI started successfully with tunnel URL: ${url}`); - - if (!browser || !page) { - throw new Error('Browser instance is not available'); - } - - context.log(`Navigating to ${url}`); - await page.goto(url); - - context.log('Waiting for the workbench to load'); - await page.waitForSelector('.monaco-workbench'); - - context.log('Selecting GitHub Account'); - await page.locator('span.monaco-highlighted-label', { hasText: 'GitHub' }).click(); - - context.log('Clicking Allow on confirmation dialog'); - const popup = page.waitForEvent('popup'); - await page.getByRole('button', { name: 'Allow' }).click(); - - await auth.runAuthorizeFlow(await popup); - - context.log('Waiting for connection to be established'); - await page.getByRole('button', { name: `remote ${tunnelId}` }).waitFor({ timeout: 5 * 60 * 1000 }); - - await test.run(page); - - context.log('Closing browser'); - await browser.close(); - - test.validate(); - return true; - } - } - ); } } diff --git a/test/sanity/src/devTunnel.test.ts b/test/sanity/src/devTunnel.test.ts new file mode 100644 index 0000000000000..d2737df6badd2 --- /dev/null +++ b/test/sanity/src/devTunnel.test.ts @@ -0,0 +1,146 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Browser, Page } from 'playwright'; +import { TestContext } from './context.js'; +import { GitHubAuth } from './githubAuth.js'; +import { UITest } from './uiTest.js'; + +export function setup(context: TestContext) { + context.test('dev-tunnel-alpine-arm64', ['alpine', 'arm64', 'browser', 'github-account'], async () => { + const dir = await context.downloadAndUnpack('cli-alpine-arm64'); + const entryPoint = context.getCliEntryPoint(dir); + await testCliApp(entryPoint); + }); + + context.test('dev-tunnel-alpine-x64', ['alpine', 'x64', 'browser', 'github-account'], async () => { + const dir = await context.downloadAndUnpack('cli-alpine-x64'); + const entryPoint = context.getCliEntryPoint(dir); + await testCliApp(entryPoint); + }); + + context.test('dev-tunnel-darwin-arm64', ['darwin', 'arm64', 'browser', 'github-account'], async () => { + const dir = await context.downloadAndUnpack('cli-darwin-arm64'); + context.validateAllCodesignSignatures(dir); + const entryPoint = context.getCliEntryPoint(dir); + await testCliApp(entryPoint); + }); + + context.test('dev-tunnel-darwin-x64', ['darwin', 'x64', 'browser', 'github-account'], async () => { + const dir = await context.downloadAndUnpack('cli-darwin-x64'); + context.validateAllCodesignSignatures(dir); + const entryPoint = context.getCliEntryPoint(dir); + await testCliApp(entryPoint); + }); + + context.test('dev-tunnel-linux-arm64', ['linux', 'arm64', 'browser', 'github-account'], async () => { + const dir = await context.downloadAndUnpack('cli-linux-arm64'); + const entryPoint = context.getCliEntryPoint(dir); + await testCliApp(entryPoint); + }); + + context.test('dev-tunnel-linux-armhf', ['linux', 'arm32', 'browser', 'github-account'], async () => { + const dir = await context.downloadAndUnpack('cli-linux-armhf'); + const entryPoint = context.getCliEntryPoint(dir); + await testCliApp(entryPoint); + }); + + context.test('dev-tunnel-linux-x64', ['linux', 'x64', 'browser', 'github-account'], async () => { + const dir = await context.downloadAndUnpack('cli-linux-x64'); + const entryPoint = context.getCliEntryPoint(dir); + await testCliApp(entryPoint); + }); + + context.test('dev-tunnel-win32-arm64', ['windows', 'arm64', 'browser', 'github-account'], async () => { + const dir = await context.downloadAndUnpack('cli-win32-arm64'); + context.validateAllAuthenticodeSignatures(dir); + context.validateAllVersionInfo(dir); + const entryPoint = context.getCliEntryPoint(dir); + await testCliApp(entryPoint); + }); + + context.test('dev-tunnel-win32-x64', ['windows', 'x64', 'browser', 'github-account'], async () => { + const dir = await context.downloadAndUnpack('cli-win32-x64'); + context.validateAllAuthenticodeSignatures(dir); + context.validateAllVersionInfo(dir); + const entryPoint = context.getCliEntryPoint(dir); + await testCliApp(entryPoint); + }); + + async function testCliApp(entryPoint: string) { + const cliDataDir = context.createTempDir(); + const test = new UITest(context); + const auth = new GitHubAuth(context); + let browser: Browser | undefined; + let page: Page | undefined; + + context.log('Logging out of Dev Tunnel to ensure fresh authentication'); + context.run(entryPoint, '--cli-data-dir', cliDataDir, 'tunnel', 'user', 'logout'); + + context.log('Starting Dev Tunnel to local server using CLI'); + await context.runCliApp('CLI', entryPoint, + [ + '--cli-data-dir', cliDataDir, + 'tunnel', + '--accept-server-license-terms', + '--server-data-dir', context.createTempDir(), + '--extensions-dir', test.extensionsDir, + '--verbose' + ], + async (line) => { + const deviceCode = /To grant access .* use code ([A-Z0-9-]+)/.exec(line)?.[1]; + if (deviceCode) { + context.log(`Device code detected: ${deviceCode}, starting device flow authentication`); + browser = await context.launchBrowser(); + page = await context.getPage(browser.newPage()); + await auth.runDeviceCodeFlow(page, deviceCode); + return; + } + + const tunnelUrl = /Open this link in your browser (https?:\/\/[^\s]+)/.exec(line)?.[1]; + if (tunnelUrl) { + const tunnelId = new URL(tunnelUrl).pathname.split('/').pop()!; + const url = context.getTunnelUrl(tunnelUrl, test.workspaceDir); + context.log(`CLI started successfully with tunnel URL: ${url}`); + + if (!browser || !page) { + throw new Error('Browser instance is not available'); + } + + try { + context.log(`Navigating to ${url}`); + await page.goto(url); + + context.log('Waiting for the workbench to load'); + await page.waitForSelector('.monaco-workbench'); + + context.log('Selecting GitHub Account'); + await page.locator('span.monaco-highlighted-label', { hasText: 'GitHub' }).click(); + + context.log('Clicking Allow on confirmation dialog'); + const popup = page.waitForEvent('popup'); + await page.getByRole('button', { name: 'Allow' }).click(); + + await auth.runAuthorizeFlow(await popup); + + context.log('Waiting for connection to be established'); + await page.getByRole('button', { name: `remote ${tunnelId}` }).waitFor({ timeout: 5 * 60 * 1000 }); + } catch (error) { + await context.captureScreenshot(page); + throw error; + } + + await test.run(page); + + context.log('Closing browser'); + await browser.close(); + + test.validate(); + return true; + } + } + ); + } +} diff --git a/test/sanity/src/main.ts b/test/sanity/src/main.ts index 840364e68a52a..15b0c7dfda0d9 100644 --- a/test/sanity/src/main.ts +++ b/test/sanity/src/main.ts @@ -11,6 +11,7 @@ import { setup as setupDesktopTests } from './desktop.test.js'; import { setup as setupServerTests } from './server.test.js'; import { setup as setupServerWebTests } from './serverWeb.test.js'; import { setup as setupWSLTests } from './wsl.test.js'; +import { setup as setupDevTunnelTests } from './devTunnel.test.js'; const options = minimist(process.argv.slice(2), { string: ['commit', 'quality', 'screenshots-dir'], @@ -52,3 +53,4 @@ setupDesktopTests(context); setupServerTests(context); setupServerWebTests(context); setupWSLTests(context); +setupDevTunnelTests(context); From 3f363fe35c73b56f373856242079961f85da3c21 Mon Sep 17 00:00:00 2001 From: Dmitriy Vasyura Date: Fri, 27 Mar 2026 14:28:42 -0700 Subject: [PATCH 03/21] Capture screenshot on WSL test failure --- test/sanity/src/wsl.test.ts | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/test/sanity/src/wsl.test.ts b/test/sanity/src/wsl.test.ts index 77df84d72758f..2108c0b08e671 100644 --- a/test/sanity/src/wsl.test.ts +++ b/test/sanity/src/wsl.test.ts @@ -157,11 +157,16 @@ export function setup(context: TestContext) { try { const window = await context.getPage(app.firstWindow()); - context.log('Installing WSL extension'); - await window.getByRole('button', { name: 'Install and Reload' }).click(); - - context.log('Waiting for WSL connection'); - await window.getByText(/WSL/).waitFor(); + try { + context.log('Installing WSL extension'); + await window.getByRole('button', { name: 'Install and Reload' }).click(); + + context.log('Waiting for WSL connection'); + await window.getByText(/WSL/).waitFor(); + } catch (error) { + await context.captureScreenshot(window); + throw error; + } await test.run(window); } finally { From dda6a1a99b5baec2bb182ebdcf7e8966d479e8b6 Mon Sep 17 00:00:00 2001 From: Dmitriy Vasyura Date: Fri, 27 Mar 2026 14:30:21 -0700 Subject: [PATCH 04/21] PR feedback --- test/sanity/scripts/run-docker.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/sanity/scripts/run-docker.sh b/test/sanity/scripts/run-docker.sh index 51d4f6921323b..b91f78197d8f8 100755 --- a/test/sanity/scripts/run-docker.sh +++ b/test/sanity/scripts/run-docker.sh @@ -43,8 +43,8 @@ docker run \ --rm \ --platform "linux/$ARCH" \ --volume "$ROOT_DIR:/root" \ - ${GITHUB_ACCOUNT:+--env GITHUB_ACCOUNT="$GITHUB_ACCOUNT"} \ - ${GITHUB_PASSWORD:+--env GITHUB_PASSWORD="$GITHUB_PASSWORD"} \ + ${GITHUB_ACCOUNT:+--env GITHUB_ACCOUNT} \ + ${GITHUB_PASSWORD:+--env GITHUB_PASSWORD} \ --entrypoint sh \ "$CONTAINER" \ /root/containers/entrypoint.sh $ARGS From 8d67d2f0e23635a2a1f2d026d1febf0aad712cfc Mon Sep 17 00:00:00 2001 From: Dmitriy Vasyura Date: Fri, 27 Mar 2026 14:36:47 -0700 Subject: [PATCH 05/21] Fix CG issues --- test/sanity/package-lock.json | 60 ++++++++++++++++++----------------- test/sanity/package.json | 2 +- 2 files changed, 32 insertions(+), 30 deletions(-) diff --git a/test/sanity/package-lock.json b/test/sanity/package-lock.json index c441eb7cf188c..a7958bb929779 100644 --- a/test/sanity/package-lock.json +++ b/test/sanity/package-lock.json @@ -10,7 +10,7 @@ "license": "MIT", "dependencies": { "minimist": "^1.2.8", - "mocha": "^11.7.5", + "mocha": "^11.3.0", "mocha-junit-reporter": "^2.2.1", "node-fetch": "^3.3.2", "playwright": "^1.57.0" @@ -113,9 +113,9 @@ "license": "MIT" }, "node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.3.tgz", + "integrity": "sha512-MCV/fYJEbqx68aE58kv2cA/kiky1G8vux3OR6/jbS+jIMe/6fJWa0DTzJU7dqijOWYwHi1t29FlfYI9uytqlpA==", "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" @@ -343,9 +343,9 @@ } }, "node_modules/diff": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/diff/-/diff-7.0.0.tgz", - "integrity": "sha512-PJWHUb1RFevKCwaFA9RlG5tCd+FO5iRh9A8HEtkmBH2Li03iJriB6m6JIN4rGz3K3JLawI7/veA1xzRKP6ISBw==", + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-5.2.2.tgz", + "integrity": "sha512-vtcDfH3TOjP8UekytvnHH1o1P4FcUdt4eQ1Y+Abap1tk/OB2MWQvcwS2ClCd1zuIhc3JKOx6p3kod8Vfys3E+A==", "license": "BSD-3-Clause", "engines": { "node": ">=0.3.1" @@ -536,15 +536,6 @@ "node": ">=8" } }, - "node_modules/is-path-inside": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz", - "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/is-plain-obj": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-2.1.0.tgz", @@ -696,29 +687,28 @@ } }, "node_modules/mocha": { - "version": "11.7.5", - "resolved": "https://registry.npmjs.org/mocha/-/mocha-11.7.5.tgz", - "integrity": "sha512-mTT6RgopEYABzXWFx+GcJ+ZQ32kp4fMf0xvpZIIfSq9Z8lC/++MtcCnQ9t5FP2veYEP95FIYSvW+U9fV4xrlig==", + "version": "11.3.0", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-11.3.0.tgz", + "integrity": "sha512-J0RLIM89xi8y6l77bgbX+03PeBRDQCOVQpnwOcCN7b8hCmbh6JvGI2ZDJ5WMoHz+IaPU+S4lvTd0j51GmBAdgQ==", "license": "MIT", "dependencies": { "browser-stdout": "^1.3.1", "chokidar": "^4.0.1", "debug": "^4.3.5", - "diff": "^7.0.0", + "diff": "^5.2.0", "escape-string-regexp": "^4.0.0", "find-up": "^5.0.0", "glob": "^10.4.5", "he": "^1.2.0", - "is-path-inside": "^3.0.3", "js-yaml": "^4.1.0", "log-symbols": "^4.1.0", - "minimatch": "^9.0.5", + "minimatch": "^5.1.6", "ms": "^2.1.3", "picocolors": "^1.1.1", "serialize-javascript": "^6.0.2", "strip-json-comments": "^3.1.1", "supports-color": "^8.1.1", - "workerpool": "^9.2.0", + "workerpool": "^6.5.1", "yargs": "^17.7.2", "yargs-parser": "^21.1.1", "yargs-unparser": "^2.0.0" @@ -768,6 +758,18 @@ "node": ">=8" } }, + "node_modules/mocha/node_modules/minimatch": { + "version": "5.1.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.9.tgz", + "integrity": "sha512-7o1wEA2RyMP7Iu7GNba9vc0RWWGACJOCZBJX2GJWip0ikV+wcOsgVuY9uE8CPiyQhkGFSlhuSkZPavN7u1c2Fw==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -941,9 +943,9 @@ } }, "node_modules/serialize-javascript": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-7.0.4.tgz", - "integrity": "sha512-DuGdB+Po43Q5Jxwpzt1lhyFSYKryqoNjQSA9M92tyw0lyHIOur+XCalOUe0KTJpyqzT8+fQ5A0Jf7vCx/NKmIg==", + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-7.0.5.tgz", + "integrity": "sha512-F4LcB0UqUl1zErq+1nYEEzSHJnIwb3AF2XWB94b+afhrekOUijwooAYqFyRbjYkm2PAKBabx6oYv/xDxNi8IBw==", "license": "BSD-3-Clause", "engines": { "node": ">=20.0.0" @@ -1151,9 +1153,9 @@ } }, "node_modules/workerpool": { - "version": "9.3.4", - "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-9.3.4.tgz", - "integrity": "sha512-TmPRQYYSAnnDiEB0P/Ytip7bFGvqnSU6I2BcuSw7Hx+JSg/DsUi5ebYfc8GYaSdpuvOcEs6dXxPurOYpe9QFwg==", + "version": "6.5.1", + "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.5.1.tgz", + "integrity": "sha512-Fs4dNYcsdpYSAfVxhnl1L5zTksjvOJxtC5hzMNl+1t9B8hTJTdKDyZ5ju7ztgPy+ft9tBFXoOlDNiOT9WUXZlA==", "license": "Apache-2.0" }, "node_modules/wrap-ansi": { diff --git a/test/sanity/package.json b/test/sanity/package.json index 93d586f98e0f1..d3760ac6c71ba 100644 --- a/test/sanity/package.json +++ b/test/sanity/package.json @@ -10,7 +10,7 @@ }, "dependencies": { "minimist": "^1.2.8", - "mocha": "^11.7.5", + "mocha": "^11.3.0", "mocha-junit-reporter": "^2.2.1", "node-fetch": "^3.3.2", "playwright": "^1.57.0" From 704584846ccc9ffb62e654714f0e3b4bbdc2d142 Mon Sep 17 00:00:00 2001 From: Dmitriy Vasyura Date: Fri, 27 Mar 2026 14:51:28 -0700 Subject: [PATCH 06/21] Add retries around auth flow. --- test/sanity/src/devTunnel.test.ts | 49 ++++++++++++++++++------------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/test/sanity/src/devTunnel.test.ts b/test/sanity/src/devTunnel.test.ts index d2737df6badd2..1b842a85a5d31 100644 --- a/test/sanity/src/devTunnel.test.ts +++ b/test/sanity/src/devTunnel.test.ts @@ -109,27 +109,34 @@ export function setup(context: TestContext) { throw new Error('Browser instance is not available'); } - try { - context.log(`Navigating to ${url}`); - await page.goto(url); - - context.log('Waiting for the workbench to load'); - await page.waitForSelector('.monaco-workbench'); - - context.log('Selecting GitHub Account'); - await page.locator('span.monaco-highlighted-label', { hasText: 'GitHub' }).click(); - - context.log('Clicking Allow on confirmation dialog'); - const popup = page.waitForEvent('popup'); - await page.getByRole('button', { name: 'Allow' }).click(); - - await auth.runAuthorizeFlow(await popup); - - context.log('Waiting for connection to be established'); - await page.getByRole('button', { name: `remote ${tunnelId}` }).waitFor({ timeout: 5 * 60 * 1000 }); - } catch (error) { - await context.captureScreenshot(page); - throw error; + const maxAttempts = 3; + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + context.log(`Navigating to ${url} (attempt ${attempt}/${maxAttempts})`); + await page.goto(url); + + context.log('Waiting for the workbench to load'); + await page.waitForSelector('.monaco-workbench'); + + context.log('Selecting GitHub Account'); + await page.locator('span.monaco-highlighted-label', { hasText: 'GitHub' }).click(); + + context.log('Clicking Allow on confirmation dialog'); + const popup = page.waitForEvent('popup'); + await page.getByRole('button', { name: 'Allow' }).click(); + + await auth.runAuthorizeFlow(await popup); + + context.log('Waiting for connection to be established'); + await page.getByRole('button', { name: `remote ${tunnelId}` }).waitFor({ timeout: 5 * 60 * 1000 }); + break; + } catch (error) { + await context.captureScreenshot(page); + if (attempt === maxAttempts) { + throw error; + } + context.log(`Auth flow attempt ${attempt} failed: ${error instanceof Error ? error.message : String(error)}, retrying...`); + } } await test.run(page); From c118122ea2976985157d78a4c364796fb2b459ae Mon Sep 17 00:00:00 2001 From: Dmitriy Vasyura Date: Fri, 27 Mar 2026 14:52:06 -0700 Subject: [PATCH 07/21] Fix --- test/sanity/src/devTunnel.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/sanity/src/devTunnel.test.ts b/test/sanity/src/devTunnel.test.ts index 1b842a85a5d31..52a3eb4bb8d56 100644 --- a/test/sanity/src/devTunnel.test.ts +++ b/test/sanity/src/devTunnel.test.ts @@ -131,8 +131,8 @@ export function setup(context: TestContext) { await page.getByRole('button', { name: `remote ${tunnelId}` }).waitFor({ timeout: 5 * 60 * 1000 }); break; } catch (error) { - await context.captureScreenshot(page); if (attempt === maxAttempts) { + await context.captureScreenshot(page); throw error; } context.log(`Auth flow attempt ${attempt} failed: ${error instanceof Error ? error.message : String(error)}, retrying...`); From 350263cf90d6fd6df5601c791315ef720a1ce6e1 Mon Sep 17 00:00:00 2001 From: Dmitriy Vasyura Date: Fri, 27 Mar 2026 14:59:01 -0700 Subject: [PATCH 08/21] Fixes --- test/sanity/src/devTunnel.test.ts | 131 +++++++++++++++--------------- 1 file changed, 64 insertions(+), 67 deletions(-) diff --git a/test/sanity/src/devTunnel.test.ts b/test/sanity/src/devTunnel.test.ts index 52a3eb4bb8d56..3102c5faeabc4 100644 --- a/test/sanity/src/devTunnel.test.ts +++ b/test/sanity/src/devTunnel.test.ts @@ -3,7 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { Browser, Page } from 'playwright'; +import { Page } from 'playwright'; import { TestContext } from './context.js'; import { GitHubAuth } from './githubAuth.js'; import { UITest } from './uiTest.js'; @@ -71,83 +71,80 @@ export function setup(context: TestContext) { async function testCliApp(entryPoint: string) { const cliDataDir = context.createTempDir(); - const test = new UITest(context); - const auth = new GitHubAuth(context); - let browser: Browser | undefined; - let page: Page | undefined; - context.log('Logging out of Dev Tunnel to ensure fresh authentication'); context.run(entryPoint, '--cli-data-dir', cliDataDir, 'tunnel', 'user', 'logout'); - context.log('Starting Dev Tunnel to local server using CLI'); - await context.runCliApp('CLI', entryPoint, - [ - '--cli-data-dir', cliDataDir, - 'tunnel', - '--accept-server-license-terms', - '--server-data-dir', context.createTempDir(), - '--extensions-dir', test.extensionsDir, - '--verbose' - ], - async (line) => { - const deviceCode = /To grant access .* use code ([A-Z0-9-]+)/.exec(line)?.[1]; - if (deviceCode) { - context.log(`Device code detected: ${deviceCode}, starting device flow authentication`); - browser = await context.launchBrowser(); - page = await context.getPage(browser.newPage()); - await auth.runDeviceCodeFlow(page, deviceCode); - return; + const test = new UITest(context); + const auth = new GitHubAuth(context); + const browser = await context.launchBrowser(); + try { + const page = await context.getPage(browser.newPage()); + context.log('Starting Dev Tunnel to local server using CLI'); + await context.runCliApp('CLI', entryPoint, + [ + '--cli-data-dir', cliDataDir, + 'tunnel', + '--accept-server-license-terms', + '--server-data-dir', context.createTempDir(), + '--extensions-dir', test.extensionsDir, + '--verbose' + ], + async (line) => { + const deviceCode = /To grant access .* use code ([A-Z0-9-]+)/.exec(line)?.[1]; + if (deviceCode) { + context.log(`Device code detected: ${deviceCode}, starting device flow authentication`); + await auth.runDeviceCodeFlow(page, deviceCode); + return; + } + + const tunnelUrl = /Open this link in your browser (https?:\/\/[^\s]+)/.exec(line)?.[1]; + if (tunnelUrl) { + await connectToTunnel(tunnelUrl, page, test, auth); + await test.run(page); + test.validate(); + return true; + } } + ); + } finally { + context.log('Closing browser'); + await browser.close(); + } + } - const tunnelUrl = /Open this link in your browser (https?:\/\/[^\s]+)/.exec(line)?.[1]; - if (tunnelUrl) { - const tunnelId = new URL(tunnelUrl).pathname.split('/').pop()!; - const url = context.getTunnelUrl(tunnelUrl, test.workspaceDir); - context.log(`CLI started successfully with tunnel URL: ${url}`); + async function connectToTunnel(tunnelUrl: string, page: Page, test: UITest, auth: GitHubAuth) { + const tunnelId = new URL(tunnelUrl).pathname.split('/').pop()!; + const url = context.getTunnelUrl(tunnelUrl, test.workspaceDir); + context.log(`CLI started successfully with tunnel URL: ${url}`); - if (!browser || !page) { - throw new Error('Browser instance is not available'); - } + const maxAttempts = 3; + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + context.log(`Navigating to ${url} (attempt ${attempt}/${maxAttempts})`); + await page.goto(url); - const maxAttempts = 3; - for (let attempt = 1; attempt <= maxAttempts; attempt++) { - try { - context.log(`Navigating to ${url} (attempt ${attempt}/${maxAttempts})`); - await page.goto(url); - - context.log('Waiting for the workbench to load'); - await page.waitForSelector('.monaco-workbench'); - - context.log('Selecting GitHub Account'); - await page.locator('span.monaco-highlighted-label', { hasText: 'GitHub' }).click(); - - context.log('Clicking Allow on confirmation dialog'); - const popup = page.waitForEvent('popup'); - await page.getByRole('button', { name: 'Allow' }).click(); - - await auth.runAuthorizeFlow(await popup); - - context.log('Waiting for connection to be established'); - await page.getByRole('button', { name: `remote ${tunnelId}` }).waitFor({ timeout: 5 * 60 * 1000 }); - break; - } catch (error) { - if (attempt === maxAttempts) { - await context.captureScreenshot(page); - throw error; - } - context.log(`Auth flow attempt ${attempt} failed: ${error instanceof Error ? error.message : String(error)}, retrying...`); - } - } + context.log('Waiting for the workbench to load'); + await page.waitForSelector('.monaco-workbench'); + + context.log('Selecting GitHub Account'); + await page.locator('span.monaco-highlighted-label', { hasText: 'GitHub' }).click(); - await test.run(page); + context.log('Clicking Allow on confirmation dialog'); + const popup = page.waitForEvent('popup'); + await page.getByRole('button', { name: 'Allow' }).click(); - context.log('Closing browser'); - await browser.close(); + await auth.runAuthorizeFlow(await popup); - test.validate(); - return true; + context.log('Waiting for connection to be established'); + await page.getByRole('button', { name: `remote ${tunnelId}` }).waitFor({ timeout: 5 * 60 * 1000 }); + return; + } catch (error) { + if (attempt === maxAttempts) { + await context.captureScreenshot(page); + throw error; } + context.log(`Auth flow attempt ${attempt} failed: ${error instanceof Error ? error.message : String(error)}, retrying...`); } - ); + } } } From a5c2050f9e9fb493057a2625311f85b456649abd Mon Sep 17 00:00:00 2001 From: Dmitriy Vasyura Date: Fri, 27 Mar 2026 14:59:29 -0700 Subject: [PATCH 09/21] Revert "Fix CG issues" This reverts commit 8d67d2f0e23635a2a1f2d026d1febf0aad712cfc. --- test/sanity/package-lock.json | 60 +++++++++++++++++------------------ test/sanity/package.json | 2 +- 2 files changed, 30 insertions(+), 32 deletions(-) diff --git a/test/sanity/package-lock.json b/test/sanity/package-lock.json index a7958bb929779..c441eb7cf188c 100644 --- a/test/sanity/package-lock.json +++ b/test/sanity/package-lock.json @@ -10,7 +10,7 @@ "license": "MIT", "dependencies": { "minimist": "^1.2.8", - "mocha": "^11.3.0", + "mocha": "^11.7.5", "mocha-junit-reporter": "^2.2.1", "node-fetch": "^3.3.2", "playwright": "^1.57.0" @@ -113,9 +113,9 @@ "license": "MIT" }, "node_modules/brace-expansion": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.3.tgz", - "integrity": "sha512-MCV/fYJEbqx68aE58kv2cA/kiky1G8vux3OR6/jbS+jIMe/6fJWa0DTzJU7dqijOWYwHi1t29FlfYI9uytqlpA==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" @@ -343,9 +343,9 @@ } }, "node_modules/diff": { - "version": "5.2.2", - "resolved": "https://registry.npmjs.org/diff/-/diff-5.2.2.tgz", - "integrity": "sha512-vtcDfH3TOjP8UekytvnHH1o1P4FcUdt4eQ1Y+Abap1tk/OB2MWQvcwS2ClCd1zuIhc3JKOx6p3kod8Vfys3E+A==", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-7.0.0.tgz", + "integrity": "sha512-PJWHUb1RFevKCwaFA9RlG5tCd+FO5iRh9A8HEtkmBH2Li03iJriB6m6JIN4rGz3K3JLawI7/veA1xzRKP6ISBw==", "license": "BSD-3-Clause", "engines": { "node": ">=0.3.1" @@ -536,6 +536,15 @@ "node": ">=8" } }, + "node_modules/is-path-inside": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz", + "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/is-plain-obj": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-2.1.0.tgz", @@ -687,28 +696,29 @@ } }, "node_modules/mocha": { - "version": "11.3.0", - "resolved": "https://registry.npmjs.org/mocha/-/mocha-11.3.0.tgz", - "integrity": "sha512-J0RLIM89xi8y6l77bgbX+03PeBRDQCOVQpnwOcCN7b8hCmbh6JvGI2ZDJ5WMoHz+IaPU+S4lvTd0j51GmBAdgQ==", + "version": "11.7.5", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-11.7.5.tgz", + "integrity": "sha512-mTT6RgopEYABzXWFx+GcJ+ZQ32kp4fMf0xvpZIIfSq9Z8lC/++MtcCnQ9t5FP2veYEP95FIYSvW+U9fV4xrlig==", "license": "MIT", "dependencies": { "browser-stdout": "^1.3.1", "chokidar": "^4.0.1", "debug": "^4.3.5", - "diff": "^5.2.0", + "diff": "^7.0.0", "escape-string-regexp": "^4.0.0", "find-up": "^5.0.0", "glob": "^10.4.5", "he": "^1.2.0", + "is-path-inside": "^3.0.3", "js-yaml": "^4.1.0", "log-symbols": "^4.1.0", - "minimatch": "^5.1.6", + "minimatch": "^9.0.5", "ms": "^2.1.3", "picocolors": "^1.1.1", "serialize-javascript": "^6.0.2", "strip-json-comments": "^3.1.1", "supports-color": "^8.1.1", - "workerpool": "^6.5.1", + "workerpool": "^9.2.0", "yargs": "^17.7.2", "yargs-parser": "^21.1.1", "yargs-unparser": "^2.0.0" @@ -758,18 +768,6 @@ "node": ">=8" } }, - "node_modules/mocha/node_modules/minimatch": { - "version": "5.1.9", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.9.tgz", - "integrity": "sha512-7o1wEA2RyMP7Iu7GNba9vc0RWWGACJOCZBJX2GJWip0ikV+wcOsgVuY9uE8CPiyQhkGFSlhuSkZPavN7u1c2Fw==", - "license": "ISC", - "dependencies": { - "brace-expansion": "^2.0.1" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -943,9 +941,9 @@ } }, "node_modules/serialize-javascript": { - "version": "7.0.5", - "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-7.0.5.tgz", - "integrity": "sha512-F4LcB0UqUl1zErq+1nYEEzSHJnIwb3AF2XWB94b+afhrekOUijwooAYqFyRbjYkm2PAKBabx6oYv/xDxNi8IBw==", + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-7.0.4.tgz", + "integrity": "sha512-DuGdB+Po43Q5Jxwpzt1lhyFSYKryqoNjQSA9M92tyw0lyHIOur+XCalOUe0KTJpyqzT8+fQ5A0Jf7vCx/NKmIg==", "license": "BSD-3-Clause", "engines": { "node": ">=20.0.0" @@ -1153,9 +1151,9 @@ } }, "node_modules/workerpool": { - "version": "6.5.1", - "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.5.1.tgz", - "integrity": "sha512-Fs4dNYcsdpYSAfVxhnl1L5zTksjvOJxtC5hzMNl+1t9B8hTJTdKDyZ5ju7ztgPy+ft9tBFXoOlDNiOT9WUXZlA==", + "version": "9.3.4", + "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-9.3.4.tgz", + "integrity": "sha512-TmPRQYYSAnnDiEB0P/Ytip7bFGvqnSU6I2BcuSw7Hx+JSg/DsUi5ebYfc8GYaSdpuvOcEs6dXxPurOYpe9QFwg==", "license": "Apache-2.0" }, "node_modules/wrap-ansi": { diff --git a/test/sanity/package.json b/test/sanity/package.json index d3760ac6c71ba..93d586f98e0f1 100644 --- a/test/sanity/package.json +++ b/test/sanity/package.json @@ -10,7 +10,7 @@ }, "dependencies": { "minimist": "^1.2.8", - "mocha": "^11.3.0", + "mocha": "^11.7.5", "mocha-junit-reporter": "^2.2.1", "node-fetch": "^3.3.2", "playwright": "^1.57.0" From 6c72a80300a59ca909693110025c52a96c01d79a Mon Sep 17 00:00:00 2001 From: Dmitriy Vasyura Date: Fri, 27 Mar 2026 15:24:28 -0700 Subject: [PATCH 10/21] Fix --- test/sanity/src/devTunnel.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/sanity/src/devTunnel.test.ts b/test/sanity/src/devTunnel.test.ts index 3102c5faeabc4..93e8eef2a7748 100644 --- a/test/sanity/src/devTunnel.test.ts +++ b/test/sanity/src/devTunnel.test.ts @@ -139,11 +139,12 @@ export function setup(context: TestContext) { await page.getByRole('button', { name: `remote ${tunnelId}` }).waitFor({ timeout: 5 * 60 * 1000 }); return; } catch (error) { + await context.captureScreenshot(page); if (attempt === maxAttempts) { - await context.captureScreenshot(page); throw error; + } else { + context.log(`Auth flow attempt ${attempt} failed: ${error instanceof Error ? error.message : String(error)}, retrying...`); } - context.log(`Auth flow attempt ${attempt} failed: ${error instanceof Error ? error.message : String(error)}, retrying...`); } } } From f79184b0a796254ae44c402fdd231a7f74b4593f Mon Sep 17 00:00:00 2001 From: Dmitriy Vasyura Date: Fri, 27 Mar 2026 15:40:21 -0700 Subject: [PATCH 11/21] Fixes --- test/sanity/src/devTunnel.test.ts | 55 ++++++++++++++++++------------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/test/sanity/src/devTunnel.test.ts b/test/sanity/src/devTunnel.test.ts index 93e8eef2a7748..72c17cadf55e3 100644 --- a/test/sanity/src/devTunnel.test.ts +++ b/test/sanity/src/devTunnel.test.ts @@ -70,6 +70,10 @@ export function setup(context: TestContext) { }); async function testCliApp(entryPoint: string) { + if (context.options.downloadOnly) { + return; + } + const cliDataDir = context.createTempDir(); context.log('Logging out of Dev Tunnel to ensure fresh authentication'); context.run(entryPoint, '--cli-data-dir', cliDataDir, 'tunnel', 'user', 'logout'); @@ -117,35 +121,42 @@ export function setup(context: TestContext) { const url = context.getTunnelUrl(tunnelUrl, test.workspaceDir); context.log(`CLI started successfully with tunnel URL: ${url}`); - const maxAttempts = 3; - for (let attempt = 1; attempt <= maxAttempts; attempt++) { - try { - context.log(`Navigating to ${url} (attempt ${attempt}/${maxAttempts})`); - await page.goto(url); + context.log(`Navigating to ${url}`); + await page.goto(url); - context.log('Waiting for the workbench to load'); - await page.waitForSelector('.monaco-workbench'); + context.log('Waiting for the workbench to load'); + await page.waitForSelector('.monaco-workbench'); - context.log('Selecting GitHub Account'); - await page.locator('span.monaco-highlighted-label', { hasText: 'GitHub' }).click(); + context.log('Selecting GitHub Account'); + await page.locator('span.monaco-highlighted-label', { hasText: 'GitHub' }).click(); - context.log('Clicking Allow on confirmation dialog'); - const popup = page.waitForEvent('popup'); - await page.getByRole('button', { name: 'Allow' }).click(); + context.log('Clicking Allow on confirmation dialog'); + const popup = page.waitForEvent('popup'); + await page.getByRole('button', { name: 'Allow' }).click(); - await auth.runAuthorizeFlow(await popup); + await auth.runAuthorizeFlow(await popup); - context.log('Waiting for connection to be established'); - await page.getByRole('button', { name: `remote ${tunnelId}` }).waitFor({ timeout: 5 * 60 * 1000 }); + context.log('Waiting for connection to be established'); + const remoteButton = page.getByRole('button', { name: `remote ${tunnelId}` }); + const reloadButton = page.getByRole('button', { name: 'Reload' }); + const maxReloads = 5; + for (let attempt = 1; ; attempt++) { + const result = await Promise.race([ + remoteButton.waitFor({ timeout: 5 * 60 * 1000 }).then(() => 'connected' as const), + reloadButton.waitFor().then(() => 'error' as const), + ]); + + if (result === 'connected') { return; - } catch (error) { - await context.captureScreenshot(page); - if (attempt === maxAttempts) { - throw error; - } else { - context.log(`Auth flow attempt ${attempt} failed: ${error instanceof Error ? error.message : String(error)}, retrying...`); - } } + + await context.captureScreenshot(page); + if (attempt >= maxReloads) { + throw new Error(`Workbench failed to connect after ${maxReloads} reload attempts`); + } + + context.log(`Error dialog detected (attempt ${attempt}/${maxReloads}), clicking Reload`); + await reloadButton.click(); } } } From 475aeb56e8c4f96f26338292a5335e30c6cd7376 Mon Sep 17 00:00:00 2001 From: Dmitriy Vasyura Date: Fri, 27 Mar 2026 15:49:22 -0700 Subject: [PATCH 12/21] Fixes --- test/sanity/src/context.ts | 4 +++- test/sanity/src/main.ts | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test/sanity/src/context.ts b/test/sanity/src/context.ts index 02d9434037596..f32c69311474f 100644 --- a/test/sanity/src/context.ts +++ b/test/sanity/src/context.ts @@ -40,6 +40,7 @@ export class TestContext { private readonly wslTempDirs = new Set(); private nextPort = 3010; private currentTestName: string | undefined; + private screenshotCounter = 0; public constructor(public readonly options: Readonly<{ quality: 'stable' | 'insider' | 'exploration'; @@ -92,6 +93,7 @@ export class TestContext { const self = this; return test(name, async function () { self.currentTestName = name; + self.screenshotCounter = 0; self.log(`Starting test: ${name}`); const homeDir = os.homedir(); @@ -1133,7 +1135,7 @@ export class TestContext { const screenshotDir = this.options.screenshotsDir ?? path.join(this.osTempDir, 'vscode-sanity-screenshots'); fs.mkdirSync(screenshotDir, { recursive: true }); const sanitizedName = this.currentTestName.replace(/[^a-zA-Z0-9_-]/g, '_'); - const screenshotPath = path.join(screenshotDir, `${sanitizedName}.png`); + const screenshotPath = path.join(screenshotDir, `${sanitizedName}-${++this.screenshotCounter}.png`); await page.screenshot({ path: screenshotPath, fullPage: true }); this.log(`Screenshot saved to: ${screenshotPath}`); } catch (e) { diff --git a/test/sanity/src/main.ts b/test/sanity/src/main.ts index 15b0c7dfda0d9..e1026eb041291 100644 --- a/test/sanity/src/main.ts +++ b/test/sanity/src/main.ts @@ -16,7 +16,7 @@ import { setup as setupDevTunnelTests } from './devTunnel.test.js'; const options = minimist(process.argv.slice(2), { string: ['commit', 'quality', 'screenshots-dir'], boolean: ['cleanup', 'verbose', 'signing-check', 'headless', 'detection'], - alias: { commit: 'c', quality: 'q', verbose: 'v' }, + alias: { commit: 'c', quality: 'q', verbose: 'v', 'screenshots-dir': 's' }, default: { cleanup: true, verbose: false, 'signing-check': true, headless: true, 'detection': true }, }); From 2fc3c19b6c4d5ac761e518ddd6bf201fed59b465 Mon Sep 17 00:00:00 2001 From: Dmitriy Vasyura Date: Sat, 28 Mar 2026 18:22:35 -0700 Subject: [PATCH 13/21] Capture screenshots. --- test/sanity/src/devTunnel.test.ts | 68 +++++++++++++++++-------------- 1 file changed, 37 insertions(+), 31 deletions(-) diff --git a/test/sanity/src/devTunnel.test.ts b/test/sanity/src/devTunnel.test.ts index 72c17cadf55e3..7f71b98d192ac 100644 --- a/test/sanity/src/devTunnel.test.ts +++ b/test/sanity/src/devTunnel.test.ts @@ -117,46 +117,52 @@ export function setup(context: TestContext) { } async function connectToTunnel(tunnelUrl: string, page: Page, test: UITest, auth: GitHubAuth) { - const tunnelId = new URL(tunnelUrl).pathname.split('/').pop()!; - const url = context.getTunnelUrl(tunnelUrl, test.workspaceDir); - context.log(`CLI started successfully with tunnel URL: ${url}`); + try { + const tunnelId = new URL(tunnelUrl).pathname.split('/').pop()!; + const url = context.getTunnelUrl(tunnelUrl, test.workspaceDir); + context.log(`CLI started successfully with tunnel URL: ${url}`); - context.log(`Navigating to ${url}`); - await page.goto(url); + context.log(`Navigating to ${url}`); + await page.goto(url); - context.log('Waiting for the workbench to load'); - await page.waitForSelector('.monaco-workbench'); + context.log('Waiting for the workbench to load'); + await page.waitForSelector('.monaco-workbench'); - context.log('Selecting GitHub Account'); - await page.locator('span.monaco-highlighted-label', { hasText: 'GitHub' }).click(); + context.log('Selecting GitHub Account'); + await page.locator('span.monaco-highlighted-label', { hasText: 'GitHub' }).click(); - context.log('Clicking Allow on confirmation dialog'); - const popup = page.waitForEvent('popup'); - await page.getByRole('button', { name: 'Allow' }).click(); + context.log('Clicking Allow on confirmation dialog'); + const popup = page.waitForEvent('popup'); + await page.getByRole('button', { name: 'Allow' }).click(); - await auth.runAuthorizeFlow(await popup); + await auth.runAuthorizeFlow(await popup); - context.log('Waiting for connection to be established'); - const remoteButton = page.getByRole('button', { name: `remote ${tunnelId}` }); - const reloadButton = page.getByRole('button', { name: 'Reload' }); - const maxReloads = 5; - for (let attempt = 1; ; attempt++) { - const result = await Promise.race([ - remoteButton.waitFor({ timeout: 5 * 60 * 1000 }).then(() => 'connected' as const), - reloadButton.waitFor().then(() => 'error' as const), - ]); + context.log('Waiting for connection to be established'); + const remoteButton = page.getByRole('button', { name: `remote ${tunnelId}` }); + const reloadButton = page.getByRole('button', { name: 'Reload' }); + const maxReloads = 5; + for (let attempt = 1; ; attempt++) { + const result = await Promise.race([ + remoteButton.waitFor({ timeout: 5 * 60 * 1000 }).then(() => 'connected' as const), + reloadButton.waitFor().then(() => 'error' as const), + ]); - if (result === 'connected') { - return; - } + if (result === 'connected') { + return; + } - await context.captureScreenshot(page); - if (attempt >= maxReloads) { - throw new Error(`Workbench failed to connect after ${maxReloads} reload attempts`); - } + await context.captureScreenshot(page); + if (attempt >= maxReloads) { + throw new Error(`Workbench failed to connect after ${maxReloads} reload attempts`); + } - context.log(`Error dialog detected (attempt ${attempt}/${maxReloads}), clicking Reload`); - await reloadButton.click(); + context.log(`Error dialog detected (attempt ${attempt}/${maxReloads}), clicking Reload`); + await context.captureScreenshot(page); + await reloadButton.click(); + } + } catch (error) { + await context.captureScreenshot(page); + throw error; } } } From 2c3e86002d6bbdb5bdcc1c5e0bee4b40844da921 Mon Sep 17 00:00:00 2001 From: Dmitriy Vasyura Date: Sat, 28 Mar 2026 18:43:16 -0700 Subject: [PATCH 14/21] Save error screenshot --- test/sanity/src/devTunnel.test.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/sanity/src/devTunnel.test.ts b/test/sanity/src/devTunnel.test.ts index 7f71b98d192ac..5b71e851bc5ac 100644 --- a/test/sanity/src/devTunnel.test.ts +++ b/test/sanity/src/devTunnel.test.ts @@ -101,6 +101,11 @@ export function setup(context: TestContext) { return; } + if (/Error getting authorization/.test(line)) { + context.log('Error during authentication, capturing screenshot'); + await context.captureScreenshot(page); + } + const tunnelUrl = /Open this link in your browser (https?:\/\/[^\s]+)/.exec(line)?.[1]; if (tunnelUrl) { await connectToTunnel(tunnelUrl, page, test, auth); From d451f1df3e48d2d8fcf874e0c1efdb366e462e0b Mon Sep 17 00:00:00 2001 From: Dmitriy Vasyura Date: Sat, 28 Mar 2026 19:01:52 -0700 Subject: [PATCH 15/21] Avoid throttling --- test/sanity/src/devTunnel.test.ts | 6 +----- test/sanity/src/githubAuth.ts | 24 ++++++++++++++++-------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/test/sanity/src/devTunnel.test.ts b/test/sanity/src/devTunnel.test.ts index 5b71e851bc5ac..f2b57532746f1 100644 --- a/test/sanity/src/devTunnel.test.ts +++ b/test/sanity/src/devTunnel.test.ts @@ -83,6 +83,7 @@ export function setup(context: TestContext) { const browser = await context.launchBrowser(); try { const page = await context.getPage(browser.newPage()); + await auth.signIn(page); context.log('Starting Dev Tunnel to local server using CLI'); await context.runCliApp('CLI', entryPoint, [ @@ -101,11 +102,6 @@ export function setup(context: TestContext) { return; } - if (/Error getting authorization/.test(line)) { - context.log('Error during authentication, capturing screenshot'); - await context.captureScreenshot(page); - } - const tunnelUrl = /Open this link in your browser (https?:\/\/[^\s]+)/.exec(line)?.[1]; if (tunnelUrl) { await connectToTunnel(tunnelUrl, page, test, auth); diff --git a/test/sanity/src/githubAuth.ts b/test/sanity/src/githubAuth.ts index 0a1844f7e2bd4..2bec42416e508 100644 --- a/test/sanity/src/githubAuth.ts +++ b/test/sanity/src/githubAuth.ts @@ -16,25 +16,33 @@ export class GitHubAuth { public constructor(private readonly context: TestContext) { } /** - * Runs GitHub device authentication flow in a browser. + * Signs in to GitHub so the browser session is authenticated. * @param page Page to use. - * @param code Device authentication code to use. */ - public async runDeviceCodeFlow(page: Page, code: string) { + public async signIn(page: Page) { if (!this.username || !this.password) { this.context.error('GITHUB_ACCOUNT and GITHUB_PASSWORD environment variables must be set'); } - this.context.log(`Running GitHub device flow with code ${code}`); - await page.goto('https://github.com/login/device'); + this.context.log('Signing in to GitHub'); + await page.goto('https://github.com/login'); - this.context.log('Filling in GitHub credentials'); await page.getByLabel('Username or email address').fill(this.username); await page.getByLabel('Password').fill(this.password); await page.getByRole('button', { name: 'Sign in', exact: true }).click(); - this.context.log('Confirming device activation'); - await page.getByRole('button', { name: 'Continue' }).click(); + await page.waitForURL('https://github.com/**'); + this.context.log('GitHub sign-in complete'); + } + + /** + * Runs GitHub device authentication flow in a browser. + * @param page Page to use. + * @param code Device authentication code to use. + */ + public async runDeviceCodeFlow(page: Page, code: string) { + this.context.log(`Running GitHub device flow with code ${code}`); + await page.goto('https://github.com/login/device'); this.context.log('Entering device code'); const codeChars = code.replace(/-/g, ''); From e452e5f05bd0852af1040c9a6824715a70bd17bc Mon Sep 17 00:00:00 2001 From: Dmitriy Vasyura Date: Sat, 28 Mar 2026 19:12:28 -0700 Subject: [PATCH 16/21] Screenshots --- test/sanity/src/githubAuth.ts | 56 +++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 19 deletions(-) diff --git a/test/sanity/src/githubAuth.ts b/test/sanity/src/githubAuth.ts index 2bec42416e508..9c7dd872d68c0 100644 --- a/test/sanity/src/githubAuth.ts +++ b/test/sanity/src/githubAuth.ts @@ -24,15 +24,21 @@ export class GitHubAuth { this.context.error('GITHUB_ACCOUNT and GITHUB_PASSWORD environment variables must be set'); } - this.context.log('Signing in to GitHub'); - await page.goto('https://github.com/login'); + try { + this.context.log('Signing in to GitHub'); + await page.goto('https://github.com/login'); - await page.getByLabel('Username or email address').fill(this.username); - await page.getByLabel('Password').fill(this.password); - await page.getByRole('button', { name: 'Sign in', exact: true }).click(); + await page.getByLabel('Username or email address').fill(this.username); + await page.getByLabel('Password').fill(this.password); + await page.getByRole('button', { name: 'Sign in', exact: true }).click(); - await page.waitForURL('https://github.com/**'); - this.context.log('GitHub sign-in complete'); + await page.waitForURL('https://github.com/**'); + this.context.log('GitHub sign-in complete'); + } catch (error) { + this.context.log('Error during GitHub sign-in, capturing screenshot'); + await this.context.captureScreenshot(page); + throw error; + } } /** @@ -41,18 +47,24 @@ export class GitHubAuth { * @param code Device authentication code to use. */ public async runDeviceCodeFlow(page: Page, code: string) { - this.context.log(`Running GitHub device flow with code ${code}`); - await page.goto('https://github.com/login/device'); + try { + this.context.log(`Running GitHub device flow with code ${code}`); + await page.goto('https://github.com/login/device'); - this.context.log('Entering device code'); - const codeChars = code.replace(/-/g, ''); - for (let i = 0; i < codeChars.length; i++) { - await page.getByRole('textbox').nth(i).fill(codeChars[i]); - } - await page.getByRole('button', { name: 'Continue' }).click(); + this.context.log('Entering device code'); + const codeChars = code.replace(/-/g, ''); + for (let i = 0; i < codeChars.length; i++) { + await page.getByRole('textbox').nth(i).fill(codeChars[i]); + } + await page.getByRole('button', { name: 'Continue' }).click(); - this.context.log('Authorizing device'); - await page.getByRole('button', { name: 'Authorize' }).click(); + this.context.log('Authorizing device'); + await page.getByRole('button', { name: 'Authorize' }).click(); + } catch (error) { + this.context.log('Error during device code flow, capturing screenshot'); + await this.context.captureScreenshot(page); + throw error; + } } /** @@ -60,7 +72,13 @@ export class GitHubAuth { * @param page Page to use. */ public async runAuthorizeFlow(page: Page) { - this.context.log(`Authorizing app at ${page.url()}`); - await page.getByRole('button', { name: 'Continue' }).click(); + try { + this.context.log(`Authorizing app at ${page.url()}`); + await page.getByRole('button', { name: 'Continue' }).click(); + } catch (error) { + this.context.log('Error during authorization, capturing screenshot'); + await this.context.captureScreenshot(page); + throw error; + } } } From d3a8f98768185ff710c6c43f1dd327fa6c6cb4bc Mon Sep 17 00:00:00 2001 From: Dmitriy Vasyura Date: Sat, 28 Mar 2026 19:21:19 -0700 Subject: [PATCH 17/21] Fix device flow --- test/sanity/src/githubAuth.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/sanity/src/githubAuth.ts b/test/sanity/src/githubAuth.ts index 9c7dd872d68c0..b5f546f82ae8a 100644 --- a/test/sanity/src/githubAuth.ts +++ b/test/sanity/src/githubAuth.ts @@ -51,6 +51,9 @@ export class GitHubAuth { this.context.log(`Running GitHub device flow with code ${code}`); await page.goto('https://github.com/login/device'); + this.context.log('Confirming signed-in account'); + await page.getByRole('button', { name: 'Continue' }).click(); + this.context.log('Entering device code'); const codeChars = code.replace(/-/g, ''); for (let i = 0; i < codeChars.length; i++) { From 6053287247c50926256aa71f21d16c1b3ad713d9 Mon Sep 17 00:00:00 2001 From: Dmitriy Vasyura Date: Sat, 28 Mar 2026 19:36:21 -0700 Subject: [PATCH 18/21] Update --- test/sanity/src/devTunnel.test.ts | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/test/sanity/src/devTunnel.test.ts b/test/sanity/src/devTunnel.test.ts index f2b57532746f1..bbfef12d247c9 100644 --- a/test/sanity/src/devTunnel.test.ts +++ b/test/sanity/src/devTunnel.test.ts @@ -139,29 +139,9 @@ export function setup(context: TestContext) { await auth.runAuthorizeFlow(await popup); context.log('Waiting for connection to be established'); - const remoteButton = page.getByRole('button', { name: `remote ${tunnelId}` }); - const reloadButton = page.getByRole('button', { name: 'Reload' }); - const maxReloads = 5; - for (let attempt = 1; ; attempt++) { - const result = await Promise.race([ - remoteButton.waitFor({ timeout: 5 * 60 * 1000 }).then(() => 'connected' as const), - reloadButton.waitFor().then(() => 'error' as const), - ]); - - if (result === 'connected') { - return; - } - - await context.captureScreenshot(page); - if (attempt >= maxReloads) { - throw new Error(`Workbench failed to connect after ${maxReloads} reload attempts`); - } - - context.log(`Error dialog detected (attempt ${attempt}/${maxReloads}), clicking Reload`); - await context.captureScreenshot(page); - await reloadButton.click(); - } + await page.getByRole('button', { name: `remote ${tunnelId}` }).waitFor({ timeout: 5 * 60 * 1000 }); } catch (error) { + context.log('Error during tunnel connection, capturing screenshot'); await context.captureScreenshot(page); throw error; } From dacc8c425bb20a0ef571835f18fb1b1ca407d512 Mon Sep 17 00:00:00 2001 From: Dmitriy Vasyura Date: Sat, 28 Mar 2026 20:03:14 -0700 Subject: [PATCH 19/21] Updates --- test/sanity/src/devTunnel.test.ts | 5 +++++ test/sanity/src/githubAuth.ts | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/test/sanity/src/devTunnel.test.ts b/test/sanity/src/devTunnel.test.ts index bbfef12d247c9..6bef813595c62 100644 --- a/test/sanity/src/devTunnel.test.ts +++ b/test/sanity/src/devTunnel.test.ts @@ -9,6 +9,9 @@ import { GitHubAuth } from './githubAuth.js'; import { UITest } from './uiTest.js'; export function setup(context: TestContext) { + /* + TODO: @dmitrivMS Reenable other platforms once throttling issues with GitHub account are resolved. + context.test('dev-tunnel-alpine-arm64', ['alpine', 'arm64', 'browser', 'github-account'], async () => { const dir = await context.downloadAndUnpack('cli-alpine-arm64'); const entryPoint = context.getCliEntryPoint(dir); @@ -53,6 +56,8 @@ export function setup(context: TestContext) { await testCliApp(entryPoint); }); + */ + context.test('dev-tunnel-win32-arm64', ['windows', 'arm64', 'browser', 'github-account'], async () => { const dir = await context.downloadAndUnpack('cli-win32-arm64'); context.validateAllAuthenticodeSignatures(dir); diff --git a/test/sanity/src/githubAuth.ts b/test/sanity/src/githubAuth.ts index b5f546f82ae8a..c9d55b56d0567 100644 --- a/test/sanity/src/githubAuth.ts +++ b/test/sanity/src/githubAuth.ts @@ -63,6 +63,11 @@ export class GitHubAuth { this.context.log('Authorizing device'); await page.getByRole('button', { name: 'Authorize' }).click(); + + if (await page.getByRole('heading', { name: 'Too many requests' }).isVisible()) { + await this.context.captureScreenshot(page); + this.context.error('GitHub rate limit hit'); + } } catch (error) { this.context.log('Error during device code flow, capturing screenshot'); await this.context.captureScreenshot(page); From fba184a74f1dd6d3d99e63781d51019fdc4f9591 Mon Sep 17 00:00:00 2001 From: Dmitriy Vasyura Date: Sat, 28 Mar 2026 20:11:17 -0700 Subject: [PATCH 20/21] Updates --- test/sanity/src/devTunnel.test.ts | 28 ++++++++++++++-------------- test/sanity/src/githubAuth.ts | 5 ----- 2 files changed, 14 insertions(+), 19 deletions(-) diff --git a/test/sanity/src/devTunnel.test.ts b/test/sanity/src/devTunnel.test.ts index 6bef813595c62..6ed1d50f0ea75 100644 --- a/test/sanity/src/devTunnel.test.ts +++ b/test/sanity/src/devTunnel.test.ts @@ -24,20 +24,6 @@ export function setup(context: TestContext) { await testCliApp(entryPoint); }); - context.test('dev-tunnel-darwin-arm64', ['darwin', 'arm64', 'browser', 'github-account'], async () => { - const dir = await context.downloadAndUnpack('cli-darwin-arm64'); - context.validateAllCodesignSignatures(dir); - const entryPoint = context.getCliEntryPoint(dir); - await testCliApp(entryPoint); - }); - - context.test('dev-tunnel-darwin-x64', ['darwin', 'x64', 'browser', 'github-account'], async () => { - const dir = await context.downloadAndUnpack('cli-darwin-x64'); - context.validateAllCodesignSignatures(dir); - const entryPoint = context.getCliEntryPoint(dir); - await testCliApp(entryPoint); - }); - context.test('dev-tunnel-linux-arm64', ['linux', 'arm64', 'browser', 'github-account'], async () => { const dir = await context.downloadAndUnpack('cli-linux-arm64'); const entryPoint = context.getCliEntryPoint(dir); @@ -58,6 +44,20 @@ export function setup(context: TestContext) { */ + context.test('dev-tunnel-darwin-arm64', ['darwin', 'arm64', 'browser', 'github-account'], async () => { + const dir = await context.downloadAndUnpack('cli-darwin-arm64'); + context.validateAllCodesignSignatures(dir); + const entryPoint = context.getCliEntryPoint(dir); + await testCliApp(entryPoint); + }); + + context.test('dev-tunnel-darwin-x64', ['darwin', 'x64', 'browser', 'github-account'], async () => { + const dir = await context.downloadAndUnpack('cli-darwin-x64'); + context.validateAllCodesignSignatures(dir); + const entryPoint = context.getCliEntryPoint(dir); + await testCliApp(entryPoint); + }); + context.test('dev-tunnel-win32-arm64', ['windows', 'arm64', 'browser', 'github-account'], async () => { const dir = await context.downloadAndUnpack('cli-win32-arm64'); context.validateAllAuthenticodeSignatures(dir); diff --git a/test/sanity/src/githubAuth.ts b/test/sanity/src/githubAuth.ts index c9d55b56d0567..b5f546f82ae8a 100644 --- a/test/sanity/src/githubAuth.ts +++ b/test/sanity/src/githubAuth.ts @@ -63,11 +63,6 @@ export class GitHubAuth { this.context.log('Authorizing device'); await page.getByRole('button', { name: 'Authorize' }).click(); - - if (await page.getByRole('heading', { name: 'Too many requests' }).isVisible()) { - await this.context.captureScreenshot(page); - this.context.error('GitHub rate limit hit'); - } } catch (error) { this.context.log('Error during device code flow, capturing screenshot'); await this.context.captureScreenshot(page); From cb6bdd75746e25b95aee806db234e8e83470d17b Mon Sep 17 00:00:00 2001 From: Dmitriy Vasyura Date: Mon, 30 Mar 2026 18:36:12 -0700 Subject: [PATCH 21/21] Recombine device flow --- test/sanity/src/devTunnel.test.ts | 1 - test/sanity/src/githubAuth.ts | 29 ++++++----------------------- 2 files changed, 6 insertions(+), 24 deletions(-) diff --git a/test/sanity/src/devTunnel.test.ts b/test/sanity/src/devTunnel.test.ts index 6ed1d50f0ea75..3c54432d6680c 100644 --- a/test/sanity/src/devTunnel.test.ts +++ b/test/sanity/src/devTunnel.test.ts @@ -88,7 +88,6 @@ export function setup(context: TestContext) { const browser = await context.launchBrowser(); try { const page = await context.getPage(browser.newPage()); - await auth.signIn(page); context.log('Starting Dev Tunnel to local server using CLI'); await context.runCliApp('CLI', entryPoint, [ diff --git a/test/sanity/src/githubAuth.ts b/test/sanity/src/githubAuth.ts index b5f546f82ae8a..420ca94f7e678 100644 --- a/test/sanity/src/githubAuth.ts +++ b/test/sanity/src/githubAuth.ts @@ -16,41 +16,24 @@ export class GitHubAuth { public constructor(private readonly context: TestContext) { } /** - * Signs in to GitHub so the browser session is authenticated. + * Runs GitHub device authentication flow in a browser, signing in first. * @param page Page to use. + * @param code Device authentication code to use. */ - public async signIn(page: Page) { + public async runDeviceCodeFlow(page: Page, code: string) { if (!this.username || !this.password) { this.context.error('GITHUB_ACCOUNT and GITHUB_PASSWORD environment variables must be set'); } try { - this.context.log('Signing in to GitHub'); - await page.goto('https://github.com/login'); + this.context.log(`Running GitHub device flow with code ${code}`); + await page.goto('https://github.com/login/device'); + this.context.log('Signing in to GitHub'); await page.getByLabel('Username or email address').fill(this.username); await page.getByLabel('Password').fill(this.password); await page.getByRole('button', { name: 'Sign in', exact: true }).click(); - await page.waitForURL('https://github.com/**'); - this.context.log('GitHub sign-in complete'); - } catch (error) { - this.context.log('Error during GitHub sign-in, capturing screenshot'); - await this.context.captureScreenshot(page); - throw error; - } - } - - /** - * Runs GitHub device authentication flow in a browser. - * @param page Page to use. - * @param code Device authentication code to use. - */ - public async runDeviceCodeFlow(page: Page, code: string) { - try { - this.context.log(`Running GitHub device flow with code ${code}`); - await page.goto('https://github.com/login/device'); - this.context.log('Confirming signed-in account'); await page.getByRole('button', { name: 'Continue' }).click();