Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Smoke test lifecycle changes #137969

Merged
merged 12 commits into from Nov 28, 2021
Merged

Smoke test lifecycle changes #137969

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
ee80a36
rewrite teardown
bpasero Nov 27, 2021
f29daf4
:lipstick:
bpasero Nov 27, 2021
0a80c79
avoid spam errors
bpasero Nov 27, 2021
ba1f4ae
:lipstick:
bpasero Nov 27, 2021
29d022e
split spawning browser from electron
bpasero Nov 27, 2021
a45d777
await copyExtension
bpasero Nov 27, 2021
cdd7b0b
cleanup
bpasero Nov 27, 2021
d3e98f0
refactor methods
bpasero Nov 27, 2021
aa13df6
cleanup
bpasero Nov 27, 2021
641961b
cleanup
bpasero Nov 27, 2021
103ef76
cleanup
bpasero Nov 28, 2021
150118a
:up: deps
bpasero Nov 28, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
6 changes: 3 additions & 3 deletions test/automation/package.json
View file
Open in desktop
Expand Up @@ -22,15 +22,15 @@
"dependencies": {
"mkdirp": "^1.0.4",
"ncp": "^2.0.0",
"tmp": "0.1.0",
"tmp": "0.2.1",
"tree-kill": "1.2.2",
"vscode-uri": "^2.0.3"
"vscode-uri": "3.0.2"
},
"devDependencies": {
"@types/mkdirp": "^1.0.1",
"@types/ncp": "2.0.1",
"@types/node": "14.x",
"@types/tmp": "0.1.0",
"@types/tmp": "0.2.2",
"cpx2": "3.0.0",
"npm-run-all": "^4.1.5",
"typescript": "^4.3.2",
Expand Down
26 changes: 12 additions & 14 deletions test/automation/src/application.ts
View file
Open in desktop
Expand Up @@ -24,24 +24,19 @@ export interface ApplicationOptions extends SpawnOptions {

export class Application {

private _code: Code | undefined;
private _workbench: Workbench | undefined;

constructor(private options: ApplicationOptions) {
this._userDataPath = options.userDataDir;
this._workspacePathOrFolder = options.workspacePath;
}

get quality(): Quality {
return this.options.quality;
}
private _code: Code | undefined;
get code(): Code { return this._code!; }

get code(): Code {
return this._code!;
}
private _workbench: Workbench | undefined;
get workbench(): Workbench { return this._workbench!; }

get workbench(): Workbench {
return this._workbench!;
get quality(): Quality {
return this.options.quality;
}

get logger(): Logger {
Expand Down Expand Up @@ -88,9 +83,11 @@ export class Application {

async stop(): Promise<any> {
if (this._code) {
await this._code.exit();
this._code.dispose();
this._code = undefined;
try {
await this._code.exit();
} finally {
this._code = undefined;
}
}
}

Expand All @@ -102,6 +99,7 @@ export class Application {
if (this.options.log) {
this.logger.log('*** Screenshot recorded:', screenshotPath);
}

fs.writeFileSync(screenshotPath, buffer);
}
}
Expand Down
244 changes: 38 additions & 206 deletions test/automation/src/code.ts
View file
Open in desktop
Expand Up @@ -4,90 +4,16 @@
*--------------------------------------------------------------------------------------------*/

import * as path from 'path';
import * as cp from 'child_process';
import * as os from 'os';
import * as fs from 'fs';
import * as mkdirp from 'mkdirp';
import { tmpName } from 'tmp';
import { IDriver, connect as connectElectronDriver, IDisposable, IElement, Thenable, ILocalizedStrings, ILocaleInfo } from './driver';
import { connect as connectPlaywrightDriver, launch } from './playwrightDriver';
import * as cp from 'child_process';
import { IDriver, IDisposable, IElement, Thenable, ILocalizedStrings, ILocaleInfo } from './driver';
import { launch as launchElectron } from './electronDriver';
import { launch as launchPlaywright } from './playwrightDriver';
import { Logger } from './logger';
import { ncp } from 'ncp';
import { URI } from 'vscode-uri';
import { copyExtension } from './extensions';

const repoPath = path.join(__dirname, '../../..');

function getDevElectronPath(): string {
const buildPath = path.join(repoPath, '.build');
const product = require(path.join(repoPath, 'product.json'));

switch (process.platform) {
case 'darwin':
return path.join(buildPath, 'electron', `${product.nameLong}.app`, 'Contents', 'MacOS', 'Electron');
case 'linux':
return path.join(buildPath, 'electron', `${product.applicationName}`);
case 'win32':
return path.join(buildPath, 'electron', `${product.nameShort}.exe`);
default:
throw new Error('Unsupported platform.');
}
}

function getBuildElectronPath(root: string): string {
switch (process.platform) {
case 'darwin':
return path.join(root, 'Contents', 'MacOS', 'Electron');
case 'linux': {
const product = require(path.join(root, 'resources', 'app', 'product.json'));
return path.join(root, product.applicationName);
}
case 'win32': {
const product = require(path.join(root, 'resources', 'app', 'product.json'));
return path.join(root, `${product.nameShort}.exe`);
}
default:
throw new Error('Unsupported platform.');
}
}

function getDevOutPath(): string {
return path.join(repoPath, 'out');
}

function getBuildOutPath(root: string): string {
switch (process.platform) {
case 'darwin':
return path.join(root, 'Contents', 'Resources', 'app', 'out');
default:
return path.join(root, 'resources', 'app', 'out');
}
}

async function connect(connectDriver: typeof connectElectronDriver | typeof connectPlaywrightDriver, child: cp.ChildProcess | undefined, outPath: string, handlePath: string, logger: Logger): Promise<Code> {
let errCount = 0;

while (true) {
try {
const { client, driver } = await connectDriver(outPath, handlePath);
return new Code(client, driver, logger, child?.pid);
} catch (err) {
if (++errCount > 50) {
if (child) {
child.kill();
}
throw err;
}

// retry
await new Promise(resolve => setTimeout(resolve, 100));
}
}
}

// Kill all running instances, when dead
const instances = new Set<cp.ChildProcess>();
process.once('exit', () => instances.forEach(code => code.kill()));

export interface SpawnOptions {
codePath?: string;
workspacePath: string;
Expand All @@ -103,109 +29,29 @@ export interface SpawnOptions {
browser?: 'chromium' | 'webkit' | 'firefox';
}

async function createDriverHandle(): Promise<string> {
if ('win32' === os.platform()) {
const name = [...Array(15)].map(() => Math.random().toString(36)[3]).join('');
return `\\\\.\\pipe\\${name}`;
} else {
return await new Promise<string>((c, e) => tmpName((err, handlePath) => err ? e(err) : c(handlePath)));
}
}

export async function spawn(options: SpawnOptions): Promise<Code> {
const handle = await createDriverHandle();

let child: cp.ChildProcess | undefined;

copyExtension(options.extensionsPath, 'vscode-notebook-tests');
await copyExtension(repoPath, options.extensionsPath, 'vscode-notebook-tests');

// Browser smoke tests
if (options.web) {
await launch(options.userDataDir, options.workspacePath, options.codePath, options.extensionsPath, Boolean(options.verbose));
return connect(connectPlaywrightDriver.bind(connectPlaywrightDriver, options), child, '', handle, options.logger);
}

const env = { ...process.env };
const codePath = options.codePath;
const logsPath = path.join(repoPath, '.build', 'logs', options.remote ? 'smoke-tests-remote' : 'smoke-tests');
const outPath = codePath ? getBuildOutPath(codePath) : getDevOutPath();

const args = [
options.workspacePath,
'--skip-release-notes',
'--skip-welcome',
'--disable-telemetry',
'--no-cached-data',
'--disable-updates',
'--disable-keytar',
'--disable-crash-reporter',
'--disable-workspace-trust',
`--extensions-dir=${options.extensionsPath}`,
`--user-data-dir=${options.userDataDir}`,
`--logsPath=${logsPath}`,
'--driver', handle
];

if (process.platform === 'linux') {
args.push('--disable-gpu'); // Linux has trouble in VMs to render properly with GPU enabled
}

if (options.remote) {
// Replace workspace path with URI
args[0] = `--${options.workspacePath.endsWith('.code-workspace') ? 'file' : 'folder'}-uri=vscode-remote://test+test/${URI.file(options.workspacePath).path}`;

if (codePath) {
// running against a build: copy the test resolver extension
copyExtension(options.extensionsPath, 'vscode-test-resolver');
}
args.push('--enable-proposed-api=vscode.vscode-test-resolver');
const remoteDataDir = `${options.userDataDir}-server`;
mkdirp.sync(remoteDataDir);

if (codePath) {
// running against a build: copy the test resolver extension into remote extensions dir
const remoteExtensionsDir = path.join(remoteDataDir, 'extensions');
mkdirp.sync(remoteExtensionsDir);
copyExtension(remoteExtensionsDir, 'vscode-notebook-tests');
}

env['TESTRESOLVER_DATA_FOLDER'] = remoteDataDir;
env['TESTRESOLVER_LOGS_FOLDER'] = path.join(logsPath, 'server');
}

const spawnOptions: cp.SpawnOptions = { env };

args.push('--enable-proposed-api=vscode.vscode-notebook-tests');

if (!codePath) {
args.unshift(repoPath);
return spawnBrowser(options);
}

if (options.verbose) {
args.push('--driver-verbose');
spawnOptions.stdio = ['ignore', 'inherit', 'inherit'];
}
// Electron smoke tests
return spawnElectron(options);
}

if (options.log) {
args.push('--log', options.log);
}
async function spawnBrowser(options: SpawnOptions): Promise<Code> {
const { serverProcess, client, driver } = await launchPlaywright(options.codePath, options.userDataDir, options.extensionsPath, options.workspacePath, Boolean(options.verbose), options);

if (options.extraArgs) {
args.push(...options.extraArgs);
}

const electronPath = codePath ? getBuildElectronPath(codePath) : getDevElectronPath();
child = cp.spawn(electronPath, args, spawnOptions);
instances.add(child);
child.once('exit', () => instances.delete(child!));
return connect(connectElectronDriver, child, outPath, handle, options.logger);
return new Code(client, driver, options.logger, serverProcess);
}

async function copyExtension(extensionsPath: string, extId: string): Promise<void> {
const dest = path.join(extensionsPath, extId);
if (!fs.existsSync(dest)) {
const orig = path.join(repoPath, 'extensions', extId);
await new Promise<void>((c, e) => ncp(orig, dest, err => err ? e(err) : c()));
}
async function spawnElectron(options: SpawnOptions): Promise<Code> {
const { electronProcess, client, driver } = await launchElectron(options.codePath, options.userDataDir, options.extensionsPath, options.workspacePath, Boolean(options.verbose), Boolean(options.remote), options.log, options.extraArgs);

return new Code(client, driver, options.logger, electronProcess);
}

async function poll<T>(
Expand All @@ -222,7 +68,7 @@ async function poll<T>(
if (trial > retryCount) {
console.error('** Timeout!');
console.error(lastError);
console.error(`Timeout: ${timeoutMessage} after ${(retryCount * retryInterval) / 1000} seconds.`);
console.error(`*** Timeout: ${timeoutMessage} after ${(retryCount * retryInterval) / 1000} seconds.`);
throw new Error(`Timeout: ${timeoutMessage} after ${(retryCount * retryInterval) / 1000} seconds.`);
}

Expand Down Expand Up @@ -252,7 +98,7 @@ export class Code {
private client: IDisposable,
driver: IDriver,
readonly logger: Logger,
private readonly pid: number | undefined
private readonly mainProcess: cp.ChildProcess
) {
this.driver = new Proxy(driver, {
get(target, prop, receiver) {
Expand Down Expand Up @@ -292,49 +138,35 @@ export class Code {
let done = false;

// Start the exit flow via driver
const exitPromise = this.driver.exitApplication().then(veto => {
this.driver.exitApplication().then(veto => {
if (veto) {
done = true;
reject(new Error('Smoke test exit call resulted in unexpected veto'));
}
});

// If we know the `pid` of the smoke tested application
// use that as way to detect the exit of the application
const pid = this.pid;
if (typeof pid === 'number') {
(async () => {
let killCounter = 0;
while (!done) {
killCounter++;

if (killCounter > 40) {
done = true;
reject(new Error('Smoke test exit call did not terminate main process after 20s, giving up'));
}

try {
process.kill(pid, 0); // throws an exception if the main process doesn't exist anymore.
await new Promise(resolve => setTimeout(resolve, 500));
} catch (error) {
done = true;
resolve();
}
// Await the exit of the application
(async () => {
let retries = 0;
while (!done) {
retries++;

if (retries > 40) {
done = true;
reject(new Error('Smoke test exit call did not terminate process after 20s, giving up'));
}
})();
}

// Otherwise await the exit promise (web).
else {
(async () => {
try {
await exitPromise;
resolve();
process.kill(this.mainProcess.pid!, 0); // throws an exception if the process doesn't exist anymore.
await new Promise(resolve => setTimeout(resolve, 500));
} catch (error) {
reject(new Error(`Smoke test exit call resulted in error: ${error}`));
done = true;
resolve();
}
})();
}
}
})();
}).finally(() => {
this.dispose();
});
}

Expand Down