Puppeteer E2E test: Multi-page in-browser parallelism

.github/workflows/ci.yml

@@ -58,8 +58,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ windows-latest, ubuntu-latest ]
-        CI: [ 0, 1, 2, 3, 4, 5, 6, 7 ]
+        os: [ windows-latest, ubuntu-latest, macos-latest ]
+        CI: [ 0, 1, 2 ]
     env:
       CI: ${{ matrix.CI }}
       FORCE_COLOR: 1

test/e2e/puppeteer.js

@@ -7,6 +7,35 @@ import jimp from 'jimp';
 import * as fs from 'fs/promises';
 import fetch from 'node-fetch';
 
+class PromiseQueue {
+
+	constructor( func, ...args ) {
+
+		this.func = func.bind( this, ...args );
+		this.promises = [];
+
+	}
+
+	add( ...args ) {
+
+		const promise = this.func( ...args );
+		this.promises.push( promise );
+		promise.then( () => this.promises.splice( this.promises.indexOf( promise ), 1 ) );
+
+	}
+
+	async waitForAll() {
+
+		while ( this.promises.length > 0 ) {
+
+			await Promise.all( this.promises );
+
+		}
+
+	}
+
+}
+
 /* CONFIG VARIABLES START */
 
 const idleTime = 3; // 3 seconds - for how long there should be no network requests
@@ -28,6 +57,7 @@ const exceptionList = [
 	'webgl_worker_offscreencanvas', // in a worker, not robust
 
 	// TODO: most of these can be fixed just by increasing idleTime and parseTime
+	'webgl_buffergeometry_glbufferattribute',
 	'webgl_lensflares',
 	'webgl_lines_sphere',
 	'webgl_loader_imagebitmap',
@@ -36,6 +66,7 @@ const exceptionList = [
 	'webgl_morphtargets_face',
 	'webgl_nodes_materials_standard',
 	'webgl_postprocessing_crossfade',
+	'webgl_postprocessing_dof2',
 	'webgl_raymarching_reflect',
 	'webgl_renderer_pathtracer',
 	'webgl_shadowmap_progressive',
@@ -56,14 +87,18 @@ const LAST_REVISION_URLS = {
 
 const port = 1234;
 const pixelThreshold = 0.1; // threshold error in one pixel
-const maxFailedPixels = 0.05; // at most 5% failed pixels
+const maxFailedPixels = 0.05; // at most 5% different pixels
 
 const networkTimeout = 30; // 30 seconds, set to 0 to disable
 const renderTimeout = 1.5; // 1.5 seconds, set to 0 to disable
 
 const numAttempts = 3; // perform 3 progressive attempts before failing
 
-const numCIJobs = 8; // GitHub Actions run the script in 8 threads
+const numPages = 16; // use 16 browser pages
+
+const numCIJobs = 3; // GitHub Actions run the script in 3 threads
+
+const multiPageStrategy = process.platform !== 'darwin'; // whether to use multiple pages or multiple browsers, true if multiple pages
 
 const width = 400;
 const height = 250;
@@ -74,7 +109,7 @@ console.red = msg => console.log( chalk.red( msg ) );
 console.yellow = msg => console.log( chalk.yellow( msg ) );
 console.green = msg => console.log( chalk.green( msg ) );
 
-let browser;
+const browsers = [];
 
 /* Launch server */
 
@@ -131,47 +166,69 @@ async function main() {
 
 	const { executablePath } = await downloadLatestChromium();
 
-	/* Launch browser */
+	/* Launch browsers */
+
+	const numThreads = Math.min( numPages, files.length );
 
 	const flags = [ '--hide-scrollbars', '--enable-unsafe-webgpu' ];
 	flags.push( '--enable-features=Vulkan', '--use-gl=swiftshader', '--use-angle=swiftshader', '--use-vulkan=swiftshader', '--use-webgpu-adapter=swiftshader' );
 	// if ( process.platform === 'linux' ) flags.push( '--enable-features=Vulkan,UseSkiaRenderer', '--use-vulkan=native', '--disable-vulkan-surface', '--disable-features=VaapiVideoDecoder', '--ignore-gpu-blocklist', '--use-angle=vulkan' );
 
 	const viewport = { width: width * viewScale, height: height * viewScale };
 
-	browser = await puppeteer.launch( {
-		executablePath,
-		headless: ! process.env.VISIBLE,
-		args: flags,
-		defaultViewport: viewport,
-		handleSIGINT: false
-	} );
+	for ( let i = 0; i < multiPageStrategy ? 1 : numThreads; i++ ) {
 
-	// this line is intended to stop the script if the browser (in headful mode) is closed by user (while debugging)
-	// browser.on( 'targetdestroyed', target => ( target.type() === 'other' ) ? close() : null );
-	// for some reason it randomly stops the script after about ~30 screenshots processed
+		browsers.push( await puppeteer.launch( {
+			executablePath,
+			headless: ! process.env.VISIBLE,
+			args: flags,
+			defaultViewport: viewport,
+			handleSIGINT: false
+		} ) );
+
+		// this line is intended to stop the script if the browser (in headful mode) is closed by user (while debugging)
+		// browser.on( 'targetdestroyed', target => ( target.type() === 'other' ) ? close() : null );
+		// for some reason it randomly stops the script after about ~30 screenshots processed
+
+	}
 
 	/* Prepare injections */
 
 	const cleanPage = await fs.readFile( 'test/e2e/clean-page.js', 'utf8' );
 	const injection = await fs.readFile( 'test/e2e/deterministic-injection.js', 'utf8' );
 	const build = ( await fs.readFile( 'build/three.module.js', 'utf8' ) ).replace( /Math\.random\(\) \* 0xffffffff/g, 'Math._random() * 0xffffffff' );
 
-	/* Prepare page */
+	/* Prepare pages */
 
 	const errorMessagesCache = [];
 
-	const page = ( await browser.pages() )[ 0 ];
-	await preparePage( page, injection, build, errorMessagesCache );
+	let pages;
+
+	if ( multiPageStrategy ) {
+
+		const browser = browsers[ 0 ];
+		pages = await browser.pages();
+		while ( pages.length < numThreads ) pages.push( await browser.newPage() );
+
+	} else {
+
+		pages = browsers.map( browser => ( await browser.pages() )[ 0 ] );
+
+	}
+
+	for ( const page of pages ) await preparePage( page, injection, build, errorMessagesCache );
 
 	/* Loop for each file */
 
 	const failedScreenshots = [];
 
-	for ( const file of files ) await makeAttempt( page, failedScreenshots, cleanPage, isMakeScreenshot, file );
+	const queue = new PromiseQueue( makeAttempt, pages, failedScreenshots, cleanPage, isMakeScreenshot );
+	for ( const file of files ) queue.add( file );
+	await queue.waitForAll();
 
 	/* Finish */
 
+	failedScreenshots.sort();
 	const list = failedScreenshots.join( ' ' );
 
 	if ( isMakeScreenshot && failedScreenshots.length ) {
@@ -263,9 +320,9 @@ async function preparePage( page, injection, build, errorMessages ) {
 
 		text = file + ': ' + text.replace( /\[\.WebGL-(.+?)\] /g, '' );
 
-		if ( errorMessages.includes( text ) ) {
+		if ( text === `${ file }: JSHandle@error` ) {
 
-			return;
+			text = `${ file }: Unknown error`;
 
 		}
 
@@ -275,6 +332,12 @@ async function preparePage( page, injection, build, errorMessages ) {
 
 		}
 
+		if ( errorMessages.includes( text ) ) {
+
+			return;
+
+		}
+
 		errorMessages.push( text );
 
 		if ( type === 'warning' ) {
@@ -323,13 +386,33 @@ async function preparePage( page, injection, build, errorMessages ) {
 
 }
 
-async function makeAttempt( page, failedScreenshots, cleanPage, isMakeScreenshot, file, attemptID = 0 ) {
+async function makeAttempt( pages, failedScreenshots, cleanPage, isMakeScreenshot, file, attemptID = 0 ) {
 
 	const timeoutCoefficient = attemptID + 1;
 
+	const page = await new Promise( ( resolve, reject ) => {
+
+		const interval = setInterval( () => {
+
+			for ( const page of pages ) {
+
+				if ( page.file === undefined ) {
+
+					page.file = file; // acquire lock
+					clearInterval( interval );
+					resolve( page );
+					break;
+
+				}
+
+			}
+
+		}, 100 );
+
+	} );
+
 	try {
 
-		page.file = file;
 		page.pageSize = 0;
 		page.error = undefined;
 
@@ -403,7 +486,7 @@ async function makeAttempt( page, failedScreenshots, cleanPage, isMakeScreenshot
 
 				console.yellow( `Render timeout exceeded in file ${ file }` );
 
-			} */
+			} */ // TODO: fix this
 
 		}
 
@@ -482,20 +565,26 @@ async function makeAttempt( page, failedScreenshots, cleanPage, isMakeScreenshot
 
 		} else {
 
-			console.yellow( `${ e }, another attempt...` );
-			await makeAttempt( page, failedScreenshots, cleanPage, isMakeScreenshot, file, attemptID + 1 );
+			if ( ! e.message.includes( 'TimeoutError' ) ) { // TODO: fix this
+
+				console.yellow( `${ e }, another attempt...` );
+
+			}
+			this.add( file, attemptID + 1 );
 
 		}
 
 	}
 
+	page.file = undefined; // release lock
+
 }
 
 function close( exitCode = 1 ) {
 
 	console.log( 'Closing...' );
 
-	if ( browser !== undefined ) browser.close();
+	browsers.forEach( browser => browser.close() );
 	server.close();
 	process.exit( exitCode );