From a9d19c6a30f9da309008fc6cca108e91dc238c76 Mon Sep 17 00:00:00 2001 From: Ann Yuan Date: Tue, 8 Dec 2020 10:39:51 -0500 Subject: [PATCH 01/31] fix --- tfjs-backend-webgl/src/backend_webgl.ts | 16 ++++++++++++++++ tfjs-backend-webgl/src/texture_manager.ts | 7 ++++--- tfjs-backend-webgl/src/webgl.ts | 11 ++++++++++- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/tfjs-backend-webgl/src/backend_webgl.ts b/tfjs-backend-webgl/src/backend_webgl.ts index e7cf36b6e6..6a05e32bb7 100644 --- a/tfjs-backend-webgl/src/backend_webgl.ts +++ b/tfjs-backend-webgl/src/backend_webgl.ts @@ -166,6 +166,22 @@ export class MathBackendWebGL extends KernelBackend { this.pendingDeletes; } + writeTexture( + texture: WebGLTexture, shape: number[], dtype: DataType, + texShape: [number, number]): DataId { + const dataId = {}; + this.texData.set(dataId, { + shape, + dtype, + usage: TextureUsage.UPLOAD, + texture, + texShape, + refCount: 1, + complexParentRefCount: 0 + }); + return dataId; + } + write(values: BackendValues, shape: number[], dtype: DataType): DataId { if (env().getBool('WEBGL_CHECK_NUMERICAL_PROBLEMS') || env().getBool('DEBUG')) { diff --git a/tfjs-backend-webgl/src/texture_manager.ts b/tfjs-backend-webgl/src/texture_manager.ts index c7bc90f32c..f14a4b852a 100644 --- a/tfjs-backend-webgl/src/texture_manager.ts +++ b/tfjs-backend-webgl/src/texture_manager.ts @@ -119,9 +119,10 @@ export class TextureManager { const texList = this.usedTextures[shapeKey]; const texIndex = texList.indexOf(texture); if (texIndex < 0) { - throw new Error( - 'Cannot release a texture that was never provided by this ' + - 'texture manager'); + return; + // throw new Error( + // 'Cannot release a texture that was never provided by this ' + + // 'texture manager'); } texList.splice(texIndex, 1); this.log(); diff --git a/tfjs-backend-webgl/src/webgl.ts b/tfjs-backend-webgl/src/webgl.ts index 48b29b9ece..22275755fa 100644 --- a/tfjs-backend-webgl/src/webgl.ts +++ b/tfjs-backend-webgl/src/webgl.ts @@ -15,8 +15,9 @@ * ============================================================================= */ -import {env} from '@tensorflow/tfjs-core'; +import {engine, env, Tensor} from '@tensorflow/tfjs-core'; +import {MathBackendWebGL} from './backend_webgl'; import * as gpgpu_util from './gpgpu_util'; import * as webgl_util from './webgl_util'; @@ -35,3 +36,11 @@ export {gpgpu_util, webgl_util}; export function forceHalfFloat(): void { env().set('WEBGL_FORCE_F16_TEXTURES', true); } + +export function createTensorFromTexture( + texture: WebGLTexture, shape: number[], + texShape: [number, number]): Tensor { + const backend = engine().backend as MathBackendWebGL; + const dataId = backend.writeTexture(texture, shape, 'float32', texShape); + return engine().makeTensorFromDataId(dataId, shape, 'float32', backend); +} From 2b03c7e056fa079d595ca5256fe2e1da85789a3b Mon Sep 17 00:00:00 2001 From: Ann Yuan Date: Tue, 8 Dec 2020 11:38:45 -0500 Subject: [PATCH 02/31] fix --- tfjs-backend-webgl/src/backend_webgl.ts | 3 ++- tfjs-backend-webgl/src/texture_manager.ts | 31 ++++++++++++++++++++--- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/tfjs-backend-webgl/src/backend_webgl.ts b/tfjs-backend-webgl/src/backend_webgl.ts index 6a05e32bb7..b7b79c33b4 100644 --- a/tfjs-backend-webgl/src/backend_webgl.ts +++ b/tfjs-backend-webgl/src/backend_webgl.ts @@ -173,12 +173,13 @@ export class MathBackendWebGL extends KernelBackend { this.texData.set(dataId, { shape, dtype, - usage: TextureUsage.UPLOAD, + usage: TextureUsage.RENDER, texture, texShape, refCount: 1, complexParentRefCount: 0 }); + this.textureManager.registerRenderTexture(texture, texShape); return dataId; } diff --git a/tfjs-backend-webgl/src/texture_manager.ts b/tfjs-backend-webgl/src/texture_manager.ts index f14a4b852a..c326c24adb 100644 --- a/tfjs-backend-webgl/src/texture_manager.ts +++ b/tfjs-backend-webgl/src/texture_manager.ts @@ -33,6 +33,30 @@ export class TextureManager { constructor(private gpgpu: GPGPUContext) {} + registerRenderTexture(texture: WebGLTexture, shape: [number, number]) { + const usage = TextureUsage.RENDER; + const isPacked = false; + const physicalTexType = getPhysicalFromLogicalTextureType(usage, isPacked); + + const shapeKey = getKeyFromTextureShape(shape, physicalTexType, isPacked); + if (!(shapeKey in this.freeTextures)) { + this.freeTextures[shapeKey] = []; + } + if (!(shapeKey in this.usedTextures)) { + this.usedTextures[shapeKey] = []; + } + + const texBytes = computeBytes( + shape, physicalTexType, this.gpgpu.gl, this.gpgpu.textureConfig, + isPacked); + + this.usedTextures[shapeKey].push(texture); + + this.numUsedTextures++; + this._numBytesAllocated += texBytes; + this.log(); + } + acquireTexture( shapeRC: [number, number], usage: TextureUsage, isPacked: boolean): WebGLTexture { @@ -119,10 +143,9 @@ export class TextureManager { const texList = this.usedTextures[shapeKey]; const texIndex = texList.indexOf(texture); if (texIndex < 0) { - return; - // throw new Error( - // 'Cannot release a texture that was never provided by this ' + - // 'texture manager'); + throw new Error( + 'Cannot release a texture that was never provided by this ' + + 'texture manager'); } texList.splice(texIndex, 1); this.log(); From 2158e76d7c1d2b0753993b56390ced04f4f3834f Mon Sep 17 00:00:00 2001 From: Ann Yuan Date: Tue, 8 Dec 2020 12:26:10 -0500 Subject: [PATCH 03/31] add shelll --- tfjs-backend-webgl/src/backend_webgl_test.ts | 80 +++++++++++++++++--- 1 file changed, 70 insertions(+), 10 deletions(-) diff --git a/tfjs-backend-webgl/src/backend_webgl_test.ts b/tfjs-backend-webgl/src/backend_webgl_test.ts index 3abb9d7f78..af92215978 100644 --- a/tfjs-backend-webgl/src/backend_webgl_test.ts +++ b/tfjs-backend-webgl/src/backend_webgl_test.ts @@ -32,6 +32,76 @@ function decodeStrings(bytes: Uint8Array[]): string[] { return bytes.map(b => decodeString(b)); } +const WEBGL1_ENVS = { + flags: {'WEBGL_VERSION': 1}, + predicate: WEBGL_ENVS.predicate +}; + +const WEBGL2_ENVS = { + flags: {'WEBGL_VERSION': 2}, + predicate: WEBGL_ENVS.predicate +}; + +describeWithFlags('create tensor from texture', WEBGL2_ENVS, () => { + let gpgpu: GPGPUContext; + + beforeEach(() => { + gpgpu = new GPGPUContext(); + }); + + afterEach(() => { + gpgpu.dispose(); + }); + + fit('basic f32', () => { + // Create a texture. + const gl = gpgpu.gl; + const texture = gl.createTexture(); + const tex2d = gl.TEXTURE_2D; + gl.bindTexture(tex2d, texture); + gl.texParameteri(tex2d, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(tex2d, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + gl.texParameteri(tex2d, gl.TEXTURE_MIN_FILTER, gl.NEAREST); + gl.texParameteri(tex2d, gl.TEXTURE_MAG_FILTER, gl.NEAREST); + gl.texImage2D( + tex2d, 0, internalFormat, width, height, 0, textureFormat, textureType, + null); + + // Create tensor from texture. + + // Perform basic operation on texture. + + // Verify that the result is correct. + }); + + fit('basic f16', + () => { + + }); +}); + +describeWithFlags('create tensor from texture', WEBGL1_ENVS, () => { + let gpgpu: GPGPUContext; + + beforeEach(() => { + gpgpu = new GPGPUContext(); + }); + + afterEach(() => { + gpgpu.dispose(); + }); + + fit('basic f32', + () => { + + }); + + fit('f16', + () => { + + }); +}); + const RENDER_FLOAT32_ENVS = { flags: {'WEBGL_RENDER_FLOAT32_ENABLED': true}, predicate: WEBGL_ENVS.predicate @@ -442,16 +512,6 @@ describeWithFlags('debug on webgl', WEBGL_ENVS, () => { }); }); -const WEBGL1_ENVS = { - flags: {'WEBGL_VERSION': 1}, - predicate: WEBGL_ENVS.predicate -}; - -const WEBGL2_ENVS = { - flags: {'WEBGL_VERSION': 2}, - predicate: WEBGL_ENVS.predicate -}; - describeWithFlags('computeBytes counts bytes correctly', WEBGL1_ENVS, () => { it('for all physical texture types', () => { const gpgpu = new GPGPUContext(); From b67ebf485412ab699f91628295c1528dca2e1649 Mon Sep 17 00:00:00 2001 From: Ann Yuan Date: Wed, 9 Dec 2020 09:59:18 -0500 Subject: [PATCH 04/31] poc --- tfjs-backend-webgl/src/backend_webgl.ts | 1 + tfjs-backend-webgl/src/backend_webgl_test.ts | 25 +++++++++++++------- tfjs-backend-webgl/src/base.ts | 7 ++++-- tfjs-backend-webgl/src/texture_manager.ts | 6 ++--- tfjs-backend-webgl/src/webgl.ts | 4 ++-- 5 files changed, 27 insertions(+), 16 deletions(-) diff --git a/tfjs-backend-webgl/src/backend_webgl.ts b/tfjs-backend-webgl/src/backend_webgl.ts index b7b79c33b4..ad66880bde 100644 --- a/tfjs-backend-webgl/src/backend_webgl.ts +++ b/tfjs-backend-webgl/src/backend_webgl.ts @@ -176,6 +176,7 @@ export class MathBackendWebGL extends KernelBackend { usage: TextureUsage.RENDER, texture, texShape, + isPacked: false, refCount: 1, complexParentRefCount: 0 }); diff --git a/tfjs-backend-webgl/src/backend_webgl_test.ts b/tfjs-backend-webgl/src/backend_webgl_test.ts index af92215978..c119215388 100644 --- a/tfjs-backend-webgl/src/backend_webgl_test.ts +++ b/tfjs-backend-webgl/src/backend_webgl_test.ts @@ -55,9 +55,16 @@ describeWithFlags('create tensor from texture', WEBGL2_ENVS, () => { fit('basic f32', () => { // Create a texture. + const width = 3; + const height = 4; + const gl = gpgpu.gl; const texture = gl.createTexture(); const tex2d = gl.TEXTURE_2D; + const internalFormat = (gl as any).R32F; + const textureFormat = (gl as any).RED; + const textureType = (gl as any).FLOAT; + gl.bindTexture(tex2d, texture); gl.texParameteri(tex2d, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); gl.texParameteri(tex2d, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); @@ -74,10 +81,10 @@ describeWithFlags('create tensor from texture', WEBGL2_ENVS, () => { // Verify that the result is correct. }); - fit('basic f16', - () => { + // fit('basic f16', + // () => { - }); + // }); }); describeWithFlags('create tensor from texture', WEBGL1_ENVS, () => { @@ -91,15 +98,15 @@ describeWithFlags('create tensor from texture', WEBGL1_ENVS, () => { gpgpu.dispose(); }); - fit('basic f32', - () => { + // fit('basic f32', + // () => { - }); + // }); - fit('f16', - () => { + // fit('f16', + // () => { - }); + // }); }); const RENDER_FLOAT32_ENVS = { diff --git a/tfjs-backend-webgl/src/base.ts b/tfjs-backend-webgl/src/base.ts index f2412eab16..3e07813623 100644 --- a/tfjs-backend-webgl/src/base.ts +++ b/tfjs-backend-webgl/src/base.ts @@ -29,5 +29,8 @@ if (device_util.isBrowser()) { export * from './webgl'; // Export forceHalfFlost under webgl namespace for the union bundle. -import {forceHalfFloat} from './webgl'; -export const webgl = {forceHalfFloat}; +import {forceHalfFloat, createTensorFromTexture} from './webgl'; +export const webgl = { + forceHalfFloat, + createTensorFromTexture +}; diff --git a/tfjs-backend-webgl/src/texture_manager.ts b/tfjs-backend-webgl/src/texture_manager.ts index c326c24adb..baa26bc000 100644 --- a/tfjs-backend-webgl/src/texture_manager.ts +++ b/tfjs-backend-webgl/src/texture_manager.ts @@ -33,12 +33,12 @@ export class TextureManager { constructor(private gpgpu: GPGPUContext) {} - registerRenderTexture(texture: WebGLTexture, shape: [number, number]) { + registerRenderTexture(texture: WebGLTexture, shapeRC: [number, number]) { const usage = TextureUsage.RENDER; const isPacked = false; const physicalTexType = getPhysicalFromLogicalTextureType(usage, isPacked); - const shapeKey = getKeyFromTextureShape(shape, physicalTexType, isPacked); + const shapeKey = getKeyFromTextureShape(shapeRC, physicalTexType, isPacked); if (!(shapeKey in this.freeTextures)) { this.freeTextures[shapeKey] = []; } @@ -47,7 +47,7 @@ export class TextureManager { } const texBytes = computeBytes( - shape, physicalTexType, this.gpgpu.gl, this.gpgpu.textureConfig, + shapeRC, physicalTexType, this.gpgpu.gl, this.gpgpu.textureConfig, isPacked); this.usedTextures[shapeKey].push(texture); diff --git a/tfjs-backend-webgl/src/webgl.ts b/tfjs-backend-webgl/src/webgl.ts index 22275755fa..60dd034865 100644 --- a/tfjs-backend-webgl/src/webgl.ts +++ b/tfjs-backend-webgl/src/webgl.ts @@ -39,8 +39,8 @@ export function forceHalfFloat(): void { export function createTensorFromTexture( texture: WebGLTexture, shape: number[], - texShape: [number, number]): Tensor { + texShapeRC: [number, number]): Tensor { const backend = engine().backend as MathBackendWebGL; - const dataId = backend.writeTexture(texture, shape, 'float32', texShape); + const dataId = backend.writeTexture(texture, shape, 'float32', texShapeRC); return engine().makeTensorFromDataId(dataId, shape, 'float32', backend); } From 60863c0d33a89bf481e8767c9f2a94b1c5bc34e6 Mon Sep 17 00:00:00 2001 From: Ann Yuan Date: Wed, 9 Dec 2020 10:12:49 -0500 Subject: [PATCH 05/31] fix --- tfjs-backend-webgl/src/backend_webgl_test.ts | 29 ++++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/tfjs-backend-webgl/src/backend_webgl_test.ts b/tfjs-backend-webgl/src/backend_webgl_test.ts index c119215388..1bd647ea74 100644 --- a/tfjs-backend-webgl/src/backend_webgl_test.ts +++ b/tfjs-backend-webgl/src/backend_webgl_test.ts @@ -21,6 +21,7 @@ import {engine, test_util, util} from '@tensorflow/tfjs-core'; import {describeWithFlags} from '@tensorflow/tfjs-core/dist/jasmine_util'; const {expectArraysClose, expectArraysEqual} = test_util; const {decodeString} = util; +import {createTensorFromTexture} from './base'; import {getBinaryCache, MathBackendWebGL, WebGLMemoryInfo, WebGLTimingInfo} from './backend_webgl'; import {computeBytes} from './texture_manager'; @@ -42,6 +43,11 @@ const WEBGL2_ENVS = { predicate: WEBGL_ENVS.predicate }; +const RENDER_FLOAT32_ENVS = { + flags: {'WEBGL_RENDER_FLOAT32_ENABLED': true}, + predicate: WEBGL_ENVS.predicate +}; + describeWithFlags('create tensor from texture', WEBGL2_ENVS, () => { let gpgpu: GPGPUContext; @@ -53,8 +59,7 @@ describeWithFlags('create tensor from texture', WEBGL2_ENVS, () => { gpgpu.dispose(); }); - fit('basic f32', () => { - // Create a texture. + fit('basic f32', async () => { const width = 3; const height = 4; @@ -64,6 +69,8 @@ describeWithFlags('create tensor from texture', WEBGL2_ENVS, () => { const internalFormat = (gl as any).R32F; const textureFormat = (gl as any).RED; const textureType = (gl as any).FLOAT; + const dataForUpload = + new Float32Array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]); gl.bindTexture(tex2d, texture); gl.texParameteri(tex2d, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); @@ -72,13 +79,16 @@ describeWithFlags('create tensor from texture', WEBGL2_ENVS, () => { gl.texParameteri(tex2d, gl.TEXTURE_MAG_FILTER, gl.NEAREST); gl.texImage2D( tex2d, 0, internalFormat, width, height, 0, textureFormat, textureType, - null); - - // Create tensor from texture. + dataForUpload); - // Perform basic operation on texture. + const logicalShape = [height, width]; + const physicalShape: [number, number] = [height, width]; + const a = createTensorFromTexture(texture, logicalShape, physicalShape); + const b = tf.mul(a, 2); - // Verify that the result is correct. + expect(b.shape).toEqual(logicalShape); + expectArraysClose( + await b.data(), [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22]); }); // fit('basic f16', @@ -109,11 +119,6 @@ describeWithFlags('create tensor from texture', WEBGL1_ENVS, () => { // }); }); -const RENDER_FLOAT32_ENVS = { - flags: {'WEBGL_RENDER_FLOAT32_ENABLED': true}, - predicate: WEBGL_ENVS.predicate -}; - describeWithFlags('forced f16 render', RENDER_FLOAT32_ENVS, () => { beforeAll(() => { tf.env().set('WEBGL_RENDER_FLOAT32_ENABLED', false); From e2a21cb509202e7c20727a0fa5c85734460ec0a3 Mon Sep 17 00:00:00 2001 From: Ann Yuan Date: Wed, 9 Dec 2020 10:24:50 -0500 Subject: [PATCH 06/31] fit --- tfjs-backend-webgl/src/backend_webgl_test.ts | 66 +++++++++++--------- tfjs-backend-webgl/src/base.ts | 3 +- 2 files changed, 37 insertions(+), 32 deletions(-) diff --git a/tfjs-backend-webgl/src/backend_webgl_test.ts b/tfjs-backend-webgl/src/backend_webgl_test.ts index 1bd647ea74..8d626cfcbe 100644 --- a/tfjs-backend-webgl/src/backend_webgl_test.ts +++ b/tfjs-backend-webgl/src/backend_webgl_test.ts @@ -48,18 +48,26 @@ const RENDER_FLOAT32_ENVS = { predicate: WEBGL_ENVS.predicate }; -describeWithFlags('create tensor from texture', WEBGL2_ENVS, () => { - let gpgpu: GPGPUContext; - - beforeEach(() => { - gpgpu = new GPGPUContext(); - }); - - afterEach(() => { - gpgpu.dispose(); - }); +const WEBGL_2_F32_ENVS = { + flags: {'WEBGL_VERSION': 2, 'WEBGL_RENDER_FLOAT32_ENABLED': true}, + predicate: WEBGL_ENVS.predicate +}; +const WEBGL_2_F16_ENVS = { + flags: {'WEBGL_VERSION': 2, 'WEBGL_RENDER_FLOAT32_ENABLED': false}, + predicate: WEBGL_ENVS.predicate +}; +const WEBGL_1_F32_ENVS = { + flags: {'WEBGL_VERSION': 1, 'WEBGL_RENDER_FLOAT32_ENABLED': true}, + predicate: WEBGL_ENVS.predicate +}; +const WEBGL_1_F16_ENVS = { + flags: {'WEBGL_VERSION': 1, 'WEBGL_RENDER_FLOAT32_ENABLED': false}, + predicate: WEBGL_ENVS.predicate +}; - fit('basic f32', async () => { +describeWithFlags('create tensor from texture', WEBGL_2_F32_ENVS, () => { + fit('basic usage', async () => { + const gpgpu = new GPGPUContext(); const width = 3; const height = 4; @@ -86,37 +94,33 @@ describeWithFlags('create tensor from texture', WEBGL2_ENVS, () => { const a = createTensorFromTexture(texture, logicalShape, physicalShape); const b = tf.mul(a, 2); + gpgpu.dispose(); + expect(b.shape).toEqual(logicalShape); expectArraysClose( await b.data(), [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22]); }); - - // fit('basic f16', - // () => { - - // }); }); -describeWithFlags('create tensor from texture', WEBGL1_ENVS, () => { - let gpgpu: GPGPUContext; - - beforeEach(() => { - gpgpu = new GPGPUContext(); - }); +describeWithFlags('create tensor from texture', WEBGL_2_F16_ENVS, () => { + fit('basic usage', + async () => { - afterEach(() => { - gpgpu.dispose(); - }); + }); +}); - // fit('basic f32', - // () => { +describeWithFlags('create tensor from texture', WEBGL_1_F32_ENVS, () => { + fit('basic usage', + async () => { - // }); + }); +}); - // fit('f16', - // () => { +describeWithFlags('create tensor from texture', WEBGL_1_F16_ENVS, () => { + fit('basic usage', + async () => { - // }); + }); }); describeWithFlags('forced f16 render', RENDER_FLOAT32_ENVS, () => { diff --git a/tfjs-backend-webgl/src/base.ts b/tfjs-backend-webgl/src/base.ts index 3e07813623..50a2807a08 100644 --- a/tfjs-backend-webgl/src/base.ts +++ b/tfjs-backend-webgl/src/base.ts @@ -28,7 +28,8 @@ if (device_util.isBrowser()) { // Export webgl utilities export * from './webgl'; -// Export forceHalfFlost under webgl namespace for the union bundle. +// Export forceHalfFloat and createTensorFromTexture under webgl namespace for +// the union bundle. import {forceHalfFloat, createTensorFromTexture} from './webgl'; export const webgl = { forceHalfFloat, From 0beab1661fea16fe4057252eda1b1119d775576a Mon Sep 17 00:00:00 2001 From: Ann Yuan Date: Wed, 9 Dec 2020 10:55:01 -0500 Subject: [PATCH 07/31] add basic --- tfjs-backend-webgl/src/backend_webgl_test.ts | 108 +++++++++++++++++-- 1 file changed, 99 insertions(+), 9 deletions(-) diff --git a/tfjs-backend-webgl/src/backend_webgl_test.ts b/tfjs-backend-webgl/src/backend_webgl_test.ts index 8d626cfcbe..11aee6f400 100644 --- a/tfjs-backend-webgl/src/backend_webgl_test.ts +++ b/tfjs-backend-webgl/src/backend_webgl_test.ts @@ -103,24 +103,114 @@ describeWithFlags('create tensor from texture', WEBGL_2_F32_ENVS, () => { }); describeWithFlags('create tensor from texture', WEBGL_2_F16_ENVS, () => { - fit('basic usage', - async () => { + fit('basic usage', async () => { + const gpgpu = new GPGPUContext(); + const width = 3; + const height = 4; + + const gl = gpgpu.gl; + const texture = gl.createTexture(); + const tex2d = gl.TEXTURE_2D; + const internalFormat = (gl as any).R16F; + const textureFormat = (gl as any).RED; + const textureType = (gl as any).HALF_FLOAT; + const dataForUpload = + new Float32Array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]); + + gl.bindTexture(tex2d, texture); + gl.texParameteri(tex2d, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(tex2d, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + gl.texParameteri(tex2d, gl.TEXTURE_MIN_FILTER, gl.NEAREST); + gl.texParameteri(tex2d, gl.TEXTURE_MAG_FILTER, gl.NEAREST); + gl.texImage2D( + tex2d, 0, internalFormat, width, height, 0, textureFormat, textureType, + dataForUpload); + + const logicalShape = [height, width]; + const physicalShape: [number, number] = [height, width]; + const a = createTensorFromTexture(texture, logicalShape, physicalShape); + const b = tf.mul(a, 2); + + gpgpu.dispose(); - }); + expect(b.shape).toEqual(logicalShape); + expectArraysClose( + await b.data(), [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22]); + }); }); describeWithFlags('create tensor from texture', WEBGL_1_F32_ENVS, () => { - fit('basic usage', - async () => { + fit('basic usage', async () => { + const gpgpu = new GPGPUContext(); + const width = 3; + const height = 4; - }); + const gl = gpgpu.gl; + const texture = gl.createTexture(); + const tex2d = gl.TEXTURE_2D; + const internalFormat = gl.RGBA; + const textureFormat = gl.RGBA; + const textureType = gl.FLOAT; + const dataForUpload = + new Float32Array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]); + + gl.bindTexture(tex2d, texture); + gl.texParameteri(tex2d, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(tex2d, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + gl.texParameteri(tex2d, gl.TEXTURE_MIN_FILTER, gl.NEAREST); + gl.texParameteri(tex2d, gl.TEXTURE_MAG_FILTER, gl.NEAREST); + gl.texImage2D( + tex2d, 0, internalFormat, width, height, 0, textureFormat, textureType, + dataForUpload); + + const logicalShape = [height, width]; + const physicalShape: [number, number] = [height, width]; + const a = createTensorFromTexture(texture, logicalShape, physicalShape); + const b = tf.mul(a, 2); + + gpgpu.dispose(); + + expect(b.shape).toEqual(logicalShape); + expectArraysClose( + await b.data(), [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22]); + }); }); describeWithFlags('create tensor from texture', WEBGL_1_F16_ENVS, () => { - fit('basic usage', - async () => { + fit('basic usage', async () => { + const gpgpu = new GPGPUContext(); + const width = 3; + const height = 4; - }); + const gl = gpgpu.gl; + const texture = gl.createTexture(); + const tex2d = gl.TEXTURE_2D; + const internalFormat = gl.RGBA; + const textureFormat = gl.RGBA; + const textureType = gl.FLOAT; + const dataForUpload = + new Float32Array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]); + + gl.bindTexture(tex2d, texture); + gl.texParameteri(tex2d, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(tex2d, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + gl.texParameteri(tex2d, gl.TEXTURE_MIN_FILTER, gl.NEAREST); + gl.texParameteri(tex2d, gl.TEXTURE_MAG_FILTER, gl.NEAREST); + gl.texImage2D( + tex2d, 0, internalFormat, width, height, 0, textureFormat, textureType, + dataForUpload); + + const logicalShape = [height, width]; + const physicalShape: [number, number] = [height, width]; + const a = createTensorFromTexture(texture, logicalShape, physicalShape); + const b = tf.mul(a, 2); + + gpgpu.dispose(); + + expect(b.shape).toEqual(logicalShape); + expectArraysClose( + await b.data(), [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22]); + }); }); describeWithFlags('forced f16 render', RENDER_FLOAT32_ENVS, () => { From 60662905db9db96161c7cac2fd52f45461973d1c Mon Sep 17 00:00:00 2001 From: Ann Yuan Date: Thu, 10 Dec 2020 08:37:04 -0500 Subject: [PATCH 08/31] fix test --- tfjs-backend-webgl/src/backend_webgl_test.ts | 80 ++++++++------------ tsconfig.json | 2 +- 2 files changed, 34 insertions(+), 48 deletions(-) diff --git a/tfjs-backend-webgl/src/backend_webgl_test.ts b/tfjs-backend-webgl/src/backend_webgl_test.ts index 11aee6f400..01072c48aa 100644 --- a/tfjs-backend-webgl/src/backend_webgl_test.ts +++ b/tfjs-backend-webgl/src/backend_webgl_test.ts @@ -48,14 +48,6 @@ const RENDER_FLOAT32_ENVS = { predicate: WEBGL_ENVS.predicate }; -const WEBGL_2_F32_ENVS = { - flags: {'WEBGL_VERSION': 2, 'WEBGL_RENDER_FLOAT32_ENABLED': true}, - predicate: WEBGL_ENVS.predicate -}; -const WEBGL_2_F16_ENVS = { - flags: {'WEBGL_VERSION': 2, 'WEBGL_RENDER_FLOAT32_ENABLED': false}, - predicate: WEBGL_ENVS.predicate -}; const WEBGL_1_F32_ENVS = { flags: {'WEBGL_VERSION': 1, 'WEBGL_RENDER_FLOAT32_ENABLED': true}, predicate: WEBGL_ENVS.predicate @@ -65,8 +57,8 @@ const WEBGL_1_F16_ENVS = { predicate: WEBGL_ENVS.predicate }; -describeWithFlags('create tensor from texture', WEBGL_2_F32_ENVS, () => { - fit('basic usage', async () => { +describeWithFlags('create tensor from texture', WEBGL2_ENVS, () => { + it('basic usage', async () => { const gpgpu = new GPGPUContext(); const width = 3; const height = 4; @@ -94,53 +86,45 @@ describeWithFlags('create tensor from texture', WEBGL_2_F32_ENVS, () => { const a = createTensorFromTexture(texture, logicalShape, physicalShape); const b = tf.mul(a, 2); - gpgpu.dispose(); - expect(b.shape).toEqual(logicalShape); expectArraysClose( await b.data(), [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22]); + + gpgpu.dispose(); }); -}); -describeWithFlags('create tensor from texture', WEBGL_2_F16_ENVS, () => { - fit('basic usage', async () => { - const gpgpu = new GPGPUContext(); + it('force f16', async () => { + // Unlike in the F32 test, rather than creating a texture from scratch, we + // must extract the output texture from an operation because we cannot + // upload Float16 data directly to the GPU. + const webglForceF16FlagSaved = tf.env().getBool('WEBGL_FORCE_F16_TEXTURES'); + const webglPackedFlagSaved = tf.env().getBool('WEBGL_PACK'); + tf.env().set('WEBGL_FORCE_F16_TEXTURES', true); + tf.env().set( + 'WEBGL_PACK', false); // so the output of relu is an unpacked texture. + const width = 3; const height = 4; - const gl = gpgpu.gl; - const texture = gl.createTexture(); - const tex2d = gl.TEXTURE_2D; - const internalFormat = (gl as any).R16F; - const textureFormat = (gl as any).RED; - const textureType = (gl as any).HALF_FLOAT; - const dataForUpload = - new Float32Array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]); - - gl.bindTexture(tex2d, texture); - gl.texParameteri(tex2d, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); - gl.texParameteri(tex2d, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); - gl.texParameteri(tex2d, gl.TEXTURE_MIN_FILTER, gl.NEAREST); - gl.texParameteri(tex2d, gl.TEXTURE_MAG_FILTER, gl.NEAREST); - gl.texImage2D( - tex2d, 0, internalFormat, width, height, 0, textureFormat, textureType, - dataForUpload); - - const logicalShape = [height, width]; + const logicalShape: [number, number] = [height, width]; const physicalShape: [number, number] = [height, width]; - const a = createTensorFromTexture(texture, logicalShape, physicalShape); - const b = tf.mul(a, 2); - - gpgpu.dispose(); + const a = tf.tensor2d([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], logicalShape); + const b = tf.relu(a); + const bTexture = (tf.backend() as MathBackendWebGL).getTexture(b.dataId); + const c = createTensorFromTexture(bTexture, logicalShape, physicalShape); + const d = tf.mul(c, 2); - expect(b.shape).toEqual(logicalShape); + expect(d.shape).toEqual(logicalShape); expectArraysClose( - await b.data(), [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22]); + await d.data(), [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22]); + + tf.env().set('WEBGL_FORCE_F16_TEXTURES', webglForceF16FlagSaved); + tf.env().set('WEBGL_PACK', webglPackedFlagSaved); }); }); describeWithFlags('create tensor from texture', WEBGL_1_F32_ENVS, () => { - fit('basic usage', async () => { + it('basic usage', async () => { const gpgpu = new GPGPUContext(); const width = 3; const height = 4; @@ -167,17 +151,18 @@ describeWithFlags('create tensor from texture', WEBGL_1_F32_ENVS, () => { const physicalShape: [number, number] = [height, width]; const a = createTensorFromTexture(texture, logicalShape, physicalShape); const b = tf.mul(a, 2); + b.print(); gpgpu.dispose(); expect(b.shape).toEqual(logicalShape); - expectArraysClose( - await b.data(), [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22]); + // expectArraysClose( + // await b.data(), [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22]); }); }); describeWithFlags('create tensor from texture', WEBGL_1_F16_ENVS, () => { - fit('basic usage', async () => { + it('basic usage', async () => { const gpgpu = new GPGPUContext(); const width = 3; const height = 4; @@ -204,12 +189,13 @@ describeWithFlags('create tensor from texture', WEBGL_1_F16_ENVS, () => { const physicalShape: [number, number] = [height, width]; const a = createTensorFromTexture(texture, logicalShape, physicalShape); const b = tf.mul(a, 2); + b.print(); gpgpu.dispose(); expect(b.shape).toEqual(logicalShape); - expectArraysClose( - await b.data(), [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22]); + // expectArraysClose( + // await b.data(), [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22]); }); }); diff --git a/tsconfig.json b/tsconfig.json index a261f8b6f5..4347043a17 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -22,7 +22,7 @@ "pretty": true, "noFallthroughCasesInSwitch": true, "allowUnreachableCode": false, - "incremental": true + "incremental": false }, "include": [ "src/" From 3a3d995c9f6cbca253a3ec5e2452a2d4861acce5 Mon Sep 17 00:00:00 2001 From: Ann Yuan Date: Thu, 10 Dec 2020 09:33:00 -0500 Subject: [PATCH 09/31] comm --- tfjs-backend-webgl/src/webgl.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tfjs-backend-webgl/src/webgl.ts b/tfjs-backend-webgl/src/webgl.ts index 60dd034865..e6ee11088e 100644 --- a/tfjs-backend-webgl/src/webgl.ts +++ b/tfjs-backend-webgl/src/webgl.ts @@ -44,3 +44,16 @@ export function createTensorFromTexture( const dataId = backend.writeTexture(texture, shape, 'float32', texShapeRC); return engine().makeTensorFromDataId(dataId, shape, 'float32', backend); } + +// type TensorFromTextureConfig = { +// texture: WebGLTexture, +// shape: number[], +// texShapeRC: [number, number], +// internalFormat: any, +// textureFormat: any, +// textureType: any +// }; + +// export function createTensorFromTexture2(): Tensor { + +// } From 945a12eef423e865ff0154e73910568f090c1c35 Mon Sep 17 00:00:00 2001 From: Ann Yuan Date: Thu, 10 Dec 2020 14:25:39 -0500 Subject: [PATCH 10/31] add test --- tfjs-backend-webgl/src/backend_webgl_test.ts | 21 ++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/tfjs-backend-webgl/src/backend_webgl_test.ts b/tfjs-backend-webgl/src/backend_webgl_test.ts index 01072c48aa..17a8c60e00 100644 --- a/tfjs-backend-webgl/src/backend_webgl_test.ts +++ b/tfjs-backend-webgl/src/backend_webgl_test.ts @@ -124,19 +124,24 @@ describeWithFlags('create tensor from texture', WEBGL2_ENVS, () => { }); describeWithFlags('create tensor from texture', WEBGL_1_F32_ENVS, () => { - it('basic usage', async () => { + fit('basic usage', async () => { const gpgpu = new GPGPUContext(); const width = 3; const height = 4; + const logicalShape: [number, number] = [height, width]; + const physicalShape: [number, number] = [height, width]; + const gl = gpgpu.gl; const texture = gl.createTexture(); const tex2d = gl.TEXTURE_2D; const internalFormat = gl.RGBA; const textureFormat = gl.RGBA; const textureType = gl.FLOAT; - const dataForUpload = - new Float32Array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]); + const dataForUpload = new Float32Array([ + 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 5, 0, 0, 0, + 6, 0, 0, 0, 7, 0, 0, 0, 8, 0, 0, 0, 9, 0, 0, 0, 10, 0, 0, 0, 11, 0, 0, 0, + ]); gl.bindTexture(tex2d, texture); gl.texParameteri(tex2d, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); @@ -147,17 +152,13 @@ describeWithFlags('create tensor from texture', WEBGL_1_F32_ENVS, () => { tex2d, 0, internalFormat, width, height, 0, textureFormat, textureType, dataForUpload); - const logicalShape = [height, width]; - const physicalShape: [number, number] = [height, width]; const a = createTensorFromTexture(texture, logicalShape, physicalShape); const b = tf.mul(a, 2); - b.print(); - - gpgpu.dispose(); expect(b.shape).toEqual(logicalShape); - // expectArraysClose( - // await b.data(), [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22]); + expectArraysClose( + await b.data(), [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22]); + gpgpu.dispose(); }); }); From c7ed1a343e67d20f74deec41b1cb02c36985f5ff Mon Sep 17 00:00:00 2001 From: Ann Yuan Date: Thu, 10 Dec 2020 14:59:46 -0500 Subject: [PATCH 11/31] fix --- tfjs-backend-webgl/src/backend_webgl_test.ts | 63 +++++++------------- 1 file changed, 21 insertions(+), 42 deletions(-) diff --git a/tfjs-backend-webgl/src/backend_webgl_test.ts b/tfjs-backend-webgl/src/backend_webgl_test.ts index 17a8c60e00..5aa6012c76 100644 --- a/tfjs-backend-webgl/src/backend_webgl_test.ts +++ b/tfjs-backend-webgl/src/backend_webgl_test.ts @@ -48,15 +48,6 @@ const RENDER_FLOAT32_ENVS = { predicate: WEBGL_ENVS.predicate }; -const WEBGL_1_F32_ENVS = { - flags: {'WEBGL_VERSION': 1, 'WEBGL_RENDER_FLOAT32_ENABLED': true}, - predicate: WEBGL_ENVS.predicate -}; -const WEBGL_1_F16_ENVS = { - flags: {'WEBGL_VERSION': 1, 'WEBGL_RENDER_FLOAT32_ENABLED': false}, - predicate: WEBGL_ENVS.predicate -}; - describeWithFlags('create tensor from texture', WEBGL2_ENVS, () => { it('basic usage', async () => { const gpgpu = new GPGPUContext(); @@ -97,11 +88,15 @@ describeWithFlags('create tensor from texture', WEBGL2_ENVS, () => { // Unlike in the F32 test, rather than creating a texture from scratch, we // must extract the output texture from an operation because we cannot // upload Float16 data directly to the GPU. + const webglForceF16FlagSaved = tf.env().getBool('WEBGL_FORCE_F16_TEXTURES'); const webglPackedFlagSaved = tf.env().getBool('WEBGL_PACK'); tf.env().set('WEBGL_FORCE_F16_TEXTURES', true); - tf.env().set( - 'WEBGL_PACK', false); // so the output of relu is an unpacked texture. + + // We must set `WEBGL_PACK` to false because createTensorFromTexture only + // accepts unpacked textures so this ensures the output texture (`bTexture` + // below) is unpacked. + tf.env().set('WEBGL_PACK', false); const width = 3; const height = 4; @@ -123,7 +118,7 @@ describeWithFlags('create tensor from texture', WEBGL2_ENVS, () => { }); }); -describeWithFlags('create tensor from texture', WEBGL_1_F32_ENVS, () => { +describeWithFlags('create tensor from texture', WEBGL1_ENVS, () => { fit('basic usage', async () => { const gpgpu = new GPGPUContext(); const width = 3; @@ -160,43 +155,27 @@ describeWithFlags('create tensor from texture', WEBGL_1_F32_ENVS, () => { await b.data(), [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22]); gpgpu.dispose(); }); -}); -describeWithFlags('create tensor from texture', WEBGL_1_F16_ENVS, () => { - it('basic usage', async () => { - const gpgpu = new GPGPUContext(); + it('chained', async () => { + const webglPackedFlagSaved = tf.env().getBool('WEBGL_PACK'); + tf.env().set('WEBGL_PACK', false); + const width = 3; const height = 4; - const gl = gpgpu.gl; - const texture = gl.createTexture(); - const tex2d = gl.TEXTURE_2D; - const internalFormat = gl.RGBA; - const textureFormat = gl.RGBA; - const textureType = gl.FLOAT; - const dataForUpload = - new Float32Array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]); - - gl.bindTexture(tex2d, texture); - gl.texParameteri(tex2d, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); - gl.texParameteri(tex2d, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); - gl.texParameteri(tex2d, gl.TEXTURE_MIN_FILTER, gl.NEAREST); - gl.texParameteri(tex2d, gl.TEXTURE_MAG_FILTER, gl.NEAREST); - gl.texImage2D( - tex2d, 0, internalFormat, width, height, 0, textureFormat, textureType, - dataForUpload); - - const logicalShape = [height, width]; + const logicalShape: [number, number] = [height, width]; const physicalShape: [number, number] = [height, width]; - const a = createTensorFromTexture(texture, logicalShape, physicalShape); - const b = tf.mul(a, 2); - b.print(); + const a = tf.tensor2d([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], logicalShape); + const b = tf.relu(a); + const bTexture = (tf.backend() as MathBackendWebGL).getTexture(b.dataId); + const c = createTensorFromTexture(bTexture, logicalShape, physicalShape); + const d = tf.mul(c, 2); - gpgpu.dispose(); + expect(d.shape).toEqual(logicalShape); + expectArraysClose( + await d.data(), [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22]); - expect(b.shape).toEqual(logicalShape); - // expectArraysClose( - // await b.data(), [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22]); + tf.env().set('WEBGL_PACK', webglPackedFlagSaved); }); }); From d069c64a7df5e539baa5153becb1f6d940843f8e Mon Sep 17 00:00:00 2001 From: Ann Yuan Date: Thu, 10 Dec 2020 15:12:23 -0500 Subject: [PATCH 12/31] tests --- tfjs-backend-webgl/src/backend_webgl_test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tfjs-backend-webgl/src/backend_webgl_test.ts b/tfjs-backend-webgl/src/backend_webgl_test.ts index 5aa6012c76..1c9266d41f 100644 --- a/tfjs-backend-webgl/src/backend_webgl_test.ts +++ b/tfjs-backend-webgl/src/backend_webgl_test.ts @@ -119,7 +119,7 @@ describeWithFlags('create tensor from texture', WEBGL2_ENVS, () => { }); describeWithFlags('create tensor from texture', WEBGL1_ENVS, () => { - fit('basic usage', async () => { + it('basic usage', async () => { const gpgpu = new GPGPUContext(); const width = 3; const height = 4; From f1274bf2fcaa9dce6d7af5f863c448a1ec5bfab4 Mon Sep 17 00:00:00 2001 From: Ann Yuan Date: Fri, 11 Dec 2020 07:04:54 -0500 Subject: [PATCH 13/31] add method --- tfjs-backend-webgl/src/gpgpu_util.ts | 70 +++++++++++++++++++++++----- tfjs-backend-webgl/src/webgl.ts | 54 ++++++++++++++++----- 2 files changed, 102 insertions(+), 22 deletions(-) diff --git a/tfjs-backend-webgl/src/gpgpu_util.ts b/tfjs-backend-webgl/src/gpgpu_util.ts index ca8492a2b8..9d3fae2326 100644 --- a/tfjs-backend-webgl/src/gpgpu_util.ts +++ b/tfjs-backend-webgl/src/gpgpu_util.ts @@ -81,15 +81,25 @@ export function getInternalFormatForFloat32MatrixTexture( return textureConfig.internalFormatFloat; } +export function getTextureParamsForFloat32MatrixTexture( + gl: WebGLRenderingContext, + textureConfig: TextureConfig): TextureCreationParams { + return { + internalFormat: getInternalFormatForFloat32MatrixTexture(textureConfig), + textureFormat: textureConfig.textureFormatFloat, + textureType: gl.FLOAT + }; +} + export function createFloat32MatrixTexture( gl: WebGLRenderingContext, rows: number, columns: number, textureConfig: TextureConfig): WebGLTexture { const [width, height] = tex_util.getUnpackedMatrixTextureShapeWidthHeight(rows, columns); + const {internalFormat, textureFormat, textureType} = + getTextureParamsForFloat32MatrixTexture(gl, textureConfig); return createAndConfigureTexture( - gl, width, height, - getInternalFormatForFloat32MatrixTexture(textureConfig), - textureConfig.textureFormatFloat, gl.FLOAT); + gl, width, height, internalFormat, textureFormat, textureType); } export function getInternalFormatForFloat16MatrixTexture( @@ -97,15 +107,25 @@ export function getInternalFormatForFloat16MatrixTexture( return textureConfig.internalFormatHalfFloat; } +export function getTextureParamsForFloat16MatrixTexture( + gl: WebGLRenderingContext, + textureConfig: TextureConfig): TextureCreationParams { + return { + internalFormat: getInternalFormatForFloat16MatrixTexture(textureConfig), + textureFormat: textureConfig.textureFormatFloat, + textureType: textureConfig.textureTypeHalfFloat + }; +} + export function createFloat16MatrixTexture( gl: WebGLRenderingContext, rows: number, columns: number, textureConfig: TextureConfig): WebGLTexture { const [width, height] = tex_util.getUnpackedMatrixTextureShapeWidthHeight(rows, columns); + const {internalFormat, textureFormat, textureType} = + getTextureParamsForFloat16PackedMatrixTexture(gl, textureConfig); return createAndConfigureTexture( - gl, width, height, - getInternalFormatForFloat16MatrixTexture(textureConfig), - textureConfig.textureFormatFloat, textureConfig.textureTypeHalfFloat); + gl, width, height, internalFormat, textureFormat, textureType); } export function getInternalFormatForUnsignedBytesMatrixTexture( @@ -129,14 +149,25 @@ export function getInternalFormatForPackedMatrixTexture( return textureConfig.internalFormatPackedFloat; } +export function getTextureParamsForPackedMatrixTexture( + gl: WebGLRenderingContext, + textureConfig: TextureConfig): TextureCreationParams { + return { + internalFormat: getInternalFormatForPackedMatrixTexture(textureConfig), + textureFormat: gl.RGBA, + textureType: gl.FLOAT + }; +} + export function createPackedMatrixTexture( gl: WebGLRenderingContext, rows: number, columns: number, textureConfig: TextureConfig): WebGLTexture { const [width, height] = tex_util.getPackedMatrixTextureShapeWidthHeight(rows, columns); + const {internalFormat, textureFormat, textureType} = + getTextureParamsForPackedMatrixTexture(gl, textureConfig); return createAndConfigureTexture( - gl, width, height, getInternalFormatForPackedMatrixTexture(textureConfig), - gl.RGBA, gl.FLOAT); + gl, width, height, internalFormat, textureFormat, textureType); } export function getInternalFormatForFloat16PackedMatrixTexture( @@ -144,15 +175,32 @@ export function getInternalFormatForFloat16PackedMatrixTexture( return textureConfig.internalFormatPackedHalfFloat; } +export type TextureCreationParams = { + internalFormat: number, + textureFormat: number, + textureType: number +}; + +export function getTextureParamsForFloat16PackedMatrixTexture( + gl: WebGLRenderingContext, + textureConfig: TextureConfig): TextureCreationParams { + return { + internalFormat: + getInternalFormatForFloat16PackedMatrixTexture(textureConfig), + textureFormat: gl.RGBA, + textureType: textureConfig.textureTypeHalfFloat + }; +} + export function createFloat16PackedMatrixTexture( gl: WebGLRenderingContext, rows: number, columns: number, textureConfig: TextureConfig): WebGLTexture { const [width, height] = tex_util.getPackedMatrixTextureShapeWidthHeight(rows, columns); + const {internalFormat, textureFormat, textureType} = + getTextureParamsForFloat16PackedMatrixTexture(gl, textureConfig); return createAndConfigureTexture( - gl, width, height, - getInternalFormatForFloat16PackedMatrixTexture(textureConfig), gl.RGBA, - textureConfig.textureTypeHalfFloat); + gl, width, height, internalFormat, textureFormat, textureType); } export function bindVertexProgramAttributeStreams( diff --git a/tfjs-backend-webgl/src/webgl.ts b/tfjs-backend-webgl/src/webgl.ts index e6ee11088e..d00a81a2fb 100644 --- a/tfjs-backend-webgl/src/webgl.ts +++ b/tfjs-backend-webgl/src/webgl.ts @@ -15,10 +15,11 @@ * ============================================================================= */ -import {engine, env, Tensor} from '@tensorflow/tfjs-core'; +import {DataType, engine, env, Tensor, util} from '@tensorflow/tfjs-core'; import {MathBackendWebGL} from './backend_webgl'; import * as gpgpu_util from './gpgpu_util'; +import {getTextureConfig} from './tex_util'; import * as webgl_util from './webgl_util'; export {MathBackendWebGL, WebGLMemoryInfo, WebGLTimingInfo} from './backend_webgl'; @@ -45,15 +46,46 @@ export function createTensorFromTexture( return engine().makeTensorFromDataId(dataId, shape, 'float32', backend); } -// type TensorFromTextureConfig = { -// texture: WebGLTexture, -// shape: number[], -// texShapeRC: [number, number], -// internalFormat: any, -// textureFormat: any, -// textureType: any -// }; +type TensorFromTextureConfig = { + texture: WebGLTexture, + shape: number[], + dtype: DataType, + texShapeRC: [number, number], + internalFormat: number, + textureFormat: number, + textureType: number +}; -// export function createTensorFromTexture2(): Tensor { +export function createTensorFromTexture2({ + texture, + shape, + dtype, + texShapeRC, + internalFormat, + textureFormat, + textureType +}: TensorFromTextureConfig): Tensor { + const backend = engine().backend as MathBackendWebGL; + const gl = backend.gpgpu.gl; + const texConfig = backend.gpgpu.textureConfig; + let params: gpgpu_util.TextureCreationParams; + + if (env().getBool('WEBGL_RENDER_FLOAT32_ENABLED') === true) { + params = gpgpu_util.getTextureParamsForFloat32MatrixTexture(gl, texConfig); + } else { + params = gpgpu_util.getTextureParamsForFloat16MatrixTexture(gl, texConfig); + } -// } + util.assert( + internalFormat === params.internalFormat, + () => `The internalFormat must be ${params.internalFormat}.`); + util.assert( + textureFormat === params.textureFormat, + () => `The textureFormat must be ${params.textureFormat}.`); + util.assert( + textureType === params.textureType, + () => `The textureType must be ${params.textureType}.`); + + const dataId = backend.writeTexture(texture, shape, dtype, texShapeRC); + return engine().makeTensorFromDataId(dataId, shape, dtype, backend); +} From 5cef63d93deb4cec9cb4f7372a40c2d20bc4f686 Mon Sep 17 00:00:00 2001 From: Ann Yuan Date: Fri, 11 Dec 2020 07:14:50 -0500 Subject: [PATCH 14/31] invoke --- tfjs-backend-webgl/src/backend_webgl_test.ts | 58 +++++++++++++++++--- tfjs-backend-webgl/src/webgl.ts | 10 +--- 2 files changed, 52 insertions(+), 16 deletions(-) diff --git a/tfjs-backend-webgl/src/backend_webgl_test.ts b/tfjs-backend-webgl/src/backend_webgl_test.ts index 1c9266d41f..432cbf6e0d 100644 --- a/tfjs-backend-webgl/src/backend_webgl_test.ts +++ b/tfjs-backend-webgl/src/backend_webgl_test.ts @@ -57,9 +57,11 @@ describeWithFlags('create tensor from texture', WEBGL2_ENVS, () => { const gl = gpgpu.gl; const texture = gl.createTexture(); const tex2d = gl.TEXTURE_2D; - const internalFormat = (gl as any).R32F; - const textureFormat = (gl as any).RED; - const textureType = (gl as any).FLOAT; + // tslint:disable-next-line:no-any + const glany = gl as any; + const internalFormat = glany.R32F; + const textureFormat = glany.RED; + const textureType = glany.FLOAT; const dataForUpload = new Float32Array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]); @@ -74,7 +76,15 @@ describeWithFlags('create tensor from texture', WEBGL2_ENVS, () => { const logicalShape = [height, width]; const physicalShape: [number, number] = [height, width]; - const a = createTensorFromTexture(texture, logicalShape, physicalShape); + const a = createTensorFromTexture({ + texture, + shape: logicalShape, + dtype: 'float32', + texShapeRC: physicalShape, + internalFormat, + textureFormat, + textureType + }); const b = tf.mul(a, 2); expect(b.shape).toEqual(logicalShape); @@ -85,6 +95,11 @@ describeWithFlags('create tensor from texture', WEBGL2_ENVS, () => { }); it('force f16', async () => { + const gpgpu = new GPGPUContext(); + const gl = gpgpu.gl; + // tslint:disable-next-line:no-any + const glany = gl as any; + // Unlike in the F32 test, rather than creating a texture from scratch, we // must extract the output texture from an operation because we cannot // upload Float16 data directly to the GPU. @@ -106,7 +121,15 @@ describeWithFlags('create tensor from texture', WEBGL2_ENVS, () => { const a = tf.tensor2d([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], logicalShape); const b = tf.relu(a); const bTexture = (tf.backend() as MathBackendWebGL).getTexture(b.dataId); - const c = createTensorFromTexture(bTexture, logicalShape, physicalShape); + const c = createTensorFromTexture({ + texture: bTexture, + shape: logicalShape, + dtype: 'float32', + texShapeRC: physicalShape, + internalFormat: glany.R16F, + textureFormat: glany.RED, + textureType: glany.HALF_FLOAT + }); const d = tf.mul(c, 2); expect(d.shape).toEqual(logicalShape); @@ -115,6 +138,8 @@ describeWithFlags('create tensor from texture', WEBGL2_ENVS, () => { tf.env().set('WEBGL_FORCE_F16_TEXTURES', webglForceF16FlagSaved); tf.env().set('WEBGL_PACK', webglPackedFlagSaved); + + gpgpu.dispose(); }); }); @@ -147,7 +172,15 @@ describeWithFlags('create tensor from texture', WEBGL1_ENVS, () => { tex2d, 0, internalFormat, width, height, 0, textureFormat, textureType, dataForUpload); - const a = createTensorFromTexture(texture, logicalShape, physicalShape); + const a = createTensorFromTexture({ + texture, + shape: logicalShape, + texShapeRC: physicalShape, + dtype: 'float32', + internalFormat, + textureFormat, + textureType + }); const b = tf.mul(a, 2); expect(b.shape).toEqual(logicalShape); @@ -157,6 +190,9 @@ describeWithFlags('create tensor from texture', WEBGL1_ENVS, () => { }); it('chained', async () => { + const gpgpu = new GPGPUContext(); + const gl = gpgpu.gl; + const webglPackedFlagSaved = tf.env().getBool('WEBGL_PACK'); tf.env().set('WEBGL_PACK', false); @@ -168,7 +204,15 @@ describeWithFlags('create tensor from texture', WEBGL1_ENVS, () => { const a = tf.tensor2d([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], logicalShape); const b = tf.relu(a); const bTexture = (tf.backend() as MathBackendWebGL).getTexture(b.dataId); - const c = createTensorFromTexture(bTexture, logicalShape, physicalShape); + const c = createTensorFromTexture({ + texture: bTexture, + shape: logicalShape, + texShapeRC: physicalShape, + dtype: 'float32', + internalFormat: gl.RGBA, + textureFormat: gl.RGBA, + textureType: gl.FLOAT + }); const d = tf.mul(c, 2); expect(d.shape).toEqual(logicalShape); diff --git a/tfjs-backend-webgl/src/webgl.ts b/tfjs-backend-webgl/src/webgl.ts index d00a81a2fb..2248a67688 100644 --- a/tfjs-backend-webgl/src/webgl.ts +++ b/tfjs-backend-webgl/src/webgl.ts @@ -38,14 +38,6 @@ export function forceHalfFloat(): void { env().set('WEBGL_FORCE_F16_TEXTURES', true); } -export function createTensorFromTexture( - texture: WebGLTexture, shape: number[], - texShapeRC: [number, number]): Tensor { - const backend = engine().backend as MathBackendWebGL; - const dataId = backend.writeTexture(texture, shape, 'float32', texShapeRC); - return engine().makeTensorFromDataId(dataId, shape, 'float32', backend); -} - type TensorFromTextureConfig = { texture: WebGLTexture, shape: number[], @@ -56,7 +48,7 @@ type TensorFromTextureConfig = { textureType: number }; -export function createTensorFromTexture2({ +export function createTensorFromTexture({ texture, shape, dtype, From 1211db35d1665face5e88f60717343b50acaae52 Mon Sep 17 00:00:00 2001 From: Ann Yuan Date: Mon, 14 Dec 2020 09:17:59 -0500 Subject: [PATCH 15/31] add comments --- tfjs-backend-webgl/src/backend_webgl.ts | 3 +++ tfjs-backend-webgl/src/backend_webgl_test.ts | 14 ++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/tfjs-backend-webgl/src/backend_webgl.ts b/tfjs-backend-webgl/src/backend_webgl.ts index ad66880bde..14bb1e4cd5 100644 --- a/tfjs-backend-webgl/src/backend_webgl.ts +++ b/tfjs-backend-webgl/src/backend_webgl.ts @@ -166,6 +166,8 @@ export class MathBackendWebGL extends KernelBackend { this.pendingDeletes; } + // Writes a new entry to the data store with a WebGL texture, and registers it + // to the texture manager. writeTexture( texture: WebGLTexture, shape: number[], dtype: DataType, texShape: [number, number]): DataId { @@ -180,6 +182,7 @@ export class MathBackendWebGL extends KernelBackend { refCount: 1, complexParentRefCount: 0 }); + this.textureManager.registerRenderTexture(texture, texShape); return dataId; } diff --git a/tfjs-backend-webgl/src/backend_webgl_test.ts b/tfjs-backend-webgl/src/backend_webgl_test.ts index 432cbf6e0d..94c0f2881e 100644 --- a/tfjs-backend-webgl/src/backend_webgl_test.ts +++ b/tfjs-backend-webgl/src/backend_webgl_test.ts @@ -50,6 +50,10 @@ const RENDER_FLOAT32_ENVS = { describeWithFlags('create tensor from texture', WEBGL2_ENVS, () => { it('basic usage', async () => { + // In this test we create a WebGL texture using the GL context from the + // WebGL backend. Then we create a tensor from that texture, and ensure that + // we can perform a TF operation on that tensor and get the expected result. + const gpgpu = new GPGPUContext(); const width = 3; const height = 4; @@ -95,15 +99,15 @@ describeWithFlags('create tensor from texture', WEBGL2_ENVS, () => { }); it('force f16', async () => { + // Unlike in the basic usage test, rather than creating a texture from + // scratch, we must extract the output texture from an operation because we + // cannot upload Float16 data directly to the GPU. + const gpgpu = new GPGPUContext(); const gl = gpgpu.gl; // tslint:disable-next-line:no-any const glany = gl as any; - // Unlike in the F32 test, rather than creating a texture from scratch, we - // must extract the output texture from an operation because we cannot - // upload Float16 data directly to the GPU. - const webglForceF16FlagSaved = tf.env().getBool('WEBGL_FORCE_F16_TEXTURES'); const webglPackedFlagSaved = tf.env().getBool('WEBGL_PACK'); tf.env().set('WEBGL_FORCE_F16_TEXTURES', true); @@ -158,6 +162,8 @@ describeWithFlags('create tensor from texture', WEBGL1_ENVS, () => { const internalFormat = gl.RGBA; const textureFormat = gl.RGBA; const textureType = gl.FLOAT; + // WebGL 1 does not accept gl.RED as an internalFormat, so we have to + // upload values for the unused channels as well. const dataForUpload = new Float32Array([ 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 5, 0, 0, 0, 6, 0, 0, 0, 7, 0, 0, 0, 8, 0, 0, 0, 9, 0, 0, 0, 10, 0, 0, 0, 11, 0, 0, 0, From 0331d6072545f30f87104b2f02415add8b548cb1 Mon Sep 17 00:00:00 2001 From: Ann Yuan Date: Mon, 14 Dec 2020 09:18:52 -0500 Subject: [PATCH 16/31] undo --- tsconfig.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tsconfig.json b/tsconfig.json index 4347043a17..a261f8b6f5 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -22,7 +22,7 @@ "pretty": true, "noFallthroughCasesInSwitch": true, "allowUnreachableCode": false, - "incremental": false + "incremental": true }, "include": [ "src/" From 0a37a8ee74944c09aa961da2528771944d9b8fe4 Mon Sep 17 00:00:00 2001 From: Ann Yuan Date: Mon, 14 Dec 2020 09:35:29 -0500 Subject: [PATCH 17/31] docs --- tfjs-backend-webgl/src/webgl.ts | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tfjs-backend-webgl/src/webgl.ts b/tfjs-backend-webgl/src/webgl.ts index 2248a67688..240a12745a 100644 --- a/tfjs-backend-webgl/src/webgl.ts +++ b/tfjs-backend-webgl/src/webgl.ts @@ -48,6 +48,23 @@ type TensorFromTextureConfig = { textureType: number }; +/** + * Create a tensor out of an existing WebGL texture. + * + * @param obj An object with the following properties: + * @param texture The WebGL texture to create a tensor from. + * @param shape The logical shape of the texture. + * @param dtype The dtype of the tensor to be created. + * @param texShapeRC The physical dimensions of the texture expressed as [rows, + * columns]. + * @param internalFormat The internalFormat of the texture provided to + * gl.texImage2D during texture creation. + * @param textureFormat The textureFormat of the texture provided to + * gl.texImage2D during texture creation. + * @param textureType The textureType of the texture provided to gl.texImage2D + * during texture creation. + * @doc {heading: 'Environment', namespace: 'webgl'} + */ export function createTensorFromTexture({ texture, shape, @@ -57,6 +74,15 @@ export function createTensorFromTexture({ textureFormat, textureType }: TensorFromTextureConfig): Tensor { + // OpenGL / WebGL do not make it possible to query textures for their + // properties (physical dimensions, internalFormat, etc.), therefore we ask + // the user to provide this information in order to validate their texture. + + // StackOverflow posts confirming that this information cannot be queried: + // https://stackoverflow.com/questions/30140178/opengl-es-2-0-get-texture-size-and-other-info + // https://stackoverflow.com/questions/26315021/is-there-a-way-to-retrieve-the-dimensions-of-a-texture-after-binding-with-gl-bin + // https://stackoverflow.com/questions/46387922/how-to-check-a-texture-is-2d-texture-or-cube-texture-in-webgl + const backend = engine().backend as MathBackendWebGL; const gl = backend.gpgpu.gl; const texConfig = backend.gpgpu.textureConfig; @@ -68,6 +94,8 @@ export function createTensorFromTexture({ params = gpgpu_util.getTextureParamsForFloat16MatrixTexture(gl, texConfig); } + // Ensure that the properties of the texture match the expectations of the + // WebGL backend. util.assert( internalFormat === params.internalFormat, () => `The internalFormat must be ${params.internalFormat}.`); From cb033e0d3228a0e1325dca4f7389aa3ecd653a0b Mon Sep 17 00:00:00 2001 From: Ann Yuan Date: Mon, 14 Dec 2020 09:42:33 -0500 Subject: [PATCH 18/31] oops --- tfjs-backend-webgl/src/webgl.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/tfjs-backend-webgl/src/webgl.ts b/tfjs-backend-webgl/src/webgl.ts index 240a12745a..84ce0c113c 100644 --- a/tfjs-backend-webgl/src/webgl.ts +++ b/tfjs-backend-webgl/src/webgl.ts @@ -19,7 +19,6 @@ import {DataType, engine, env, Tensor, util} from '@tensorflow/tfjs-core'; import {MathBackendWebGL} from './backend_webgl'; import * as gpgpu_util from './gpgpu_util'; -import {getTextureConfig} from './tex_util'; import * as webgl_util from './webgl_util'; export {MathBackendWebGL, WebGLMemoryInfo, WebGLTimingInfo} from './backend_webgl'; From 700f1aa177887503d503b90ccb774179d43cd05a Mon Sep 17 00:00:00 2001 From: Ann Yuan Date: Mon, 14 Dec 2020 10:22:25 -0500 Subject: [PATCH 19/31] fix --- tfjs-backend-webgl/src/backend_webgl_test.ts | 25 +++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/tfjs-backend-webgl/src/backend_webgl_test.ts b/tfjs-backend-webgl/src/backend_webgl_test.ts index 94c0f2881e..8b47d6b5b0 100644 --- a/tfjs-backend-webgl/src/backend_webgl_test.ts +++ b/tfjs-backend-webgl/src/backend_webgl_test.ts @@ -103,20 +103,25 @@ describeWithFlags('create tensor from texture', WEBGL2_ENVS, () => { // scratch, we must extract the output texture from an operation because we // cannot upload Float16 data directly to the GPU. - const gpgpu = new GPGPUContext(); - const gl = gpgpu.gl; - // tslint:disable-next-line:no-any - const glany = gl as any; + // We clean up explicitly so that we have full control over the environment + // flags during texture initialization / disposal. + tf.engine().startScope(); - const webglForceF16FlagSaved = tf.env().getBool('WEBGL_FORCE_F16_TEXTURES'); + const webglRenderF32EnabledFlagSaved = + tf.env().getBool('WEBGL_RENDER_FLOAT32_ENABLED'); const webglPackedFlagSaved = tf.env().getBool('WEBGL_PACK'); - tf.env().set('WEBGL_FORCE_F16_TEXTURES', true); + tf.env().set('WEBGL_RENDER_FLOAT32_ENABLED', false); // We must set `WEBGL_PACK` to false because createTensorFromTexture only // accepts unpacked textures so this ensures the output texture (`bTexture` // below) is unpacked. tf.env().set('WEBGL_PACK', false); + const gpgpu = new GPGPUContext(); + const gl = gpgpu.gl; + // tslint:disable-next-line:no-any + const glany = gl as any; + const width = 3; const height = 4; @@ -140,10 +145,12 @@ describeWithFlags('create tensor from texture', WEBGL2_ENVS, () => { expectArraysClose( await d.data(), [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22]); - tf.env().set('WEBGL_FORCE_F16_TEXTURES', webglForceF16FlagSaved); - tf.env().set('WEBGL_PACK', webglPackedFlagSaved); - gpgpu.dispose(); + + tf.env().set( + 'WEBGL_RENDER_FLOAT32_ENABLED', webglRenderF32EnabledFlagSaved); + tf.env().set('WEBGL_PACK', webglPackedFlagSaved); + tf.engine().startScope(); }); }); From f4c755c0e8c49636d7dc28a4417e0440123992d3 Mon Sep 17 00:00:00 2001 From: Ann Yuan Date: Thu, 17 Dec 2020 13:27:46 -0500 Subject: [PATCH 20/31] add example --- tfjs-backend-webgl/src/webgl.ts | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/tfjs-backend-webgl/src/webgl.ts b/tfjs-backend-webgl/src/webgl.ts index 84ce0c113c..a6c2bb5c22 100644 --- a/tfjs-backend-webgl/src/webgl.ts +++ b/tfjs-backend-webgl/src/webgl.ts @@ -50,8 +50,37 @@ type TensorFromTextureConfig = { /** * Create a tensor out of an existing WebGL texture. * + * ```js + * // Example for WebGL2: + * const gl = tf.backend().gpgpu.gl; + * const texture = gl.createTexture(); + * const tex2d = gl.TEXTURE_2D; + * const width = 3; + * const height = 4; + * + * gl.bindTexture(tex2d, texture); + * gl.texParameteri(tex2d, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + * gl.texParameteri(tex2d, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + * gl.texParameteri(tex2d, gl.TEXTURE_MIN_FILTER, gl.NEAREST); + * gl.texParameteri(tex2d, gl.TEXTURE_MAG_FILTER, gl.NEAREST); + * gl.texImage2D( + * tex2d, 0, gl.R32F, // internalFormat + * width, height, 0, + * gl.RED, // textureFormat + * gl.FLOAT, // textureType + * new Float32Array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]) // data + * ); + * + * const logicalShape = [height, width]; + * const physicalShape = [height, width]; + * const a = tf.webgl.createTensorFromTexture(texture, logicalShape, + * physicalShape); + * + * ``` + * * @param obj An object with the following properties: - * @param texture The WebGL texture to create a tensor from. + * @param texture The WebGL texture to create a tensor from. The texture must + * be unpacked - each texel should only store a single value. * @param shape The logical shape of the texture. * @param dtype The dtype of the tensor to be created. * @param texShapeRC The physical dimensions of the texture expressed as [rows, @@ -77,7 +106,7 @@ export function createTensorFromTexture({ // properties (physical dimensions, internalFormat, etc.), therefore we ask // the user to provide this information in order to validate their texture. - // StackOverflow posts confirming that this information cannot be queried: + // References that this information cannot be queried: // https://stackoverflow.com/questions/30140178/opengl-es-2-0-get-texture-size-and-other-info // https://stackoverflow.com/questions/26315021/is-there-a-way-to-retrieve-the-dimensions-of-a-texture-after-binding-with-gl-bin // https://stackoverflow.com/questions/46387922/how-to-check-a-texture-is-2d-texture-or-cube-texture-in-webgl From 33064e38333544b975c09578235461bfb6668e17 Mon Sep 17 00:00:00 2001 From: Ann Yuan Date: Thu, 17 Dec 2020 16:21:58 -0500 Subject: [PATCH 21/31] fix --- tfjs-backend-webgl/src/backend_webgl_test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tfjs-backend-webgl/src/backend_webgl_test.ts b/tfjs-backend-webgl/src/backend_webgl_test.ts index 8b47d6b5b0..86519ea268 100644 --- a/tfjs-backend-webgl/src/backend_webgl_test.ts +++ b/tfjs-backend-webgl/src/backend_webgl_test.ts @@ -147,10 +147,10 @@ describeWithFlags('create tensor from texture', WEBGL2_ENVS, () => { gpgpu.dispose(); + tf.engine().endScope(); tf.env().set( 'WEBGL_RENDER_FLOAT32_ENABLED', webglRenderF32EnabledFlagSaved); tf.env().set('WEBGL_PACK', webglPackedFlagSaved); - tf.engine().startScope(); }); }); From 2914b0cb820871000631a0a5667c8f4aa9d95e9d Mon Sep 17 00:00:00 2001 From: Ann Yuan Date: Thu, 17 Dec 2020 16:37:52 -0500 Subject: [PATCH 22/31] add enum --- tfjs-backend-webgl/src/webgl.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tfjs-backend-webgl/src/webgl.ts b/tfjs-backend-webgl/src/webgl.ts index a6c2bb5c22..cb1e21de7e 100644 --- a/tfjs-backend-webgl/src/webgl.ts +++ b/tfjs-backend-webgl/src/webgl.ts @@ -42,9 +42,13 @@ type TensorFromTextureConfig = { shape: number[], dtype: DataType, texShapeRC: [number, number], - internalFormat: number, - textureFormat: number, - textureType: number + internalFormat: WebGL2RenderingContext['R32F']|WebGL2RenderingContext['R16F']| + WebGL2RenderingContext['RGBA16F']|WebGL2RenderingContext['RGBA32F']| + WebGLRenderingContext['RGBA'], + textureFormat: WebGL2RenderingContext['RED']|WebGLRenderingContext['RGBA'], + textureType: WebGL2RenderingContext['HALF_FLOAT']| + WebGL2RenderingContext['FLOAT']|WebGLRenderingContext['FLOAT']| + OES_texture_half_float }; /** From f5e1d7ae8f11ae6a107adad3d168ca621aa183eb Mon Sep 17 00:00:00 2001 From: Ann Yuan Date: Thu, 17 Dec 2020 16:46:56 -0500 Subject: [PATCH 23/31] define types --- tfjs-backend-webgl/src/tex_util.ts | 25 +++++++++++++++++-------- tfjs-backend-webgl/src/webgl.ts | 11 ++++------- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/tfjs-backend-webgl/src/tex_util.ts b/tfjs-backend-webgl/src/tex_util.ts index c4119f7db3..2df81ee835 100644 --- a/tfjs-backend-webgl/src/tex_util.ts +++ b/tfjs-backend-webgl/src/tex_util.ts @@ -182,24 +182,33 @@ export interface TextureConfig { textureTypeFloat: number; } +export type WebGLTextureInternalFormat = WebGL2RenderingContext['R32F']| + WebGL2RenderingContext['R16F']|WebGL2RenderingContext['RGBA16F']| + WebGL2RenderingContext['RGBA32F']|WebGLRenderingContext['RGBA']; +export type WebGLTextureFormat = + WebGL2RenderingContext['RED']|WebGLRenderingContext['RGBA']; +export type WebGLTextureType = + WebGL2RenderingContext['HALF_FLOAT']|WebGL2RenderingContext['FLOAT']| + WebGLRenderingContext['FLOAT']|OES_texture_half_float['HALF_FLOAT_OES']; + export function getTextureConfig( // tslint:disable-next-line:no-any gl: WebGLRenderingContext, textureHalfFloatExtension?: any): TextureConfig { // tslint:disable-next-line:no-any const glany = gl as any; - let internalFormatFloat: number; - let internalFormatHalfFloat: number; - let internalFormatPackedHalfFloat: number; - let internalFormatPackedFloat: number; - let textureFormatFloat: number; + let internalFormatFloat: WebGLTextureInternalFormat; + let internalFormatHalfFloat: WebGLTextureInternalFormat; + let internalFormatPackedHalfFloat: WebGLTextureInternalFormat; + let internalFormatPackedFloat: WebGLTextureInternalFormat; + let textureFormatFloat: WebGLTextureFormat; - let downloadTextureFormat: number; + let downloadTextureFormat: WebGLTextureFormat; let downloadUnpackNumChannels: number; let defaultNumChannels: number; - let textureTypeHalfFloat: number; - let textureTypeFloat: number; + let textureTypeHalfFloat: WebGLTextureType; + let textureTypeFloat: WebGLTextureType; if (env().getNumber('WEBGL_VERSION') === 2) { internalFormatFloat = glany.R32F; diff --git a/tfjs-backend-webgl/src/webgl.ts b/tfjs-backend-webgl/src/webgl.ts index cb1e21de7e..74662a02db 100644 --- a/tfjs-backend-webgl/src/webgl.ts +++ b/tfjs-backend-webgl/src/webgl.ts @@ -19,6 +19,7 @@ import {DataType, engine, env, Tensor, util} from '@tensorflow/tfjs-core'; import {MathBackendWebGL} from './backend_webgl'; import * as gpgpu_util from './gpgpu_util'; +import {WebGLTextureFormat, WebGLTextureInternalFormat, WebGLTextureType} from './tex_util'; import * as webgl_util from './webgl_util'; export {MathBackendWebGL, WebGLMemoryInfo, WebGLTimingInfo} from './backend_webgl'; @@ -42,13 +43,9 @@ type TensorFromTextureConfig = { shape: number[], dtype: DataType, texShapeRC: [number, number], - internalFormat: WebGL2RenderingContext['R32F']|WebGL2RenderingContext['R16F']| - WebGL2RenderingContext['RGBA16F']|WebGL2RenderingContext['RGBA32F']| - WebGLRenderingContext['RGBA'], - textureFormat: WebGL2RenderingContext['RED']|WebGLRenderingContext['RGBA'], - textureType: WebGL2RenderingContext['HALF_FLOAT']| - WebGL2RenderingContext['FLOAT']|WebGLRenderingContext['FLOAT']| - OES_texture_half_float + internalFormat: WebGLTextureInternalFormat, + textureFormat: WebGLTextureFormat, + textureType: WebGLTextureType }; /** From 84fe0ae25a7ae8281a6d7abccc3ef3c77bb0441a Mon Sep 17 00:00:00 2001 From: Ann Yuan Date: Thu, 17 Dec 2020 16:48:58 -0500 Subject: [PATCH 24/31] fix --- tfjs-backend-webgl/src/backend_webgl_test.ts | 49 ++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/tfjs-backend-webgl/src/backend_webgl_test.ts b/tfjs-backend-webgl/src/backend_webgl_test.ts index 86519ea268..b1a75c2c14 100644 --- a/tfjs-backend-webgl/src/backend_webgl_test.ts +++ b/tfjs-backend-webgl/src/backend_webgl_test.ts @@ -98,6 +98,55 @@ describeWithFlags('create tensor from texture', WEBGL2_ENVS, () => { gpgpu.dispose(); }); + it('logical and physical shapes do not match', async () => { + // In this test we create a WebGL texture using the GL context from the + // WebGL backend. Then we create a tensor from that texture, and ensure that + // we can perform a TF operation on that tensor and get the expected result. + + const gpgpu = new GPGPUContext(); + const width = 3; + const height = 4; + + const gl = gpgpu.gl; + const texture = gl.createTexture(); + const tex2d = gl.TEXTURE_2D; + // tslint:disable-next-line:no-any + const glany = gl as any; + const internalFormat = glany.R32F; + const textureFormat = glany.RED; + const textureType = glany.FLOAT; + const dataForUpload = + new Float32Array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]); + + gl.bindTexture(tex2d, texture); + gl.texParameteri(tex2d, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(tex2d, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + gl.texParameteri(tex2d, gl.TEXTURE_MIN_FILTER, gl.NEAREST); + gl.texParameteri(tex2d, gl.TEXTURE_MAG_FILTER, gl.NEAREST); + gl.texImage2D( + tex2d, 0, internalFormat, width, height, 0, textureFormat, textureType, + dataForUpload); + + const logicalShape = [2, 6]; + const physicalShape: [number, number] = [height, width]; + const a = createTensorFromTexture({ + texture, + shape: logicalShape, + dtype: 'float32', + texShapeRC: physicalShape, + internalFormat, + textureFormat, + textureType + }); + const b = tf.mul(a, 2); + + expect(b.shape).toEqual(logicalShape); + expectArraysClose( + await b.data(), [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22]); + + gpgpu.dispose(); + }); + it('force f16', async () => { // Unlike in the basic usage test, rather than creating a texture from // scratch, we must extract the output texture from an operation because we From cc95c5d29cffe4cd844e69d263e48218a933e4c2 Mon Sep 17 00:00:00 2001 From: Ann Yuan Date: Mon, 21 Dec 2020 11:01:53 -0500 Subject: [PATCH 25/31] add test --- tfjs-backend-webgl/src/backend_webgl_test.ts | 51 ++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/tfjs-backend-webgl/src/backend_webgl_test.ts b/tfjs-backend-webgl/src/backend_webgl_test.ts index b1a75c2c14..c235c2b086 100644 --- a/tfjs-backend-webgl/src/backend_webgl_test.ts +++ b/tfjs-backend-webgl/src/backend_webgl_test.ts @@ -147,6 +147,57 @@ describeWithFlags('create tensor from texture', WEBGL2_ENVS, () => { gpgpu.dispose(); }); + it('physical texture has empty entries', async () => { + // In this test we create a WebGL texture using the GL context from the + // WebGL backend. Then we create a tensor from that texture, and ensure that + // we can perform a TF operation on that tensor and get the expected result. + + const gpgpu = new GPGPUContext(); + const width = 3; + const height = 5; + + const gl = gpgpu.gl; + const texture = gl.createTexture(); + const tex2d = gl.TEXTURE_2D; + // tslint:disable-next-line:no-any + const glany = gl as any; + const internalFormat = glany.R32F; + const textureFormat = glany.RED; + const textureType = glany.FLOAT; + + const dataForUpload = new Float32Array(width * height); + const data = new Float32Array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]); + dataForUpload.set(data); + + gl.bindTexture(tex2d, texture); + gl.texParameteri(tex2d, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(tex2d, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + gl.texParameteri(tex2d, gl.TEXTURE_MIN_FILTER, gl.NEAREST); + gl.texParameteri(tex2d, gl.TEXTURE_MAG_FILTER, gl.NEAREST); + gl.texImage2D( + tex2d, 0, internalFormat, width, height, 0, textureFormat, textureType, + dataForUpload); + + const logicalShape = [1, 13]; + const physicalShape: [number, number] = [height, width]; + const a = createTensorFromTexture({ + texture, + shape: logicalShape, + dtype: 'float32', + texShapeRC: physicalShape, + internalFormat, + textureFormat, + textureType + }); + const b = tf.mul(a, 2); + + expect(b.shape).toEqual(logicalShape); + expectArraysClose( + await b.data(), [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24]); + + gpgpu.dispose(); + }); + it('force f16', async () => { // Unlike in the basic usage test, rather than creating a texture from // scratch, we must extract the output texture from an operation because we From 08f9f86b5efacae1db3bd69cece325afdbd38ce2 Mon Sep 17 00:00:00 2001 From: Ann Yuan Date: Mon, 21 Dec 2020 11:26:33 -0500 Subject: [PATCH 26/31] add docs --- tfjs-backend-webgl/src/webgl.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tfjs-backend-webgl/src/webgl.ts b/tfjs-backend-webgl/src/webgl.ts index 74662a02db..3cf756efda 100644 --- a/tfjs-backend-webgl/src/webgl.ts +++ b/tfjs-backend-webgl/src/webgl.ts @@ -49,7 +49,12 @@ type TensorFromTextureConfig = { }; /** - * Create a tensor out of an existing WebGL texture. + * Create a tensor out of an existing WebGL texture. The texture is added to the + * WebGL texture registry. + * + * This can speed up applications that include a preprocessing step on the GPU - + * in this case you would be able to upload the GPU output directly to TF.js, + * rather than first downloading the values. * * ```js * // Example for WebGL2: @@ -81,7 +86,10 @@ type TensorFromTextureConfig = { * * @param obj An object with the following properties: * @param texture The WebGL texture to create a tensor from. The texture must - * be unpacked - each texel should only store a single value. + * be unpacked - each texel should only store a single value. The flattened + * values of the tensor will be read from left to right, and top to bottom. The + * texture can have empty texels at the end, but the values must be written + * densely - all empty texels must be contiguous. * @param shape The logical shape of the texture. * @param dtype The dtype of the tensor to be created. * @param texShapeRC The physical dimensions of the texture expressed as [rows, From 9b5500f1a32c5cb86cce46bf8663aceee2f2c216 Mon Sep 17 00:00:00 2001 From: Ann Yuan Date: Mon, 21 Dec 2020 11:29:40 -0500 Subject: [PATCH 27/31] fix --- tfjs-backend-webgl/src/webgl.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tfjs-backend-webgl/src/webgl.ts b/tfjs-backend-webgl/src/webgl.ts index 3cf756efda..54c2dd83ed 100644 --- a/tfjs-backend-webgl/src/webgl.ts +++ b/tfjs-backend-webgl/src/webgl.ts @@ -89,7 +89,7 @@ type TensorFromTextureConfig = { * be unpacked - each texel should only store a single value. The flattened * values of the tensor will be read from left to right, and top to bottom. The * texture can have empty texels at the end, but the values must be written - * densely - all empty texels must be contiguous. + * densely - in other words, all empty texels must be contiguous. * @param shape The logical shape of the texture. * @param dtype The dtype of the tensor to be created. * @param texShapeRC The physical dimensions of the texture expressed as [rows, @@ -114,8 +114,8 @@ export function createTensorFromTexture({ // OpenGL / WebGL do not make it possible to query textures for their // properties (physical dimensions, internalFormat, etc.), therefore we ask // the user to provide this information in order to validate their texture. - - // References that this information cannot be queried: + // This information cannot be queried. + // References: // https://stackoverflow.com/questions/30140178/opengl-es-2-0-get-texture-size-and-other-info // https://stackoverflow.com/questions/26315021/is-there-a-way-to-retrieve-the-dimensions-of-a-texture-after-binding-with-gl-bin // https://stackoverflow.com/questions/46387922/how-to-check-a-texture-is-2d-texture-or-cube-texture-in-webgl From 93ace06b5badfc992d323fc8665df15a6d2f4693 Mon Sep 17 00:00:00 2001 From: Ann Yuan Date: Mon, 21 Dec 2020 11:45:51 -0500 Subject: [PATCH 28/31] fix --- tfjs-backend-webgl/src/webgl.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tfjs-backend-webgl/src/webgl.ts b/tfjs-backend-webgl/src/webgl.ts index 54c2dd83ed..405a9be208 100644 --- a/tfjs-backend-webgl/src/webgl.ts +++ b/tfjs-backend-webgl/src/webgl.ts @@ -49,12 +49,13 @@ type TensorFromTextureConfig = { }; /** - * Create a tensor out of an existing WebGL texture. The texture is added to the - * WebGL texture registry. + * Create a TF.js tensor out of an existing WebGL texture. The texture is added + * to the WebGL texture registry. * - * This can speed up applications that include a preprocessing step on the GPU - - * in this case you would be able to upload the GPU output directly to TF.js, - * rather than first downloading the values. + * This makes it possible for TF.js applications to avoid GPU / CPU sync. For + * example, if your application includes a preprocessing step on the GPU, you + * could upload the GPU output directly to TF.js, rather than first downloading + * the values. * * ```js * // Example for WebGL2: From 1084500bc4462c4acfbda6a2304f2d3e628036f3 Mon Sep 17 00:00:00 2001 From: Ann Yuan Date: Tue, 29 Dec 2020 12:22:31 -0500 Subject: [PATCH 29/31] add postprocessing example --- tfjs-backend-webgl/src/webgl.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tfjs-backend-webgl/src/webgl.ts b/tfjs-backend-webgl/src/webgl.ts index 405a9be208..34dc994c93 100644 --- a/tfjs-backend-webgl/src/webgl.ts +++ b/tfjs-backend-webgl/src/webgl.ts @@ -85,6 +85,14 @@ type TensorFromTextureConfig = { * * ``` * + * For postprocessing on the GPU, it's possible to retrieve the texture backing + * any tensor by calling the WebGL backend's `getTexture` method like so: + * + * ```js + * const a = tf.tensor1d([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]); + * const tex = tf.backend().getTexture(a.dataId); + * ``` + * * @param obj An object with the following properties: * @param texture The WebGL texture to create a tensor from. The texture must * be unpacked - each texel should only store a single value. The flattened @@ -115,7 +123,6 @@ export function createTensorFromTexture({ // OpenGL / WebGL do not make it possible to query textures for their // properties (physical dimensions, internalFormat, etc.), therefore we ask // the user to provide this information in order to validate their texture. - // This information cannot be queried. // References: // https://stackoverflow.com/questions/30140178/opengl-es-2-0-get-texture-size-and-other-info // https://stackoverflow.com/questions/26315021/is-there-a-way-to-retrieve-the-dimensions-of-a-texture-after-binding-with-gl-bin From 39ec6c37004864f85dda2568bfccb76f3eed674b Mon Sep 17 00:00:00 2001 From: Ann Yuan Date: Thu, 11 Feb 2021 15:06:34 -0500 Subject: [PATCH 30/31] undo --- tfjs-backend-webgl/src/backend_webgl.ts | 731 +++++++++++++++--------- 1 file changed, 473 insertions(+), 258 deletions(-) diff --git a/tfjs-backend-webgl/src/backend_webgl.ts b/tfjs-backend-webgl/src/backend_webgl.ts index 9c4a4caea3..a8d2ea9580 100644 --- a/tfjs-backend-webgl/src/backend_webgl.ts +++ b/tfjs-backend-webgl/src/backend_webgl.ts @@ -16,32 +16,58 @@ */ // Import webgl flags. -import './flags_webgl'; - -import * as tf from '@tensorflow/tfjs-core'; -import {backend_util, BackendValues, buffer, DataId, DataStorage, DataType, DataValues, engine, env, kernel_impls, KernelBackend, MemoryInfo, NumericDataType, Rank, RecursiveArray, scalar, ShapeMap, Tensor, Tensor2D, TensorBuffer, TensorInfo, tidy, TimingInfo, TypedArray, util} from '@tensorflow/tfjs-core'; - -import {getWebGLContext} from './canvas_util'; -import {DecodeMatrixProgram} from './decode_matrix_gpu'; -import {DecodeMatrixPackedProgram} from './decode_matrix_packed_gpu'; -import {EncodeFloatProgram} from './encode_float_gpu'; -import {EncodeFloatPackedProgram} from './encode_float_packed_gpu'; -import {EncodeMatrixProgram} from './encode_matrix_gpu'; -import {EncodeMatrixPackedProgram} from './encode_matrix_packed_gpu'; -import {GPGPUContext} from './gpgpu_context'; -import * as gpgpu_math from './gpgpu_math'; -import {GPGPUBinary, GPGPUProgram, TensorData} from './gpgpu_math'; -import {simpleAbsImplCPU} from './kernel_utils/shared'; -import {PackProgram} from './pack_gpu'; -import {ReshapePackedProgram} from './reshape_packed_gpu'; -import * as tex_util from './tex_util'; -import {TextureData, TextureUsage} from './tex_util'; -import {TextureManager} from './texture_manager'; -import * as unary_op from './unaryop_gpu'; -import {UnaryOpProgram} from './unaryop_gpu'; -import {UnaryOpPackedProgram} from './unaryop_packed_gpu'; -import {UnpackProgram} from './unpack_gpu'; -import * as webgl_util from './webgl_util'; +import "./flags_webgl"; + +import * as tf from "@tensorflow/tfjs-core"; +import { + backend_util, + BackendValues, + buffer, + DataId, + DataStorage, + DataType, + DataValues, + engine, + env, + kernel_impls, + KernelBackend, + MemoryInfo, + NumericDataType, + Rank, + RecursiveArray, + scalar, + ShapeMap, + Tensor, + Tensor2D, + TensorBuffer, + TensorInfo, + tidy, + TimingInfo, + TypedArray, + util, +} from "@tensorflow/tfjs-core"; + +import { getWebGLContext } from "./canvas_util"; +import { DecodeMatrixProgram } from "./decode_matrix_gpu"; +import { DecodeMatrixPackedProgram } from "./decode_matrix_packed_gpu"; +import { EncodeFloatProgram } from "./encode_float_gpu"; +import { EncodeFloatPackedProgram } from "./encode_float_packed_gpu"; +import { EncodeMatrixProgram } from "./encode_matrix_gpu"; +import { EncodeMatrixPackedProgram } from "./encode_matrix_packed_gpu"; +import { GPGPUContext } from "./gpgpu_context"; +import * as gpgpu_math from "./gpgpu_math"; +import { GPGPUBinary, GPGPUProgram, TensorData } from "./gpgpu_math"; +import { simpleAbsImplCPU } from "./kernel_utils/shared"; +import { PackProgram } from "./pack_gpu"; +import { ReshapePackedProgram } from "./reshape_packed_gpu"; +import * as tex_util from "./tex_util"; +import { TextureData, TextureUsage } from "./tex_util"; +import { TextureManager } from "./texture_manager"; +import * as unary_op from "./unaryop_gpu"; +import { UnaryOpProgram } from "./unaryop_gpu"; +import { UnaryOpPackedProgram } from "./unaryop_packed_gpu"; +import { UnpackProgram } from "./unpack_gpu"; +import * as webgl_util from "./webgl_util"; const whereImpl = kernel_impls.whereImpl; @@ -49,10 +75,11 @@ export const EPSILON_FLOAT32 = 1e-7; export const EPSILON_FLOAT16 = 1e-4; type KernelInfo = { - name: string; query: Promise; + name: string; + query: Promise; }; -export type TimerNode = RecursiveArray|KernelInfo; +export type TimerNode = RecursiveArray | KernelInfo; export interface CPUTimerQuery { startMs: number; endMs?: number; @@ -74,7 +101,9 @@ export interface WebGLTimingInfo extends TimingInfo { downloadWaitMs: number; } -const binaryCaches: {[webGLVersion: string]: {[key: string]: GPGPUBinary}} = {}; +const binaryCaches: { + [webGLVersion: string]: { [key: string]: GPGPUBinary }; +} = {}; export function getBinaryCache(webGLVersion: number) { if (webGLVersion in binaryCaches) { @@ -94,11 +123,16 @@ const CPU_HANDOFF_SIZE_THRESHOLD = 128; const BEFORE_PAGING_CONSTANT = 600; function numMBBeforeWarning(): number { if (env().global.screen == null) { - return 1024; // 1 GB. + return 1024; // 1 GB. } - return (env().global.screen.height * env().global.screen.width * - window.devicePixelRatio) * - BEFORE_PAGING_CONSTANT / 1024 / 1024; + return ( + (env().global.screen.height * + env().global.screen.width * + window.devicePixelRatio * + BEFORE_PAGING_CONSTANT) / + 1024 / + 1024 + ); } export class MathBackendWebGL extends KernelBackend { @@ -120,7 +154,7 @@ export class MathBackendWebGL extends KernelBackend { dataRefCount = new WeakMap(); private numBytesInGPU = 0; - private canvas: HTMLCanvasElement|OffscreenCanvas; + private canvas: HTMLCanvasElement | OffscreenCanvas; private programTimersStack: TimerNode[]; private activeTimers: TimerNode[]; @@ -131,10 +165,10 @@ export class MathBackendWebGL extends KernelBackend { private cpuBackend: KernelBackend; // Number of bits of precision of this backend. - private floatPrecisionValue: 32|16; + private floatPrecisionValue: 32 | 16; private textureManager: TextureManager; - private binaryCache: {[key: string]: GPGPUBinary}; + private binaryCache: { [key: string]: GPGPUBinary }; private gpgpuCreatedLocally: boolean; private numMBBeforeWarning: number; private warnedAboutMemory = false; @@ -142,13 +176,13 @@ export class MathBackendWebGL extends KernelBackend { constructor(gpgpu?: GPGPUContext) { super(); - if (!env().getBool('HAS_WEBGL')) { - throw new Error('WebGL is not supported on this device'); + if (!env().getBool("HAS_WEBGL")) { + throw new Error("WebGL is not supported on this device"); } if (gpgpu == null) { - const gl = getWebGLContext(env().getNumber('WEBGL_VERSION')); - this.binaryCache = getBinaryCache(env().getNumber('WEBGL_VERSION')); + const gl = getWebGLContext(env().getNumber("WEBGL_VERSION")); + this.binaryCache = getBinaryCache(env().getNumber("WEBGL_VERSION")); this.gpgpu = new GPGPUContext(gl); this.canvas = gl.canvas; this.gpgpuCreatedLocally = true; @@ -165,16 +199,21 @@ export class MathBackendWebGL extends KernelBackend { } numDataIds() { - return this.texData.numDataIds() + - (this.cpuBackend ? this.cpuBackend.numDataIds() : 0) - - this.pendingDeletes; + return ( + this.texData.numDataIds() + + (this.cpuBackend ? this.cpuBackend.numDataIds() : 0) - + this.pendingDeletes + ); } // Writes a new entry to the data store with a WebGL texture, and registers it // to the texture manager. writeTexture( - texture: WebGLTexture, shape: number[], dtype: DataType, - texShape: [number, number]): DataId { + texture: WebGLTexture, + shape: number[], + dtype: DataType, + texShape: [number, number] + ): DataId { const dataId = {}; this.texData.set(dataId, { shape, @@ -184,7 +223,6 @@ export class MathBackendWebGL extends KernelBackend { texShape, isPacked: false, refCount: 1, - complexParentRefCount: 0 }); this.textureManager.registerRenderTexture(texture, texShape); @@ -192,19 +230,26 @@ export class MathBackendWebGL extends KernelBackend { } write(values: BackendValues, shape: number[], dtype: DataType): DataId { - if (env().getBool('WEBGL_CHECK_NUMERICAL_PROBLEMS') || - env().getBool('DEBUG')) { + if ( + env().getBool("WEBGL_CHECK_NUMERICAL_PROBLEMS") || + env().getBool("DEBUG") + ) { this.checkNumericalProblems(values); } - if (dtype === 'complex64' && values != null) { + if (dtype === "complex64" && values != null) { throw new Error( - `Cannot write to a complex64 dtype. ` + - `Please use tf.complex(real, imag).`); + `Cannot write to a complex64 dtype. ` + + `Please use tf.complex(real, imag).` + ); } - const dataId = {id: this.nextDataId()}; - this.texData.set( - dataId, - {shape, dtype, values, usage: TextureUsage.UPLOAD, refCount: 1}); + const dataId = { id: this.nextDataId() }; + this.texData.set(dataId, { + shape, + dtype, + values, + usage: TextureUsage.UPLOAD, + refCount: 1, + }); return dataId; } @@ -232,18 +277,28 @@ export class MathBackendWebGL extends KernelBackend { } move( - dataId: DataId, values: BackendValues, shape: number[], dtype: DataType, - refCount: number): void { - if (env().getBool('DEBUG')) { + dataId: DataId, + values: BackendValues, + shape: number[], + dtype: DataType, + refCount: number + ): void { + if (env().getBool("DEBUG")) { this.checkNumericalProblems(values); } - if (dtype === 'complex64') { + if (dtype === "complex64") { throw new Error( - `Cannot write to a complex64 dtype. ` + - `Please use tf.complex(real, imag).`); + `Cannot write to a complex64 dtype. ` + + `Please use tf.complex(real, imag).` + ); } - this.texData.set( - dataId, {shape, dtype, values, usage: TextureUsage.UPLOAD, refCount}); + this.texData.set(dataId, { + shape, + dtype, + values, + usage: TextureUsage.UPLOAD, + refCount, + }); } disposeIntermediateTensorInfo(tensorInfo: TensorInfo): void { @@ -252,7 +307,14 @@ export class MathBackendWebGL extends KernelBackend { readSync(dataId: DataId): BackendValues { const texData = this.texData.get(dataId); - const {values, dtype, complexTensorInfos, slice, shape, isPacked} = texData; + const { + values, + dtype, + complexTensorInfos, + slice, + shape, + isPacked, + } = texData; // The presence of `slice` indicates this tensor is a shallow slice of a // different tensor, and is using that original tensor's texture. Run @@ -264,8 +326,11 @@ export class MathBackendWebGL extends KernelBackend { } else { program = new UnaryOpProgram(shape, unary_op.CLONE); } - const res = - this.runWebGLProgram(program, [{dataId, shape, dtype}], dtype); + const res = this.runWebGLProgram( + program, + [{ dataId, shape, dtype }], + dtype + ); const data = this.readSync(res.dataId); this.disposeIntermediateTensorInfo(res); return data; @@ -273,7 +338,7 @@ export class MathBackendWebGL extends KernelBackend { if (values != null) { return this.convertAndCacheOnCPU(dataId); } - if (dtype === 'string') { + if (dtype === "string") { return values; } const shouldTimeProgram = this.activeTimers != null; @@ -283,11 +348,13 @@ export class MathBackendWebGL extends KernelBackend { } let result: Float32Array; - if (dtype === 'complex64') { - const realValues = - this.readSync(complexTensorInfos.real.dataId) as Float32Array; - const imagValues = - this.readSync(complexTensorInfos.imag.dataId) as Float32Array; + if (dtype === "complex64") { + const realValues = this.readSync( + complexTensorInfos.real.dataId + ) as Float32Array; + const imagValues = this.readSync( + complexTensorInfos.imag.dataId + ) as Float32Array; result = backend_util.mergeRealAndImagArrays(realValues, imagValues); } else { result = this.getValuesFromTexture(dataId); @@ -302,10 +369,17 @@ export class MathBackendWebGL extends KernelBackend { async read(dataId: DataId): Promise { if (this.pendingRead.has(dataId)) { const subscribers = this.pendingRead.get(dataId); - return new Promise(resolve => subscribers.push(resolve)); + return new Promise((resolve) => subscribers.push(resolve)); } const texData = this.texData.get(dataId); - const {values, shape, slice, dtype, complexTensorInfos, isPacked} = texData; + const { + values, + shape, + slice, + dtype, + complexTensorInfos, + isPacked, + } = texData; // The presence of `slice` indicates this tensor is a shallow slice of a // different tensor, and is using that original tensor's texture. Run @@ -317,8 +391,11 @@ export class MathBackendWebGL extends KernelBackend { } else { program = new UnaryOpProgram(shape, unary_op.CLONE); } - const res = - this.runWebGLProgram(program, [{dataId, shape, dtype}], dtype); + const res = this.runWebGLProgram( + program, + [{ dataId, shape, dtype }], + dtype + ); const data = this.read(res.dataId); this.disposeIntermediateTensorInfo(res); return data; @@ -328,44 +405,51 @@ export class MathBackendWebGL extends KernelBackend { return this.convertAndCacheOnCPU(dataId); } - if (!env().getBool('WEBGL_DOWNLOAD_FLOAT_ENABLED') && - env().getNumber('WEBGL_VERSION') === 2) { + if ( + !env().getBool("WEBGL_DOWNLOAD_FLOAT_ENABLED") && + env().getNumber("WEBGL_VERSION") === 2 + ) { throw new Error( - `tensor.data() with WEBGL_DOWNLOAD_FLOAT_ENABLED=false and ` + - `WEBGL_VERSION=2 not yet supported.`); + `tensor.data() with WEBGL_DOWNLOAD_FLOAT_ENABLED=false and ` + + `WEBGL_VERSION=2 not yet supported.` + ); } let buffer = null; let tmpDownloadTarget: TensorInfo; - if (dtype !== 'complex64' && env().get('WEBGL_BUFFER_SUPPORTED')) { + if (dtype !== "complex64" && env().get("WEBGL_BUFFER_SUPPORTED")) { // Possibly copy the texture into a buffer before inserting a fence. tmpDownloadTarget = this.decode(dataId); const tmpData = this.texData.get(tmpDownloadTarget.dataId); buffer = this.gpgpu.createBufferFromTexture( - tmpData.texture, ...tex_util.getDenseTexShape(shape)); + tmpData.texture, + ...tex_util.getDenseTexShape(shape) + ); } this.pendingRead.set(dataId, []); - if (dtype !== 'complex64') { + if (dtype !== "complex64") { // Create a fence and wait for it to resolve. await this.gpgpu.createAndWaitForFence(); } // Download the values from the GPU. let vals: Float32Array; - if (dtype === 'complex64') { + if (dtype === "complex64") { const ps = await Promise.all([ this.read(complexTensorInfos.real.dataId), - this.read(complexTensorInfos.imag.dataId) + this.read(complexTensorInfos.imag.dataId), ]); const realValues = ps[0]; const imagValues = ps[1]; vals = backend_util.mergeRealAndImagArrays( - realValues as Float32Array, imagValues as Float32Array); + realValues as Float32Array, + imagValues as Float32Array + ); } else if (buffer == null) { vals = this.getValuesFromTexture(dataId); } else { @@ -381,7 +465,7 @@ export class MathBackendWebGL extends KernelBackend { this.pendingRead.delete(dataId); // Notify all pending reads. - subscribers.forEach(resolve => resolve(dTypeVals)); + subscribers.forEach((resolve) => resolve(dTypeVals)); if (this.pendingDisposal.has(dataId)) { this.pendingDisposal.delete(dataId); if (this.disposeData(dataId)) { @@ -395,16 +479,19 @@ export class MathBackendWebGL extends KernelBackend { bufferSync(t: TensorInfo): TensorBuffer { const data = this.readSync(t.dataId); let decodedData = data as DataValues; - if (t.dtype === 'string') { + if (t.dtype === "string") { try { // Decode the bytes into string. - decodedData = (data as Uint8Array[]).map(d => util.decodeString(d)); + decodedData = (data as Uint8Array[]).map((d) => util.decodeString(d)); } catch { - throw new Error('Failed to decode encoded string bytes into utf-8'); + throw new Error("Failed to decode encoded string bytes into utf-8"); } } - return buffer(t.shape as ShapeMap[R], t.dtype, decodedData) as - TensorBuffer; + return buffer( + t.shape as ShapeMap[R], + t.dtype, + decodedData + ) as TensorBuffer; } private checkNumericalProblems(values: BackendValues): void { @@ -414,11 +501,12 @@ export class MathBackendWebGL extends KernelBackend { for (let i = 0; i < values.length; i++) { const num = values[i] as number; if (!webgl_util.canBeRepresented(num)) { - if (env().getBool('WEBGL_RENDER_FLOAT32_CAPABLE')) { + if (env().getBool("WEBGL_RENDER_FLOAT32_CAPABLE")) { throw Error( - `The value ${num} cannot be represented with your ` + + `The value ${num} cannot be represented with your ` + `current settings. Consider enabling float32 rendering: ` + - `'tf.env().set('WEBGL_RENDER_FLOAT32_ENABLED', true);'`); + `'tf.env().set('WEBGL_RENDER_FLOAT32_ENABLED', true);'` + ); } throw Error(`The value ${num} cannot be represented on this device.`); } @@ -426,15 +514,17 @@ export class MathBackendWebGL extends KernelBackend { } private getValuesFromTexture(dataId: DataId): Float32Array { - const {shape, dtype, isPacked} = this.texData.get(dataId); + const { shape, dtype, isPacked } = this.texData.get(dataId); const size = util.sizeFromShape(shape); - if (env().getBool('WEBGL_DOWNLOAD_FLOAT_ENABLED')) { + if (env().getBool("WEBGL_DOWNLOAD_FLOAT_ENABLED")) { const tmpTarget = this.decode(dataId); const tmpData = this.texData.get(tmpTarget.dataId); const vals = this.gpgpu - .downloadMatrixFromPackedTexture( - tmpData.texture, ...tex_util.getDenseTexShape(shape)) - .subarray(0, size); + .downloadMatrixFromPackedTexture( + tmpData.texture, + ...tex_util.getDenseTexShape(shape) + ) + .subarray(0, size); this.disposeIntermediateTensorInfo(tmpTarget); @@ -442,27 +532,33 @@ export class MathBackendWebGL extends KernelBackend { } const shouldUsePackedProgram = - env().getBool('WEBGL_PACK') && isPacked === true; - const outputShape = - shouldUsePackedProgram ? webgl_util.getShapeAs3D(shape) : shape; - const program = shouldUsePackedProgram ? - new EncodeFloatPackedProgram(outputShape as [number, number, number]) : - new EncodeFloatProgram(outputShape); + env().getBool("WEBGL_PACK") && isPacked === true; + const outputShape = shouldUsePackedProgram + ? webgl_util.getShapeAs3D(shape) + : shape; + const program = shouldUsePackedProgram + ? new EncodeFloatPackedProgram(outputShape as [number, number, number]) + : new EncodeFloatProgram(outputShape); const output = this.runWebGLProgram( - program, [{shape: outputShape, dtype, dataId}], 'float32'); + program, + [{ shape: outputShape, dtype, dataId }], + "float32" + ); const tmpData = this.texData.get(output.dataId); - const vals = - this.gpgpu - .downloadByteEncodedFloatMatrixFromOutputTexture( - tmpData.texture, tmpData.texShape[0], tmpData.texShape[1]) - .subarray(0, size); + const vals = this.gpgpu + .downloadByteEncodedFloatMatrixFromOutputTexture( + tmpData.texture, + tmpData.texShape[0], + tmpData.texShape[1] + ) + .subarray(0, size); this.disposeIntermediateTensorInfo(output); return vals; } timerAvailable(): boolean { - return env().getNumber('WEBGL_DISJOINT_QUERY_TIMER_EXTENSION_RELIABLE') > 0; + return env().getNumber("WEBGL_DISJOINT_QUERY_TIMER_EXTENSION_RELIABLE") > 0; } async time(f: () => void): Promise { @@ -481,12 +577,12 @@ export class MathBackendWebGL extends KernelBackend { f(); // needing to split these up because util.flatten only accepts certain types - const flattenedActiveTimerQueries = - util.flatten(this.activeTimers.map((d: KernelInfo) => d.query)) - .filter(d => d != null); - const flattenedActiveTimerNames = - util.flatten(this.activeTimers.map((d: KernelInfo) => d.name)) - .filter(d => d != null); + const flattenedActiveTimerQueries = util + .flatten(this.activeTimers.map((d: KernelInfo) => d.query)) + .filter((d) => d != null); + const flattenedActiveTimerNames = util + .flatten(this.activeTimers.map((d: KernelInfo) => d.name)) + .filter((d) => d != null); this.activeTimers = oldActiveTimers; @@ -498,20 +594,21 @@ export class MathBackendWebGL extends KernelBackend { uploadWaitMs: this.uploadWaitMs, downloadWaitMs: this.downloadWaitMs, kernelMs: null, - wallMs: null // will be filled by the engine + wallMs: null, // will be filled by the engine }; - if (env().getNumber('WEBGL_DISJOINT_QUERY_TIMER_EXTENSION_RELIABLE') > 0) { + if (env().getNumber("WEBGL_DISJOINT_QUERY_TIMER_EXTENSION_RELIABLE") > 0) { const kernelMs = await Promise.all(flattenedActiveTimerQueries); - res['kernelMs'] = util.sum(kernelMs); - res['getExtraProfileInfo'] = () => - kernelMs.map((d, i) => ({name: flattenedActiveTimerNames[i], ms: d})) - .map(d => `${d.name}: ${d.ms}`) - .join(', '); + res["kernelMs"] = util.sum(kernelMs); + res["getExtraProfileInfo"] = () => + kernelMs + .map((d, i) => ({ name: flattenedActiveTimerNames[i], ms: d })) + .map((d) => `${d.name}: ${d.ms}`) + .join(", "); } else { - res['kernelMs'] = { - error: 'WebGL query timers are not supported in this environment.' + res["kernelMs"] = { + error: "WebGL query timers are not supported in this environment.", }; } @@ -524,19 +621,21 @@ export class MathBackendWebGL extends KernelBackend { unreliable: false, numBytesInGPU: this.numBytesInGPU, numBytesInGPUAllocated: this.textureManager.numBytesAllocated, - numBytesInGPUFree: this.textureManager.numBytesFree + numBytesInGPUFree: this.textureManager.numBytesFree, } as WebGLMemoryInfo; } - private startTimer(): WebGLQuery|CPUTimerQuery { - if (env().getNumber('WEBGL_DISJOINT_QUERY_TIMER_EXTENSION_RELIABLE') > 0) { + private startTimer(): WebGLQuery | CPUTimerQuery { + if (env().getNumber("WEBGL_DISJOINT_QUERY_TIMER_EXTENSION_RELIABLE") > 0) { return this.gpgpu.beginQuery(); } - return {startMs: util.now(), endMs: null}; + return { startMs: util.now(), endMs: null }; } - private endTimer(query: WebGLQuery|CPUTimerQuery): WebGLQuery|CPUTimerQuery { - if (env().getNumber('WEBGL_DISJOINT_QUERY_TIMER_EXTENSION_RELIABLE') > 0) { + private endTimer( + query: WebGLQuery | CPUTimerQuery + ): WebGLQuery | CPUTimerQuery { + if (env().getNumber("WEBGL_DISJOINT_QUERY_TIMER_EXTENSION_RELIABLE") > 0) { this.gpgpu.endQuery(); return query; } @@ -544,8 +643,10 @@ export class MathBackendWebGL extends KernelBackend { return query; } - private async getQueryTime(query: WebGLQuery|CPUTimerQuery): Promise { - if (env().getNumber('WEBGL_DISJOINT_QUERY_TIMER_EXTENSION_RELIABLE') > 0) { + private async getQueryTime( + query: WebGLQuery | CPUTimerQuery + ): Promise { + if (env().getNumber("WEBGL_DISJOINT_QUERY_TIMER_EXTENSION_RELIABLE") > 0) { return this.gpgpu.waitForQueryAndGetTime(query as WebGLQuery); } const timerQuery = query as CPUTimerQuery; @@ -594,7 +695,7 @@ export class MathBackendWebGL extends KernelBackend { } this.releaseGPUData(dataId); - const {complexTensorInfos} = this.texData.get(dataId); + const { complexTensorInfos } = this.texData.get(dataId); if (complexTensorInfos != null) { this.disposeData(complexTensorInfos.real.dataId, force); this.disposeData(complexTensorInfos.imag.dataId, force); @@ -606,9 +707,15 @@ export class MathBackendWebGL extends KernelBackend { } private releaseGPUData(dataId: DataId): void { - const {texture, dtype, texShape, usage, isPacked, slice} = - this.texData.get(dataId); - const key = slice && slice.origDataId || dataId; + const { + texture, + dtype, + texShape, + usage, + isPacked, + slice, + } = this.texData.get(dataId); + const key = (slice && slice.origDataId) || dataId; const refCount = this.dataRefCount.get(key); if (refCount > 1) { @@ -641,13 +748,13 @@ export class MathBackendWebGL extends KernelBackend { return this.texData.get(dataId); } - private getCPUBackend(): KernelBackend|null { - if (!env().getBool('WEBGL_CPU_FORWARD')) { + private getCPUBackend(): KernelBackend | null { + if (!env().getBool("WEBGL_CPU_FORWARD")) { return null; } if (this.cpuBackend == null) { - this.cpuBackend = engine().findBackend('cpu'); + this.cpuBackend = engine().findBackend("cpu"); } return this.cpuBackend; @@ -661,24 +768,33 @@ export class MathBackendWebGL extends KernelBackend { sustainable strategy for optimizing backend execution of ops. */ shouldExecuteOnCPU( - inputs: TensorInfo[], - sizeThreshold = CPU_HANDOFF_SIZE_THRESHOLD): boolean { + inputs: TensorInfo[], + sizeThreshold = CPU_HANDOFF_SIZE_THRESHOLD + ): boolean { const cpuBackend = this.getCPUBackend(); - if (!env().getBool('IS_TEST') && !this.warnedAboutCPUBackend && - cpuBackend == null) { + if ( + !env().getBool("IS_TEST") && + !this.warnedAboutCPUBackend && + cpuBackend == null + ) { console.warn( - 'Your application contains ops that are small enough to be ' + - 'executed on the CPU backend, however the CPU backend cannot ' + - 'be found. Consider importing the CPU backend ' + - '(@tensorflow/tfjs-backend-cpu) for better performance.'); + "Your application contains ops that are small enough to be " + + "executed on the CPU backend, however the CPU backend cannot " + + "be found. Consider importing the CPU backend " + + "(@tensorflow/tfjs-backend-cpu) for better performance." + ); this.warnedAboutCPUBackend = true; } - return cpuBackend != null && - inputs.every( - input => this.texData.get(input.dataId).texture == null && - util.sizeFromShape(input.shape) < sizeThreshold); + return ( + cpuBackend != null && + inputs.every( + (input) => + this.texData.get(input.dataId).texture == null && + util.sizeFromShape(input.shape) < sizeThreshold + ) + ); } getGPGPUContext(): GPGPUContext { @@ -687,8 +803,9 @@ export class MathBackendWebGL extends KernelBackend { where(condition: Tensor): Tensor2D { backend_util.warn( - 'tf.where() in webgl locks the UI thread. ' + - 'Call tf.whereAsync() instead'); + "tf.where() in webgl locks the UI thread. " + + "Call tf.whereAsync() instead" + ); const condVals = condition.dataSync(); return whereImpl(condition.shape, condVals); } @@ -697,7 +814,10 @@ export class MathBackendWebGL extends KernelBackend { const program = new UnaryOpPackedProgram(x.shape, op); const outInfo = this.compileAndRun(program, [x], dtype); return engine().makeTensorFromDataId( - outInfo.dataId, outInfo.shape, outInfo.dtype); + outInfo.dataId, + outInfo.shape, + outInfo.dtype + ); } // TODO(msoulanille) remove this once the backend has been modularized @@ -705,30 +825,41 @@ export class MathBackendWebGL extends KernelBackend { // Also remove the op from unary_op. abs(x: T): T { // TODO: handle cases when x is complex. - if (this.shouldExecuteOnCPU([x]) && x.dtype !== 'complex64') { - const outValues = - simpleAbsImplCPU(this.texData.get(x.dataId).values as TypedArray); + if (this.shouldExecuteOnCPU([x]) && x.dtype !== "complex64") { + const outValues = simpleAbsImplCPU( + this.texData.get(x.dataId).values as TypedArray + ); return this.makeOutput(x.shape, x.dtype, outValues); } - if (env().getBool('WEBGL_PACK_UNARY_OPERATIONS')) { + if (env().getBool("WEBGL_PACK_UNARY_OPERATIONS")) { return this.packedUnaryOp(x, unary_op.ABS, x.dtype) as T; } const program = new UnaryOpProgram(x.shape, unary_op.ABS); const outInfo = this.compileAndRun(program, [x]); return engine().makeTensorFromDataId( - outInfo.dataId, outInfo.shape, outInfo.dtype) as T; + outInfo.dataId, + outInfo.shape, + outInfo.dtype + ) as T; } makeTensorInfo( - shape: number[], dtype: DataType, - values?: BackendValues|string[]): TensorInfo { + shape: number[], + dtype: DataType, + values?: BackendValues | string[] + ): TensorInfo { let dataId; - if (dtype === 'string' && values != null && values.length > 0 && - util.isString(values[0])) { - const encodedValues = - (values as {} as string[]).map(d => util.encodeString(d)); + if ( + dtype === "string" && + values != null && + values.length > 0 && + util.isString(values[0]) + ) { + const encodedValues = ((values as {}) as string[]).map((d) => + util.encodeString(d) + ); dataId = this.write(encodedValues, shape, dtype); } else { @@ -736,12 +867,15 @@ export class MathBackendWebGL extends KernelBackend { } this.texData.get(dataId).usage = null; - return {dataId, shape, dtype}; + return { dataId, shape, dtype }; } private makeOutput( - shape: number[], dtype: DataType, values?: BackendValues): T { - const {dataId} = this.makeTensorInfo(shape, dtype, values); + shape: number[], + dtype: DataType, + values?: BackendValues + ): T { + const { dataId } = this.makeTensorInfo(shape, dtype, values); return engine().makeTensorFromDataId(dataId, shape, dtype, this) as T; } @@ -754,37 +888,49 @@ export class MathBackendWebGL extends KernelBackend { const program = new PackProgram(input.shape); const preventEagerUnpackingOutput = true; return this.runWebGLProgram( - program, [input], input.dtype, null /* customSetup */, - preventEagerUnpackingOutput); + program, + [input], + input.dtype, + null /* customSetup */, + preventEagerUnpackingOutput + ); } private packedReshape(input: TensorInfo, afterShape: number[]): TensorInfo { const input3DShape = [ webgl_util.getBatchDim(input.shape), - ...webgl_util.getRowsCols(input.shape) + ...webgl_util.getRowsCols(input.shape), ] as [number, number, number]; const input3D: TensorInfo = { dtype: input.dtype, shape: input3DShape, - dataId: input.dataId + dataId: input.dataId, }; const afterShapeAs3D = [ - webgl_util.getBatchDim(afterShape), ...webgl_util.getRowsCols(afterShape) + webgl_util.getBatchDim(afterShape), + ...webgl_util.getRowsCols(afterShape), ] as [number, number, number]; const program = new ReshapePackedProgram(afterShapeAs3D, input3DShape); const preventEagerUnpackingOfOutput = true; const output = this.runWebGLProgram( - program, [input3D], input.dtype, null /* customSetup */, - preventEagerUnpackingOfOutput); - return {dataId: output.dataId, shape: afterShape, dtype: output.dtype}; + program, + [input3D], + input.dtype, + null /* customSetup */, + preventEagerUnpackingOfOutput + ); + return { dataId: output.dataId, shape: afterShape, dtype: output.dtype }; } private decode(dataId: DataId): TensorInfo { const texData = this.texData.get(dataId); - const {isPacked, shape, dtype} = texData; - const shapeAs3D = - webgl_util.getShapeAs3D(shape) as [number, number, number]; + const { isPacked, shape, dtype } = texData; + const shapeAs3D = webgl_util.getShapeAs3D(shape) as [ + number, + number, + number + ]; let program; if (isPacked) { program = new DecodeMatrixPackedProgram(shapeAs3D); @@ -793,15 +939,22 @@ export class MathBackendWebGL extends KernelBackend { } const preventEagerUnpackingOfOutput = true; const out = this.runWebGLProgram( - program, [{shape: shapeAs3D, dtype, dataId}], dtype, - null /* customSetup */, preventEagerUnpackingOfOutput); - return {dtype, shape, dataId: out.dataId}; + program, + [{ shape: shapeAs3D, dtype, dataId }], + dtype, + null /* customSetup */, + preventEagerUnpackingOfOutput + ); + return { dtype, shape, dataId: out.dataId }; } runWebGLProgram( - program: GPGPUProgram, inputs: TensorInfo[], outputDtype: DataType, - customSetup?: (gpgpu: GPGPUContext, webGLProgram: WebGLProgram) => void, - preventEagerUnpackingOfOutput = false): TensorInfo { + program: GPGPUProgram, + inputs: TensorInfo[], + outputDtype: DataType, + customSetup?: (gpgpu: GPGPUContext, webGLProgram: WebGLProgram) => void, + preventEagerUnpackingOfOutput = false + ): TensorInfo { const output = this.makeTensorInfo(program.outputShape, outputDtype); const outData = this.texData.get(output.dataId); if (program.packedOutput) { @@ -813,7 +966,7 @@ export class MathBackendWebGL extends KernelBackend { // so it doesn't get assigned later according to our typical packing // scheme wherein a single texel can only contain values from adjacent // rows/cols. - outData.texShape = texelShape.map(d => d * 2) as [number, number]; + outData.texShape = texelShape.map((d) => d * 2) as [number, number]; } if (program.outTexUsage != null) { outData.usage = program.outTexUsage; @@ -821,26 +974,31 @@ export class MathBackendWebGL extends KernelBackend { if (util.sizeFromShape(output.shape) === 0) { // Short-circuit the computation since the result is empty (has 0 in its // shape). - outData.values = - util.getTypedArrayFromDType(output.dtype as 'float32', 0); + outData.values = util.getTypedArrayFromDType( + output.dtype as "float32", + 0 + ); return output; } const dataToDispose: TensorInfo[] = []; - const inputsData: TensorData[] = inputs.map(input => { - if (input.dtype === 'complex64') { + const inputsData: TensorData[] = inputs.map((input) => { + if (input.dtype === "complex64") { throw new Error( - `GPGPUProgram does not support complex64 input. For complex64 ` + + `GPGPUProgram does not support complex64 input. For complex64 ` + `dtypes, please separate the program into real and imaginary ` + - `parts.`); + `parts.` + ); } let texData = this.texData.get(input.dataId); if (texData.texture == null) { - if (!program.packedInputs && - util.sizeFromShape(input.shape) <= - env().getNumber('WEBGL_SIZE_UPLOAD_UNIFORM')) { + if ( + !program.packedInputs && + util.sizeFromShape(input.shape) <= + env().getNumber("WEBGL_SIZE_UPLOAD_UNIFORM") + ) { // Upload small tensors that live on the CPU as uniforms, not as // textures. Do this only when the environment supports 32bit floats // due to problems when comparing 16bit floats with 32bit floats. @@ -850,7 +1008,7 @@ export class MathBackendWebGL extends KernelBackend { shape: input.shape, texData: null, isUniform: true, - uniformValues: texData.values as TypedArray + uniformValues: texData.values as TypedArray, }; } @@ -861,13 +1019,15 @@ export class MathBackendWebGL extends KernelBackend { texData.shape = input.shape; } } else if (!!texData.isPacked !== !!program.packedInputs) { - input = texData.isPacked ? this.unpackTensor(input) : - this.packTensor(input); + input = texData.isPacked + ? this.unpackTensor(input) + : this.packTensor(input); dataToDispose.push(input); texData = this.texData.get(input.dataId); } else if ( - texData.isPacked && - !webgl_util.isReshapeFree(texData.shape, input.shape)) { + texData.isPacked && + !webgl_util.isReshapeFree(texData.shape, input.shape) + ) { // This is a special case where a texture exists for a tensor // but the shapes are incompatible (due to packing constraints) because // the tensor did not have a chance to go through the packed reshape @@ -887,36 +1047,53 @@ export class MathBackendWebGL extends KernelBackend { } this.uploadToGPU(input.dataId); - return {shape: input.shape, texData, isUniform: false}; + return { shape: input.shape, texData, isUniform: false }; }); this.uploadToGPU(output.dataId); - const outputData: - TensorData = {shape: output.shape, texData: outData, isUniform: false}; + const outputData: TensorData = { + shape: output.shape, + texData: outData, + isUniform: false, + }; const key = gpgpu_math.makeShaderKey(program, inputsData, outputData); const binary = this.getAndSaveBinary(key, () => { return gpgpu_math.compileProgram( - this.gpgpu, program, inputsData, outputData); + this.gpgpu, + program, + inputsData, + outputData + ); }); const shouldTimeProgram = this.activeTimers != null; - let query: WebGLQuery|CPUTimerQuery; + let query: WebGLQuery | CPUTimerQuery; if (shouldTimeProgram) { query = this.startTimer(); } gpgpu_math.runProgram( - this.gpgpu, binary, inputsData, outputData, customSetup); + this.gpgpu, + binary, + inputsData, + outputData, + customSetup + ); - dataToDispose.forEach(info => this.disposeIntermediateTensorInfo(info)); + dataToDispose.forEach((info) => this.disposeIntermediateTensorInfo(info)); if (shouldTimeProgram) { query = this.endTimer(query); - this.activeTimers.push( - {name: program.constructor.name, query: this.getQueryTime(query)}); + this.activeTimers.push({ + name: program.constructor.name, + query: this.getQueryTime(query), + }); } - if (!env().getBool('WEBGL_LAZILY_UNPACK') && outData.isPacked && - preventEagerUnpackingOfOutput === false) { + if ( + !env().getBool("WEBGL_LAZILY_UNPACK") && + outData.isPacked && + preventEagerUnpackingOfOutput === false + ) { const unpacked = this.unpackTensor(output); this.disposeIntermediateTensorInfo(output); return unpacked; @@ -925,18 +1102,27 @@ export class MathBackendWebGL extends KernelBackend { } compileAndRun( - program: GPGPUProgram, inputs: TensorInfo[], outputDtype?: DataType, - customSetup?: (gpgpu: GPGPUContext, webGLProgram: WebGLProgram) => void, - preventEagerUnpackingOfOutput = false): TensorInfo { + program: GPGPUProgram, + inputs: TensorInfo[], + outputDtype?: DataType, + customSetup?: (gpgpu: GPGPUContext, webGLProgram: WebGLProgram) => void, + preventEagerUnpackingOfOutput = false + ): TensorInfo { outputDtype = outputDtype || inputs[0].dtype; const outInfo = this.runWebGLProgram( - program, inputs, outputDtype, customSetup, - preventEagerUnpackingOfOutput); + program, + inputs, + outputDtype, + customSetup, + preventEagerUnpackingOfOutput + ); return outInfo; } - private getAndSaveBinary(key: string, getBinary: () => GPGPUBinary): - GPGPUBinary { + private getAndSaveBinary( + key: string, + getBinary: () => GPGPUBinary + ): GPGPUBinary { if (!(key in this.binaryCache)) { this.binaryCache[key] = getBinary(); } @@ -955,17 +1141,19 @@ export class MathBackendWebGL extends KernelBackend { } // Avoid disposing the compiled webgl programs during unit testing because // it slows down test execution. - if (!env().getBool('IS_TEST')) { + if (!env().getBool("IS_TEST")) { const allKeys = Object.keys(this.binaryCache); - allKeys.forEach(key => { + allKeys.forEach((key) => { this.gpgpu.deleteProgram(this.binaryCache[key].webGLProgram); delete this.binaryCache[key]; }); } this.textureManager.dispose(); - if (this.canvas != null && - (typeof (HTMLCanvasElement) !== 'undefined' && - this.canvas instanceof HTMLCanvasElement)) { + if ( + this.canvas != null && + typeof HTMLCanvasElement !== "undefined" && + this.canvas instanceof HTMLCanvasElement + ) { this.canvas.remove(); } else { this.canvas = null; @@ -977,16 +1165,16 @@ export class MathBackendWebGL extends KernelBackend { this.disposed = true; } - floatPrecision(): 16|32 { + floatPrecision(): 16 | 32 { if (this.floatPrecisionValue == null) { this.floatPrecisionValue = tidy(() => { - if (!env().get('WEBGL_RENDER_FLOAT32_ENABLED')) { + if (!env().get("WEBGL_RENDER_FLOAT32_ENABLED")) { // Momentarily switching DEBUG flag to false so we don't throw an // error trying to upload a small value. - const debugFlag = env().getBool('DEBUG'); - env().set('DEBUG', false); + const debugFlag = env().getBool("DEBUG"); + env().set("DEBUG", false); const underflowCheckValue = this.abs(scalar(1e-8)).dataSync()[0]; - env().set('DEBUG', debugFlag); + env().set("DEBUG", debugFlag); if (underflowCheckValue > 0) { return 32; @@ -1005,7 +1193,7 @@ export class MathBackendWebGL extends KernelBackend { uploadToGPU(dataId: DataId): void { const texData = this.texData.get(dataId); - const {shape, dtype, values, texture, usage, isPacked} = texData; + const { shape, dtype, values, texture, usage, isPacked } = texData; if (texture != null) { // Array is already on GPU. No-op. @@ -1027,36 +1215,53 @@ export class MathBackendWebGL extends KernelBackend { const shapeAs3D = webgl_util.getShapeAs3D(shape); let program; - let width = texShape[1], height = texShape[0]; + let width = texShape[1], + height = texShape[0]; const isByteArray = values instanceof Uint8Array; if (isPacked) { [width, height] = tex_util.getPackedMatrixTextureShapeWidthHeight( - texShape[0], texShape[1]); + texShape[0], + texShape[1] + ); program = new EncodeMatrixPackedProgram( - shapeAs3D, [height, width], isByteArray); + shapeAs3D, + [height, width], + isByteArray + ); } else { - program = - new EncodeMatrixProgram(shapeAs3D, [height, width], isByteArray); + program = new EncodeMatrixProgram( + shapeAs3D, + [height, width], + isByteArray + ); } const tempDenseInputHandle = this.makeTensorInfo([height, width], dtype); if (isByteArray) { this.texData.get(tempDenseInputHandle.dataId).usage = - TextureUsage.PIXELS; + TextureUsage.PIXELS; } else { this.texData.get(tempDenseInputHandle.dataId).usage = - TextureUsage.UPLOAD; + TextureUsage.UPLOAD; } this.gpgpu.uploadDenseMatrixToTexture( - this.getTexture(tempDenseInputHandle.dataId), width, height, - values as TypedArray); + this.getTexture(tempDenseInputHandle.dataId), + width, + height, + values as TypedArray + ); // We want the output to remain packed regardless of the value of // WEBGL_PACK. const preventEagerUnpacking = true; const encodedOutputTarget = this.runWebGLProgram( - program, [tempDenseInputHandle], dtype, null, preventEagerUnpacking); + program, + [tempDenseInputHandle], + dtype, + null, + preventEagerUnpacking + ); // Have the original texture assume the identity of the encoded output. const outputTexData = this.texData.get(encodedOutputTarget.dataId); @@ -1079,30 +1284,38 @@ export class MathBackendWebGL extends KernelBackend { } } - private convertAndCacheOnCPU(dataId: DataId, float32Values?: Float32Array): - TypedArray { + private convertAndCacheOnCPU( + dataId: DataId, + float32Values?: Float32Array + ): TypedArray { const texData = this.texData.get(dataId); - const {dtype} = texData; + const { dtype } = texData; this.releaseGPUData(dataId); if (float32Values != null) { - texData.values = float32ToTypedArray(float32Values, dtype as 'float32'); + texData.values = float32ToTypedArray(float32Values, dtype as "float32"); } return texData.values as TypedArray; } private acquireTexture( - texShape: [number, number], texType: TextureUsage, dtype: DataType, - isPacked: boolean): WebGLTexture { + texShape: [number, number], + texType: TextureUsage, + dtype: DataType, + isPacked: boolean + ): WebGLTexture { this.numBytesInGPU += this.computeBytes(texShape, dtype); - if (!this.warnedAboutMemory && - this.numBytesInGPU > this.numMBBeforeWarning * 1024 * 1024) { + if ( + !this.warnedAboutMemory && + this.numBytesInGPU > this.numMBBeforeWarning * 1024 * 1024 + ) { const mb = (this.numBytesInGPU / 1024 / 1024).toFixed(2); this.warnedAboutMemory = true; console.warn( - `High memory usage in GPU: ${mb} MB, ` + - `most likely due to a memory leak`); + `High memory usage in GPU: ${mb} MB, ` + + `most likely due to a memory leak` + ); } return this.textureManager.acquireTexture(texShape, texType, isPacked); } @@ -1113,12 +1326,14 @@ export class MathBackendWebGL extends KernelBackend { } function float32ToTypedArray( - a: Float32Array, dtype: D): tf.DataTypeMap[D] { - if (dtype === 'float32' || dtype === 'complex64') { + a: Float32Array, + dtype: D +): tf.DataTypeMap[D] { + if (dtype === "float32" || dtype === "complex64") { return a as tf.DataTypeMap[D]; - } else if (dtype === 'int32' || dtype === 'bool') { - const result = (dtype === 'int32') ? new Int32Array(a.length) : - new Uint8Array(a.length); + } else if (dtype === "int32" || dtype === "bool") { + const result = + dtype === "int32" ? new Int32Array(a.length) : new Uint8Array(a.length); for (let i = 0; i < result.length; ++i) { result[i] = Math.round(a[i]); } From bb1d13978340fed5a667ef975e800969d1037f5b Mon Sep 17 00:00:00 2001 From: Ann Yuan Date: Thu, 11 Feb 2021 15:09:27 -0500 Subject: [PATCH 31/31] undo format --- tfjs-backend-webgl/src/backend_webgl.ts | 732 +++++++++--------------- 1 file changed, 258 insertions(+), 474 deletions(-) diff --git a/tfjs-backend-webgl/src/backend_webgl.ts b/tfjs-backend-webgl/src/backend_webgl.ts index a8d2ea9580..1a8568e0e0 100644 --- a/tfjs-backend-webgl/src/backend_webgl.ts +++ b/tfjs-backend-webgl/src/backend_webgl.ts @@ -16,58 +16,32 @@ */ // Import webgl flags. -import "./flags_webgl"; - -import * as tf from "@tensorflow/tfjs-core"; -import { - backend_util, - BackendValues, - buffer, - DataId, - DataStorage, - DataType, - DataValues, - engine, - env, - kernel_impls, - KernelBackend, - MemoryInfo, - NumericDataType, - Rank, - RecursiveArray, - scalar, - ShapeMap, - Tensor, - Tensor2D, - TensorBuffer, - TensorInfo, - tidy, - TimingInfo, - TypedArray, - util, -} from "@tensorflow/tfjs-core"; - -import { getWebGLContext } from "./canvas_util"; -import { DecodeMatrixProgram } from "./decode_matrix_gpu"; -import { DecodeMatrixPackedProgram } from "./decode_matrix_packed_gpu"; -import { EncodeFloatProgram } from "./encode_float_gpu"; -import { EncodeFloatPackedProgram } from "./encode_float_packed_gpu"; -import { EncodeMatrixProgram } from "./encode_matrix_gpu"; -import { EncodeMatrixPackedProgram } from "./encode_matrix_packed_gpu"; -import { GPGPUContext } from "./gpgpu_context"; -import * as gpgpu_math from "./gpgpu_math"; -import { GPGPUBinary, GPGPUProgram, TensorData } from "./gpgpu_math"; -import { simpleAbsImplCPU } from "./kernel_utils/shared"; -import { PackProgram } from "./pack_gpu"; -import { ReshapePackedProgram } from "./reshape_packed_gpu"; -import * as tex_util from "./tex_util"; -import { TextureData, TextureUsage } from "./tex_util"; -import { TextureManager } from "./texture_manager"; -import * as unary_op from "./unaryop_gpu"; -import { UnaryOpProgram } from "./unaryop_gpu"; -import { UnaryOpPackedProgram } from "./unaryop_packed_gpu"; -import { UnpackProgram } from "./unpack_gpu"; -import * as webgl_util from "./webgl_util"; +import './flags_webgl'; + +import * as tf from '@tensorflow/tfjs-core'; +import {backend_util, BackendValues, buffer, DataId, DataStorage, DataType, DataValues, engine, env, kernel_impls, KernelBackend, MemoryInfo, NumericDataType, Rank, RecursiveArray, scalar, ShapeMap, Tensor, Tensor2D, TensorBuffer, TensorInfo, tidy, TimingInfo, TypedArray, util} from '@tensorflow/tfjs-core'; + +import {getWebGLContext} from './canvas_util'; +import {DecodeMatrixProgram} from './decode_matrix_gpu'; +import {DecodeMatrixPackedProgram} from './decode_matrix_packed_gpu'; +import {EncodeFloatProgram} from './encode_float_gpu'; +import {EncodeFloatPackedProgram} from './encode_float_packed_gpu'; +import {EncodeMatrixProgram} from './encode_matrix_gpu'; +import {EncodeMatrixPackedProgram} from './encode_matrix_packed_gpu'; +import {GPGPUContext} from './gpgpu_context'; +import * as gpgpu_math from './gpgpu_math'; +import {GPGPUBinary, GPGPUProgram, TensorData} from './gpgpu_math'; +import {simpleAbsImplCPU} from './kernel_utils/shared'; +import {PackProgram} from './pack_gpu'; +import {ReshapePackedProgram} from './reshape_packed_gpu'; +import * as tex_util from './tex_util'; +import {TextureData, TextureUsage} from './tex_util'; +import {TextureManager} from './texture_manager'; +import * as unary_op from './unaryop_gpu'; +import {UnaryOpProgram} from './unaryop_gpu'; +import {UnaryOpPackedProgram} from './unaryop_packed_gpu'; +import {UnpackProgram} from './unpack_gpu'; +import * as webgl_util from './webgl_util'; const whereImpl = kernel_impls.whereImpl; @@ -75,11 +49,10 @@ export const EPSILON_FLOAT32 = 1e-7; export const EPSILON_FLOAT16 = 1e-4; type KernelInfo = { - name: string; - query: Promise; + name: string; query: Promise; }; -export type TimerNode = RecursiveArray | KernelInfo; +export type TimerNode = RecursiveArray|KernelInfo; export interface CPUTimerQuery { startMs: number; endMs?: number; @@ -101,9 +74,7 @@ export interface WebGLTimingInfo extends TimingInfo { downloadWaitMs: number; } -const binaryCaches: { - [webGLVersion: string]: { [key: string]: GPGPUBinary }; -} = {}; +const binaryCaches: {[webGLVersion: string]: {[key: string]: GPGPUBinary}} = {}; export function getBinaryCache(webGLVersion: number) { if (webGLVersion in binaryCaches) { @@ -123,16 +94,11 @@ const CPU_HANDOFF_SIZE_THRESHOLD = 128; const BEFORE_PAGING_CONSTANT = 600; function numMBBeforeWarning(): number { if (env().global.screen == null) { - return 1024; // 1 GB. + return 1024; // 1 GB. } - return ( - (env().global.screen.height * - env().global.screen.width * - window.devicePixelRatio * - BEFORE_PAGING_CONSTANT) / - 1024 / - 1024 - ); + return (env().global.screen.height * env().global.screen.width * + window.devicePixelRatio) * + BEFORE_PAGING_CONSTANT / 1024 / 1024; } export class MathBackendWebGL extends KernelBackend { @@ -154,7 +120,7 @@ export class MathBackendWebGL extends KernelBackend { dataRefCount = new WeakMap(); private numBytesInGPU = 0; - private canvas: HTMLCanvasElement | OffscreenCanvas; + private canvas: HTMLCanvasElement|OffscreenCanvas; private programTimersStack: TimerNode[]; private activeTimers: TimerNode[]; @@ -165,10 +131,10 @@ export class MathBackendWebGL extends KernelBackend { private cpuBackend: KernelBackend; // Number of bits of precision of this backend. - private floatPrecisionValue: 32 | 16; + private floatPrecisionValue: 32|16; private textureManager: TextureManager; - private binaryCache: { [key: string]: GPGPUBinary }; + private binaryCache: {[key: string]: GPGPUBinary}; private gpgpuCreatedLocally: boolean; private numMBBeforeWarning: number; private warnedAboutMemory = false; @@ -176,13 +142,13 @@ export class MathBackendWebGL extends KernelBackend { constructor(gpgpu?: GPGPUContext) { super(); - if (!env().getBool("HAS_WEBGL")) { - throw new Error("WebGL is not supported on this device"); + if (!env().getBool('HAS_WEBGL')) { + throw new Error('WebGL is not supported on this device'); } if (gpgpu == null) { - const gl = getWebGLContext(env().getNumber("WEBGL_VERSION")); - this.binaryCache = getBinaryCache(env().getNumber("WEBGL_VERSION")); + const gl = getWebGLContext(env().getNumber('WEBGL_VERSION')); + this.binaryCache = getBinaryCache(env().getNumber('WEBGL_VERSION')); this.gpgpu = new GPGPUContext(gl); this.canvas = gl.canvas; this.gpgpuCreatedLocally = true; @@ -199,21 +165,16 @@ export class MathBackendWebGL extends KernelBackend { } numDataIds() { - return ( - this.texData.numDataIds() + - (this.cpuBackend ? this.cpuBackend.numDataIds() : 0) - - this.pendingDeletes - ); + return this.texData.numDataIds() + + (this.cpuBackend ? this.cpuBackend.numDataIds() : 0) - + this.pendingDeletes; } // Writes a new entry to the data store with a WebGL texture, and registers it // to the texture manager. writeTexture( - texture: WebGLTexture, - shape: number[], - dtype: DataType, - texShape: [number, number] - ): DataId { + texture: WebGLTexture, shape: number[], dtype: DataType, + texShape: [number, number]): DataId { const dataId = {}; this.texData.set(dataId, { shape, @@ -222,7 +183,7 @@ export class MathBackendWebGL extends KernelBackend { texture, texShape, isPacked: false, - refCount: 1, + refCount: 1 }); this.textureManager.registerRenderTexture(texture, texShape); @@ -230,26 +191,19 @@ export class MathBackendWebGL extends KernelBackend { } write(values: BackendValues, shape: number[], dtype: DataType): DataId { - if ( - env().getBool("WEBGL_CHECK_NUMERICAL_PROBLEMS") || - env().getBool("DEBUG") - ) { + if (env().getBool('WEBGL_CHECK_NUMERICAL_PROBLEMS') || + env().getBool('DEBUG')) { this.checkNumericalProblems(values); } - if (dtype === "complex64" && values != null) { + if (dtype === 'complex64' && values != null) { throw new Error( - `Cannot write to a complex64 dtype. ` + - `Please use tf.complex(real, imag).` - ); + `Cannot write to a complex64 dtype. ` + + `Please use tf.complex(real, imag).`); } - const dataId = { id: this.nextDataId() }; - this.texData.set(dataId, { - shape, - dtype, - values, - usage: TextureUsage.UPLOAD, - refCount: 1, - }); + const dataId = {id: this.nextDataId()}; + this.texData.set( + dataId, + {shape, dtype, values, usage: TextureUsage.UPLOAD, refCount: 1}); return dataId; } @@ -277,28 +231,18 @@ export class MathBackendWebGL extends KernelBackend { } move( - dataId: DataId, - values: BackendValues, - shape: number[], - dtype: DataType, - refCount: number - ): void { - if (env().getBool("DEBUG")) { + dataId: DataId, values: BackendValues, shape: number[], dtype: DataType, + refCount: number): void { + if (env().getBool('DEBUG')) { this.checkNumericalProblems(values); } - if (dtype === "complex64") { + if (dtype === 'complex64') { throw new Error( - `Cannot write to a complex64 dtype. ` + - `Please use tf.complex(real, imag).` - ); + `Cannot write to a complex64 dtype. ` + + `Please use tf.complex(real, imag).`); } - this.texData.set(dataId, { - shape, - dtype, - values, - usage: TextureUsage.UPLOAD, - refCount, - }); + this.texData.set( + dataId, {shape, dtype, values, usage: TextureUsage.UPLOAD, refCount}); } disposeIntermediateTensorInfo(tensorInfo: TensorInfo): void { @@ -307,14 +251,7 @@ export class MathBackendWebGL extends KernelBackend { readSync(dataId: DataId): BackendValues { const texData = this.texData.get(dataId); - const { - values, - dtype, - complexTensorInfos, - slice, - shape, - isPacked, - } = texData; + const {values, dtype, complexTensorInfos, slice, shape, isPacked} = texData; // The presence of `slice` indicates this tensor is a shallow slice of a // different tensor, and is using that original tensor's texture. Run @@ -326,11 +263,8 @@ export class MathBackendWebGL extends KernelBackend { } else { program = new UnaryOpProgram(shape, unary_op.CLONE); } - const res = this.runWebGLProgram( - program, - [{ dataId, shape, dtype }], - dtype - ); + const res = + this.runWebGLProgram(program, [{dataId, shape, dtype}], dtype); const data = this.readSync(res.dataId); this.disposeIntermediateTensorInfo(res); return data; @@ -338,7 +272,7 @@ export class MathBackendWebGL extends KernelBackend { if (values != null) { return this.convertAndCacheOnCPU(dataId); } - if (dtype === "string") { + if (dtype === 'string') { return values; } const shouldTimeProgram = this.activeTimers != null; @@ -348,13 +282,11 @@ export class MathBackendWebGL extends KernelBackend { } let result: Float32Array; - if (dtype === "complex64") { - const realValues = this.readSync( - complexTensorInfos.real.dataId - ) as Float32Array; - const imagValues = this.readSync( - complexTensorInfos.imag.dataId - ) as Float32Array; + if (dtype === 'complex64') { + const realValues = + this.readSync(complexTensorInfos.real.dataId) as Float32Array; + const imagValues = + this.readSync(complexTensorInfos.imag.dataId) as Float32Array; result = backend_util.mergeRealAndImagArrays(realValues, imagValues); } else { result = this.getValuesFromTexture(dataId); @@ -369,17 +301,10 @@ export class MathBackendWebGL extends KernelBackend { async read(dataId: DataId): Promise { if (this.pendingRead.has(dataId)) { const subscribers = this.pendingRead.get(dataId); - return new Promise((resolve) => subscribers.push(resolve)); + return new Promise(resolve => subscribers.push(resolve)); } const texData = this.texData.get(dataId); - const { - values, - shape, - slice, - dtype, - complexTensorInfos, - isPacked, - } = texData; + const {values, shape, slice, dtype, complexTensorInfos, isPacked} = texData; // The presence of `slice` indicates this tensor is a shallow slice of a // different tensor, and is using that original tensor's texture. Run @@ -391,11 +316,8 @@ export class MathBackendWebGL extends KernelBackend { } else { program = new UnaryOpProgram(shape, unary_op.CLONE); } - const res = this.runWebGLProgram( - program, - [{ dataId, shape, dtype }], - dtype - ); + const res = + this.runWebGLProgram(program, [{dataId, shape, dtype}], dtype); const data = this.read(res.dataId); this.disposeIntermediateTensorInfo(res); return data; @@ -405,51 +327,44 @@ export class MathBackendWebGL extends KernelBackend { return this.convertAndCacheOnCPU(dataId); } - if ( - !env().getBool("WEBGL_DOWNLOAD_FLOAT_ENABLED") && - env().getNumber("WEBGL_VERSION") === 2 - ) { + if (!env().getBool('WEBGL_DOWNLOAD_FLOAT_ENABLED') && + env().getNumber('WEBGL_VERSION') === 2) { throw new Error( - `tensor.data() with WEBGL_DOWNLOAD_FLOAT_ENABLED=false and ` + - `WEBGL_VERSION=2 not yet supported.` - ); + `tensor.data() with WEBGL_DOWNLOAD_FLOAT_ENABLED=false and ` + + `WEBGL_VERSION=2 not yet supported.`); } let buffer = null; let tmpDownloadTarget: TensorInfo; - if (dtype !== "complex64" && env().get("WEBGL_BUFFER_SUPPORTED")) { + if (dtype !== 'complex64' && env().get('WEBGL_BUFFER_SUPPORTED')) { // Possibly copy the texture into a buffer before inserting a fence. tmpDownloadTarget = this.decode(dataId); const tmpData = this.texData.get(tmpDownloadTarget.dataId); buffer = this.gpgpu.createBufferFromTexture( - tmpData.texture, - ...tex_util.getDenseTexShape(shape) - ); + tmpData.texture, ...tex_util.getDenseTexShape(shape)); } this.pendingRead.set(dataId, []); - if (dtype !== "complex64") { + if (dtype !== 'complex64') { // Create a fence and wait for it to resolve. await this.gpgpu.createAndWaitForFence(); } // Download the values from the GPU. let vals: Float32Array; - if (dtype === "complex64") { + if (dtype === 'complex64') { const ps = await Promise.all([ this.read(complexTensorInfos.real.dataId), - this.read(complexTensorInfos.imag.dataId), + this.read(complexTensorInfos.imag.dataId) ]); const realValues = ps[0]; const imagValues = ps[1]; vals = backend_util.mergeRealAndImagArrays( - realValues as Float32Array, - imagValues as Float32Array - ); + realValues as Float32Array, imagValues as Float32Array); } else if (buffer == null) { vals = this.getValuesFromTexture(dataId); } else { @@ -465,7 +380,7 @@ export class MathBackendWebGL extends KernelBackend { this.pendingRead.delete(dataId); // Notify all pending reads. - subscribers.forEach((resolve) => resolve(dTypeVals)); + subscribers.forEach(resolve => resolve(dTypeVals)); if (this.pendingDisposal.has(dataId)) { this.pendingDisposal.delete(dataId); if (this.disposeData(dataId)) { @@ -479,19 +394,16 @@ export class MathBackendWebGL extends KernelBackend { bufferSync(t: TensorInfo): TensorBuffer { const data = this.readSync(t.dataId); let decodedData = data as DataValues; - if (t.dtype === "string") { + if (t.dtype === 'string') { try { // Decode the bytes into string. - decodedData = (data as Uint8Array[]).map((d) => util.decodeString(d)); + decodedData = (data as Uint8Array[]).map(d => util.decodeString(d)); } catch { - throw new Error("Failed to decode encoded string bytes into utf-8"); + throw new Error('Failed to decode encoded string bytes into utf-8'); } } - return buffer( - t.shape as ShapeMap[R], - t.dtype, - decodedData - ) as TensorBuffer; + return buffer(t.shape as ShapeMap[R], t.dtype, decodedData) as + TensorBuffer; } private checkNumericalProblems(values: BackendValues): void { @@ -501,12 +413,11 @@ export class MathBackendWebGL extends KernelBackend { for (let i = 0; i < values.length; i++) { const num = values[i] as number; if (!webgl_util.canBeRepresented(num)) { - if (env().getBool("WEBGL_RENDER_FLOAT32_CAPABLE")) { + if (env().getBool('WEBGL_RENDER_FLOAT32_CAPABLE')) { throw Error( - `The value ${num} cannot be represented with your ` + + `The value ${num} cannot be represented with your ` + `current settings. Consider enabling float32 rendering: ` + - `'tf.env().set('WEBGL_RENDER_FLOAT32_ENABLED', true);'` - ); + `'tf.env().set('WEBGL_RENDER_FLOAT32_ENABLED', true);'`); } throw Error(`The value ${num} cannot be represented on this device.`); } @@ -514,17 +425,15 @@ export class MathBackendWebGL extends KernelBackend { } private getValuesFromTexture(dataId: DataId): Float32Array { - const { shape, dtype, isPacked } = this.texData.get(dataId); + const {shape, dtype, isPacked} = this.texData.get(dataId); const size = util.sizeFromShape(shape); - if (env().getBool("WEBGL_DOWNLOAD_FLOAT_ENABLED")) { + if (env().getBool('WEBGL_DOWNLOAD_FLOAT_ENABLED')) { const tmpTarget = this.decode(dataId); const tmpData = this.texData.get(tmpTarget.dataId); const vals = this.gpgpu - .downloadMatrixFromPackedTexture( - tmpData.texture, - ...tex_util.getDenseTexShape(shape) - ) - .subarray(0, size); + .downloadMatrixFromPackedTexture( + tmpData.texture, ...tex_util.getDenseTexShape(shape)) + .subarray(0, size); this.disposeIntermediateTensorInfo(tmpTarget); @@ -532,33 +441,27 @@ export class MathBackendWebGL extends KernelBackend { } const shouldUsePackedProgram = - env().getBool("WEBGL_PACK") && isPacked === true; - const outputShape = shouldUsePackedProgram - ? webgl_util.getShapeAs3D(shape) - : shape; - const program = shouldUsePackedProgram - ? new EncodeFloatPackedProgram(outputShape as [number, number, number]) - : new EncodeFloatProgram(outputShape); + env().getBool('WEBGL_PACK') && isPacked === true; + const outputShape = + shouldUsePackedProgram ? webgl_util.getShapeAs3D(shape) : shape; + const program = shouldUsePackedProgram ? + new EncodeFloatPackedProgram(outputShape as [number, number, number]) : + new EncodeFloatProgram(outputShape); const output = this.runWebGLProgram( - program, - [{ shape: outputShape, dtype, dataId }], - "float32" - ); + program, [{shape: outputShape, dtype, dataId}], 'float32'); const tmpData = this.texData.get(output.dataId); - const vals = this.gpgpu - .downloadByteEncodedFloatMatrixFromOutputTexture( - tmpData.texture, - tmpData.texShape[0], - tmpData.texShape[1] - ) - .subarray(0, size); + const vals = + this.gpgpu + .downloadByteEncodedFloatMatrixFromOutputTexture( + tmpData.texture, tmpData.texShape[0], tmpData.texShape[1]) + .subarray(0, size); this.disposeIntermediateTensorInfo(output); return vals; } timerAvailable(): boolean { - return env().getNumber("WEBGL_DISJOINT_QUERY_TIMER_EXTENSION_RELIABLE") > 0; + return env().getNumber('WEBGL_DISJOINT_QUERY_TIMER_EXTENSION_RELIABLE') > 0; } async time(f: () => void): Promise { @@ -577,12 +480,12 @@ export class MathBackendWebGL extends KernelBackend { f(); // needing to split these up because util.flatten only accepts certain types - const flattenedActiveTimerQueries = util - .flatten(this.activeTimers.map((d: KernelInfo) => d.query)) - .filter((d) => d != null); - const flattenedActiveTimerNames = util - .flatten(this.activeTimers.map((d: KernelInfo) => d.name)) - .filter((d) => d != null); + const flattenedActiveTimerQueries = + util.flatten(this.activeTimers.map((d: KernelInfo) => d.query)) + .filter(d => d != null); + const flattenedActiveTimerNames = + util.flatten(this.activeTimers.map((d: KernelInfo) => d.name)) + .filter(d => d != null); this.activeTimers = oldActiveTimers; @@ -594,21 +497,20 @@ export class MathBackendWebGL extends KernelBackend { uploadWaitMs: this.uploadWaitMs, downloadWaitMs: this.downloadWaitMs, kernelMs: null, - wallMs: null, // will be filled by the engine + wallMs: null // will be filled by the engine }; - if (env().getNumber("WEBGL_DISJOINT_QUERY_TIMER_EXTENSION_RELIABLE") > 0) { + if (env().getNumber('WEBGL_DISJOINT_QUERY_TIMER_EXTENSION_RELIABLE') > 0) { const kernelMs = await Promise.all(flattenedActiveTimerQueries); - res["kernelMs"] = util.sum(kernelMs); - res["getExtraProfileInfo"] = () => - kernelMs - .map((d, i) => ({ name: flattenedActiveTimerNames[i], ms: d })) - .map((d) => `${d.name}: ${d.ms}`) - .join(", "); + res['kernelMs'] = util.sum(kernelMs); + res['getExtraProfileInfo'] = () => + kernelMs.map((d, i) => ({name: flattenedActiveTimerNames[i], ms: d})) + .map(d => `${d.name}: ${d.ms}`) + .join(', '); } else { - res["kernelMs"] = { - error: "WebGL query timers are not supported in this environment.", + res['kernelMs'] = { + error: 'WebGL query timers are not supported in this environment.' }; } @@ -621,21 +523,19 @@ export class MathBackendWebGL extends KernelBackend { unreliable: false, numBytesInGPU: this.numBytesInGPU, numBytesInGPUAllocated: this.textureManager.numBytesAllocated, - numBytesInGPUFree: this.textureManager.numBytesFree, + numBytesInGPUFree: this.textureManager.numBytesFree } as WebGLMemoryInfo; } - private startTimer(): WebGLQuery | CPUTimerQuery { - if (env().getNumber("WEBGL_DISJOINT_QUERY_TIMER_EXTENSION_RELIABLE") > 0) { + private startTimer(): WebGLQuery|CPUTimerQuery { + if (env().getNumber('WEBGL_DISJOINT_QUERY_TIMER_EXTENSION_RELIABLE') > 0) { return this.gpgpu.beginQuery(); } - return { startMs: util.now(), endMs: null }; + return {startMs: util.now(), endMs: null}; } - private endTimer( - query: WebGLQuery | CPUTimerQuery - ): WebGLQuery | CPUTimerQuery { - if (env().getNumber("WEBGL_DISJOINT_QUERY_TIMER_EXTENSION_RELIABLE") > 0) { + private endTimer(query: WebGLQuery|CPUTimerQuery): WebGLQuery|CPUTimerQuery { + if (env().getNumber('WEBGL_DISJOINT_QUERY_TIMER_EXTENSION_RELIABLE') > 0) { this.gpgpu.endQuery(); return query; } @@ -643,10 +543,8 @@ export class MathBackendWebGL extends KernelBackend { return query; } - private async getQueryTime( - query: WebGLQuery | CPUTimerQuery - ): Promise { - if (env().getNumber("WEBGL_DISJOINT_QUERY_TIMER_EXTENSION_RELIABLE") > 0) { + private async getQueryTime(query: WebGLQuery|CPUTimerQuery): Promise { + if (env().getNumber('WEBGL_DISJOINT_QUERY_TIMER_EXTENSION_RELIABLE') > 0) { return this.gpgpu.waitForQueryAndGetTime(query as WebGLQuery); } const timerQuery = query as CPUTimerQuery; @@ -695,7 +593,7 @@ export class MathBackendWebGL extends KernelBackend { } this.releaseGPUData(dataId); - const { complexTensorInfos } = this.texData.get(dataId); + const {complexTensorInfos} = this.texData.get(dataId); if (complexTensorInfos != null) { this.disposeData(complexTensorInfos.real.dataId, force); this.disposeData(complexTensorInfos.imag.dataId, force); @@ -707,15 +605,9 @@ export class MathBackendWebGL extends KernelBackend { } private releaseGPUData(dataId: DataId): void { - const { - texture, - dtype, - texShape, - usage, - isPacked, - slice, - } = this.texData.get(dataId); - const key = (slice && slice.origDataId) || dataId; + const {texture, dtype, texShape, usage, isPacked, slice} = + this.texData.get(dataId); + const key = slice && slice.origDataId || dataId; const refCount = this.dataRefCount.get(key); if (refCount > 1) { @@ -748,13 +640,13 @@ export class MathBackendWebGL extends KernelBackend { return this.texData.get(dataId); } - private getCPUBackend(): KernelBackend | null { - if (!env().getBool("WEBGL_CPU_FORWARD")) { + private getCPUBackend(): KernelBackend|null { + if (!env().getBool('WEBGL_CPU_FORWARD')) { return null; } if (this.cpuBackend == null) { - this.cpuBackend = engine().findBackend("cpu"); + this.cpuBackend = engine().findBackend('cpu'); } return this.cpuBackend; @@ -768,33 +660,24 @@ export class MathBackendWebGL extends KernelBackend { sustainable strategy for optimizing backend execution of ops. */ shouldExecuteOnCPU( - inputs: TensorInfo[], - sizeThreshold = CPU_HANDOFF_SIZE_THRESHOLD - ): boolean { + inputs: TensorInfo[], + sizeThreshold = CPU_HANDOFF_SIZE_THRESHOLD): boolean { const cpuBackend = this.getCPUBackend(); - if ( - !env().getBool("IS_TEST") && - !this.warnedAboutCPUBackend && - cpuBackend == null - ) { + if (!env().getBool('IS_TEST') && !this.warnedAboutCPUBackend && + cpuBackend == null) { console.warn( - "Your application contains ops that are small enough to be " + - "executed on the CPU backend, however the CPU backend cannot " + - "be found. Consider importing the CPU backend " + - "(@tensorflow/tfjs-backend-cpu) for better performance." - ); + 'Your application contains ops that are small enough to be ' + + 'executed on the CPU backend, however the CPU backend cannot ' + + 'be found. Consider importing the CPU backend ' + + '(@tensorflow/tfjs-backend-cpu) for better performance.'); this.warnedAboutCPUBackend = true; } - return ( - cpuBackend != null && - inputs.every( - (input) => - this.texData.get(input.dataId).texture == null && - util.sizeFromShape(input.shape) < sizeThreshold - ) - ); + return cpuBackend != null && + inputs.every( + input => this.texData.get(input.dataId).texture == null && + util.sizeFromShape(input.shape) < sizeThreshold); } getGPGPUContext(): GPGPUContext { @@ -803,9 +686,8 @@ export class MathBackendWebGL extends KernelBackend { where(condition: Tensor): Tensor2D { backend_util.warn( - "tf.where() in webgl locks the UI thread. " + - "Call tf.whereAsync() instead" - ); + 'tf.where() in webgl locks the UI thread. ' + + 'Call tf.whereAsync() instead'); const condVals = condition.dataSync(); return whereImpl(condition.shape, condVals); } @@ -814,10 +696,7 @@ export class MathBackendWebGL extends KernelBackend { const program = new UnaryOpPackedProgram(x.shape, op); const outInfo = this.compileAndRun(program, [x], dtype); return engine().makeTensorFromDataId( - outInfo.dataId, - outInfo.shape, - outInfo.dtype - ); + outInfo.dataId, outInfo.shape, outInfo.dtype); } // TODO(msoulanille) remove this once the backend has been modularized @@ -825,41 +704,30 @@ export class MathBackendWebGL extends KernelBackend { // Also remove the op from unary_op. abs(x: T): T { // TODO: handle cases when x is complex. - if (this.shouldExecuteOnCPU([x]) && x.dtype !== "complex64") { - const outValues = simpleAbsImplCPU( - this.texData.get(x.dataId).values as TypedArray - ); + if (this.shouldExecuteOnCPU([x]) && x.dtype !== 'complex64') { + const outValues = + simpleAbsImplCPU(this.texData.get(x.dataId).values as TypedArray); return this.makeOutput(x.shape, x.dtype, outValues); } - if (env().getBool("WEBGL_PACK_UNARY_OPERATIONS")) { + if (env().getBool('WEBGL_PACK_UNARY_OPERATIONS')) { return this.packedUnaryOp(x, unary_op.ABS, x.dtype) as T; } const program = new UnaryOpProgram(x.shape, unary_op.ABS); const outInfo = this.compileAndRun(program, [x]); return engine().makeTensorFromDataId( - outInfo.dataId, - outInfo.shape, - outInfo.dtype - ) as T; + outInfo.dataId, outInfo.shape, outInfo.dtype) as T; } makeTensorInfo( - shape: number[], - dtype: DataType, - values?: BackendValues | string[] - ): TensorInfo { + shape: number[], dtype: DataType, + values?: BackendValues|string[]): TensorInfo { let dataId; - if ( - dtype === "string" && - values != null && - values.length > 0 && - util.isString(values[0]) - ) { - const encodedValues = ((values as {}) as string[]).map((d) => - util.encodeString(d) - ); + if (dtype === 'string' && values != null && values.length > 0 && + util.isString(values[0])) { + const encodedValues = + (values as {} as string[]).map(d => util.encodeString(d)); dataId = this.write(encodedValues, shape, dtype); } else { @@ -867,15 +735,12 @@ export class MathBackendWebGL extends KernelBackend { } this.texData.get(dataId).usage = null; - return { dataId, shape, dtype }; + return {dataId, shape, dtype}; } private makeOutput( - shape: number[], - dtype: DataType, - values?: BackendValues - ): T { - const { dataId } = this.makeTensorInfo(shape, dtype, values); + shape: number[], dtype: DataType, values?: BackendValues): T { + const {dataId} = this.makeTensorInfo(shape, dtype, values); return engine().makeTensorFromDataId(dataId, shape, dtype, this) as T; } @@ -888,49 +753,37 @@ export class MathBackendWebGL extends KernelBackend { const program = new PackProgram(input.shape); const preventEagerUnpackingOutput = true; return this.runWebGLProgram( - program, - [input], - input.dtype, - null /* customSetup */, - preventEagerUnpackingOutput - ); + program, [input], input.dtype, null /* customSetup */, + preventEagerUnpackingOutput); } private packedReshape(input: TensorInfo, afterShape: number[]): TensorInfo { const input3DShape = [ webgl_util.getBatchDim(input.shape), - ...webgl_util.getRowsCols(input.shape), + ...webgl_util.getRowsCols(input.shape) ] as [number, number, number]; const input3D: TensorInfo = { dtype: input.dtype, shape: input3DShape, - dataId: input.dataId, + dataId: input.dataId }; const afterShapeAs3D = [ - webgl_util.getBatchDim(afterShape), - ...webgl_util.getRowsCols(afterShape), + webgl_util.getBatchDim(afterShape), ...webgl_util.getRowsCols(afterShape) ] as [number, number, number]; const program = new ReshapePackedProgram(afterShapeAs3D, input3DShape); const preventEagerUnpackingOfOutput = true; const output = this.runWebGLProgram( - program, - [input3D], - input.dtype, - null /* customSetup */, - preventEagerUnpackingOfOutput - ); - return { dataId: output.dataId, shape: afterShape, dtype: output.dtype }; + program, [input3D], input.dtype, null /* customSetup */, + preventEagerUnpackingOfOutput); + return {dataId: output.dataId, shape: afterShape, dtype: output.dtype}; } private decode(dataId: DataId): TensorInfo { const texData = this.texData.get(dataId); - const { isPacked, shape, dtype } = texData; - const shapeAs3D = webgl_util.getShapeAs3D(shape) as [ - number, - number, - number - ]; + const {isPacked, shape, dtype} = texData; + const shapeAs3D = + webgl_util.getShapeAs3D(shape) as [number, number, number]; let program; if (isPacked) { program = new DecodeMatrixPackedProgram(shapeAs3D); @@ -939,22 +792,15 @@ export class MathBackendWebGL extends KernelBackend { } const preventEagerUnpackingOfOutput = true; const out = this.runWebGLProgram( - program, - [{ shape: shapeAs3D, dtype, dataId }], - dtype, - null /* customSetup */, - preventEagerUnpackingOfOutput - ); - return { dtype, shape, dataId: out.dataId }; + program, [{shape: shapeAs3D, dtype, dataId}], dtype, + null /* customSetup */, preventEagerUnpackingOfOutput); + return {dtype, shape, dataId: out.dataId}; } runWebGLProgram( - program: GPGPUProgram, - inputs: TensorInfo[], - outputDtype: DataType, - customSetup?: (gpgpu: GPGPUContext, webGLProgram: WebGLProgram) => void, - preventEagerUnpackingOfOutput = false - ): TensorInfo { + program: GPGPUProgram, inputs: TensorInfo[], outputDtype: DataType, + customSetup?: (gpgpu: GPGPUContext, webGLProgram: WebGLProgram) => void, + preventEagerUnpackingOfOutput = false): TensorInfo { const output = this.makeTensorInfo(program.outputShape, outputDtype); const outData = this.texData.get(output.dataId); if (program.packedOutput) { @@ -966,7 +812,7 @@ export class MathBackendWebGL extends KernelBackend { // so it doesn't get assigned later according to our typical packing // scheme wherein a single texel can only contain values from adjacent // rows/cols. - outData.texShape = texelShape.map((d) => d * 2) as [number, number]; + outData.texShape = texelShape.map(d => d * 2) as [number, number]; } if (program.outTexUsage != null) { outData.usage = program.outTexUsage; @@ -974,31 +820,26 @@ export class MathBackendWebGL extends KernelBackend { if (util.sizeFromShape(output.shape) === 0) { // Short-circuit the computation since the result is empty (has 0 in its // shape). - outData.values = util.getTypedArrayFromDType( - output.dtype as "float32", - 0 - ); + outData.values = + util.getTypedArrayFromDType(output.dtype as 'float32', 0); return output; } const dataToDispose: TensorInfo[] = []; - const inputsData: TensorData[] = inputs.map((input) => { - if (input.dtype === "complex64") { + const inputsData: TensorData[] = inputs.map(input => { + if (input.dtype === 'complex64') { throw new Error( - `GPGPUProgram does not support complex64 input. For complex64 ` + + `GPGPUProgram does not support complex64 input. For complex64 ` + `dtypes, please separate the program into real and imaginary ` + - `parts.` - ); + `parts.`); } let texData = this.texData.get(input.dataId); if (texData.texture == null) { - if ( - !program.packedInputs && - util.sizeFromShape(input.shape) <= - env().getNumber("WEBGL_SIZE_UPLOAD_UNIFORM") - ) { + if (!program.packedInputs && + util.sizeFromShape(input.shape) <= + env().getNumber('WEBGL_SIZE_UPLOAD_UNIFORM')) { // Upload small tensors that live on the CPU as uniforms, not as // textures. Do this only when the environment supports 32bit floats // due to problems when comparing 16bit floats with 32bit floats. @@ -1008,7 +849,7 @@ export class MathBackendWebGL extends KernelBackend { shape: input.shape, texData: null, isUniform: true, - uniformValues: texData.values as TypedArray, + uniformValues: texData.values as TypedArray }; } @@ -1019,15 +860,13 @@ export class MathBackendWebGL extends KernelBackend { texData.shape = input.shape; } } else if (!!texData.isPacked !== !!program.packedInputs) { - input = texData.isPacked - ? this.unpackTensor(input) - : this.packTensor(input); + input = texData.isPacked ? this.unpackTensor(input) : + this.packTensor(input); dataToDispose.push(input); texData = this.texData.get(input.dataId); } else if ( - texData.isPacked && - !webgl_util.isReshapeFree(texData.shape, input.shape) - ) { + texData.isPacked && + !webgl_util.isReshapeFree(texData.shape, input.shape)) { // This is a special case where a texture exists for a tensor // but the shapes are incompatible (due to packing constraints) because // the tensor did not have a chance to go through the packed reshape @@ -1047,53 +886,36 @@ export class MathBackendWebGL extends KernelBackend { } this.uploadToGPU(input.dataId); - return { shape: input.shape, texData, isUniform: false }; + return {shape: input.shape, texData, isUniform: false}; }); this.uploadToGPU(output.dataId); - const outputData: TensorData = { - shape: output.shape, - texData: outData, - isUniform: false, - }; + const outputData: + TensorData = {shape: output.shape, texData: outData, isUniform: false}; const key = gpgpu_math.makeShaderKey(program, inputsData, outputData); const binary = this.getAndSaveBinary(key, () => { return gpgpu_math.compileProgram( - this.gpgpu, - program, - inputsData, - outputData - ); + this.gpgpu, program, inputsData, outputData); }); const shouldTimeProgram = this.activeTimers != null; - let query: WebGLQuery | CPUTimerQuery; + let query: WebGLQuery|CPUTimerQuery; if (shouldTimeProgram) { query = this.startTimer(); } gpgpu_math.runProgram( - this.gpgpu, - binary, - inputsData, - outputData, - customSetup - ); + this.gpgpu, binary, inputsData, outputData, customSetup); - dataToDispose.forEach((info) => this.disposeIntermediateTensorInfo(info)); + dataToDispose.forEach(info => this.disposeIntermediateTensorInfo(info)); if (shouldTimeProgram) { query = this.endTimer(query); - this.activeTimers.push({ - name: program.constructor.name, - query: this.getQueryTime(query), - }); + this.activeTimers.push( + {name: program.constructor.name, query: this.getQueryTime(query)}); } - if ( - !env().getBool("WEBGL_LAZILY_UNPACK") && - outData.isPacked && - preventEagerUnpackingOfOutput === false - ) { + if (!env().getBool('WEBGL_LAZILY_UNPACK') && outData.isPacked && + preventEagerUnpackingOfOutput === false) { const unpacked = this.unpackTensor(output); this.disposeIntermediateTensorInfo(output); return unpacked; @@ -1102,27 +924,18 @@ export class MathBackendWebGL extends KernelBackend { } compileAndRun( - program: GPGPUProgram, - inputs: TensorInfo[], - outputDtype?: DataType, - customSetup?: (gpgpu: GPGPUContext, webGLProgram: WebGLProgram) => void, - preventEagerUnpackingOfOutput = false - ): TensorInfo { + program: GPGPUProgram, inputs: TensorInfo[], outputDtype?: DataType, + customSetup?: (gpgpu: GPGPUContext, webGLProgram: WebGLProgram) => void, + preventEagerUnpackingOfOutput = false): TensorInfo { outputDtype = outputDtype || inputs[0].dtype; const outInfo = this.runWebGLProgram( - program, - inputs, - outputDtype, - customSetup, - preventEagerUnpackingOfOutput - ); + program, inputs, outputDtype, customSetup, + preventEagerUnpackingOfOutput); return outInfo; } - private getAndSaveBinary( - key: string, - getBinary: () => GPGPUBinary - ): GPGPUBinary { + private getAndSaveBinary(key: string, getBinary: () => GPGPUBinary): + GPGPUBinary { if (!(key in this.binaryCache)) { this.binaryCache[key] = getBinary(); } @@ -1141,19 +954,17 @@ export class MathBackendWebGL extends KernelBackend { } // Avoid disposing the compiled webgl programs during unit testing because // it slows down test execution. - if (!env().getBool("IS_TEST")) { + if (!env().getBool('IS_TEST')) { const allKeys = Object.keys(this.binaryCache); - allKeys.forEach((key) => { + allKeys.forEach(key => { this.gpgpu.deleteProgram(this.binaryCache[key].webGLProgram); delete this.binaryCache[key]; }); } this.textureManager.dispose(); - if ( - this.canvas != null && - typeof HTMLCanvasElement !== "undefined" && - this.canvas instanceof HTMLCanvasElement - ) { + if (this.canvas != null && + (typeof (HTMLCanvasElement) !== 'undefined' && + this.canvas instanceof HTMLCanvasElement)) { this.canvas.remove(); } else { this.canvas = null; @@ -1165,16 +976,16 @@ export class MathBackendWebGL extends KernelBackend { this.disposed = true; } - floatPrecision(): 16 | 32 { + floatPrecision(): 16|32 { if (this.floatPrecisionValue == null) { this.floatPrecisionValue = tidy(() => { - if (!env().get("WEBGL_RENDER_FLOAT32_ENABLED")) { + if (!env().get('WEBGL_RENDER_FLOAT32_ENABLED')) { // Momentarily switching DEBUG flag to false so we don't throw an // error trying to upload a small value. - const debugFlag = env().getBool("DEBUG"); - env().set("DEBUG", false); + const debugFlag = env().getBool('DEBUG'); + env().set('DEBUG', false); const underflowCheckValue = this.abs(scalar(1e-8)).dataSync()[0]; - env().set("DEBUG", debugFlag); + env().set('DEBUG', debugFlag); if (underflowCheckValue > 0) { return 32; @@ -1193,7 +1004,7 @@ export class MathBackendWebGL extends KernelBackend { uploadToGPU(dataId: DataId): void { const texData = this.texData.get(dataId); - const { shape, dtype, values, texture, usage, isPacked } = texData; + const {shape, dtype, values, texture, usage, isPacked} = texData; if (texture != null) { // Array is already on GPU. No-op. @@ -1215,53 +1026,36 @@ export class MathBackendWebGL extends KernelBackend { const shapeAs3D = webgl_util.getShapeAs3D(shape); let program; - let width = texShape[1], - height = texShape[0]; + let width = texShape[1], height = texShape[0]; const isByteArray = values instanceof Uint8Array; if (isPacked) { [width, height] = tex_util.getPackedMatrixTextureShapeWidthHeight( - texShape[0], - texShape[1] - ); + texShape[0], texShape[1]); program = new EncodeMatrixPackedProgram( - shapeAs3D, - [height, width], - isByteArray - ); + shapeAs3D, [height, width], isByteArray); } else { - program = new EncodeMatrixProgram( - shapeAs3D, - [height, width], - isByteArray - ); + program = + new EncodeMatrixProgram(shapeAs3D, [height, width], isByteArray); } const tempDenseInputHandle = this.makeTensorInfo([height, width], dtype); if (isByteArray) { this.texData.get(tempDenseInputHandle.dataId).usage = - TextureUsage.PIXELS; + TextureUsage.PIXELS; } else { this.texData.get(tempDenseInputHandle.dataId).usage = - TextureUsage.UPLOAD; + TextureUsage.UPLOAD; } this.gpgpu.uploadDenseMatrixToTexture( - this.getTexture(tempDenseInputHandle.dataId), - width, - height, - values as TypedArray - ); + this.getTexture(tempDenseInputHandle.dataId), width, height, + values as TypedArray); // We want the output to remain packed regardless of the value of // WEBGL_PACK. const preventEagerUnpacking = true; const encodedOutputTarget = this.runWebGLProgram( - program, - [tempDenseInputHandle], - dtype, - null, - preventEagerUnpacking - ); + program, [tempDenseInputHandle], dtype, null, preventEagerUnpacking); // Have the original texture assume the identity of the encoded output. const outputTexData = this.texData.get(encodedOutputTarget.dataId); @@ -1284,38 +1078,30 @@ export class MathBackendWebGL extends KernelBackend { } } - private convertAndCacheOnCPU( - dataId: DataId, - float32Values?: Float32Array - ): TypedArray { + private convertAndCacheOnCPU(dataId: DataId, float32Values?: Float32Array): + TypedArray { const texData = this.texData.get(dataId); - const { dtype } = texData; + const {dtype} = texData; this.releaseGPUData(dataId); if (float32Values != null) { - texData.values = float32ToTypedArray(float32Values, dtype as "float32"); + texData.values = float32ToTypedArray(float32Values, dtype as 'float32'); } return texData.values as TypedArray; } private acquireTexture( - texShape: [number, number], - texType: TextureUsage, - dtype: DataType, - isPacked: boolean - ): WebGLTexture { + texShape: [number, number], texType: TextureUsage, dtype: DataType, + isPacked: boolean): WebGLTexture { this.numBytesInGPU += this.computeBytes(texShape, dtype); - if ( - !this.warnedAboutMemory && - this.numBytesInGPU > this.numMBBeforeWarning * 1024 * 1024 - ) { + if (!this.warnedAboutMemory && + this.numBytesInGPU > this.numMBBeforeWarning * 1024 * 1024) { const mb = (this.numBytesInGPU / 1024 / 1024).toFixed(2); this.warnedAboutMemory = true; console.warn( - `High memory usage in GPU: ${mb} MB, ` + - `most likely due to a memory leak` - ); + `High memory usage in GPU: ${mb} MB, ` + + `most likely due to a memory leak`); } return this.textureManager.acquireTexture(texShape, texType, isPacked); } @@ -1326,14 +1112,12 @@ export class MathBackendWebGL extends KernelBackend { } function float32ToTypedArray( - a: Float32Array, - dtype: D -): tf.DataTypeMap[D] { - if (dtype === "float32" || dtype === "complex64") { + a: Float32Array, dtype: D): tf.DataTypeMap[D] { + if (dtype === 'float32' || dtype === 'complex64') { return a as tf.DataTypeMap[D]; - } else if (dtype === "int32" || dtype === "bool") { - const result = - dtype === "int32" ? new Int32Array(a.length) : new Uint8Array(a.length); + } else if (dtype === 'int32' || dtype === 'bool') { + const result = (dtype === 'int32') ? new Int32Array(a.length) : + new Uint8Array(a.length); for (let i = 0; i < result.length; ++i) { result[i] = Math.round(a[i]); }