From ba3bc1bc463a40139a7eb9f6eb666feb1ff1c693 Mon Sep 17 00:00:00 2001 From: Ann Yuan Date: Tue, 3 Sep 2019 13:39:28 -0400 Subject: [PATCH 1/3] fused elu --- tfjs-core/src/backends/cpu/backend_cpu.ts | 2 ++ tfjs-core/src/backends/webgl/backend_webgl.ts | 2 ++ tfjs-core/src/ops/fused_util.ts | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tfjs-core/src/backends/cpu/backend_cpu.ts b/tfjs-core/src/backends/cpu/backend_cpu.ts index 0715afc7192..85b2314b701 100644 --- a/tfjs-core/src/backends/cpu/backend_cpu.ts +++ b/tfjs-core/src/backends/cpu/backend_cpu.ts @@ -54,6 +54,8 @@ function mapActivation( return backend.linear(x); } else if (activation === 'relu') { return backend.relu(x); + } else if (activation === 'elu') { + return backend.elu(x); } else if (activation === 'prelu') { return backend.prelu(x, preluActivationWeights); } diff --git a/tfjs-core/src/backends/webgl/backend_webgl.ts b/tfjs-core/src/backends/webgl/backend_webgl.ts index 6dc0e2475ec..0c865b027ce 100644 --- a/tfjs-core/src/backends/webgl/backend_webgl.ts +++ b/tfjs-core/src/backends/webgl/backend_webgl.ts @@ -176,6 +176,8 @@ function mapActivationToShaderProgram( return unary_packed_op.RELU; } return unary_op.RELU; + } else if (activation === 'elu') { + return unary_op.ELU; } else if (activation === 'prelu') { if (packed) { return binaryop_packed_gpu.PRELU; diff --git a/tfjs-core/src/ops/fused_util.ts b/tfjs-core/src/ops/fused_util.ts index 0848d328253..a38a3ada9fc 100644 --- a/tfjs-core/src/ops/fused_util.ts +++ b/tfjs-core/src/ops/fused_util.ts @@ -17,7 +17,7 @@ import {Tensor, Tensor3D} from '../tensor'; -export type Activation = 'linear'|'relu'|'prelu'; +export type Activation = 'linear'|'relu'|'prelu'|'elu'; export type FusedBatchMatMulConfig = { a: Tensor3D, From 50c37a80a2232379dd68b6542e4383bfc4bd0439 Mon Sep 17 00:00:00 2001 From: Ann Yuan Date: Tue, 3 Sep 2019 16:32:34 -0400 Subject: [PATCH 2/3] add tests --- tfjs-core/src/backends/webgl/backend_webgl.ts | 6 ++ .../src/backends/webgl/unaryop_packed_gpu.ts | 11 ++++ tfjs-core/src/ops/fused_test.ts | 59 +++++++++++++++++++ 3 files changed, 76 insertions(+) diff --git a/tfjs-core/src/backends/webgl/backend_webgl.ts b/tfjs-core/src/backends/webgl/backend_webgl.ts index 0c865b027ce..e1a54826dff 100644 --- a/tfjs-core/src/backends/webgl/backend_webgl.ts +++ b/tfjs-core/src/backends/webgl/backend_webgl.ts @@ -177,6 +177,9 @@ function mapActivationToShaderProgram( } return unary_op.RELU; } else if (activation === 'elu') { + if (packed) { + return unary_packed_op.ELU; + } return unary_op.ELU; } else if (activation === 'prelu') { if (packed) { @@ -1746,6 +1749,9 @@ export class MathBackendWebGL implements KernelBackend { } elu(x: T): T { + if (ENV.getBool('WEBGL_PACK_UNARY_OPERATIONS')) { + return this.packedUnaryOp(x, unary_packed_op.ELU, x.dtype) as T; + } const program = new UnaryOpProgram(x.shape, unary_op.ELU); return this.compileAndRun(program, [x]); } diff --git a/tfjs-core/src/backends/webgl/unaryop_packed_gpu.ts b/tfjs-core/src/backends/webgl/unaryop_packed_gpu.ts index 810eac0dcd6..225f840eb7a 100644 --- a/tfjs-core/src/backends/webgl/unaryop_packed_gpu.ts +++ b/tfjs-core/src/backends/webgl/unaryop_packed_gpu.ts @@ -42,6 +42,17 @@ export const RELU = ` return result; `; +export const ELU = ` + vec4 result; + + result.r = (x.r >= 0.0) ? x.r : (exp(x.r) - 1.0); + result.g = (x.g >= 0.0) ? x.g : (exp(x.g) - 1.0); + result.b = (x.b >= 0.0) ? x.b : (exp(x.b) - 1.0); + result.a = (x.a >= 0.0) ? x.a : (exp(x.a) - 1.0); + + return result; +`; + export class UnaryOpPackedProgram implements GPGPUProgram { variableNames = ['A']; userCode: string; diff --git a/tfjs-core/src/ops/fused_test.ts b/tfjs-core/src/ops/fused_test.ts index 660079778bc..6749d81e48f 100644 --- a/tfjs-core/src/ops/fused_test.ts +++ b/tfjs-core/src/ops/fused_test.ts @@ -43,6 +43,19 @@ describeWithFlags('fused matmul', ALL_ENVS, () => { expectArraysClose(await c.data(), [0, 8, 0, 20]); }); + it('A x B with elu', async () => { + const a = tf.tensor2d([1, 2, 3, 4, 5, 6], [2, 3]); + const b = tf.tensor2d([0, 1, -3, 2, 2, 1], [3, 2]); + const transposeA = false; + const transposeB = false; + + const c = tf.fused.matMul( + {a, b, transposeA, transposeB, bias: null, activation: 'elu'}); + + expect(c.shape).toEqual([2, 2]); + expectArraysClose(await c.data(), [0, 8, -0.9502, 20]); + }); + it('A x B with prelu', async () => { const a = tf.tensor2d([1, 2, 3, 4, 5, 6], [2, 3]); const b = tf.tensor2d([0, 1, -3, 2, 2, 1], [3, 2]); @@ -106,6 +119,21 @@ describeWithFlags('fused matmul', ALL_ENVS, () => { expectArraysClose(await d.data(), [1, 9, 0, 21]); }); + it('A x B with elu and broadcasted bias', async () => { + const a = tf.tensor2d([1, 2, 3, 4, 5, 6], [2, 3]); + const b = tf.tensor2d([0, 1, -3, 2, 2, 1], [3, 2]); + const c = tf.tensor1d([1, 1]); + const act: tf.fused.Activation = 'elu'; + const transposeA = false; + const transposeB = false; + + const d = tf.fused.matMul( + {a, b, transposeA, transposeB, bias: c, activation: act}); + + expect(d.shape).toEqual([2, 2]); + expectArraysClose(await d.data(), [1, 9, -0.8647, 21]); + }); + it('A x B with relu and broadcasted bias different rank', async () => { const a = tf.tensor3d([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], [2, 2, 3]); const b = tf.tensor3d([0, 1, -3, 2, 2, 1, 0, 1, -3, 2, 2, 1], [2, 3, 2]); @@ -318,6 +346,37 @@ describeWithFlags('fused conv2d', ALL_ENVS, () => { expectArraysClose(await result.data(), expected); }); + it('basic with elu', async () => { + const inputDepth = 2; + const inShape: [number, number, number, number] = [2, 2, 2, inputDepth]; + const outputDepth = 2; + const fSize = 1; + const pad = 0; + const stride = 1; + + const x = tf.tensor4d( + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], inShape); + const w = + tf.tensor4d([-1, 1, -2, 0.5], [fSize, fSize, inputDepth, outputDepth]); + + const result = tf.fused.conv2d({ + x, + filter: w, + strides: stride, + pad, + dataFormat: 'NHWC', + dilations: [1, 1], + activation: 'elu' + }); + expect(result.shape).toEqual([2, 2, 2, 2]); + const expected = [ + -0.99326, 2, -0.99998, 5, -0.9999999, 8, -1, 11, -1, 14, -1, 17, -1, 20, + -1, 23 + ]; + + expectArraysClose(await result.data(), expected); + }); + it('basic with prelu', async () => { const inputDepth = 2; const inShape: [number, number, number, number] = [2, 2, 2, inputDepth]; From 20c3994cc573acb5f7c50566fa7c0be4cf5e96bd Mon Sep 17 00:00:00 2001 From: Ann Yuan Date: Tue, 3 Sep 2019 16:46:00 -0400 Subject: [PATCH 3/3] remove precision --- tfjs-core/src/ops/fused_test.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tfjs-core/src/ops/fused_test.ts b/tfjs-core/src/ops/fused_test.ts index 6749d81e48f..bf8673c8acb 100644 --- a/tfjs-core/src/ops/fused_test.ts +++ b/tfjs-core/src/ops/fused_test.ts @@ -369,10 +369,8 @@ describeWithFlags('fused conv2d', ALL_ENVS, () => { activation: 'elu' }); expect(result.shape).toEqual([2, 2, 2, 2]); - const expected = [ - -0.99326, 2, -0.99998, 5, -0.9999999, 8, -1, 11, -1, 14, -1, 17, -1, 20, - -1, 23 - ]; + const expected = + [-0.99326, 2, -1, 5, -1, 8, -1, 11, -1, 14, -1, 17, -1, 20, -1, 23]; expectArraysClose(await result.data(), expected); });