From a88a81b52ce1f76a460b8080128805f8fe6f70a0 Mon Sep 17 00:00:00 2001 From: Jing Jin <8752427+jinjingforever@users.noreply.github.com> Date: Thu, 24 Sep 2020 11:24:10 -0700 Subject: [PATCH 1/6] save --- tfjs-backend-cpu/src/backend_cpu.ts | 12 ++-- tfjs-backend-cpu/src/kernels/MaxPool.ts | 72 ++++++++++++++++++++ tfjs-backend-cpu/src/register_all_kernels.ts | 2 + 3 files changed, 80 insertions(+), 6 deletions(-) create mode 100644 tfjs-backend-cpu/src/kernels/MaxPool.ts diff --git a/tfjs-backend-cpu/src/backend_cpu.ts b/tfjs-backend-cpu/src/backend_cpu.ts index d9ecd0bb6be..1a0a492afca 100644 --- a/tfjs-backend-cpu/src/backend_cpu.ts +++ b/tfjs-backend-cpu/src/backend_cpu.ts @@ -1529,12 +1529,12 @@ export class MathBackendCPU extends KernelBackend { .slice(sliceBeginCoords, sliceSize) as T; } - maxPool(x: Tensor4D, convInfo: backend_util.Conv2DInfo): Tensor4D { - assertNotComplex(x, 'maxPool'); - const xValues = this.readSync(x.dataId) as TypedArray; - return pool(xValues, x.shape, x.dtype, x.strides, convInfo, 'max') - .toTensor() as Tensor4D; - } + // maxPool(x: Tensor4D, convInfo: backend_util.Conv2DInfo): Tensor4D { + // assertNotComplex(x, 'maxPool'); + // const xValues = this.readSync(x.dataId) as TypedArray; + // return pool(xValues, x.shape, x.dtype, x.strides, convInfo, 'max') + // .toTensor() as Tensor4D; + // } maxPoolBackprop( dy: Tensor4D, x: Tensor4D, y: Tensor4D, diff --git a/tfjs-backend-cpu/src/kernels/MaxPool.ts b/tfjs-backend-cpu/src/kernels/MaxPool.ts new file mode 100644 index 00000000000..56407e05b64 --- /dev/null +++ b/tfjs-backend-cpu/src/kernels/MaxPool.ts @@ -0,0 +1,72 @@ +/** + * @license + * Copyright 2020 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ +import {backend_util, KernelConfig, KernelFunc, MaxPool, MaxPoolAttrs, MaxPoolInputs, TensorInfo, TypedArray, util} from '@tensorflow/tfjs-core'; + +import {MathBackendCPU} from '../backend_cpu'; +import {assertNotComplex} from '../cpu_util'; +import {pool} from '../utils/pool_utils'; + +export function maxPool( + args: + {inputs: MaxPoolInputs, backend: MathBackendCPU, attrs: MaxPoolAttrs}): + TensorInfo { + const {inputs, backend, attrs} = args; + const {x} = inputs; + assertNotComplex(x, 'maxPool'); + const {filterSize, strides, pad, dimRoundingMode} = attrs; + const dilations = 1; + + const xRank = x.shape.length; + util.assert( + xRank === 4, + () => `Error in maxPool: input must be rank 4 but got rank ${ + x.shape.length}}.`); + util.assert( + backend_util.eitherStridesOrDilationsAreOne(strides, dilations), + () => 'Error in maxPool: Either strides or dilations must be 1. ' + + `Got strides ${strides} and dilations '${dilations}'`); + if (dimRoundingMode != null) { + util.assert( + util.isInt(pad as number), + () => `Error in maxPool: pad must be an integer when using, ` + + `dimRoundingMode ${dimRoundingMode} but got pad ${pad}.`); + } + + const convInfo = backend_util.computePool2DInfo( + x.shape as [number, number, number, number], filterSize, strides, + dilations, pad, dimRoundingMode); + let y: TensorInfo; + + if (convInfo.filterWidth === 1 && convInfo.filterHeight === 1 && + util.arraysEqual(convInfo.inShape, convInfo.outShape)) { + y = {...x}; + backend.incRef(x.dataId); + } else { + const xValues = backend.data.get(x.dataId).values as TypedArray; + const strides = util.computeStrides(x.shape); + const buffer = pool(xValues, x.shape, x.dtype, strides, convInfo, 'max'); + y = backend.makeTensorInfo( + convInfo.outShape, x.dtype, buffer.values as TypedArray); + } + return y; +} + +export const maxPoolConfig: KernelConfig = { + kernelName: MaxPool, + backendName: 'cpu', + kernelFunc: maxPool as {} as KernelFunc +}; diff --git a/tfjs-backend-cpu/src/register_all_kernels.ts b/tfjs-backend-cpu/src/register_all_kernels.ts index 499a271523b..8f2730aa8f4 100644 --- a/tfjs-backend-cpu/src/register_all_kernels.ts +++ b/tfjs-backend-cpu/src/register_all_kernels.ts @@ -56,6 +56,7 @@ import {logConfig} from './kernels/Log'; import {log1pConfig} from './kernels/Log1p'; import {logicalNotConfig} from './kernels/LogicalNot'; import {maxConfig} from './kernels/Max'; +import {maxPoolConfig} from './kernels/MaxPool'; import {maxPoolWithArgmaxConfig} from './kernels/MaxPoolWithArgmax'; import {multiplyConfig} from './kernels/Multiply'; import {nonMaxSuppressionV4Config} from './kernels/NonMaxSuppressionV4'; @@ -123,6 +124,7 @@ const kernelConfigs: KernelConfig[] = [ logConfig, log1pConfig, logicalNotConfig, + maxPoolConfig, maxPoolWithArgmaxConfig, maxConfig, multiplyConfig, From b3140c8df40fb6a628ee4091a1919855f78a4b84 Mon Sep 17 00:00:00 2001 From: Jing Jin <8752427+jinjingforever@users.noreply.github.com> Date: Thu, 24 Sep 2020 13:16:20 -0700 Subject: [PATCH 2/6] save --- tfjs-backend-cpu/src/backend_cpu.ts | 192 ++++++------------ tfjs-backend-cpu/src/kernels/AvgPool.ts | 73 +++++++ .../src/kernels/AvgPoolBackprop.ts | 103 ++++++++++ tfjs-backend-cpu/src/kernels/MaxPool.ts | 8 +- .../src/kernels/MaxPoolBackprop.ts | 123 +++++++++++ tfjs-backend-cpu/src/register_all_kernels.ts | 6 + 6 files changed, 371 insertions(+), 134 deletions(-) create mode 100644 tfjs-backend-cpu/src/kernels/AvgPool.ts create mode 100644 tfjs-backend-cpu/src/kernels/AvgPoolBackprop.ts create mode 100644 tfjs-backend-cpu/src/kernels/MaxPoolBackprop.ts diff --git a/tfjs-backend-cpu/src/backend_cpu.ts b/tfjs-backend-cpu/src/backend_cpu.ts index 1a0a492afca..64c2c10051b 100644 --- a/tfjs-backend-cpu/src/backend_cpu.ts +++ b/tfjs-backend-cpu/src/backend_cpu.ts @@ -16,7 +16,7 @@ */ import * as tf from '@tensorflow/tfjs-core'; -import {backend_util, BackendTimingInfo, buffer, DataStorage, DataType, DataValues, engine, env, kernel_impls, KernelBackend, max, NumericDataType, Rank, Scalar, ShapeMap, slice_util, Tensor, Tensor1D, Tensor2D, Tensor3D, Tensor4D, Tensor5D, TensorBuffer, TensorInfo, TypedArray, upcastType, util} from '@tensorflow/tfjs-core'; +import {backend_util, BackendTimingInfo, DataStorage, DataType, DataValues, engine, env, kernel_impls, KernelBackend, max, NumericDataType, Rank, Scalar, ShapeMap, slice_util, Tensor, Tensor1D, Tensor2D, Tensor3D, Tensor4D, Tensor5D, TensorBuffer, TensorInfo, TypedArray, upcastType, util} from '@tensorflow/tfjs-core'; const nonMaxSuppressionV3Impl = kernel_impls.nonMaxSuppressionV3Impl; const split = kernel_impls.split; @@ -25,7 +25,6 @@ const topkImpl = kernel_impls.topkImpl; const whereImpl = kernel_impls.whereImpl; import * as seedrandom from 'seedrandom'; import {assertNotComplex} from './cpu_util'; -import {maxPoolPositions, pool} from './utils/pool_utils'; interface DataId {} @@ -1529,128 +1528,62 @@ export class MathBackendCPU extends KernelBackend { .slice(sliceBeginCoords, sliceSize) as T; } - // maxPool(x: Tensor4D, convInfo: backend_util.Conv2DInfo): Tensor4D { - // assertNotComplex(x, 'maxPool'); - // const xValues = this.readSync(x.dataId) as TypedArray; - // return pool(xValues, x.shape, x.dtype, x.strides, convInfo, 'max') - // .toTensor() as Tensor4D; + // avgPoolBackprop(dy: Tensor4D, x: Tensor4D, convInfo: + // backend_util.Conv2DInfo): + // Tensor4D { + // assertNotComplex([dy, x], 'avgPoolBackprop'); + + // const strideHeight = convInfo.strideHeight; + // const strideWidth = convInfo.strideWidth; + // const filterHeight = convInfo.filterHeight; + // const filterWidth = convInfo.filterWidth; + // const dilationHeight = convInfo.dilationHeight; + // const dilationWidth = convInfo.dilationWidth; + // const effectiveFilterHeight = convInfo.effectiveFilterHeight; + // const effectiveFilterWidth = convInfo.effectiveFilterWidth; + // const padLeft = effectiveFilterWidth - 1 - convInfo.padInfo.left; + // const padTop = effectiveFilterHeight - 1 - convInfo.padInfo.top; + // const dx = tf.buffer(x.shape, 'float32'); + + // const avgMultiplier = 1 / (filterHeight * filterWidth); + + // const dyBuf = this.bufferSync(dy); + + // for (let b = 0; b < convInfo.batchSize; ++b) { + // for (let d = 0; d < convInfo.inChannels; ++d) { + // for (let dxR = 0; dxR < convInfo.inHeight; ++dxR) { + // for (let dxC = 0; dxC < convInfo.inWidth; ++dxC) { + // // Shader code begins. + // const dyRCorner = dxR - padTop; + // const dyCCorner = dxC - padLeft; + // let dotProd = 0; + // for (let wR = 0; wR < effectiveFilterHeight; wR += + // dilationHeight) { + // const dyR = (dyRCorner + wR) / strideHeight; + // if (dyR < 0 || dyR >= convInfo.outHeight || + // Math.floor(dyR) !== dyR) { + // continue; + // } + // for (let wC = 0; wC < effectiveFilterWidth; wC += + // dilationWidth) { + // const dyC = (dyCCorner + wC) / strideWidth; + // if (dyC < 0 || dyC >= convInfo.outWidth || + // Math.floor(dyC) !== dyC) { + // continue; + // } + + // const pixel = dyBuf.get(b, dyR, dyC, d); + // dotProd += pixel; + // } + // } + // dx.set(dotProd * avgMultiplier, b, dxR, dxC, d); + // } + // } + // } + // } + // return dx.toTensor(); // } - maxPoolBackprop( - dy: Tensor4D, x: Tensor4D, y: Tensor4D, - convInfo: backend_util.Conv2DInfo): Tensor4D { - assertNotComplex([x, y], 'maxPoolBackprop'); - - const xValues = this.readSync(x.dataId) as TypedArray; - const maxPosBuf = buffer( - convInfo.outShape, x.dtype, - maxPoolPositions(xValues, x.shape, x.dtype, convInfo).values); - const strideHeight = convInfo.strideHeight; - const strideWidth = convInfo.strideWidth; - const dilationHeight = convInfo.dilationHeight; - const dilationWidth = convInfo.dilationWidth; - const effectiveFilterHeight = convInfo.effectiveFilterHeight; - const effectiveFilterWidth = convInfo.effectiveFilterWidth; - const padLeft = effectiveFilterWidth - 1 - convInfo.padInfo.left; - const padTop = effectiveFilterHeight - 1 - convInfo.padInfo.top; - const dx = tf.buffer(x.shape, 'float32'); - - const dyBuf = this.bufferSync(dy); - - for (let b = 0; b < convInfo.batchSize; ++b) { - for (let d = 0; d < convInfo.inChannels; ++d) { - for (let dxR = 0; dxR < convInfo.inHeight; ++dxR) { - for (let dxC = 0; dxC < convInfo.inWidth; ++dxC) { - // Shader code begins. - const dyRCorner = dxR - padTop; - const dyCCorner = dxC - padLeft; - let dotProd = 0; - for (let wR = 0; wR < effectiveFilterHeight; wR += dilationHeight) { - const dyR = (dyRCorner + wR) / strideHeight; - if (dyR < 0 || dyR >= convInfo.outHeight || - Math.floor(dyR) !== dyR) { - continue; - } - for (let wC = 0; wC < effectiveFilterWidth; wC += dilationWidth) { - const dyC = (dyCCorner + wC) / strideWidth; - if (dyC < 0 || dyC >= convInfo.outWidth || - Math.floor(dyC) !== dyC) { - continue; - } - const maxPos = effectiveFilterHeight * effectiveFilterWidth - - 1 - (maxPosBuf.get(b, dyR, dyC, d) as number); - const curPos = wR * effectiveFilterWidth + wC; - - const mask = maxPos === curPos ? 1 : 0; - if (mask === 0) { - continue; - } - - const pixel = dyBuf.get(b, dyR, dyC, d); - dotProd += pixel * mask; - } - } - dx.set(dotProd, b, dxR, dxC, d); - } - } - } - } - return dx.toTensor(); - } - - avgPoolBackprop(dy: Tensor4D, x: Tensor4D, convInfo: backend_util.Conv2DInfo): - Tensor4D { - assertNotComplex([dy, x], 'avgPoolBackprop'); - - const strideHeight = convInfo.strideHeight; - const strideWidth = convInfo.strideWidth; - const filterHeight = convInfo.filterHeight; - const filterWidth = convInfo.filterWidth; - const dilationHeight = convInfo.dilationHeight; - const dilationWidth = convInfo.dilationWidth; - const effectiveFilterHeight = convInfo.effectiveFilterHeight; - const effectiveFilterWidth = convInfo.effectiveFilterWidth; - const padLeft = effectiveFilterWidth - 1 - convInfo.padInfo.left; - const padTop = effectiveFilterHeight - 1 - convInfo.padInfo.top; - const dx = tf.buffer(x.shape, 'float32'); - - const avgMultiplier = 1 / (filterHeight * filterWidth); - - const dyBuf = this.bufferSync(dy); - - for (let b = 0; b < convInfo.batchSize; ++b) { - for (let d = 0; d < convInfo.inChannels; ++d) { - for (let dxR = 0; dxR < convInfo.inHeight; ++dxR) { - for (let dxC = 0; dxC < convInfo.inWidth; ++dxC) { - // Shader code begins. - const dyRCorner = dxR - padTop; - const dyCCorner = dxC - padLeft; - let dotProd = 0; - for (let wR = 0; wR < effectiveFilterHeight; wR += dilationHeight) { - const dyR = (dyRCorner + wR) / strideHeight; - if (dyR < 0 || dyR >= convInfo.outHeight || - Math.floor(dyR) !== dyR) { - continue; - } - for (let wC = 0; wC < effectiveFilterWidth; wC += dilationWidth) { - const dyC = (dyCCorner + wC) / strideWidth; - if (dyC < 0 || dyC >= convInfo.outWidth || - Math.floor(dyC) !== dyC) { - continue; - } - - const pixel = dyBuf.get(b, dyR, dyC, d); - dotProd += pixel; - } - } - dx.set(dotProd * avgMultiplier, b, dxR, dxC, d); - } - } - } - } - return dx.toTensor(); - } - private pool3d( x: Tensor5D, convInfo: backend_util.Conv3DInfo, poolType: 'max'|'avg'): Tensor5D { @@ -2005,14 +1938,13 @@ export class MathBackendCPU extends KernelBackend { return dx.toTensor(); } - avgPool(x: Tensor4D, convInfo: backend_util.Conv2DInfo): Tensor4D { - assertNotComplex(x, 'avgPool'); - assertNotComplex(x, 'maxPool'); - const xValues = this.readSync(x.dataId) as TypedArray; - return pool(xValues, x.shape, x.dtype, x.strides, convInfo, 'avg') - .toTensor() - .toFloat() as Tensor4D; - } + // avgPool(x: Tensor4D, convInfo: backend_util.Conv2DInfo): Tensor4D { + // assertNotComplex(x, 'avgPool'); + // const xValues = this.readSync(x.dataId) as TypedArray; + // return pool(xValues, x.shape, x.dtype, x.strides, convInfo, 'avg') + // .toTensor() + // .toFloat() as Tensor4D; + // } resizeBilinear( x: Tensor4D, newHeight: number, newWidth: number, diff --git a/tfjs-backend-cpu/src/kernels/AvgPool.ts b/tfjs-backend-cpu/src/kernels/AvgPool.ts new file mode 100644 index 00000000000..abe9306d15f --- /dev/null +++ b/tfjs-backend-cpu/src/kernels/AvgPool.ts @@ -0,0 +1,73 @@ +/** + * @license + * Copyright 2020 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ +import {AvgPool, AvgPoolAttrs, AvgPoolInputs, backend_util, KernelConfig, KernelFunc, TensorInfo, TypedArray, util} from '@tensorflow/tfjs-core'; + +import {MathBackendCPU} from '../backend_cpu'; +import {assertNotComplex} from '../cpu_util'; +import {pool} from '../utils/pool_utils'; + +export function avgPool( + args: + {inputs: AvgPoolInputs, backend: MathBackendCPU, attrs: AvgPoolAttrs}): + TensorInfo { + const {inputs, backend, attrs} = args; + const {x} = inputs; + assertNotComplex(x, 'avgPool'); + const {filterSize, strides, pad, dimRoundingMode} = attrs; + const dilations = 1; + + util.assert( + backend_util.eitherStridesOrDilationsAreOne(strides, dilations), + () => 'Error in avgPool: Either strides or dilations must be 1. ' + + `Got strides ${strides} and dilations '${dilations}'`); + + const xRank = x.shape.length; + util.assert( + xRank === 4, + () => `Error in avgPool: input must be rank 4 but got rank ${ + x.shape.length}}.`); + if (dimRoundingMode != null) { + util.assert( + util.isInt(pad as number), + () => `Error in avgPool: pad must be an integer when using, ` + + `dimRoundingMode ${dimRoundingMode} but got pad ${pad}.`); + } + + const convInfo = backend_util.computePool2DInfo( + x.shape as [number, number, number, number], filterSize, strides, + dilations, pad, dimRoundingMode); + let res: TensorInfo; + + if (convInfo.filterWidth === 1 && convInfo.filterHeight === 1 && + util.arraysEqual(convInfo.inShape, convInfo.outShape)) { + res = {...x}; + backend.incRef(x.dataId); + } else { + const xValues = backend.data.get(x.dataId).values as TypedArray; + const strides = util.computeStrides(x.shape); + const buffer = pool(xValues, x.shape, x.dtype, strides, convInfo, 'avg'); + res = backend.makeTensorInfo( + convInfo.outShape, x.dtype, buffer.values as TypedArray); + } + return res; +} + +export const avgPoolConfig: KernelConfig = { + kernelName: AvgPool, + backendName: 'cpu', + kernelFunc: avgPool as {} as KernelFunc +}; diff --git a/tfjs-backend-cpu/src/kernels/AvgPoolBackprop.ts b/tfjs-backend-cpu/src/kernels/AvgPoolBackprop.ts new file mode 100644 index 00000000000..2671308b594 --- /dev/null +++ b/tfjs-backend-cpu/src/kernels/AvgPoolBackprop.ts @@ -0,0 +1,103 @@ +/** + * @license + * Copyright 2020 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ +import {AvgPoolBackprop, AvgPoolBackpropAttrs, AvgPoolBackpropInputs, backend_util, buffer, KernelConfig, KernelFunc, Rank, TensorInfo, util} from '@tensorflow/tfjs-core'; + +import {MathBackendCPU} from '../backend_cpu'; +import {assertNotComplex} from '../cpu_util'; + +export function avgPoolBackprop(args: { + inputs: AvgPoolBackpropInputs, + backend: MathBackendCPU, + attrs: AvgPoolBackpropAttrs +}): TensorInfo { + const {inputs, backend, attrs} = args; + const {dy, input} = inputs; + const x = input; + assertNotComplex([dy, input], 'avgPoolBackprop'); + const {filterSize, strides, pad} = attrs; + + const inputRank = input.shape.length; + const dyRank = dy.shape.length; + util.assert( + dyRank === 4, + () => `Error in avgPoolBackprop: dy must be rank 4 but got rank ` + + `${dyRank}.`); + util.assert( + inputRank === 4, + () => `Error in avgPoolBackprop: input must be rank 4 but got rank ` + + `${inputRank}.`); + + const convInfo = backend_util.computePool2DInfo( + x.shape as [number, number, number, number], filterSize, strides, + 1 /* dilations */, pad); + const strideHeight = convInfo.strideHeight; + const strideWidth = convInfo.strideWidth; + const filterHeight = convInfo.filterHeight; + const filterWidth = convInfo.filterWidth; + const dilationHeight = convInfo.dilationHeight; + const dilationWidth = convInfo.dilationWidth; + const effectiveFilterHeight = convInfo.effectiveFilterHeight; + const effectiveFilterWidth = convInfo.effectiveFilterWidth; + const padLeft = effectiveFilterWidth - 1 - convInfo.padInfo.left; + const padTop = effectiveFilterHeight - 1 - convInfo.padInfo.top; + const dx = + buffer(x.shape as [number, number, number, number], 'float32'); + + const avgMultiplier = 1 / (filterHeight * filterWidth); + + const dyData = backend.data.get(dy.dataId).values as Float32Array; + const dyBuf = buffer( + dy.shape as [number, number, number, number], 'float32', dyData); + + for (let b = 0; b < convInfo.batchSize; ++b) { + for (let d = 0; d < convInfo.inChannels; ++d) { + for (let dxR = 0; dxR < convInfo.inHeight; ++dxR) { + for (let dxC = 0; dxC < convInfo.inWidth; ++dxC) { + // Shader code begins. + const dyRCorner = dxR - padTop; + const dyCCorner = dxC - padLeft; + let dotProd = 0; + for (let wR = 0; wR < effectiveFilterHeight; wR += dilationHeight) { + const dyR = (dyRCorner + wR) / strideHeight; + if (dyR < 0 || dyR >= convInfo.outHeight || + Math.floor(dyR) !== dyR) { + continue; + } + for (let wC = 0; wC < effectiveFilterWidth; wC += dilationWidth) { + const dyC = (dyCCorner + wC) / strideWidth; + if (dyC < 0 || dyC >= convInfo.outWidth || + Math.floor(dyC) !== dyC) { + continue; + } + + const pixel = dyBuf.get(b, dyR, dyC, d); + dotProd += pixel; + } + } + dx.set(dotProd * avgMultiplier, b, dxR, dxC, d); + } + } + } + } + return backend.makeTensorInfo(dx.shape, dx.dtype, dx.values); +} + +export const avgPoolBackpropConfig: KernelConfig = { + kernelName: AvgPoolBackprop, + backendName: 'cpu', + kernelFunc: avgPoolBackprop as {} as KernelFunc +}; diff --git a/tfjs-backend-cpu/src/kernels/MaxPool.ts b/tfjs-backend-cpu/src/kernels/MaxPool.ts index 56407e05b64..9a44e44343a 100644 --- a/tfjs-backend-cpu/src/kernels/MaxPool.ts +++ b/tfjs-backend-cpu/src/kernels/MaxPool.ts @@ -49,20 +49,20 @@ export function maxPool( const convInfo = backend_util.computePool2DInfo( x.shape as [number, number, number, number], filterSize, strides, dilations, pad, dimRoundingMode); - let y: TensorInfo; + let res: TensorInfo; if (convInfo.filterWidth === 1 && convInfo.filterHeight === 1 && util.arraysEqual(convInfo.inShape, convInfo.outShape)) { - y = {...x}; + res = {...x}; backend.incRef(x.dataId); } else { const xValues = backend.data.get(x.dataId).values as TypedArray; const strides = util.computeStrides(x.shape); const buffer = pool(xValues, x.shape, x.dtype, strides, convInfo, 'max'); - y = backend.makeTensorInfo( + res = backend.makeTensorInfo( convInfo.outShape, x.dtype, buffer.values as TypedArray); } - return y; + return res; } export const maxPoolConfig: KernelConfig = { diff --git a/tfjs-backend-cpu/src/kernels/MaxPoolBackprop.ts b/tfjs-backend-cpu/src/kernels/MaxPoolBackprop.ts new file mode 100644 index 00000000000..95c631ca2a0 --- /dev/null +++ b/tfjs-backend-cpu/src/kernels/MaxPoolBackprop.ts @@ -0,0 +1,123 @@ +/** + * @license + * Copyright 2020 Google LLC. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ +import {backend_util, buffer, KernelConfig, KernelFunc, MaxPoolBackprop, MaxPoolBackpropAttrs, MaxPoolBackpropInputs, Rank, TensorInfo, TypedArray, util} from '@tensorflow/tfjs-core'; + +import {MathBackendCPU} from '../backend_cpu'; +import {assertNotComplex} from '../cpu_util'; +import {maxPoolPositions} from '../utils/pool_utils'; + +export function maxPoolBackprop(args: { + inputs: MaxPoolBackpropInputs, + backend: MathBackendCPU, + attrs: MaxPoolBackpropAttrs +}): TensorInfo { + const {inputs, backend, attrs} = args; + const {dy, input, output} = inputs; + const x = input; + assertNotComplex([input, output], 'maxPoolBackprop'); + const {filterSize, strides, pad, dimRoundingMode} = attrs; + + const inputRank = input.shape.length; + const dyRank = dy.shape.length; + util.assert( + inputRank === dyRank, + () => `Rank of input (${inputRank}) does not match rank of dy ` + + `(${dyRank})`); + + util.assert( + dyRank === 4, + () => `Error in maxPoolBackprop: dy must be rank 4 but got rank ` + + `${dyRank}.`); + util.assert( + inputRank === 4, + () => `Error in maxPoolBackprop: input must be rank 4 but got rank ` + + `${inputRank}.`); + if (dimRoundingMode != null) { + util.assert( + util.isInt(pad as number), + () => `Error in maxPoolBackprop: pad must be an integer when using, ` + + `dimRoundingMode ${dimRoundingMode} but got pad ${pad}.`); + } + + const convInfo = backend_util.computePool2DInfo( + x.shape as [number, number, number, number], filterSize, strides, + 1 /* dilations */, pad, dimRoundingMode); + const xValues = backend.data.get(x.dataId).values as TypedArray; + const maxPosBuf = buffer( + convInfo.outShape, x.dtype, + maxPoolPositions(xValues, x.shape, x.dtype, convInfo).values); + const strideHeight = convInfo.strideHeight; + const strideWidth = convInfo.strideWidth; + const dilationHeight = convInfo.dilationHeight; + const dilationWidth = convInfo.dilationWidth; + const effectiveFilterHeight = convInfo.effectiveFilterHeight; + const effectiveFilterWidth = convInfo.effectiveFilterWidth; + const padLeft = effectiveFilterWidth - 1 - convInfo.padInfo.left; + const padTop = effectiveFilterHeight - 1 - convInfo.padInfo.top; + const dx = + buffer(x.shape as [number, number, number, number], 'float32'); + + const dyData = backend.data.get(dy.dataId).values as Float32Array; + const dyBuf = buffer( + dy.shape as [number, number, number, number], 'float32', dyData); + + for (let b = 0; b < convInfo.batchSize; ++b) { + for (let d = 0; d < convInfo.inChannels; ++d) { + for (let dxR = 0; dxR < convInfo.inHeight; ++dxR) { + for (let dxC = 0; dxC < convInfo.inWidth; ++dxC) { + // Shader code begins. + const dyRCorner = dxR - padTop; + const dyCCorner = dxC - padLeft; + let dotProd = 0; + for (let wR = 0; wR < effectiveFilterHeight; wR += dilationHeight) { + const dyR = (dyRCorner + wR) / strideHeight; + if (dyR < 0 || dyR >= convInfo.outHeight || + Math.floor(dyR) !== dyR) { + continue; + } + for (let wC = 0; wC < effectiveFilterWidth; wC += dilationWidth) { + const dyC = (dyCCorner + wC) / strideWidth; + if (dyC < 0 || dyC >= convInfo.outWidth || + Math.floor(dyC) !== dyC) { + continue; + } + const maxPos = effectiveFilterHeight * effectiveFilterWidth - 1 - + (maxPosBuf.get(b, dyR, dyC, d) as number); + const curPos = wR * effectiveFilterWidth + wC; + + const mask = maxPos === curPos ? 1 : 0; + if (mask === 0) { + continue; + } + + const pixel = dyBuf.get(b, dyR, dyC, d); + dotProd += pixel * mask; + } + } + dx.set(dotProd, b, dxR, dxC, d); + } + } + } + } + return backend.makeTensorInfo(dx.shape, dx.dtype, dx.values); +} + +export const maxPoolBackpropConfig: KernelConfig = { + kernelName: MaxPoolBackprop, + backendName: 'cpu', + kernelFunc: maxPoolBackprop as {} as KernelFunc +}; diff --git a/tfjs-backend-cpu/src/register_all_kernels.ts b/tfjs-backend-cpu/src/register_all_kernels.ts index 8f2730aa8f4..bec5c0cd270 100644 --- a/tfjs-backend-cpu/src/register_all_kernels.ts +++ b/tfjs-backend-cpu/src/register_all_kernels.ts @@ -27,6 +27,8 @@ import {asinConfig} from './kernels/Asin'; import {asinhConfig} from './kernels/Asinh'; import {atanConfig} from './kernels/Atan'; import {atanhConfig} from './kernels/Atanh'; +import {avgPoolConfig} from './kernels/AvgPool'; +import {avgPoolBackpropConfig} from './kernels/AvgPoolBackprop'; import {batchNormConfig} from './kernels/BatchNorm'; import {castConfig} from './kernels/Cast'; import {ceilConfig} from './kernels/Ceil'; @@ -57,6 +59,7 @@ import {log1pConfig} from './kernels/Log1p'; import {logicalNotConfig} from './kernels/LogicalNot'; import {maxConfig} from './kernels/Max'; import {maxPoolConfig} from './kernels/MaxPool'; +import {maxPoolBackpropConfig} from './kernels/MaxPoolBackprop'; import {maxPoolWithArgmaxConfig} from './kernels/MaxPoolWithArgmax'; import {multiplyConfig} from './kernels/Multiply'; import {nonMaxSuppressionV4Config} from './kernels/NonMaxSuppressionV4'; @@ -96,6 +99,8 @@ const kernelConfigs: KernelConfig[] = [ asinhConfig, atanConfig, atanhConfig, + avgPoolConfig, + avgPoolBackpropConfig, batchNormConfig, castConfig, ceilConfig, @@ -125,6 +130,7 @@ const kernelConfigs: KernelConfig[] = [ log1pConfig, logicalNotConfig, maxPoolConfig, + maxPoolBackpropConfig, maxPoolWithArgmaxConfig, maxConfig, multiplyConfig, From f623dddcf36cac48a59aee5a6f4d2787ab5ed627 Mon Sep 17 00:00:00 2001 From: Jing Jin <8752427+jinjingforever@users.noreply.github.com> Date: Thu, 24 Sep 2020 13:17:31 -0700 Subject: [PATCH 3/6] save --- tfjs-backend-cpu/src/backend_cpu.ts | 56 ----------------------------- 1 file changed, 56 deletions(-) diff --git a/tfjs-backend-cpu/src/backend_cpu.ts b/tfjs-backend-cpu/src/backend_cpu.ts index 64c2c10051b..5915f803aa2 100644 --- a/tfjs-backend-cpu/src/backend_cpu.ts +++ b/tfjs-backend-cpu/src/backend_cpu.ts @@ -1528,62 +1528,6 @@ export class MathBackendCPU extends KernelBackend { .slice(sliceBeginCoords, sliceSize) as T; } - // avgPoolBackprop(dy: Tensor4D, x: Tensor4D, convInfo: - // backend_util.Conv2DInfo): - // Tensor4D { - // assertNotComplex([dy, x], 'avgPoolBackprop'); - - // const strideHeight = convInfo.strideHeight; - // const strideWidth = convInfo.strideWidth; - // const filterHeight = convInfo.filterHeight; - // const filterWidth = convInfo.filterWidth; - // const dilationHeight = convInfo.dilationHeight; - // const dilationWidth = convInfo.dilationWidth; - // const effectiveFilterHeight = convInfo.effectiveFilterHeight; - // const effectiveFilterWidth = convInfo.effectiveFilterWidth; - // const padLeft = effectiveFilterWidth - 1 - convInfo.padInfo.left; - // const padTop = effectiveFilterHeight - 1 - convInfo.padInfo.top; - // const dx = tf.buffer(x.shape, 'float32'); - - // const avgMultiplier = 1 / (filterHeight * filterWidth); - - // const dyBuf = this.bufferSync(dy); - - // for (let b = 0; b < convInfo.batchSize; ++b) { - // for (let d = 0; d < convInfo.inChannels; ++d) { - // for (let dxR = 0; dxR < convInfo.inHeight; ++dxR) { - // for (let dxC = 0; dxC < convInfo.inWidth; ++dxC) { - // // Shader code begins. - // const dyRCorner = dxR - padTop; - // const dyCCorner = dxC - padLeft; - // let dotProd = 0; - // for (let wR = 0; wR < effectiveFilterHeight; wR += - // dilationHeight) { - // const dyR = (dyRCorner + wR) / strideHeight; - // if (dyR < 0 || dyR >= convInfo.outHeight || - // Math.floor(dyR) !== dyR) { - // continue; - // } - // for (let wC = 0; wC < effectiveFilterWidth; wC += - // dilationWidth) { - // const dyC = (dyCCorner + wC) / strideWidth; - // if (dyC < 0 || dyC >= convInfo.outWidth || - // Math.floor(dyC) !== dyC) { - // continue; - // } - - // const pixel = dyBuf.get(b, dyR, dyC, d); - // dotProd += pixel; - // } - // } - // dx.set(dotProd * avgMultiplier, b, dxR, dxC, d); - // } - // } - // } - // } - // return dx.toTensor(); - // } - private pool3d( x: Tensor5D, convInfo: backend_util.Conv3DInfo, poolType: 'max'|'avg'): Tensor5D { From 8ff632b09172ece93421fbba550291ff3b2411e2 Mon Sep 17 00:00:00 2001 From: Jing Jin <8752427+jinjingforever@users.noreply.github.com> Date: Thu, 24 Sep 2020 13:18:48 -0700 Subject: [PATCH 4/6] save --- tfjs-backend-cpu/src/backend_cpu.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tfjs-backend-cpu/src/backend_cpu.ts b/tfjs-backend-cpu/src/backend_cpu.ts index 5915f803aa2..eced73a2599 100644 --- a/tfjs-backend-cpu/src/backend_cpu.ts +++ b/tfjs-backend-cpu/src/backend_cpu.ts @@ -1882,14 +1882,6 @@ export class MathBackendCPU extends KernelBackend { return dx.toTensor(); } - // avgPool(x: Tensor4D, convInfo: backend_util.Conv2DInfo): Tensor4D { - // assertNotComplex(x, 'avgPool'); - // const xValues = this.readSync(x.dataId) as TypedArray; - // return pool(xValues, x.shape, x.dtype, x.strides, convInfo, 'avg') - // .toTensor() - // .toFloat() as Tensor4D; - // } - resizeBilinear( x: Tensor4D, newHeight: number, newWidth: number, alignCorners: boolean): Tensor4D { From 88d0220cd72c8e471712aae68419c78bd22c9946 Mon Sep 17 00:00:00 2001 From: Jing Jin <8752427+jinjingforever@users.noreply.github.com> Date: Fri, 25 Sep 2020 11:14:29 -0700 Subject: [PATCH 5/6] use identity --- tfjs-backend-cpu/src/kernels/AvgPool.ts | 4 ++-- tfjs-backend-cpu/src/kernels/MaxPool.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tfjs-backend-cpu/src/kernels/AvgPool.ts b/tfjs-backend-cpu/src/kernels/AvgPool.ts index abe9306d15f..f8346a1a0a5 100644 --- a/tfjs-backend-cpu/src/kernels/AvgPool.ts +++ b/tfjs-backend-cpu/src/kernels/AvgPool.ts @@ -19,6 +19,7 @@ import {AvgPool, AvgPoolAttrs, AvgPoolInputs, backend_util, KernelConfig, Kernel import {MathBackendCPU} from '../backend_cpu'; import {assertNotComplex} from '../cpu_util'; import {pool} from '../utils/pool_utils'; +import {identity} from './Identity'; export function avgPool( args: @@ -54,8 +55,7 @@ export function avgPool( if (convInfo.filterWidth === 1 && convInfo.filterHeight === 1 && util.arraysEqual(convInfo.inShape, convInfo.outShape)) { - res = {...x}; - backend.incRef(x.dataId); + res = identity({inputs: {x}, backend}); } else { const xValues = backend.data.get(x.dataId).values as TypedArray; const strides = util.computeStrides(x.shape); diff --git a/tfjs-backend-cpu/src/kernels/MaxPool.ts b/tfjs-backend-cpu/src/kernels/MaxPool.ts index 9a44e44343a..d424917e362 100644 --- a/tfjs-backend-cpu/src/kernels/MaxPool.ts +++ b/tfjs-backend-cpu/src/kernels/MaxPool.ts @@ -19,6 +19,7 @@ import {backend_util, KernelConfig, KernelFunc, MaxPool, MaxPoolAttrs, MaxPoolIn import {MathBackendCPU} from '../backend_cpu'; import {assertNotComplex} from '../cpu_util'; import {pool} from '../utils/pool_utils'; +import {identity} from './Identity'; export function maxPool( args: @@ -53,8 +54,7 @@ export function maxPool( if (convInfo.filterWidth === 1 && convInfo.filterHeight === 1 && util.arraysEqual(convInfo.inShape, convInfo.outShape)) { - res = {...x}; - backend.incRef(x.dataId); + res = identity({inputs: {x}, backend}); } else { const xValues = backend.data.get(x.dataId).values as TypedArray; const strides = util.computeStrides(x.shape); From a3788973d7ebb3ef81c3e37e7f403b99db671c26 Mon Sep 17 00:00:00 2001 From: Jing Jin <8752427+jinjingforever@users.noreply.github.com> Date: Mon, 28 Sep 2020 10:31:29 -0700 Subject: [PATCH 6/6] address comments --- tfjs-backend-cpu/src/kernels/AvgPool.ts | 12 ---------- .../src/kernels/AvgPoolBackprop.ts | 13 +--------- tfjs-backend-cpu/src/kernels/MaxPool.ts | 11 --------- .../src/kernels/MaxPoolBackprop.ts | 24 +------------------ 4 files changed, 2 insertions(+), 58 deletions(-) diff --git a/tfjs-backend-cpu/src/kernels/AvgPool.ts b/tfjs-backend-cpu/src/kernels/AvgPool.ts index f8346a1a0a5..89b8523f5d1 100644 --- a/tfjs-backend-cpu/src/kernels/AvgPool.ts +++ b/tfjs-backend-cpu/src/kernels/AvgPool.ts @@ -36,18 +36,6 @@ export function avgPool( () => 'Error in avgPool: Either strides or dilations must be 1. ' + `Got strides ${strides} and dilations '${dilations}'`); - const xRank = x.shape.length; - util.assert( - xRank === 4, - () => `Error in avgPool: input must be rank 4 but got rank ${ - x.shape.length}}.`); - if (dimRoundingMode != null) { - util.assert( - util.isInt(pad as number), - () => `Error in avgPool: pad must be an integer when using, ` + - `dimRoundingMode ${dimRoundingMode} but got pad ${pad}.`); - } - const convInfo = backend_util.computePool2DInfo( x.shape as [number, number, number, number], filterSize, strides, dilations, pad, dimRoundingMode); diff --git a/tfjs-backend-cpu/src/kernels/AvgPoolBackprop.ts b/tfjs-backend-cpu/src/kernels/AvgPoolBackprop.ts index 2671308b594..08e743f1061 100644 --- a/tfjs-backend-cpu/src/kernels/AvgPoolBackprop.ts +++ b/tfjs-backend-cpu/src/kernels/AvgPoolBackprop.ts @@ -14,7 +14,7 @@ * limitations under the License. * ============================================================================= */ -import {AvgPoolBackprop, AvgPoolBackpropAttrs, AvgPoolBackpropInputs, backend_util, buffer, KernelConfig, KernelFunc, Rank, TensorInfo, util} from '@tensorflow/tfjs-core'; +import {AvgPoolBackprop, AvgPoolBackpropAttrs, AvgPoolBackpropInputs, backend_util, buffer, KernelConfig, KernelFunc, Rank, TensorInfo} from '@tensorflow/tfjs-core'; import {MathBackendCPU} from '../backend_cpu'; import {assertNotComplex} from '../cpu_util'; @@ -30,17 +30,6 @@ export function avgPoolBackprop(args: { assertNotComplex([dy, input], 'avgPoolBackprop'); const {filterSize, strides, pad} = attrs; - const inputRank = input.shape.length; - const dyRank = dy.shape.length; - util.assert( - dyRank === 4, - () => `Error in avgPoolBackprop: dy must be rank 4 but got rank ` + - `${dyRank}.`); - util.assert( - inputRank === 4, - () => `Error in avgPoolBackprop: input must be rank 4 but got rank ` + - `${inputRank}.`); - const convInfo = backend_util.computePool2DInfo( x.shape as [number, number, number, number], filterSize, strides, 1 /* dilations */, pad); diff --git a/tfjs-backend-cpu/src/kernels/MaxPool.ts b/tfjs-backend-cpu/src/kernels/MaxPool.ts index d424917e362..7c7cc43d54e 100644 --- a/tfjs-backend-cpu/src/kernels/MaxPool.ts +++ b/tfjs-backend-cpu/src/kernels/MaxPool.ts @@ -31,21 +31,10 @@ export function maxPool( const {filterSize, strides, pad, dimRoundingMode} = attrs; const dilations = 1; - const xRank = x.shape.length; - util.assert( - xRank === 4, - () => `Error in maxPool: input must be rank 4 but got rank ${ - x.shape.length}}.`); util.assert( backend_util.eitherStridesOrDilationsAreOne(strides, dilations), () => 'Error in maxPool: Either strides or dilations must be 1. ' + `Got strides ${strides} and dilations '${dilations}'`); - if (dimRoundingMode != null) { - util.assert( - util.isInt(pad as number), - () => `Error in maxPool: pad must be an integer when using, ` + - `dimRoundingMode ${dimRoundingMode} but got pad ${pad}.`); - } const convInfo = backend_util.computePool2DInfo( x.shape as [number, number, number, number], filterSize, strides, diff --git a/tfjs-backend-cpu/src/kernels/MaxPoolBackprop.ts b/tfjs-backend-cpu/src/kernels/MaxPoolBackprop.ts index 95c631ca2a0..69dc207faa1 100644 --- a/tfjs-backend-cpu/src/kernels/MaxPoolBackprop.ts +++ b/tfjs-backend-cpu/src/kernels/MaxPoolBackprop.ts @@ -14,7 +14,7 @@ * limitations under the License. * ============================================================================= */ -import {backend_util, buffer, KernelConfig, KernelFunc, MaxPoolBackprop, MaxPoolBackpropAttrs, MaxPoolBackpropInputs, Rank, TensorInfo, TypedArray, util} from '@tensorflow/tfjs-core'; +import {backend_util, buffer, KernelConfig, KernelFunc, MaxPoolBackprop, MaxPoolBackpropAttrs, MaxPoolBackpropInputs, Rank, TensorInfo, TypedArray} from '@tensorflow/tfjs-core'; import {MathBackendCPU} from '../backend_cpu'; import {assertNotComplex} from '../cpu_util'; @@ -31,28 +31,6 @@ export function maxPoolBackprop(args: { assertNotComplex([input, output], 'maxPoolBackprop'); const {filterSize, strides, pad, dimRoundingMode} = attrs; - const inputRank = input.shape.length; - const dyRank = dy.shape.length; - util.assert( - inputRank === dyRank, - () => `Rank of input (${inputRank}) does not match rank of dy ` + - `(${dyRank})`); - - util.assert( - dyRank === 4, - () => `Error in maxPoolBackprop: dy must be rank 4 but got rank ` + - `${dyRank}.`); - util.assert( - inputRank === 4, - () => `Error in maxPoolBackprop: input must be rank 4 but got rank ` + - `${inputRank}.`); - if (dimRoundingMode != null) { - util.assert( - util.isInt(pad as number), - () => `Error in maxPoolBackprop: pad must be an integer when using, ` + - `dimRoundingMode ${dimRoundingMode} but got pad ${pad}.`); - } - const convInfo = backend_util.computePool2DInfo( x.shape as [number, number, number, number], filterSize, strides, 1 /* dilations */, pad, dimRoundingMode);