From d3d3dd33b81ea7380895073ca822c5bfb9a23399 Mon Sep 17 00:00:00 2001 From: Ping Yu <4018+pyu10055@users.noreply.github.com> Date: Mon, 22 Jun 2020 16:57:26 -0700 Subject: [PATCH 1/2] added support explicit padding for conv2d ops --- .../executors/basic_math_executor_test.ts | 2 +- .../executors/convolution_executor.ts | 10 +- .../executors/convolution_executor_test.ts | 134 +++++++++++++++++- .../executors/normalization_executor_test.ts | 2 +- .../executors/slice_join_executor_test.ts | 35 ++--- .../executors/transformation_executor.ts | 15 +- .../src/operations/executors/utils.ts | 19 ++- 7 files changed, 181 insertions(+), 36 deletions(-) diff --git a/tfjs-converter/src/operations/executors/basic_math_executor_test.ts b/tfjs-converter/src/operations/executors/basic_math_executor_test.ts index c6ca5c4667c..1c38cafe597 100644 --- a/tfjs-converter/src/operations/executors/basic_math_executor_test.ts +++ b/tfjs-converter/src/operations/executors/basic_math_executor_test.ts @@ -86,7 +86,7 @@ describe('basic math', () => { node.op = 'Prod'; node.inputParams['axes'] = createNumericArrayAttrFromIndex(1); node.inputNames = ['input1', 'input2']; - const input2 = [tfc.scalar(2)]; + const input2 = [tfc.tensor1d([2])]; executeOp(node, {input1, input2}, context); expect(tfc.prod).toHaveBeenCalledWith(input1[0], [2]); diff --git a/tfjs-converter/src/operations/executors/convolution_executor.ts b/tfjs-converter/src/operations/executors/convolution_executor.ts index b3cdf93b3c9..0856d63d628 100644 --- a/tfjs-converter/src/operations/executors/convolution_executor.ts +++ b/tfjs-converter/src/operations/executors/convolution_executor.ts @@ -21,7 +21,7 @@ import {NamedTensorsMap} from '../../data/types'; import {ExecutionContext} from '../../executor/execution_context'; import {InternalOpExecutor, Node} from '../types'; -import {getParamValue} from './utils'; +import {getPadding, getParamValue} from './utils'; export const executeOp: InternalOpExecutor = (node: Node, tensorMap: NamedTensorsMap, @@ -46,7 +46,7 @@ export const executeOp: InternalOpExecutor = (node: Node, case 'Conv2D': { const stride = getParamValue('strides', node, tensorMap, context) as number[]; - const pad = getParamValue('pad', node, tensorMap, context); + const pad = getPadding(node, tensorMap, context); const dataFormat = (getParamValue('dataFormat', node, tensorMap, context) as string) .toUpperCase(); @@ -88,7 +88,7 @@ export const executeOp: InternalOpExecutor = (node: Node, } const stride = getParamValue('strides', node, tensorMap, context) as number[]; - const pad = getParamValue('pad', node, tensorMap, context); + const pad = getPadding(node, tensorMap, context); const dataFormat = (getParamValue('dataFormat', node, tensorMap, context) as string) .toUpperCase(); @@ -121,7 +121,7 @@ export const executeOp: InternalOpExecutor = (node: Node, [number, number, number, number]; const stride = getParamValue('strides', node, tensorMap, context) as number[]; - const pad = getParamValue('pad', node, tensorMap, context); + const pad = getPadding(node, tensorMap, context); return [tfc.conv2dTranspose( getParamValue('x', node, tensorMap, context) as tfc.Tensor3D | tfc.Tensor4D, @@ -132,7 +132,7 @@ export const executeOp: InternalOpExecutor = (node: Node, case 'DepthwiseConv2d': { const stride = getParamValue('strides', node, tensorMap, context) as number[]; - const pad = getParamValue('pad', node, tensorMap, context); + const pad = getPadding(node, tensorMap, context); const dilations = getParamValue('dilations', node, tensorMap, context) as number[]; const dataFormat = diff --git a/tfjs-converter/src/operations/executors/convolution_executor_test.ts b/tfjs-converter/src/operations/executors/convolution_executor_test.ts index 80df2d6ac6f..1ecec05769e 100644 --- a/tfjs-converter/src/operations/executors/convolution_executor_test.ts +++ b/tfjs-converter/src/operations/executors/convolution_executor_test.ts @@ -91,6 +91,28 @@ describe('convolution', () => { .toHaveBeenCalledWith( input1[0], input2[0], [2, 2], 'same', 'NHWC', [2, 2]); }); + it('should support explicit padding', () => { + spyOn(tfc, 'conv2d'); + node.op = 'Conv2D'; + node.inputParams['filter'] = createTensorAttr(1); + node.attrParams['strides'] = createNumericArrayAttr([1, 2, 2, 1]); + node.attrParams['pad'] = createStrAttr('explicit'); + node.attrParams['explicitPaddings'] = + createNumericArrayAttr([0, 0, 1, 1, 2, 2, 0, 0]); + node.attrParams['dataFormat'] = createStrAttr('NHWC'); + node.attrParams['dilations'] = createNumericArrayAttr([1, 2, 2, 1]); + + const input1 = [tfc.scalar(1.0)]; + const input2 = [tfc.scalar(1.0)]; + node.inputNames = ['input1', 'input2']; + + executeOp(node, {input1, input2}, context); + + expect(tfc.conv2d) + .toHaveBeenCalledWith( + input1[0], input2[0], [2, 2], [[0, 0], [1, 1], [2, 2], [0, 0]], + 'NHWC', [2, 2]); + }); }); describe('Conv2DBackpropInput', () => { it('should call tfc.conv2dTranspose', () => { @@ -111,6 +133,31 @@ describe('convolution', () => { .toHaveBeenCalledWith( input1[0], input2[0], [1, 2, 2, 2], [2, 2], 'same'); }); + it('should support explicit padding', () => { + spyOn(tfc, 'conv2dTranspose'); + node.op = 'Conv2DBackpropInput'; + node.attrParams['outputShape'] = createNumericArrayAttr([1, 2, 2, 2]); + node.inputParams['filter'] = createTensorAttr(1); + node.attrParams['strides'] = createNumericArrayAttr([1, 2, 2, 1]); + node.attrParams['pad'] = createStrAttr('explicit'); + node.attrParams['explicitPaddings'] = + createNumericArrayAttr([0, 0, 1, 1, 2, 2, 0, 0]); + + const input1 = [tfc.scalar(1.0)]; + const input2 = [tfc.scalar(1.0)]; + node.inputNames = ['input1', 'input2']; + + executeOp(node, {input1, input2}, context); + + expect(tfc.conv2dTranspose) + .toHaveBeenCalledWith( + input1[0], + input2[0], + [1, 2, 2, 2], + [2, 2], + [[0, 0], [1, 1], [2, 2], [0, 0]], + ); + }); }); describe('Conv1D', () => { it('should call tfc.conv1d', () => { @@ -155,6 +202,29 @@ describe('convolution', () => { .toHaveBeenCalledWith( input1[0], input2[0], [2, 2], 'same', 'NHWC', [2, 2]); }); + it('support explicit padding', () => { + spyOn(tfc, 'depthwiseConv2d'); + node.op = 'DepthwiseConv2d'; + node.category = 'convolution'; + node.inputParams['input'] = createTensorAttr(0); + node.inputParams['filter'] = createTensorAttr(1); + node.attrParams['strides'] = createNumericArrayAttr([1, 2, 2, 1]); + node.attrParams['pad'] = createStrAttr('explicit'); + node.attrParams['explicitPaddings'] = + createNumericArrayAttr([0, 0, 1, 1, 2, 2, 0, 0]); + node.attrParams['dataFormat'] = createStrAttr('NHWC'); + node.attrParams['dilations'] = createNumericArrayAttr([1, 2, 2, 1]); + const input1 = [tfc.scalar(1.0)]; + const input2 = [tfc.scalar(1.0)]; + node.inputNames = ['input1', 'input2']; + + executeOp(node, {input1, input2}, context); + + expect(tfc.depthwiseConv2d) + .toHaveBeenCalledWith( + input1[0], input2[0], [2, 2], [[0, 0], [1, 1], [2, 2], [0, 0]], + 'NHWC', [2, 2]); + }); }); describe('Conv3d', () => { @@ -257,7 +327,38 @@ describe('convolution', () => { preluActivationWeights: undefined }); }); + it('should support explicit padding', () => { + spyOn(tfc.fused, 'conv2d'); + node.op = '_FusedConv2D'; + node.inputParams['filter'] = createTensorAttr(1); + node.inputParams['args'] = createTensorsAttr(2, 0); + node.attrParams['fusedOps'] = createStrArrayAttr(['biasadd', 'relu']); + node.attrParams['strides'] = createNumericArrayAttr([1, 2, 2, 1]); + node.attrParams['pad'] = createStrAttr('explicit'); + node.attrParams['explicitPaddings'] = + createNumericArrayAttr([0, 0, 1, 1, 2, 2, 0, 0]); + node.attrParams['dataFormat'] = createStrAttr('NHWC'); + node.attrParams['dilations'] = createNumericArrayAttr([1, 2, 2, 1]); + node.attrParams['numArgs'] = createNumberAttr(1); + const input1 = [tfc.scalar(1.0)]; + const input2 = [tfc.scalar(2.0)]; + const input3 = [tfc.scalar(3.0)]; + + node.inputNames = ['input1', 'input2', 'input3']; + executeOp(node, {input1, input2, input3}, context); + expect(tfc.fused.conv2d).toHaveBeenCalledWith({ + x: input1[0], + filter: input2[0], + strides: [2, 2], + pad: [[0, 0], [1, 1], [2, 2], [0, 0]], + dataFormat: 'NHWC', + dilations: [2, 2], + bias: input3[0], + activation: 'relu', + preluActivationWeights: undefined + }); + }); it('with bias and prelu activation func', () => { spyOn(tfc.fused, 'conv2d'); node.op = '_FusedConv2D'; @@ -341,6 +442,38 @@ describe('convolution', () => { }); }); describe('FusedDepthwiseConv2d', () => { + it('support explicit padding', () => { + spyOn(tfc.fused, 'depthwiseConv2d'); + node.op = 'FusedDepthwiseConv2dNative'; + node.inputParams['filter'] = createTensorAttr(1); + node.inputParams['args'] = createTensorsAttr(2, 0); + node.attrParams['fusedOps'] = createStrArrayAttr(['biasadd', 'relu']); + node.attrParams['strides'] = createNumericArrayAttr([1, 2, 2, 1]); + node.attrParams['pad'] = createStrAttr('explicit'); + node.attrParams['explicitPaddings'] = + createNumericArrayAttr([0, 0, 1, 1, 2, 2, 0, 0]); + node.attrParams['dataFormat'] = createStrAttr('NHWC'); + node.attrParams['dilations'] = createNumericArrayAttr([1, 2, 2, 1]); + node.attrParams['numArgs'] = createNumberAttr(1); + const input1 = [tfc.scalar(1.0)]; + const input2 = [tfc.scalar(2.0)]; + const input3 = [tfc.scalar(3.0)]; + + node.inputNames = ['input1', 'input2', 'input3']; + executeOp(node, {input1, input2, input3}, context); + + expect(tfc.fused.depthwiseConv2d).toHaveBeenCalledWith({ + x: input1[0], + filter: input2[0], + strides: [2, 2], + pad: [[0, 0], [1, 1], [2, 2], [0, 0]], + dataFormat: 'NHWC', + dilations: [2, 2], + bias: input3[0], + activation: 'relu', + preluActivationWeights: undefined + }); + }); it('with bias and activation func', () => { spyOn(tfc.fused, 'depthwiseConv2d'); node.op = 'FusedDepthwiseConv2dNative'; @@ -371,7 +504,6 @@ describe('convolution', () => { preluActivationWeights: undefined }); }); - it('with bias and prelu activation func', () => { spyOn(tfc.fused, 'depthwiseConv2d'); node.op = 'FusedDepthwiseConv2dNative'; diff --git a/tfjs-converter/src/operations/executors/normalization_executor_test.ts b/tfjs-converter/src/operations/executors/normalization_executor_test.ts index 9b6f61eb6ee..5da2c80e1e5 100644 --- a/tfjs-converter/src/operations/executors/normalization_executor_test.ts +++ b/tfjs-converter/src/operations/executors/normalization_executor_test.ts @@ -170,7 +170,7 @@ describe('normalization', () => { node.inputParams.sparseValues = createTensorAttr(2); node.inputParams.defaultValue = createTensorAttr(3); node.inputNames = ['input1', 'input2', 'input3', 'input4']; - const input2 = [tfc.scalar(1)]; + const input2 = [tfc.tensor1d([1], 'int32')]; const input3 = [tfc.scalar(2)]; const input4 = [tfc.scalar(3)]; executeOp(node, {input1, input2, input3, input4}, context); diff --git a/tfjs-converter/src/operations/executors/slice_join_executor_test.ts b/tfjs-converter/src/operations/executors/slice_join_executor_test.ts index b9e5c4d3449..1b6852c5412 100644 --- a/tfjs-converter/src/operations/executors/slice_join_executor_test.ts +++ b/tfjs-converter/src/operations/executors/slice_join_executor_test.ts @@ -181,10 +181,10 @@ describe('slice join', () => { spyOn(tfc, 'reverse'); node.op = 'Reverse'; node.inputParams.axis = createNumericArrayAttrFromIndex(1); - node.inputNames = ['input1', 'input2']; - executeOp(node, {input1, input2}, context); + node.inputNames = ['input1', 'input4']; + executeOp(node, {input1, input4}, context); - expect(tfc.reverse).toHaveBeenCalledWith(input1[0], [2]); + expect(tfc.reverse).toHaveBeenCalledWith(input1[0], [3]); }); it('should match json def for reverse', () => { node.op = 'Reverse'; @@ -196,10 +196,10 @@ describe('slice join', () => { spyOn(tfc, 'reverse'); node.op = 'ReverseV2'; node.inputParams.axis = createNumericArrayAttrFromIndex(1); - node.inputNames = ['input1', 'input2']; - executeOp(node, {input1, input2}, context); + node.inputNames = ['input1', 'input4']; + executeOp(node, {input1, input4}, context); - expect(tfc.reverse).toHaveBeenCalledWith(input1[0], [2]); + expect(tfc.reverse).toHaveBeenCalledWith(input1[0], [3]); }); it('should match json def for reverse', () => { node.op = 'ReverseV2'; @@ -211,10 +211,10 @@ describe('slice join', () => { spyOn(tfc, 'tile'); node.op = 'Tile'; node.inputParams.reps = createNumericArrayAttrFromIndex(1); - node.inputNames = ['input1', 'input2']; - executeOp(node, {input1, input2}, context); + node.inputNames = ['input1', 'input4']; + executeOp(node, {input1, input4}, context); - expect(tfc.tile).toHaveBeenCalledWith(input1[0], [2]); + expect(tfc.tile).toHaveBeenCalledWith(input1[0], [3]); }); it('should match json def for tile', () => { node.op = 'Tile'; @@ -227,9 +227,10 @@ describe('slice join', () => { node.op = 'Slice'; node.inputParams.begin = createNumericArrayAttrFromIndex(1); node.inputParams.size = createNumericArrayAttrFromIndex(2); - node.inputNames = ['input1', 'input2', 'input3']; + const input6 = [tfc.tensor1d([2], 'int32')]; + node.inputNames = ['input1', 'input6', 'input4']; - executeOp(node, {input1, input2, input3}, context); + executeOp(node, {input1, input6, input4}, context); expect(tfc.slice).toHaveBeenCalledWith(input1[0], [2], [3]); }); @@ -251,8 +252,10 @@ describe('slice join', () => { node.attrParams.ellipsisMask = createNumberAttr(1); node.attrParams.newAxisMask = createNumberAttr(2); node.attrParams.shrinkAxisMask = createNumberAttr(3); - node.inputNames = ['input1', 'input2', 'input3', 'input4']; - executeOp(node, {input1, input2, input3, input4}, context); + node.inputNames = ['input1', 'input6', 'input7', 'input4']; + const input6 = [tfc.tensor1d([2], 'int32')]; + const input7 = [tfc.tensor1d([3], 'int32')]; + executeOp(node, {input1, input6, input7, input4}, context); expect(tfc.stridedSlice) .toHaveBeenCalledWith(input1[0], [2], [3], [3], 4, 5, 1, 2, 3); @@ -367,7 +370,7 @@ describe('slice join', () => { node.inputNames = ['input1', 'input2', 'input3']; executeOp(node, {input1, input2, input3}, context); - expect(tfc.split).toHaveBeenCalledWith(input1[0], [2], 3); + expect(tfc.split).toHaveBeenCalledWith(input1[0], 2, 3); }); it('should match json def for split', () => { node.op = 'SplitV'; @@ -383,8 +386,8 @@ describe('slice join', () => { node.inputParams.indices = createTensorAttr(0); node.inputParams.values = createTensorAttr(1); node.inputParams.shape = createNumericArrayAttrFromIndex(2); - node.inputNames = ['input1', 'input2', 'input3']; - executeOp(node, {input1, input2, input3}, context); + node.inputNames = ['input1', 'input2', 'input4']; + executeOp(node, {input1, input2, input4}, context); expect(tfc.scatterND).toHaveBeenCalledWith(input1[0], input2[0], [3]); }); diff --git a/tfjs-converter/src/operations/executors/transformation_executor.ts b/tfjs-converter/src/operations/executors/transformation_executor.ts index 96339c1562b..b91ee2e6a69 100644 --- a/tfjs-converter/src/operations/executors/transformation_executor.ts +++ b/tfjs-converter/src/operations/executors/transformation_executor.ts @@ -21,7 +21,7 @@ import {NamedTensorsMap} from '../../data/types'; import {ExecutionContext} from '../../executor/execution_context'; import {InternalOpExecutor, Node} from '../types'; -import {getParamValue, split} from './utils'; +import {getParamValue} from './utils'; export const executeOp: InternalOpExecutor = (node: Node, tensorMap: NamedTensorsMap, @@ -54,16 +54,15 @@ export const executeOp: InternalOpExecutor = (node: Node, case 'Pad': { return [tfc.pad( getParamValue('x', node, tensorMap, context) as tfc.Tensor, - split( - getParamValue('padding', node, tensorMap, context) as number[], - 2) as Array<[number, number]>, + getParamValue('padding', node, tensorMap, context) as + Array<[number, number]>, getParamValue('constantValue', node, tensorMap, context) as number)]; } case 'SpaceToBatchND': { const blockShape = getParamValue('blockShape', node, tensorMap, context) as number[]; - const paddings = split( - getParamValue('paddings', node, tensorMap, context) as number[], 2); + const paddings = + getParamValue('paddings', node, tensorMap, context) as number[][]; return [tfc.spaceToBatchND( getParamValue('x', node, tensorMap, context) as tfc.Tensor, blockShape, paddings)]; @@ -71,8 +70,8 @@ export const executeOp: InternalOpExecutor = (node: Node, case 'BatchToSpaceND': { const blockShape = getParamValue('blockShape', node, tensorMap, context) as number[]; - const crops = split( - getParamValue('crops', node, tensorMap, context) as number[], 2); + const crops = + getParamValue('crops', node, tensorMap, context) as number[][]; return [tfc.batchToSpaceND( getParamValue('x', node, tensorMap, context) as tfc.Tensor, blockShape, crops)]; diff --git a/tfjs-converter/src/operations/executors/utils.ts b/tfjs-converter/src/operations/executors/utils.ts index 3011cfad05d..a426de46a9d 100644 --- a/tfjs-converter/src/operations/executors/utils.ts +++ b/tfjs-converter/src/operations/executors/utils.ts @@ -17,6 +17,7 @@ import * as tfc from '@tensorflow/tfjs-core'; +import {toNestedArray} from '../../../../tfjs-core/src/util'; import {NamedTensorsMap} from '../../data/types'; import {ExecutionContext} from '../../executor/execution_context'; import {Node, ValueType} from '../types'; @@ -40,10 +41,11 @@ export function getParamValue( return inputs.map(name => getTensor(name, tensorMap, context)); } - const data = Array.prototype.slice.call( - getTensor(node.inputNames.slice(start)[0], tensorMap, context) - .dataSync()); - return inputParam.type === 'number' ? data[0] : data; + const tensor = + getTensor(node.inputNames.slice(start)[0], tensorMap, context); + const data = tensor.dataSync(); + return inputParam.type === 'number' ? data[0] : + toNestedArray(tensor.shape, data); } const attrParam = node.attrParams[paramName]; return attrParam && attrParam.value; @@ -121,7 +123,16 @@ export function getPadding( context: ExecutionContext): ValueType { let pad = getParamValue('pad', node, tensorMap, context); if (pad === 'explicit') { + // This is 1d array, we need to convert it to 2d array pad = getParamValue('explicitPaddings', node, tensorMap, context); + const explicitPadding: [ + [number, number], [number, number], [number, number], [number, number] + ] = [[0, 0], [0, 0], [0, 0], [0, 0]]; + for (let i = 0; i < 4; i++) { + explicitPadding[i][0] = (pad as number[])[i * 2]; + explicitPadding[i][1] = (pad as number[])[i * 2 + 1]; + } + return explicitPadding; } return pad; } From bc93a4c07cd5e1456248b0ea1d2f133cc1e6c396 Mon Sep 17 00:00:00 2001 From: Ping Yu <4018+pyu10055@users.noreply.github.com> Date: Mon, 22 Jun 2020 17:08:42 -0700 Subject: [PATCH 2/2] fix compilation error --- tfjs-converter/src/operations/executors/utils.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tfjs-converter/src/operations/executors/utils.ts b/tfjs-converter/src/operations/executors/utils.ts index a426de46a9d..55af5eafd52 100644 --- a/tfjs-converter/src/operations/executors/utils.ts +++ b/tfjs-converter/src/operations/executors/utils.ts @@ -17,7 +17,6 @@ import * as tfc from '@tensorflow/tfjs-core'; -import {toNestedArray} from '../../../../tfjs-core/src/util'; import {NamedTensorsMap} from '../../data/types'; import {ExecutionContext} from '../../executor/execution_context'; import {Node, ValueType} from '../types'; @@ -44,8 +43,9 @@ export function getParamValue( const tensor = getTensor(node.inputNames.slice(start)[0], tensorMap, context); const data = tensor.dataSync(); - return inputParam.type === 'number' ? data[0] : - toNestedArray(tensor.shape, data); + return inputParam.type === 'number' ? + data[0] : + tfc.util.toNestedArray(tensor.shape, data); } const attrParam = node.attrParams[paramName]; return attrParam && attrParam.value;