[core] Modularize batchnorm.

tfjs-backend-wasm/src/kernels/FusedBatchNorm.ts

@@ -32,19 +32,19 @@ interface BatchNormAttrs extends NamedAttrMap {
 }
 
 let wasmBatchNorm: (
-    xId: number, meanId: number, varianceId: number, offsetId: number,
-    scaleId: number, varianceEpsilon: number, outId: number) => void;
+  xId: number, meanId: number, varianceId: number, offsetId: number,
+  scaleId: number, varianceEpsilon: number, outId: number) => void;
 
 function setup(backend: BackendWasm): void {
   wasmBatchNorm = backend.wasm.cwrap(
-      'FusedBatchNorm', null /* void */,
-      ['number', 'number', 'number', 'number', 'number', 'number', 'number']);
+    'FusedBatchNorm', null /* void */,
+    ['number', 'number', 'number', 'number', 'number', 'number', 'number']);
 }
 
 function fusedBatchNorm(
-    args:
-        {backend: BackendWasm, inputs: BatchNormInputs, attrs: BatchNormAttrs}):
-    TensorInfo {
+  args:
+    {backend: BackendWasm, inputs: BatchNormInputs, attrs: BatchNormAttrs}):
+  TensorInfo {
   const {backend, inputs, attrs} = args;
   const {varianceEpsilon} = attrs;
   const {x, mean, variance, offset, scale} = inputs;
@@ -63,12 +63,12 @@ function fusedBatchNorm(
   const outId = backend.dataIdMap.get(out.dataId).id;
 
   wasmBatchNorm(
-      xId, meanId, varianceId, offsetId, scaleId, varianceEpsilon, outId);
+    xId, meanId, varianceId, offsetId, scaleId, varianceEpsilon, outId);
   return out;
 }
 
 registerKernel({
-  kernelName: 'BatchNormalization',
+  kernelName: 'FusedBatchNorm',
   backendName: 'wasm',
   setupFunc: setup,
   kernelFunc: fusedBatchNorm

tfjs-core/src/gradients/FusedBatchNorm_grad.ts

@@ -0,0 +1,111 @@
+/**
+ * @license
+ * Copyright 2020 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+import {FusedBatchNorm, FusedBatchNormAttrs} from '../kernel_names';
+import {GradConfig, NamedAttrMap} from '../kernel_registry';
+import {xAs4D} from '../ops/batchnorm_util';
+import {getReductionAxes} from '../ops/broadcast_util';
+import {add, mul, reshape, sub} from '../ops/ops';
+import {sum} from '../ops/reduction_ops';
+import {scalar} from '../ops/tensor_ops';
+import {tile} from '../ops/tile';
+import {rsqrt} from '../ops/unary_ops';
+import {Tensor, Tensor4D} from '../tensor';
+import {Rank, ShapeMap} from '../types';
+
+export const fusedBatchNormGradConfig: GradConfig = {
+  kernelName: FusedBatchNorm,
+  inputsToSave: ['x', 'mean', 'variance', 'scale'],
+  gradFunc: <R extends Rank>(
+      dy: Tensor, saved: Tensor[], attrs: NamedAttrMap) => {
+    const batchNormalizationAttrs: FusedBatchNormAttrs =
+        attrs as {} as FusedBatchNormAttrs;
+    const {varianceEpsilon} = batchNormalizationAttrs;
+    const [x, mean, variance, scale] = saved;
+
+    const x4D: Tensor4D = xAs4D(x);
+
+    const scaleValue = scale == null ? scalar(1) : scale;
+    const reductionAxes = getReductionAxes(mean.shape, x4D.shape);
+    const tileShape: number[] = [];
+    if (mean.rank === 1) {
+      for (let i = 0; i < x4D.shape.length - 1; ++i) {
+        tileShape.push(x4D.shape[i]);
+      }
+      tileShape.push(1);
+    }
+
+    const xMinusMean = sub(x, mean);
+    const dyTimesScaleValue = mul(dy, scaleValue);
+    const oneOverSqrtVariance = rsqrt(add(variance, scalar(varianceEpsilon)));
+    const minusHalfRCube = mul(
+        mul(mul(oneOverSqrtVariance, oneOverSqrtVariance), oneOverSqrtVariance),
+        scalar(-0.5));
+
+    const derX = () => {
+      if (mean.rank === 1) {
+        return reshape(
+            mul(mul(dy,
+                    tile(
+                        oneOverSqrtVariance.as4D(1, 1, 1, mean.shape[0]),
+                        tileShape)),
+                scaleValue),
+            x.shape);
+      } else {
+        return reshape(mul(mul(dy, oneOverSqrtVariance), scaleValue), x.shape);
+      }
+    };
+    const derMean = () => {
+      let meanDer =
+          mul(mul(oneOverSqrtVariance, scalar(-1)), dyTimesScaleValue);
+      if (mean.rank === 1) {
+        meanDer = sum(meanDer, reductionAxes);
+      }
+      return reshape(meanDer, mean.shape as ShapeMap[R]);
+    };
+    const derVariance = () => {
+      let varianceDer = mul(mul(minusHalfRCube, xMinusMean), dyTimesScaleValue);
+
+      if (mean.rank === 1) {
+        varianceDer = sum(varianceDer, reductionAxes);
+      }
+      return reshape(varianceDer, mean.shape as ShapeMap[R]);
+    };
+    const derScale = () => {
+      const xMinusMean2TimesRsqrt = mul(xMinusMean, oneOverSqrtVariance);
+
+      let scaleDer = mul(dy, xMinusMean2TimesRsqrt);
+      if (mean.rank === 1) {
+        scaleDer = sum(scaleDer, reductionAxes);
+      }
+      return reshape(scaleDer, mean.shape as ShapeMap[R]);
+    };
+    const derOffset = () => {
+      let offsetDer = dy;
+      if (mean.rank === 1) {
+        offsetDer = sum(offsetDer, reductionAxes);
+      }
+      return reshape(offsetDer, mean.shape as ShapeMap[R]);
+    };
+    return {
+      x: derX,
+      mean: derMean,
+      variance: derVariance,
+      scale: derScale,
+      offset: derOffset
+    };
+  }
+};

tfjs-core/src/kernel_names.ts

@@ -21,6 +21,13 @@
 import {NamedTensorInfoMap} from './kernel_registry';
 import {PixelData} from './types';
 
+export const FusedBatchNorm = 'FusedBatchNorm';
+export type FusedBatchNormInputs =
+    Pick<NamedTensorInfoMap, 'x'|'scale'|'offset'|'mean'|'variance'>;
+export interface FusedBatchNormAttrs {
+  varianceEpsilon: number;
+}
+
 export type BinaryInputs = Pick<NamedTensorInfoMap, 'a'|'b'>;
 
 export const Div = 'Div';