From e3e03c7f3c8dcbfe133416271c35741e7929603d Mon Sep 17 00:00:00 2001 From: James Bradbury Date: Thu, 21 Feb 2019 23:57:47 -0800 Subject: [PATCH 1/3] Fix Glorot uniform initialization for convolutional layers --- Sources/DeepLearning/Initializers.swift | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/Sources/DeepLearning/Initializers.swift b/Sources/DeepLearning/Initializers.swift index c3c9d14e7..0dccd3e23 100644 --- a/Sources/DeepLearning/Initializers.swift +++ b/Sources/DeepLearning/Initializers.swift @@ -129,22 +129,27 @@ public extension Tensor where Scalar: TensorFlowFloatingPoint, Scalar.RawSignificand: FixedWidthInteger { /// Performs Glorot uniform initialization for the specified shape, creating a tensor by /// randomly sampling scalar values from a uniform distribution between `-limit` and `limit`, - /// where limit is `sqrt(6 / (fanIn + fanOut))`. + /// where limit is `sqrt(6 / (fanIn + fanOut))` and `fanIn`/`fanOut` represent the number of + /// input and output features multiplied by the receptive field if present. /// /// - Parameters: /// - shape: The dimensions of the tensor. /// - generator: Random number generator to use. /// init(glorotUniform shape: TensorShape, generator: inout G) { - let fanIn = shape[shape.count - 2] - let fanOut = shape[shape.count - 1] + let spatialDimCount = shape.count - 2 + let receptiveField = shape[0 ..< spatialDimCount].contiguousSize + let fanIn = shape[shape.count - 2] * receptiveField + let fanOut = shape[shape.count - 1] * receptiveField let minusOneToOne = 2 * Tensor(randomUniform: shape, generator: &generator) - 1 self = sqrt(Scalar(6) / Scalar(fanIn + fanOut)) * minusOneToOne } /// Creates a tensor by performing Glorot uniform initialization for the specified shape, /// randomly sampling scalar values from a uniform distribution between `-limit` and `limit`, - /// where limit is `sqrt(6 / (fanIn + fanOut))`, using the default random number generator. + /// generated by the default random number generator, where limit is + /// `sqrt(6 / (fanIn + fanOut))` and fanIn/fanOut represent the number of input and output + /// features multiplied by the receptive field if present. /// /// - Parameters: /// - shape: The dimensions of the tensor. From 1d58ae3980a971d14acfbf7ec7935945b49d93ee Mon Sep 17 00:00:00 2001 From: James Bradbury Date: Fri, 22 Feb 2019 00:31:55 -0800 Subject: [PATCH 2/3] Apply suggestions from code review Co-Authored-By: jekbradbury --- Sources/DeepLearning/Initializers.swift | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Sources/DeepLearning/Initializers.swift b/Sources/DeepLearning/Initializers.swift index 0dccd3e23..1dc55c346 100644 --- a/Sources/DeepLearning/Initializers.swift +++ b/Sources/DeepLearning/Initializers.swift @@ -129,7 +129,7 @@ public extension Tensor where Scalar: TensorFlowFloatingPoint, Scalar.RawSignificand: FixedWidthInteger { /// Performs Glorot uniform initialization for the specified shape, creating a tensor by /// randomly sampling scalar values from a uniform distribution between `-limit` and `limit`, - /// where limit is `sqrt(6 / (fanIn + fanOut))` and `fanIn`/`fanOut` represent the number of + /// where limit is `sqrt(6 / (fanIn + fanOut))` and `fanIn` / `fanOut` represent the number of /// input and output features multiplied by the receptive field if present. /// /// - Parameters: @@ -138,7 +138,7 @@ public extension Tensor where Scalar: TensorFlowFloatingPoint, /// init(glorotUniform shape: TensorShape, generator: inout G) { let spatialDimCount = shape.count - 2 - let receptiveField = shape[0 ..< spatialDimCount].contiguousSize + let receptiveField = shape[0.. Date: Fri, 22 Feb 2019 00:33:11 -0800 Subject: [PATCH 3/3] Update Sources/DeepLearning/Initializers.swift --- Sources/DeepLearning/Initializers.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/DeepLearning/Initializers.swift b/Sources/DeepLearning/Initializers.swift index 1dc55c346..4539b18ea 100644 --- a/Sources/DeepLearning/Initializers.swift +++ b/Sources/DeepLearning/Initializers.swift @@ -129,7 +129,7 @@ public extension Tensor where Scalar: TensorFlowFloatingPoint, Scalar.RawSignificand: FixedWidthInteger { /// Performs Glorot uniform initialization for the specified shape, creating a tensor by /// randomly sampling scalar values from a uniform distribution between `-limit` and `limit`, - /// where limit is `sqrt(6 / (fanIn + fanOut))` and `fanIn` / `fanOut` represent the number of + /// where limit is `sqrt(6 / (fanIn + fanOut))` and `fanIn`/`fanOut` represent the number of /// input and output features multiplied by the receptive field if present. /// /// - Parameters: