From eb1f909261b4145ce05c2ac08d0041610e35e82c Mon Sep 17 00:00:00 2001 From: Tanmay Bakshi Date: Fri, 29 Mar 2019 04:47:10 -0400 Subject: [PATCH 1/9] Cleanup operations --- Sources/DeepLearning/Operators.swift | 78 ++++++++++++---------------- 1 file changed, 34 insertions(+), 44 deletions(-) diff --git a/Sources/DeepLearning/Operators.swift b/Sources/DeepLearning/Operators.swift index dfdd0d045..b954715ad 100644 --- a/Sources/DeepLearning/Operators.swift +++ b/Sources/DeepLearning/Operators.swift @@ -121,90 +121,77 @@ public extension Padding { } extension Tensor where Scalar: TensorFlowFloatingPoint { + /// TensorFlow builtin conv2d gradient helper for the input. @inlinable - @differentiable( - wrt: (filter, backpropOutput), - vjp: _vjpTFConv2DBackpropInput(_:_:_:_:_:) - ) - func _TFConv2DBackpropInput( + @differentiable(wrt: (self, filter), vjp: _vjpConv2DBackpropInput) + func conv2DBackpropInput( shape: Tensor, filter: Tensor, - backpropOutput: Tensor, strides: (Int32, Int32, Int32, Int32), padding: Padding ) -> Tensor { return Raw.conv2DBackpropInput( inputSizes: shape, filter: filter, - outBackprop: backpropOutput, + outBackprop: self, strides: [strides.0, strides.1, strides.2, strides.3], padding: padding.raw) } - - /// TensorFlow builtin conv2d gradient helper for the filter. + + /// TensorFlow builtin conv2d gradient helper for the filter. @inlinable - @differentiable( - wrt: (input, backpropOutput), - vjp: _vjpTFConv2DBackpropFilter(_:_:_:_:_:) - ) - func _TFConv2DBackpropFilter( + @differentiable(wrt: (self, input), vjp: _vjpConv2DBackpropFilter) + func conv2DBackpropFilter( input: Tensor, filterSizes: Tensor, - backpropOutput: Tensor, strides: (Int32, Int32, Int32, Int32), padding: Padding ) -> Tensor { return Raw.conv2DBackpropFilter( input, filterSizes: filterSizes, - outBackprop: backpropOutput, + outBackprop: self, strides: [strides.0, strides.1, strides.2, strides.3], padding: padding.raw) } - + @inlinable - func _vjpTFConv2DBackpropInput( + func _vjpConv2DBackpropInput( _ shape: Tensor, _ filter: Tensor, - _ backpropOutput: Tensor, _ strides: (Int32, Int32, Int32, Int32), _ padding: Padding ) -> (Tensor, (Tensor) -> (Tensor, Tensor)) { - let value = _TFConv2DBackpropInput(shape: shape, filter: filter, - backpropOutput: backpropOutput, - strides: strides, padding: padding) + let value = conv2DBackpropInput(shape: shape, filter: filter, strides: strides, + padding: padding) return (value, { v in return ( - self._TFConv2DBackpropFilter(input: v, filterSizes: shape, - backpropOutput: backpropOutput, - strides: strides, padding: padding), + self.conv2DBackpropFilter(input: v, filterSizes: shape, strides: strides, + padding: padding), v.convolved2D(withFilter: filter, strides: strides, padding: padding) ) }) } - + @inlinable - func _vjpTFConv2DBackpropFilter( + func _vjpConv2DBackpropFilter( _ input: Tensor, _ filterSizes: Tensor, - _ backpropOutput: Tensor, _ strides: (Int32, Int32, Int32, Int32), _ padding: Padding ) -> (Tensor, (Tensor) -> (Tensor, Tensor)) { - let value = _TFConv2DBackpropFilter(input: input, filterSizes: filterSizes, - backpropOutput: backpropOutput, - strides: strides, padding: padding) + let value = conv2DBackpropFilter(input: input, filterSizes: filterSizes, + strides: strides, padding: padding) return (value, { v in return ( - self._TFConv2DBackpropInput(shape: filterSizes, filter: v, - backpropOutput: backpropOutput, - strides: strides, padding: padding), + self.conv2DBackpropInput(shape: filterSizes, filter: v, strides: strides, + padding: padding), input.convolved2D(withFilter: v, strides: strides, padding: padding) ) }) } - + @inlinable func _vjpConvolved2D( filter: Tensor, @@ -215,18 +202,18 @@ extension Tensor where Scalar: TensorFlowFloatingPoint { padding: padding) return (value, { v in return ( - self._TFConv2DBackpropInput( - shape: self.shapeTensor, filter: filter, backpropOutput: v, + v.conv2DBackpropInput( + shape: self.shapeTensor, filter: filter, strides: strides, padding: padding ), - self._TFConv2DBackpropFilter( - input: self, filterSizes: filter.shapeTensor, backpropOutput: v, + v.conv2DBackpropFilter( + input: self, filterSizes: filter.shapeTensor, strides: strides, padding: padding ) ) }) } - + @inlinable func _vjpMaxPooled( kernelSize: (Int32, Int32, Int32, Int32), @@ -269,9 +256,11 @@ extension Tensor where Scalar: TensorFlowFloatingPoint { ) }) } + } -public extension Tensor where Scalar: FloatingPoint { +extension Tensor where Scalar: FloatingPoint { + /// Computes a 2-D convolution using `self` as input, with the specified /// filter, strides, and padding. /// @@ -284,8 +273,8 @@ public extension Tensor where Scalar: FloatingPoint { /// - Precondition: `filter` must have rank 4. @inlinable @inline(__always) @differentiable( - wrt: (self, filter), vjp: _vjpConvolved2D(filter:strides:padding:) - where Scalar : TensorFlowFloatingPoint + wrt: (self, filter), vjp: _vjpConvolved2D + where Scalar: TensorFlowFloatingPoint ) func convolved2D( withFilter filter: Tensor, @@ -298,7 +287,7 @@ public extension Tensor where Scalar: FloatingPoint { strides: [strides.0, strides.1, strides.2, strides.3], padding: padding.raw) } - + /// Computes a 2-D max pooling, with the specified kernel sizes, strides, and /// padding. /// @@ -350,4 +339,5 @@ public extension Tensor where Scalar: FloatingPoint { strides: [strides.0, strides.1, strides.2, strides.3], padding: padding.raw) } + } From 6601840fc0651a7e29bb1ecb7443b97cc76106a5 Mon Sep 17 00:00:00 2001 From: Tanmay Bakshi Date: Fri, 29 Mar 2019 04:47:40 -0400 Subject: [PATCH 2/9] Make extensions public --- Sources/DeepLearning/Operators.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/DeepLearning/Operators.swift b/Sources/DeepLearning/Operators.swift index b954715ad..d41be737f 100644 --- a/Sources/DeepLearning/Operators.swift +++ b/Sources/DeepLearning/Operators.swift @@ -120,7 +120,7 @@ public extension Padding { } } -extension Tensor where Scalar: TensorFlowFloatingPoint { +public extension Tensor where Scalar: TensorFlowFloatingPoint { /// TensorFlow builtin conv2d gradient helper for the input. @inlinable @@ -259,7 +259,7 @@ extension Tensor where Scalar: TensorFlowFloatingPoint { } -extension Tensor where Scalar: FloatingPoint { +public extension Tensor where Scalar: FloatingPoint { /// Computes a 2-D convolution using `self` as input, with the specified /// filter, strides, and padding. From 108fcb3f804015917e7788f9ddc567c670f9eadd Mon Sep 17 00:00:00 2001 From: Tanmay Bakshi Date: Fri, 29 Mar 2019 04:48:14 -0400 Subject: [PATCH 3/9] Add Transposed Convolution 2D Layer --- Sources/DeepLearning/Layer.swift | 128 +++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) diff --git a/Sources/DeepLearning/Layer.swift b/Sources/DeepLearning/Layer.swift index 4a3553654..bb7c3efee 100644 --- a/Sources/DeepLearning/Layer.swift +++ b/Sources/DeepLearning/Layer.swift @@ -575,6 +575,134 @@ public extension Conv2D { } } +/// A 2-D transposed convolution layer (e.g. spatial transposed convolution over images). +/// +/// This layer creates a convolution filter that is transpose-convolved with the layer input +/// to produce a tensor of outputs. +@_fixed_layout +public struct TransposedConv2D: Layer { + /// The 4-D convolution kernel. + public var filter: Tensor + /// The bias vector. + public var bias: Tensor + /// An activation function. + public typealias Activation = @differentiable (Tensor) -> Tensor + /// The element-wise activation function. + @noDerivative public let activation: Activation + /// The strides of the sliding window for spatial dimensions. + @noDerivative public let strides: (Int32, Int32) + /// The padding algorithm for convolution. + @noDerivative public let padding: Padding + @noDerivative public let paddingIndex: Int32 + + /// Creates a `TransposedConv2D` layer with the specified filter, bias, + /// activation function, strides, and padding. + /// + /// - Parameters: + /// - filter: The 4-D convolution kernel. + /// - bias: The bias vector. + /// - activation: The element-wise activation function. + /// - strides: The strides of the sliding window for spatial dimensions. + /// - padding: The padding algorithm for convolution. + public init( + filter: Tensor, + bias: Tensor, + activation: @escaping Activation, + strides: (Int, Int), + padding: Padding + ) { + self.filter = filter + self.bias = bias + self.activation = activation + (self.strides.0, self.strides.1) = (Int32(strides.0), Int32(strides.1)) + self.padding = padding + self.paddingIndex = padding == .same ? 0 : 1 + } + + /// Returns the output obtained from applying the layer to the given input. + /// + /// - Parameters: + /// - input: The input to the layer. + /// - context: The contextual information for the layer application, e.g. the current learning + /// phase. + /// - Returns: The output. + @differentiable + public func applied(to input: Tensor, in _: Context) -> Tensor { + let batchSize = input.shape[0] + let w = (input.shape[1] - (1 * paddingIndex)) * strides.0 + (filter.shape[0] * paddingIndex) + let h = (input.shape[2] - (1 * paddingIndex)) * strides.1 + (filter.shape[1] * paddingIndex) + let c = filter.shape[2] + let newShape = Tensor([batchSize, w, h, c]) + return activation(input.convolved2DBackpropInput(shape: newShape, filter: filter, + strides: (1, strides.0, strides.1, 1), + padding: padding) + bias) + } +} + +public extension TransposedConv2D { + /// Creates a `TransposedConv2D` layer with the specified filter shape, strides, padding, and + /// element-wise activation function. The filter tensor is initialized using Glorot uniform + /// initialization with the specified generator. The bias vector is initialized with zeros. + /// + /// - Parameters: + /// - filterShape: The shape of the 4-D convolution kernel. + /// - strides: The strides of the sliding window for spatial dimensions. + /// - padding: The padding algorithm for convolution. + /// - activation: The element-wise activation function. + /// - generator: The random number generator for initialization. + /// + /// - Note: Use `init(filterShape:strides:padding:activation:seed:)` for faster random + /// initialization. + init( + filterShape: (Int, Int, Int, Int), + strides: (Int, Int) = (1, 1), + padding: Padding = .valid, + activation: @escaping Activation = identity, + generator: inout G + ) { + let filterTensorShape = TensorShape([ + Int32(filterShape.0), Int32(filterShape.1), + Int32(filterShape.2), Int32(filterShape.3)]) + self.init( + filter: Tensor(glorotUniform: filterTensorShape, generator: &generator), + bias: Tensor(zeros: TensorShape([Int32(filterShape.3)])), + activation: activation, + strides: strides, + padding: padding) + } +} + +public extension TransposedConv2D { + /// Creates a `TransposedConv2D` layer with the specified filter shape, strides, padding, and + /// element-wise activation function. The filter tensor is initialized using Glorot uniform + /// initialization with the specified seed. The bias vector is initialized with zeros. + /// + /// - Parameters: + /// - filterShape: The shape of the 4-D convolution kernel. + /// - strides: The strides of the sliding window for spatial dimensions. + /// - padding: The padding algorithm for convolution. + /// - activation: The element-wise activation function. + /// - seed: The random seed for initialization. The default value is random. + init( + filterShape: (Int, Int, Int, Int), + strides: (Int, Int) = (1, 1), + padding: Padding = .valid, + activation: @escaping Activation = identity, + seed: (Int64, Int64) = (Int64.random(in: Int64.min.. Date: Fri, 29 Mar 2019 04:56:18 -0400 Subject: [PATCH 4/9] Make VJP functions `internal` --- Sources/DeepLearning/Operators.swift | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Sources/DeepLearning/Operators.swift b/Sources/DeepLearning/Operators.swift index d41be737f..05c1da280 100644 --- a/Sources/DeepLearning/Operators.swift +++ b/Sources/DeepLearning/Operators.swift @@ -34,7 +34,7 @@ public func identity(_ x: Tensor) -> Tensor { public extension Tensor where Scalar: TensorFlowFloatingPoint { // TODO: Verify that these calculations are correct. @inlinable - func _vjpBatchNormalized( + internal func _vjpBatchNormalized( alongAxis axis: Int32, offset: Tensor, scale: Tensor, @@ -157,7 +157,7 @@ public extension Tensor where Scalar: TensorFlowFloatingPoint { } @inlinable - func _vjpConv2DBackpropInput( + internal func _vjpConv2DBackpropInput( _ shape: Tensor, _ filter: Tensor, _ strides: (Int32, Int32, Int32, Int32), @@ -175,7 +175,7 @@ public extension Tensor where Scalar: TensorFlowFloatingPoint { } @inlinable - func _vjpConv2DBackpropFilter( + internal func _vjpConv2DBackpropFilter( _ input: Tensor, _ filterSizes: Tensor, _ strides: (Int32, Int32, Int32, Int32), @@ -193,7 +193,7 @@ public extension Tensor where Scalar: TensorFlowFloatingPoint { } @inlinable - func _vjpConvolved2D( + internal func _vjpConvolved2D( filter: Tensor, strides: (Int32, Int32, Int32, Int32), padding: Padding @@ -215,7 +215,7 @@ public extension Tensor where Scalar: TensorFlowFloatingPoint { } @inlinable - func _vjpMaxPooled( + internal func _vjpMaxPooled( kernelSize: (Int32, Int32, Int32, Int32), strides: (Int32, Int32, Int32, Int32), padding: Padding @@ -237,7 +237,7 @@ public extension Tensor where Scalar: TensorFlowFloatingPoint { } @inlinable - func _vjpAveragePooled( + internal func _vjpAveragePooled( kernelSize: (Int32, Int32, Int32, Int32), strides: (Int32, Int32, Int32, Int32), padding: Padding From 3a129535e39f08d2b9edacab6d460620b56b54a1 Mon Sep 17 00:00:00 2001 From: Tanmay Bakshi Date: Fri, 29 Mar 2019 05:10:01 -0400 Subject: [PATCH 5/9] Formatting --- Sources/DeepLearning/Operators.swift | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/Sources/DeepLearning/Operators.swift b/Sources/DeepLearning/Operators.swift index 05c1da280..fec709844 100644 --- a/Sources/DeepLearning/Operators.swift +++ b/Sources/DeepLearning/Operators.swift @@ -121,7 +121,6 @@ public extension Padding { } public extension Tensor where Scalar: TensorFlowFloatingPoint { - /// TensorFlow builtin conv2d gradient helper for the input. @inlinable @differentiable(wrt: (self, filter), vjp: _vjpConv2DBackpropInput) @@ -138,7 +137,7 @@ public extension Tensor where Scalar: TensorFlowFloatingPoint { strides: [strides.0, strides.1, strides.2, strides.3], padding: padding.raw) } - + /// TensorFlow builtin conv2d gradient helper for the filter. @inlinable @differentiable(wrt: (self, input), vjp: _vjpConv2DBackpropFilter) @@ -155,7 +154,7 @@ public extension Tensor where Scalar: TensorFlowFloatingPoint { strides: [strides.0, strides.1, strides.2, strides.3], padding: padding.raw) } - + @inlinable internal func _vjpConv2DBackpropInput( _ shape: Tensor, @@ -173,7 +172,7 @@ public extension Tensor where Scalar: TensorFlowFloatingPoint { ) }) } - + @inlinable internal func _vjpConv2DBackpropFilter( _ input: Tensor, @@ -191,7 +190,7 @@ public extension Tensor where Scalar: TensorFlowFloatingPoint { ) }) } - + @inlinable internal func _vjpConvolved2D( filter: Tensor, @@ -213,7 +212,7 @@ public extension Tensor where Scalar: TensorFlowFloatingPoint { ) }) } - + @inlinable internal func _vjpMaxPooled( kernelSize: (Int32, Int32, Int32, Int32), @@ -256,11 +255,9 @@ public extension Tensor where Scalar: TensorFlowFloatingPoint { ) }) } - } public extension Tensor where Scalar: FloatingPoint { - /// Computes a 2-D convolution using `self` as input, with the specified /// filter, strides, and padding. /// @@ -339,5 +336,4 @@ public extension Tensor where Scalar: FloatingPoint { strides: [strides.0, strides.1, strides.2, strides.3], padding: padding.raw) } - } From 11227dca7df33bd93da038cebc1ba89fd2fb3438 Mon Sep 17 00:00:00 2001 From: Tanmay Bakshi Date: Fri, 29 Mar 2019 05:12:12 -0400 Subject: [PATCH 6/9] Formatting --- Sources/DeepLearning/Operators.swift | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Sources/DeepLearning/Operators.swift b/Sources/DeepLearning/Operators.swift index fec709844..80ef1a30d 100644 --- a/Sources/DeepLearning/Operators.swift +++ b/Sources/DeepLearning/Operators.swift @@ -163,11 +163,11 @@ public extension Tensor where Scalar: TensorFlowFloatingPoint { _ padding: Padding ) -> (Tensor, (Tensor) -> (Tensor, Tensor)) { let value = conv2DBackpropInput(shape: shape, filter: filter, strides: strides, - padding: padding) + padding: padding) return (value, { v in return ( self.conv2DBackpropFilter(input: v, filterSizes: shape, strides: strides, - padding: padding), + padding: padding), v.convolved2D(withFilter: filter, strides: strides, padding: padding) ) }) @@ -181,11 +181,11 @@ public extension Tensor where Scalar: TensorFlowFloatingPoint { _ padding: Padding ) -> (Tensor, (Tensor) -> (Tensor, Tensor)) { let value = conv2DBackpropFilter(input: input, filterSizes: filterSizes, - strides: strides, padding: padding) + strides: strides, padding: padding) return (value, { v in return ( self.conv2DBackpropInput(shape: filterSizes, filter: v, strides: strides, - padding: padding), + padding: padding), input.convolved2D(withFilter: v, strides: strides, padding: padding) ) }) From 972e65caa38d019de83a4eed05dcd1a6fb9b3323 Mon Sep 17 00:00:00 2001 From: Tanmay Bakshi Date: Fri, 29 Mar 2019 05:13:29 -0400 Subject: [PATCH 7/9] Make `Raw` wrappers `internal` --- Sources/DeepLearning/Operators.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/DeepLearning/Operators.swift b/Sources/DeepLearning/Operators.swift index 80ef1a30d..36efdfcdf 100644 --- a/Sources/DeepLearning/Operators.swift +++ b/Sources/DeepLearning/Operators.swift @@ -124,7 +124,7 @@ public extension Tensor where Scalar: TensorFlowFloatingPoint { /// TensorFlow builtin conv2d gradient helper for the input. @inlinable @differentiable(wrt: (self, filter), vjp: _vjpConv2DBackpropInput) - func conv2DBackpropInput( + internal func conv2DBackpropInput( shape: Tensor, filter: Tensor, strides: (Int32, Int32, Int32, Int32), @@ -141,7 +141,7 @@ public extension Tensor where Scalar: TensorFlowFloatingPoint { /// TensorFlow builtin conv2d gradient helper for the filter. @inlinable @differentiable(wrt: (self, input), vjp: _vjpConv2DBackpropFilter) - func conv2DBackpropFilter( + internal func conv2DBackpropFilter( input: Tensor, filterSizes: Tensor, strides: (Int32, Int32, Int32, Int32), From ef7d4c633c95ec3fe7bade84562975761bbb291d Mon Sep 17 00:00:00 2001 From: Tanmay Bakshi Date: Fri, 29 Mar 2019 05:30:01 -0400 Subject: [PATCH 8/9] Update TransposedConv2D --- Sources/DeepLearning/Layer.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/DeepLearning/Layer.swift b/Sources/DeepLearning/Layer.swift index bb7c3efee..00fc92878 100644 --- a/Sources/DeepLearning/Layer.swift +++ b/Sources/DeepLearning/Layer.swift @@ -633,7 +633,7 @@ public struct TransposedConv2D: Layer { let h = (input.shape[2] - (1 * paddingIndex)) * strides.1 + (filter.shape[1] * paddingIndex) let c = filter.shape[2] let newShape = Tensor([batchSize, w, h, c]) - return activation(input.convolved2DBackpropInput(shape: newShape, filter: filter, + return activation(input.conv2DBackpropInput(shape: newShape, filter: filter, strides: (1, strides.0, strides.1, 1), padding: padding) + bias) } @@ -698,7 +698,7 @@ public extension TransposedConv2D { filter: Tensor(glorotUniform: filterTensorShape, seed: seed), bias: Tensor(zeros: TensorShape([Int32(filterShape.3)])), activation: activation, - strides: (Int32(strides.0), Int32(strides.1)), + strides: strides, padding: padding) } } From 5c3b07c8f5e44972772a0a3d1f529a7ff8ccf3d1 Mon Sep 17 00:00:00 2001 From: Tanmay Bakshi Date: Fri, 29 Mar 2019 05:30:30 -0400 Subject: [PATCH 9/9] Remove whitespace --- Sources/DeepLearning/Operators.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/DeepLearning/Operators.swift b/Sources/DeepLearning/Operators.swift index 36efdfcdf..51b2406a8 100644 --- a/Sources/DeepLearning/Operators.swift +++ b/Sources/DeepLearning/Operators.swift @@ -284,7 +284,7 @@ public extension Tensor where Scalar: FloatingPoint { strides: [strides.0, strides.1, strides.2, strides.3], padding: padding.raw) } - + /// Computes a 2-D max pooling, with the specified kernel sizes, strides, and /// padding. ///