From 8cf9fda8892b5d9a3a8fd56082dc05cbb29f1e88 Mon Sep 17 00:00:00 2001 From: PAWAN SASANKA AMMANAMANCHI Date: Mon, 24 Jun 2019 17:51:03 +0530 Subject: [PATCH 01/14] Fixing Transposed conv2d errors --- Sources/TensorFlow/Layers/Convolutional.swift | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Sources/TensorFlow/Layers/Convolutional.swift b/Sources/TensorFlow/Layers/Convolutional.swift index a668b30b1..c384d5b6d 100644 --- a/Sources/TensorFlow/Layers/Convolutional.swift +++ b/Sources/TensorFlow/Layers/Convolutional.swift @@ -32,7 +32,7 @@ public struct Conv1D: Layer { @noDerivative public let padding: Padding /// The dilation factor for the temporal dimension. @noDerivative public let dilation: Int - + /// Creates a `Conv1D` layer with the specified filter, bias, activation function, stride, and /// padding. /// @@ -165,7 +165,7 @@ public struct Conv2D: Layer { @noDerivative public let padding: Padding /// The dilation factor for spatials dimensions. @noDerivative public let dilations: (Int, Int) - + /// Creates a `Conv2D` layer with the specified filter, bias, activation function, strides, and /// padding. /// @@ -445,9 +445,9 @@ public struct TransposedConv2D: Layer { @differentiable public func callAsFunction(_ input: Tensor) -> Tensor { let batchSize = input.shape[0] - let w = (input.shape[1] - (1 * paddingIndex)) * + let w = (input.shape[1] - 1) * strides.0 + (filter.shape[0] * paddingIndex) - let h = (input.shape[2] - (1 * paddingIndex)) * + let h = (input.shape[2] - 1) * strides.1 + (filter.shape[1] * paddingIndex) let c = filter.shape[2] let newShape = Tensor([Int32(batchSize), Int32(w), Int32(h), Int32(c)]) From cd00fbf12a2dcd5f07a25cb091b499c47ef7c805 Mon Sep 17 00:00:00 2001 From: PAWAN SASANKA AMMANAMANCHI Date: Mon, 24 Jun 2019 17:54:00 +0530 Subject: [PATCH 02/14] adding transposed conv2d test --- Tests/TensorFlowTests/LayerTests.swift | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/Tests/TensorFlowTests/LayerTests.swift b/Tests/TensorFlowTests/LayerTests.swift index cd78b04b8..4c7661f55 100644 --- a/Tests/TensorFlowTests/LayerTests.swift +++ b/Tests/TensorFlowTests/LayerTests.swift @@ -38,7 +38,7 @@ final class LayerTests: XCTestCase { // Input shapes. let inputHeight = 2 let inputWidth = 5 - + let filter = Tensor(shape: [width, inputChannels, outputChannels], scalars: [2, 3, 4, 1, 2, 3]) let bias = Tensor([0]) @@ -116,6 +116,18 @@ final class LayerTests: XCTestCase { XCTAssertEqual(output, expected) } + func testTransposedConv2D() { + let filter = Tensor(shape: [1, 4, 2, 1], scalars: (0..<8).map(Float.init)) + let bias = Tensor([8]) + let layer = Conv2D(filter: filter, bias: bias, activation: identity, + strides: (1, 1), padding: .valid) + let input = Tensor(shape: [1, 4, 2, 2], scalars: (0..<8).map(Float.init)) + let output = layer.inferring(from: input) + let expected = Tensor(shape: [1, 4, 2, 1], + scalars: [0, 4, 4, 20, 16, 56, 40, 104]) + XCTAssertEqual(output, expected) + } + func testMaxPool1D() { let layer = MaxPool1D(poolSize: 3, stride: 1, padding: .valid) let input = Tensor([[0, 1, 2, 3, 4], [10, 11, 12, 13, 14]]).expandingShape(at: 2) @@ -256,14 +268,14 @@ final class LayerTests: XCTestCase { XCTAssertEqual(output.shape, expected) } - func testEmbedding() { - var layer = Embedding(vocabularySize: 3, embeddingSize: 5) + func testEmbedding() { + var layer = Embedding(vocabularySize: 3, embeddingSize: 5) var data = Tensor(shape: [2, 3], scalars: [0, 1, 2, 1, 2, 2]) var input = EmbeddingInput(indices: data) var output = layer.inferring(from: input) let expectedShape = TensorShape([2, 3, 5]) XCTAssertEqual(output.shape, expectedShape) - + let pretrained = Tensor(shape:[2, 2], scalars: [0.4, 0.3, 0.2, 0.1]) layer = Embedding(embeddings: pretrained) data = Tensor(shape: [2, 2], scalars: [0, 1, 1, 1]) @@ -324,6 +336,7 @@ final class LayerTests: XCTestCase { ("testConv2D", testConv2D), ("testConv2DDilation", testConv2DDilation), ("testConv3D", testConv3D), + ("testTransposedConv2D", testTransposedConv2D), ("testDepthConv2D", testDepthConv2D), ("testMaxPool1D", testMaxPool1D), ("testMaxPool2D", testMaxPool2D), From 8f44e69b92d3247c38a109917a0186c14a40e55a Mon Sep 17 00:00:00 2001 From: PAWAN SASANKA AMMANAMANCHI Date: Mon, 24 Jun 2019 18:00:29 +0530 Subject: [PATCH 03/14] updating according to parameter doc --- Sources/TensorFlow/Layers/Convolutional.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/TensorFlow/Layers/Convolutional.swift b/Sources/TensorFlow/Layers/Convolutional.swift index c384d5b6d..70818cd06 100644 --- a/Sources/TensorFlow/Layers/Convolutional.swift +++ b/Sources/TensorFlow/Layers/Convolutional.swift @@ -449,7 +449,7 @@ public struct TransposedConv2D: Layer { strides.0 + (filter.shape[0] * paddingIndex) let h = (input.shape[2] - 1) * strides.1 + (filter.shape[1] * paddingIndex) - let c = filter.shape[2] + let c = filter.shape[3] let newShape = Tensor([Int32(batchSize), Int32(w), Int32(h), Int32(c)]) return activation(conv2DBackpropInput( input, From 11505817fbf55c233351ed4087be76bb3f7c0a3a Mon Sep 17 00:00:00 2001 From: PAWAN SASANKA AMMANAMANCHI Date: Mon, 24 Jun 2019 18:08:18 +0530 Subject: [PATCH 04/14] Updating tconv2d test --- Tests/TensorFlowTests/LayerTests.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tests/TensorFlowTests/LayerTests.swift b/Tests/TensorFlowTests/LayerTests.swift index 4c7661f55..7ee2f5eeb 100644 --- a/Tests/TensorFlowTests/LayerTests.swift +++ b/Tests/TensorFlowTests/LayerTests.swift @@ -117,11 +117,11 @@ final class LayerTests: XCTestCase { } func testTransposedConv2D() { - let filter = Tensor(shape: [1, 4, 2, 1], scalars: (0..<8).map(Float.init)) + let filter = Tensor(shape: [2, 2, 2, 1], scalars: (0..<8).map(Float.init)) let bias = Tensor([8]) let layer = Conv2D(filter: filter, bias: bias, activation: identity, strides: (1, 1), padding: .valid) - let input = Tensor(shape: [1, 4, 2, 2], scalars: (0..<8).map(Float.init)) + let input = Tensor(shape: [1, 4, 2, 1], scalars: (0..<8).map(Float.init)) let output = layer.inferring(from: input) let expected = Tensor(shape: [1, 4, 2, 1], scalars: [0, 4, 4, 20, 16, 56, 40, 104]) From 86af57f3cb46ec0f3545bc7034cad1486660acea Mon Sep 17 00:00:00 2001 From: PAWAN SASANKA AMMANAMANCHI Date: Mon, 24 Jun 2019 18:09:33 +0530 Subject: [PATCH 05/14] Updating tconv2d test --- Tests/TensorFlowTests/LayerTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/TensorFlowTests/LayerTests.swift b/Tests/TensorFlowTests/LayerTests.swift index 7ee2f5eeb..918620d1a 100644 --- a/Tests/TensorFlowTests/LayerTests.swift +++ b/Tests/TensorFlowTests/LayerTests.swift @@ -117,7 +117,7 @@ final class LayerTests: XCTestCase { } func testTransposedConv2D() { - let filter = Tensor(shape: [2, 2, 2, 1], scalars: (0..<8).map(Float.init)) + let filter = Tensor(shape: [4, 2, 1, 1], scalars: (0..<8).map(Float.init)) let bias = Tensor([8]) let layer = Conv2D(filter: filter, bias: bias, activation: identity, strides: (1, 1), padding: .valid) From 6bfdeb3433e82b50ab660e6b0e4c7c8a41e03974 Mon Sep 17 00:00:00 2001 From: PAWAN SASANKA AMMANAMANCHI Date: Mon, 24 Jun 2019 23:22:46 +0530 Subject: [PATCH 06/14] test changes --- Sources/TensorFlow/Layers/Convolutional.swift | 2 +- Tests/TensorFlowTests/LayerTests.swift | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/TensorFlow/Layers/Convolutional.swift b/Sources/TensorFlow/Layers/Convolutional.swift index 70818cd06..c384d5b6d 100644 --- a/Sources/TensorFlow/Layers/Convolutional.swift +++ b/Sources/TensorFlow/Layers/Convolutional.swift @@ -449,7 +449,7 @@ public struct TransposedConv2D: Layer { strides.0 + (filter.shape[0] * paddingIndex) let h = (input.shape[2] - 1) * strides.1 + (filter.shape[1] * paddingIndex) - let c = filter.shape[3] + let c = filter.shape[2] let newShape = Tensor([Int32(batchSize), Int32(w), Int32(h), Int32(c)]) return activation(conv2DBackpropInput( input, diff --git a/Tests/TensorFlowTests/LayerTests.swift b/Tests/TensorFlowTests/LayerTests.swift index 918620d1a..0fa05f597 100644 --- a/Tests/TensorFlowTests/LayerTests.swift +++ b/Tests/TensorFlowTests/LayerTests.swift @@ -124,7 +124,7 @@ final class LayerTests: XCTestCase { let input = Tensor(shape: [1, 4, 2, 1], scalars: (0..<8).map(Float.init)) let output = layer.inferring(from: input) let expected = Tensor(shape: [1, 4, 2, 1], - scalars: [0, 4, 4, 20, 16, 56, 40, 104]) + scalars: [8, 12, 12, 28, 24, 64, 48, 112]) XCTAssertEqual(output, expected) } From 72d7c8fb17f54eb5c5fe53458f84f68d81137a63 Mon Sep 17 00:00:00 2001 From: PAWAN SASANKA AMMANAMANCHI Date: Mon, 24 Jun 2019 23:45:43 +0530 Subject: [PATCH 07/14] Some changes in the test --- Tests/TensorFlowTests/LayerTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/TensorFlowTests/LayerTests.swift b/Tests/TensorFlowTests/LayerTests.swift index 0fa05f597..ba2d8e0c2 100644 --- a/Tests/TensorFlowTests/LayerTests.swift +++ b/Tests/TensorFlowTests/LayerTests.swift @@ -120,7 +120,7 @@ final class LayerTests: XCTestCase { let filter = Tensor(shape: [4, 2, 1, 1], scalars: (0..<8).map(Float.init)) let bias = Tensor([8]) let layer = Conv2D(filter: filter, bias: bias, activation: identity, - strides: (1, 1), padding: .valid) + strides: (1, 1), padding: .same) let input = Tensor(shape: [1, 4, 2, 1], scalars: (0..<8).map(Float.init)) let output = layer.inferring(from: input) let expected = Tensor(shape: [1, 4, 2, 1], From 3527a4abc6f31e9340b7efc2f440ffff7bc15cd5 Mon Sep 17 00:00:00 2001 From: Pawan Sasanka Ammanamanchi Date: Tue, 25 Jun 2019 00:01:02 +0530 Subject: [PATCH 08/14] Merge errors --- Sources/TensorFlow/Layers/Convolutional.swift | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/Sources/TensorFlow/Layers/Convolutional.swift b/Sources/TensorFlow/Layers/Convolutional.swift index 10f6c3554..7e7dcb49e 100644 --- a/Sources/TensorFlow/Layers/Convolutional.swift +++ b/Sources/TensorFlow/Layers/Convolutional.swift @@ -32,19 +32,9 @@ public struct Conv1D: Layer { @noDerivative public let padding: Padding /// The dilation factor for the temporal dimension. @noDerivative public let dilation: Int -<<<<<<< HEAD - - /// Creates a `Conv1D` layer with the specified filter, bias, activation function, stride, and - /// padding. -||||||| merged common ancestors - - /// Creates a `Conv1D` layer with the specified filter, bias, activation function, stride, and - /// padding. -======= /// Creates a `Conv1D` layer with the specified filter, bias, activation function, stride, /// dilation and padding. ->>>>>>> 380a95c3689bcd2d55df00a28b4640a7a4937c17 /// /// - Parameters: /// - filter: The 3-D convolution kernel `[width, inputChannels, outputChannels]`. @@ -175,19 +165,9 @@ public struct Conv2D: Layer { @noDerivative public let padding: Padding /// The dilation factor for spatial dimensions. @noDerivative public let dilations: (Int, Int) -<<<<<<< HEAD - - /// Creates a `Conv2D` layer with the specified filter, bias, activation function, strides, and - /// padding. -||||||| merged common ancestors - - /// Creates a `Conv2D` layer with the specified filter, bias, activation function, strides, and - /// padding. -======= /// Creates a `Conv2D` layer with the specified filter, bias, activation function, strides, /// dilations and padding. ->>>>>>> 380a95c3689bcd2d55df00a28b4640a7a4937c17 /// /// - Parameters: /// - filter: The 4-D convolution kernel. From 32433062cbae104b32a93b65f1d50e2c7a3afe39 Mon Sep 17 00:00:00 2001 From: Pawan Sasanka Ammanamanchi Date: Tue, 25 Jun 2019 09:39:41 +0530 Subject: [PATCH 09/14] Minor error --- Tests/TensorFlowTests/LayerTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/TensorFlowTests/LayerTests.swift b/Tests/TensorFlowTests/LayerTests.swift index bce9f6a2b..a52f63e7e 100644 --- a/Tests/TensorFlowTests/LayerTests.swift +++ b/Tests/TensorFlowTests/LayerTests.swift @@ -119,7 +119,7 @@ final class LayerTests: XCTestCase { func testTransposedConv2D() { let filter = Tensor(shape: [4, 2, 1, 1], scalars: (0..<8).map(Float.init)) let bias = Tensor([8]) - let layer = Conv2D(filter: filter, bias: bias, activation: identity, + let layer = TransposedConv2D(filter: filter, bias: bias, activation: identity, strides: (1, 1), padding: .same) let input = Tensor(shape: [1, 4, 2, 1], scalars: (0..<8).map(Float.init)) let output = layer.inferring(from: input) From 0fac0ecb4b07beb74af9c17e0be25c8735d8c184 Mon Sep 17 00:00:00 2001 From: PAWAN SASANKA AMMANAMANCHI Date: Tue, 25 Jun 2019 11:33:06 +0530 Subject: [PATCH 10/14] Another transposed conv2d error --- Sources/TensorFlow/Layers/Convolutional.swift | 22 +------------------ 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/Sources/TensorFlow/Layers/Convolutional.swift b/Sources/TensorFlow/Layers/Convolutional.swift index 10f6c3554..81feb864f 100644 --- a/Sources/TensorFlow/Layers/Convolutional.swift +++ b/Sources/TensorFlow/Layers/Convolutional.swift @@ -32,19 +32,9 @@ public struct Conv1D: Layer { @noDerivative public let padding: Padding /// The dilation factor for the temporal dimension. @noDerivative public let dilation: Int -<<<<<<< HEAD - - /// Creates a `Conv1D` layer with the specified filter, bias, activation function, stride, and - /// padding. -||||||| merged common ancestors - - /// Creates a `Conv1D` layer with the specified filter, bias, activation function, stride, and - /// padding. -======= /// Creates a `Conv1D` layer with the specified filter, bias, activation function, stride, /// dilation and padding. ->>>>>>> 380a95c3689bcd2d55df00a28b4640a7a4937c17 /// /// - Parameters: /// - filter: The 3-D convolution kernel `[width, inputChannels, outputChannels]`. @@ -175,19 +165,9 @@ public struct Conv2D: Layer { @noDerivative public let padding: Padding /// The dilation factor for spatial dimensions. @noDerivative public let dilations: (Int, Int) -<<<<<<< HEAD - - /// Creates a `Conv2D` layer with the specified filter, bias, activation function, strides, and - /// padding. -||||||| merged common ancestors - - /// Creates a `Conv2D` layer with the specified filter, bias, activation function, strides, and - /// padding. -======= /// Creates a `Conv2D` layer with the specified filter, bias, activation function, strides, /// dilations and padding. ->>>>>>> 380a95c3689bcd2d55df00a28b4640a7a4937c17 /// /// - Parameters: /// - filter: The 4-D convolution kernel. @@ -418,7 +398,7 @@ public extension Conv3D { /// This layer creates a convolution filter that is transpose-convolved with the layer input /// to produce a tensor of outputs. @frozen -public struct TransposedConv2D: Layer { +public struct TransposedConv2D: Layer { /// The 4-D convolution kernel. public var filter: Tensor /// The bias vector. From a96c6f40695237c85fe7f4c8089dab8c74d23b91 Mon Sep 17 00:00:00 2001 From: PAWAN SASANKA AMMANAMANCHI Date: Tue, 25 Jun 2019 11:50:19 +0530 Subject: [PATCH 11/14] Making tensors generic over scalar type --- Sources/TensorFlow/Layers/Convolutional.swift | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Sources/TensorFlow/Layers/Convolutional.swift b/Sources/TensorFlow/Layers/Convolutional.swift index 81feb864f..a304c2a6c 100644 --- a/Sources/TensorFlow/Layers/Convolutional.swift +++ b/Sources/TensorFlow/Layers/Convolutional.swift @@ -400,11 +400,11 @@ public extension Conv3D { @frozen public struct TransposedConv2D: Layer { /// The 4-D convolution kernel. - public var filter: Tensor + public var filter: Tensor /// The bias vector. - public var bias: Tensor + public var bias: Tensor /// An activation function. - public typealias Activation = @differentiable (Tensor) -> Tensor + public typealias Activation = @differentiable (Tensor) -> Tensor /// The element-wise activation function. @noDerivative public let activation: Activation /// The strides of the sliding window for spatial dimensions. @@ -424,8 +424,8 @@ public struct TransposedConv2D: Layer { /// - strides: The strides of the sliding window for spatial dimensions. /// - padding: The padding algorithm for convolution. public init( - filter: Tensor, - bias: Tensor, + filter: Tensor, + bias: Tensor, activation: @escaping Activation = identity, strides: (Int, Int) = (1, 1), padding: Padding = .valid From 6ee8541ecba7c8efdba9e32b1279976fd80c4ba7 Mon Sep 17 00:00:00 2001 From: PAWAN SASANKA AMMANAMANCHI Date: Tue, 25 Jun 2019 12:56:38 +0530 Subject: [PATCH 12/14] Review changes --- Sources/TensorFlow/Layers/Convolutional.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/TensorFlow/Layers/Convolutional.swift b/Sources/TensorFlow/Layers/Convolutional.swift index a304c2a6c..8857c2ed9 100644 --- a/Sources/TensorFlow/Layers/Convolutional.swift +++ b/Sources/TensorFlow/Layers/Convolutional.swift @@ -404,7 +404,7 @@ public struct TransposedConv2D: Layer { /// The bias vector. public var bias: Tensor /// An activation function. - public typealias Activation = @differentiable (Tensor) -> Tensor + public typealias Activation = @differentiable (Tensor) -> Tensor /// The element-wise activation function. @noDerivative public let activation: Activation /// The strides of the sliding window for spatial dimensions. @@ -443,7 +443,7 @@ public struct TransposedConv2D: Layer { /// - Parameter input: The input to the layer. /// - Returns: The output. @differentiable - public func callAsFunction(_ input: Tensor) -> Tensor { + public func callAsFunction(_ input: Tensor) -> Tensor { let batchSize = input.shape[0] let w = (input.shape[1] - 1) * strides.0 + (filter.shape[0] * paddingIndex) From 757450bfae2e0ca44c9e1887025a968eee7c3e58 Mon Sep 17 00:00:00 2001 From: PAWAN SASANKA AMMANAMANCHI Date: Mon, 19 Aug 2019 13:49:54 +0530 Subject: [PATCH 13/14] Fixing argument order --- Sources/TensorFlow/Layers/Convolutional.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/TensorFlow/Layers/Convolutional.swift b/Sources/TensorFlow/Layers/Convolutional.swift index 02476dc3e..b607ad98a 100644 --- a/Sources/TensorFlow/Layers/Convolutional.swift +++ b/Sources/TensorFlow/Layers/Convolutional.swift @@ -377,7 +377,7 @@ public struct TransposedConv2D: Layer { /// /// - Parameters: /// - filter: A 4-D tensor of shape - /// `[width, height, input channel count, output channel count]`. + /// `[height, width, output channel count, input channel count]`. /// - bias: The bias tensor of shape `[output channel count]`. /// - activation: The element-wise activation function. /// - strides: The strides of the sliding window for spatial dimensions. From 1410c44a1f30c68a7180b9863c42ec5fea4578dc Mon Sep 17 00:00:00 2001 From: PAWAN SASANKA AMMANAMANCHI Date: Mon, 19 Aug 2019 14:10:16 +0530 Subject: [PATCH 14/14] Updating acc to review --- Sources/TensorFlow/Layers/Convolutional.swift | 6 ++--- Tests/TensorFlowTests/LayerTests.swift | 26 +++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/Sources/TensorFlow/Layers/Convolutional.swift b/Sources/TensorFlow/Layers/Convolutional.swift index b607ad98a..58875cd0d 100644 --- a/Sources/TensorFlow/Layers/Convolutional.swift +++ b/Sources/TensorFlow/Layers/Convolutional.swift @@ -404,12 +404,12 @@ public struct TransposedConv2D: Layer { @differentiable public func callAsFunction(_ input: Tensor) -> Tensor { let batchSize = input.shape[0] - let w = (input.shape[1] - 1) * + let h = (input.shape[1] - (1 * paddingIndex)) * strides.0 + (filter.shape[0] * paddingIndex) - let h = (input.shape[2] - 1) * + let w = (input.shape[2] - (1 * paddingIndex)) * strides.1 + (filter.shape[1] * paddingIndex) let c = filter.shape[2] - let newShape = Tensor([Int32(batchSize), Int32(w), Int32(h), Int32(c)]) + let newShape = Tensor([Int32(batchSize), Int32(h), Int32(w), Int32(c)]) return activation(conv2DBackpropInput( input, shape: newShape, diff --git a/Tests/TensorFlowTests/LayerTests.swift b/Tests/TensorFlowTests/LayerTests.swift index 2274dee65..4abd09aa1 100644 --- a/Tests/TensorFlowTests/LayerTests.swift +++ b/Tests/TensorFlowTests/LayerTests.swift @@ -113,28 +113,28 @@ final class LayerTests: XCTestCase { func testConv2DGradient() { let filter = Tensor(shape: [3, 3, 2, 4], scalars: (0..<72).map(Float.init)) let bias = Tensor(zeros: [4]) - let layer = Conv2D(filter: filter, - bias: bias, + let layer = Conv2D(filter: filter, + bias: bias, activation: identity, - strides: (2, 2), + strides: (2, 2), padding: .valid) let input = Tensor(shape: [2, 4, 4, 2], scalars: (0..<64).map(Float.init)) let grads = gradient( at: input, layer) { $1($0).sum() } - // The expected gradients were computed using the following Python code: + // The expected gradients were computed using the following Python code: // ``` // x = tf.reshape(tf.range(64, dtype=tf.float32), [2, 4, 4, 2]) // filter = tf.reshape(tf.range(72, dtype=tf.float32), [3, 3, 2, 4]) // bias = tf.zeros([4]) // with tf.GradientTape() as t: // t.watch([x, filter, bias]) - // y = tf.math.reduce_sum(tf.nn.conv2d(input=x, + // y = tf.math.reduce_sum(tf.nn.conv2d(input=x, // filters=filter, // strides=[1, 2, 2, 1], // data_format="NHWC", // padding="VALID") + bias) // grads = t.gradient(y, [x, filter, bias]) // ``` - XCTAssertEqual(grads.0, + XCTAssertEqual(grads.0, [[[[ 6, 22], [ 38, 54], [ 70, 86], [ 0, 0]], [[102, 118], [134, 150], [166, 182], [ 0, 0]], [[198, 214], [230, 246], [262, 278], [ 0, 0]], @@ -143,7 +143,7 @@ final class LayerTests: XCTestCase { [[102, 118], [134, 150], [166, 182], [ 0, 0]], [[198, 214], [230, 246], [262, 278], [ 0, 0]], [[ 0, 0], [ 0, 0], [ 0, 0], [ 0, 0]]]]) - XCTAssertEqual(grads.1.filter, + XCTAssertEqual(grads.1.filter, [[[[32, 32, 32, 32], [34, 34, 34, 34]], [[36, 36, 36, 36], [38, 38, 38, 38]], [[40, 40, 40, 40], [42, 42, 42, 42]]], @@ -213,7 +213,7 @@ final class LayerTests: XCTestCase { func testTransposedConv2D() { let filter = Tensor(shape: [4, 2, 1, 1], scalars: (0..<8).map(Float.init)) let bias = Tensor([8]) - let layer = TransposedConv2D(filter: filter, bias: bias, activation: identity, + let layer = TransposedConv2D(filter: filter, bias: bias, activation: identity, strides: (1, 1), padding: .same) let input = Tensor(shape: [1, 4, 2, 1], scalars: (0..<8).map(Float.init)) let output = layer.inferring(from: input) @@ -238,7 +238,7 @@ final class LayerTests: XCTestCase { scalars: [1016, 2616]) XCTAssertEqual(output, expected) } - + func testZeroPadding1D() { let input = Tensor([0.0, 1.0, 2.0]) let layer = ZeroPadding1D(padding: 2) @@ -593,7 +593,7 @@ final class LayerTests: XCTestCase { // [ 0.0, 0.0, 0.0, 0.0]]) // XCTAssertEqual(𝛁rnn.cell.bias, [ 0.2496884, 0.66947335, 0.7978788, -0.22378457]) } - + func testLSTM() { let x = Tensor(rangeFrom: 0.0, to: 0.4, stride: 0.1).rankLifted() let inputs: [Tensor] = Array(repeating: x, count: 4) @@ -721,7 +721,7 @@ final class LayerTests: XCTestCase { // lnLayer = tf.keras.layers.LayerNormalization(axis=1, epsilon=0.001) // with tf.GradientTape() as t: // t.watch(x) - // y = lnLayer(x) + // y = lnLayer(x) // z = tf.math.reduce_sum(tf.math.square(y)) // print(y, t.gradient(z, [x] + lnLayer.trainable_variables)) // ``` @@ -740,8 +740,8 @@ final class LayerTests: XCTestCase { [-0.0019815 , 0.00164783, 0.00130618, 0.00119543, -0.00216818]], accuracy: 1e-5) assertEqual( - grad.1.offset, - [-0.645803 , -5.8017054 , 0.03168535, 5.973418 , 0.44240427], + grad.1.offset, + [-0.645803 , -5.8017054 , 0.03168535, 5.973418 , 0.44240427], accuracy: 1e-5) assertEqual( grad.1.scale,