From 20e47597646e4bf8e58486ce27b966a0ebe4bf18 Mon Sep 17 00:00:00 2001 From: Brennan Saeta Date: Sun, 26 Apr 2020 16:47:04 -0700 Subject: [PATCH 1/4] [WIP]: Exploring removing `` everywhere. This change explores stuffing some extra typealiases in a protocol to obviate the need to specify `Foo` everywhere. Unfortunately, it appears that the typealiases don't override the imports when they're defined in another module. Disadvantages of this approach: it might make it more confusing for someone who is unfamiliar with this pattern to understand what is specifying the scalar parameter. Finally, in certain usages, typealiases even within the same module did not work, and had to use `Self.$TYPEALIAS` to get it to compile, even in the same module. Also, although I didn't play around with it much, the `Tensor` typealias also caused a number of problems in the typechecker. --- Models/ImageClassification/LeNet-5.swift | 32 ++++++++++++------ Models/ImageClassification/VGG.swift | 22 ++++++------ Support/SugarLayer.swift | 43 ++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 21 deletions(-) create mode 100644 Support/SugarLayer.swift diff --git a/Models/ImageClassification/LeNet-5.swift b/Models/ImageClassification/LeNet-5.swift index a4f08fd901d..8e4caa20b3c 100644 --- a/Models/ImageClassification/LeNet-5.swift +++ b/Models/ImageClassification/LeNet-5.swift @@ -13,6 +13,7 @@ // limitations under the License. import TensorFlow +import ModelSupport // Original Paper: // "Gradient-Based Learning Applied to Document Recognition" @@ -22,21 +23,32 @@ import TensorFlow // Note: this implementation connects all the feature maps in the second convolutional layer. // Additionally, ReLU is used instead of sigmoid activations. -public struct LeNet: Layer { - public var conv1 = Conv2D(filterShape: (5, 5, 1, 6), padding: .same, activation: relu) - public var pool1 = AvgPool2D(poolSize: (2, 2), strides: (2, 2)) - public var conv2 = Conv2D(filterShape: (5, 5, 6, 16), activation: relu) - public var pool2 = AvgPool2D(poolSize: (2, 2), strides: (2, 2)) - public var flatten = Flatten() - public var fc1 = Dense(inputSize: 400, outputSize: 120, activation: relu) - public var fc2 = Dense(inputSize: 120, outputSize: 84, activation: relu) - public var fc3 = Dense(inputSize: 84, outputSize: 10) + +public struct LeNet: Layer, Foo { + public var conv1 = Conv2D(filterShape: (5, 5, 1, 6), padding: .same, activation: relu) + public var pool1 = AvgPool2D(poolSize: (2, 2), strides: (2, 2)) + public var conv2 = Conv2D(filterShape: (5, 5, 6, 16), activation: relu) + public var pool2 = AvgPool2D(poolSize: (2, 2), strides: (2, 2)) + public var flatten = Flatten() + public var fc1 = Dense(inputSize: 400, outputSize: 120, activation: relu) + public var fc2 = Dense(inputSize: 120, outputSize: 84, activation: relu) + public var fc3 = Dense(inputSize: 84, outputSize: 10) public init() {} @differentiable - public func callAsFunction(_ input: Tensor) -> Tensor { + public func callAsFunction(_ input: TensorF) -> TensorF { let convolved = input.sequenced(through: conv1, pool1, conv2, pool2) return convolved.sequenced(through: flatten, fc1, fc2, fc3) } } + +// Can't move this to another module apparently... +public protocol Foo { + typealias Conv2D = TensorFlow.Conv2D + typealias AvgPool2D = TensorFlow.AvgPool2D + typealias MaxPool2D = TensorFlow.MaxPool2D + typealias Flatten = TensorFlow.Flatten + typealias Dense = TensorFlow.Dense + typealias TensorF = TensorFlow.Tensor +} diff --git a/Models/ImageClassification/VGG.swift b/Models/ImageClassification/VGG.swift index 4e1bd89c4a8..a6cefebc64f 100644 --- a/Models/ImageClassification/VGG.swift +++ b/Models/ImageClassification/VGG.swift @@ -19,38 +19,38 @@ import TensorFlow // Karen Simonyan, Andrew Zisserman // https://arxiv.org/abs/1409.1556 -public struct VGGBlock: Layer { - var blocks: [Conv2D] = [] - var maxpool = MaxPool2D(poolSize: (2, 2), strides: (2, 2)) +public struct VGGBlock: Layer, Foo { + var blocks: [Self.Conv2D] = [] + var maxpool = MaxPool2D(poolSize: (2, 2), strides: (2, 2)) public init(featureCounts: (Int, Int, Int, Int), blockCount: Int) { - self.blocks = [Conv2D(filterShape: (3, 3, featureCounts.0, featureCounts.1), + self.blocks = [Conv2D(filterShape: (3, 3, featureCounts.0, featureCounts.1), padding: .same, activation: relu)] for _ in 1..(filterShape: (3, 3, featureCounts.2, featureCounts.3), + self.blocks += [Conv2D(filterShape: (3, 3, featureCounts.2, featureCounts.3), padding: .same, activation: relu)] } } @differentiable - public func callAsFunction(_ input: Tensor) -> Tensor { + public func callAsFunction(_ input: TensorF) -> TensorF { return maxpool(blocks.differentiableReduce(input) { $1($0) }) } } -public struct VGG16: Layer { +public struct VGG16: Layer, Foo { var layer1: VGGBlock var layer2: VGGBlock var layer3: VGGBlock var layer4: VGGBlock var layer5: VGGBlock - var flatten = Flatten() - var dense1 = Dense(inputSize: 512 * 7 * 7, outputSize: 4096, activation: relu) - var dense2 = Dense(inputSize: 4096, outputSize: 4096, activation: relu) - var output: Dense + var flatten = Flatten() + var dense1 = Dense(inputSize: 512 * 7 * 7, outputSize: 4096, activation: relu) + var dense2 = Dense(inputSize: 4096, outputSize: 4096, activation: relu) + var output: Self.Dense public init(classCount: Int = 1000) { layer1 = VGGBlock(featureCounts: (3, 64, 64, 64), blockCount: 2) diff --git a/Support/SugarLayer.swift b/Support/SugarLayer.swift new file mode 100644 index 00000000000..272c6be1c3f --- /dev/null +++ b/Support/SugarLayer.swift @@ -0,0 +1,43 @@ +// Copyright 2019 The TensorFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import TensorFlow + +public protocol EasyLayers { + typealias Conv2D = TensorFlow.Conv2D + typealias AvgPool2D = TensorFlow.AvgPool2D + typealias Flatten = TensorFlow.Flatten + typealias Dense = TensorFlow.Dense + typealias TensorF = TensorFlow.Tensor +} + + +/* +// Tinkercruft... +public protocol TypedModel: Layer { + associatedtype Scalar: TensorFlowScalar + + typealias Tensor = TensorFlow.Tensor + typealias TTensor = TensorFlow.Tensor +} + +public extension TypedModel where Scalar: TensorFlowFloatingPoint { + typealias Conv2D = TensorFlow.Conv2D + typealias AvgPool2D = TensorFlow.AvgPool2D + typealias Flatten = TensorFlow.Flatten + typealias Dense = TensorFlow.Dense +} + +public protocol FloatModel: TypedModel where Scalar == Float {} +*/ \ No newline at end of file From 91bdc0625511ba9f89227fc93959aa46e88d12b3 Mon Sep 17 00:00:00 2001 From: Brennan Saeta Date: Mon, 27 Apr 2020 15:14:56 -0700 Subject: [PATCH 2/4] Fix build issues and clean up a bit. --- Models/ImageClassification/LeNet-5.swift | 14 +------------- Models/ImageClassification/VGG.swift | 5 +++-- Package.swift | 2 +- Support/SugarLayer.swift | 6 +++--- 4 files changed, 8 insertions(+), 19 deletions(-) diff --git a/Models/ImageClassification/LeNet-5.swift b/Models/ImageClassification/LeNet-5.swift index 8e4caa20b3c..31c4356cfac 100644 --- a/Models/ImageClassification/LeNet-5.swift +++ b/Models/ImageClassification/LeNet-5.swift @@ -22,9 +22,7 @@ import ModelSupport // // Note: this implementation connects all the feature maps in the second convolutional layer. // Additionally, ReLU is used instead of sigmoid activations. - - -public struct LeNet: Layer, Foo { +public struct LeNet: Layer, FloatModel { public var conv1 = Conv2D(filterShape: (5, 5, 1, 6), padding: .same, activation: relu) public var pool1 = AvgPool2D(poolSize: (2, 2), strides: (2, 2)) public var conv2 = Conv2D(filterShape: (5, 5, 6, 16), activation: relu) @@ -42,13 +40,3 @@ public struct LeNet: Layer, Foo { return convolved.sequenced(through: flatten, fc1, fc2, fc3) } } - -// Can't move this to another module apparently... -public protocol Foo { - typealias Conv2D = TensorFlow.Conv2D - typealias AvgPool2D = TensorFlow.AvgPool2D - typealias MaxPool2D = TensorFlow.MaxPool2D - typealias Flatten = TensorFlow.Flatten - typealias Dense = TensorFlow.Dense - typealias TensorF = TensorFlow.Tensor -} diff --git a/Models/ImageClassification/VGG.swift b/Models/ImageClassification/VGG.swift index a6cefebc64f..56f34403feb 100644 --- a/Models/ImageClassification/VGG.swift +++ b/Models/ImageClassification/VGG.swift @@ -13,13 +13,14 @@ // limitations under the License. import TensorFlow +import ModelSupport // Original Paper: // "Very Deep Convolutional Networks for Large-Scale Image Recognition" // Karen Simonyan, Andrew Zisserman // https://arxiv.org/abs/1409.1556 -public struct VGGBlock: Layer, Foo { +public struct VGGBlock: Layer, EasyLayers { var blocks: [Self.Conv2D] = [] var maxpool = MaxPool2D(poolSize: (2, 2), strides: (2, 2)) @@ -40,7 +41,7 @@ public struct VGGBlock: Layer, Foo { } } -public struct VGG16: Layer, Foo { +public struct VGG16: Layer, EasyLayers { var layer1: VGGBlock var layer2: VGGBlock var layer3: VGGBlock diff --git a/Package.swift b/Package.swift index e21ceb32381..1a092b35175 100644 --- a/Package.swift +++ b/Package.swift @@ -46,7 +46,7 @@ let package = Package( .target( name: "ModelSupport", dependencies: ["SwiftProtobuf", "STBImage"], path: "Support", exclude: ["STBImage"]), - .target(name: "ImageClassificationModels", path: "Models/ImageClassification"), + .target(name: "ImageClassificationModels", dependencies: ["ModelSupport"], path: "Models/ImageClassification"), .target(name: "VideoClassificationModels", path: "Models/Spatiotemporal"), .target(name: "TextModels", dependencies: ["Datasets"], path: "Models/Text"), .target(name: "RecommendationModels", path: "Models/Recommendation"), diff --git a/Support/SugarLayer.swift b/Support/SugarLayer.swift index 272c6be1c3f..37e28e9be1a 100644 --- a/Support/SugarLayer.swift +++ b/Support/SugarLayer.swift @@ -17,18 +17,19 @@ import TensorFlow public protocol EasyLayers { typealias Conv2D = TensorFlow.Conv2D typealias AvgPool2D = TensorFlow.AvgPool2D + typealias MaxPool2D = TensorFlow.MaxPool2D typealias Flatten = TensorFlow.Flatten typealias Dense = TensorFlow.Dense typealias TensorF = TensorFlow.Tensor } -/* -// Tinkercruft... +// Something closer to the actual design... public protocol TypedModel: Layer { associatedtype Scalar: TensorFlowScalar typealias Tensor = TensorFlow.Tensor + typealias TensorF = TensorFlow.Tensor typealias TTensor = TensorFlow.Tensor } @@ -40,4 +41,3 @@ public extension TypedModel where Scalar: TensorFlowFloatingPoint { } public protocol FloatModel: TypedModel where Scalar == Float {} -*/ \ No newline at end of file From ff3582c0204a529745a47c8e2d5baffa69104b71 Mon Sep 17 00:00:00 2001 From: Brennan Saeta Date: Mon, 27 Apr 2020 20:03:51 -0700 Subject: [PATCH 3/4] Clean up tinkercruft now that the builds are a bit more reliable. --- Models/ImageClassification/LeNet-5.swift | 6 ++++-- Models/ImageClassification/VGG.swift | 14 +++++++------- Support/SugarLayer.swift | 17 +++++------------ 3 files changed, 16 insertions(+), 21 deletions(-) diff --git a/Models/ImageClassification/LeNet-5.swift b/Models/ImageClassification/LeNet-5.swift index 31c4356cfac..378e0657e5f 100644 --- a/Models/ImageClassification/LeNet-5.swift +++ b/Models/ImageClassification/LeNet-5.swift @@ -22,7 +22,7 @@ import ModelSupport // // Note: this implementation connects all the feature maps in the second convolutional layer. // Additionally, ReLU is used instead of sigmoid activations. -public struct LeNet: Layer, FloatModel { +public struct LeNet: FloatModel { public var conv1 = Conv2D(filterShape: (5, 5, 1, 6), padding: .same, activation: relu) public var pool1 = AvgPool2D(poolSize: (2, 2), strides: (2, 2)) public var conv2 = Conv2D(filterShape: (5, 5, 6, 16), activation: relu) @@ -34,8 +34,10 @@ public struct LeNet: Layer, FloatModel { public init() {} + public typealias Tensor = TensorF // Lines below do not compile without this! + @differentiable - public func callAsFunction(_ input: TensorF) -> TensorF { + public func callAsFunction(_ input: Tensor) -> Tensor { let convolved = input.sequenced(through: conv1, pool1, conv2, pool2) return convolved.sequenced(through: flatten, fc1, fc2, fc3) } diff --git a/Models/ImageClassification/VGG.swift b/Models/ImageClassification/VGG.swift index 56f34403feb..9fff9e9cc4f 100644 --- a/Models/ImageClassification/VGG.swift +++ b/Models/ImageClassification/VGG.swift @@ -20,7 +20,7 @@ import ModelSupport // Karen Simonyan, Andrew Zisserman // https://arxiv.org/abs/1409.1556 -public struct VGGBlock: Layer, EasyLayers { +public struct VGGBlock: FloatModel { var blocks: [Self.Conv2D] = [] var maxpool = MaxPool2D(poolSize: (2, 2), strides: (2, 2)) @@ -41,7 +41,7 @@ public struct VGGBlock: Layer, EasyLayers { } } -public struct VGG16: Layer, EasyLayers { +public struct VGG16: FloatModel { var layer1: VGGBlock var layer2: VGGBlock var layer3: VGGBlock @@ -69,17 +69,17 @@ public struct VGG16: Layer, EasyLayers { } } -public struct VGG19: Layer { +public struct VGG19: FloatModel { var layer1: VGGBlock var layer2: VGGBlock var layer3: VGGBlock var layer4: VGGBlock var layer5: VGGBlock - var flatten = Flatten() - var dense1 = Dense(inputSize: 512 * 7 * 7, outputSize: 4096, activation: relu) - var dense2 = Dense(inputSize: 4096, outputSize: 4096, activation: relu) - var output: Dense + var flatten = Flatten() + var dense1 = Dense(inputSize: 512 * 7 * 7, outputSize: 4096, activation: relu) + var dense2 = Dense(inputSize: 4096, outputSize: 4096, activation: relu) + var output: Self.Dense public init(classCount: Int = 1000) { layer1 = VGGBlock(featureCounts: (3, 64, 64, 64), blockCount: 2) diff --git a/Support/SugarLayer.swift b/Support/SugarLayer.swift index 37e28e9be1a..9d1d7306e4f 100644 --- a/Support/SugarLayer.swift +++ b/Support/SugarLayer.swift @@ -14,21 +14,11 @@ import TensorFlow -public protocol EasyLayers { - typealias Conv2D = TensorFlow.Conv2D - typealias AvgPool2D = TensorFlow.AvgPool2D - typealias MaxPool2D = TensorFlow.MaxPool2D - typealias Flatten = TensorFlow.Flatten - typealias Dense = TensorFlow.Dense - typealias TensorF = TensorFlow.Tensor -} - - -// Something closer to the actual design... +/// A model composed of a variety of layers all using the same scalar type. public protocol TypedModel: Layer { associatedtype Scalar: TensorFlowScalar - typealias Tensor = TensorFlow.Tensor + typealias Tensor = TensorFlow.Tensor // For some reason, this doesn't work! typealias TensorF = TensorFlow.Tensor typealias TTensor = TensorFlow.Tensor } @@ -36,8 +26,11 @@ public protocol TypedModel: Layer { public extension TypedModel where Scalar: TensorFlowFloatingPoint { typealias Conv2D = TensorFlow.Conv2D typealias AvgPool2D = TensorFlow.AvgPool2D + typealias MaxPool2D = TensorFlow.MaxPool2D typealias Flatten = TensorFlow.Flatten typealias Dense = TensorFlow.Dense } +/// A model composed of a variety of layers, where each one is parameterized +/// by `Float`. public protocol FloatModel: TypedModel where Scalar == Float {} From aa895a6eb0b587bc2413ae48ad6886af04902561 Mon Sep 17 00:00:00 2001 From: Brennan Saeta Date: Mon, 27 Apr 2020 20:17:42 -0700 Subject: [PATCH 4/4] Have `TypedModule` refine `Module` instead of `Layer` to make more flexible. Additionally, add these helpers to simplify `DLRM` and `MLP` from the recommendations models package. --- Models/Recommendation/DLRM.swift | 12 +++++++----- Models/Recommendation/MLP.swift | 11 +++++++---- Package.swift | 2 +- Support/SugarLayer.swift | 7 ++++--- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/Models/Recommendation/DLRM.swift b/Models/Recommendation/DLRM.swift index 4183cc5a874..c41c1888037 100644 --- a/Models/Recommendation/DLRM.swift +++ b/Models/Recommendation/DLRM.swift @@ -13,6 +13,7 @@ // limitations under the License. import TensorFlow +import ModelSupport /// The DLRM model is parameterized to support multiple ways of combining the latent spaces of the inputs. public enum InteractionType { @@ -38,11 +39,12 @@ public enum InteractionType { /// "Deep Learning Recommendation Model for Personalization and Recommendation Systems" /// Maxim Naumov et al. /// https://arxiv.org/pdf/1906.00091.pdf -public struct DLRM: Module { +public struct DLRM: TypedModule { + public typealias Scalar = Float - public var mlpBottom: MLP - public var mlpTop: MLP - public var latentFactors: [Embedding] + public var mlpBottom: Self.MLP + public var mlpTop: Self.MLP + public var latentFactors: [Self.Embedding] @noDerivative public let nDense: Int @noDerivative public let interaction: InteractionType @@ -63,7 +65,7 @@ public struct DLRM: Module { mlpTop = MLP(dims: [topInput] + lnTop + [1], sigmoidLastLayer: true) latentFactors = lnEmb.map { embeddingSize -> Embedding in // Use a random uniform initialization to match the reference implementation. - let weights = Tensor( + let weights = Tensor( randomUniform: [embeddingSize, mSpa], lowerBound: Tensor(Float(-1.0)/Float(embeddingSize)), upperBound: Tensor(Float(1.0)/Float(embeddingSize))) diff --git a/Models/Recommendation/MLP.swift b/Models/Recommendation/MLP.swift index df4af50e88c..8842c52e0e0 100644 --- a/Models/Recommendation/MLP.swift +++ b/Models/Recommendation/MLP.swift @@ -13,10 +13,11 @@ // limitations under the License. import TensorFlow +import ModelSupport /// MLP is a multi-layer perceptron and is used as a component of the DLRM model -public struct MLP: Layer { - public var blocks: [Dense] = [] +public struct MLP: Layer, TypedModule { + public var blocks: [Self.Dense] = [] /// Randomly initializes a new multilayer perceptron from the given hyperparameters. /// @@ -35,12 +36,14 @@ public struct MLP: Layer { } @differentiable - public func callAsFunction(_ input: Tensor) -> Tensor { + public func callAsFunction(_ input: Tensor) -> Tensor { let blocksReduced = blocks.differentiableReduce(input) { last, layer in layer(last) } return blocksReduced } - } +extension TypedModule where Scalar: TensorFlowFloatingPoint { + public typealias MLP = RecommendationModels.MLP +} diff --git a/Package.swift b/Package.swift index 1a092b35175..01fa823acea 100644 --- a/Package.swift +++ b/Package.swift @@ -49,7 +49,7 @@ let package = Package( .target(name: "ImageClassificationModels", dependencies: ["ModelSupport"], path: "Models/ImageClassification"), .target(name: "VideoClassificationModels", path: "Models/Spatiotemporal"), .target(name: "TextModels", dependencies: ["Datasets"], path: "Models/Text"), - .target(name: "RecommendationModels", path: "Models/Recommendation"), + .target(name: "RecommendationModels", dependencies: ["ModelSupport"], path: "Models/Recommendation"), .target( name: "Autoencoder1D", dependencies: ["Datasets", "ModelSupport"], path: "Autoencoder/Autoencoder1D"), diff --git a/Support/SugarLayer.swift b/Support/SugarLayer.swift index 9d1d7306e4f..8332274e007 100644 --- a/Support/SugarLayer.swift +++ b/Support/SugarLayer.swift @@ -15,7 +15,7 @@ import TensorFlow /// A model composed of a variety of layers all using the same scalar type. -public protocol TypedModel: Layer { +public protocol TypedModule: Module { associatedtype Scalar: TensorFlowScalar typealias Tensor = TensorFlow.Tensor // For some reason, this doesn't work! @@ -23,14 +23,15 @@ public protocol TypedModel: Layer { typealias TTensor = TensorFlow.Tensor } -public extension TypedModel where Scalar: TensorFlowFloatingPoint { +public extension TypedModule where Scalar: TensorFlowFloatingPoint { typealias Conv2D = TensorFlow.Conv2D typealias AvgPool2D = TensorFlow.AvgPool2D typealias MaxPool2D = TensorFlow.MaxPool2D typealias Flatten = TensorFlow.Flatten typealias Dense = TensorFlow.Dense + typealias Embedding = TensorFlow.Embedding } /// A model composed of a variety of layers, where each one is parameterized /// by `Float`. -public protocol FloatModel: TypedModel where Scalar == Float {} +public protocol FloatModel: TypedModule, Layer where Scalar == Float {}