diff --git a/Sources/DeepLearning/Layer.swift b/Sources/DeepLearning/Layer.swift index 240202fa7..c20957dc8 100644 --- a/Sources/DeepLearning/Layer.swift +++ b/Sources/DeepLearning/Layer.swift @@ -101,6 +101,80 @@ public extension Layer { } } +/// Adds helpers for standard feed-forward, sequential models. +public extension Differentiable { + @differentiable(wrt: (self, l1, l2)) + func sequenced( + in context: Context, through l1: L1, _ l2: L2) + -> L2.Output + where L1.Input == Self, + L1.Output == L2.Input { + let o1 = l1.applied(to: self, in: context) + return l2.applied(to: o1, in: context) + } + + @differentiable(wrt: (self, l1, l2, l3)) + func sequenced( + in context: Context, through l1: L1, _ l2: L2, _ l3: L3) + -> L3.Output + where L1.Input == Self, + L1.Output == L2.Input, + L2.Output == L3.Input { + let o1 = l1.applied(to: self, in: context) + let o2 = l2.applied(to: o1, in: context) + return l3.applied(to: o2, in: context) + } + + @differentiable(wrt: (self, l1, l2, l3, l4)) + func sequenced( + in context: Context, through l1: L1, _ l2: L2, _ l3: L3, _ l4: L4) + -> L4.Output + where L1.Input == Self, + L1.Output == L2.Input, + L2.Output == L3.Input, + L3.Output == L4.Input { + let o1 = l1.applied(to: self, in: context) + let o2 = l2.applied(to: o1, in: context) + let o3 = l3.applied(to: o2, in: context) + return l4.applied(to: o3, in: context) + } + + @differentiable(wrt: (self, l1, l2, l3, l4, l5)) + func sequenced( + in context: Context, through l1: L1, _ l2: L2, _ l3: L3, _ l4: L4, _ l5: L5) + -> L5.Output + where L1.Input == Self, + L1.Output == L2.Input, + L2.Output == L3.Input, + L3.Output == L4.Input, + L4.Output == L5.Input { + let o1 = l1.applied(to: self, in: context) + let o2 = l2.applied(to: o1, in: context) + let o3 = l3.applied(to: o2, in: context) + let o4 = l4.applied(to: o3, in: context) + return l5.applied(to: o4, in: context) + } + + @differentiable(wrt: (self, l1, l2, l3, l4, l5, l6)) + func sequenced( + in context: Context, through l1: L1, _ l2: L2, _ l3: L3, _ l4: L4, _ l5: L5, _ l6: L6) + -> L6.Output + where L1.Input == Self, + L1.Output == L2.Input, + L2.Output == L3.Input, + L3.Output == L4.Input, + L4.Output == L5.Input, + L5.Output == L6.Input { + let o1 = l1.applied(to: self, in: context) + let o2 = l2.applied(to: o1, in: context) + let o3 = l3.applied(to: o2, in: context) + let o4 = l4.applied(to: o3, in: context) + let o5 = l5.applied(to: o4, in: context) + return l6.applied(to: o5, in: context) + } +} + + /// A mutable, shareable, owning reference to a tensor. public final class Parameter { public var value: Tensor diff --git a/Tests/DeepLearningTests/SequentialTests.swift b/Tests/DeepLearningTests/SequentialTests.swift new file mode 100644 index 000000000..6591fe320 --- /dev/null +++ b/Tests/DeepLearningTests/SequentialTests.swift @@ -0,0 +1,47 @@ +// Copyright 2019 The TensorFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import XCTest +@testable import DeepLearning + +final class SequentialTests: XCTestCase { + func testSequential() { + struct Model: Layer { + var dense1 = Dense(inputSize: 2, outputSize: 4, activation: relu) + var dense2 = Dense(inputSize: 4, outputSize: 1, activation: relu) + + @differentiable(wrt: (self, input)) + func applied(to input: Tensor, in context: Context) -> Tensor { + return input.sequenced(in: context, through: dense1, dense2) + } + } + var model = Model() + let optimizer = SGD(learningRate: 0.02, modelType: type(of: model), scalarType: Float.self) + let x: Tensor = [[0, 0], [0, 1], [1, 0], [1, 1]] + let y: Tensor = [0, 1, 1, 0] + let context = Context(learningPhase: .training) + for _ in 0..<1000 { + let 𝛁model = model.gradient { model -> Tensor in + let ŷ = model.applied(to: x, in: context) + return meanSquaredError(predicted: ŷ, expected: y) + } + optimizer.update(&model.allDifferentiableVariables, along: 𝛁model) + } + print(model.inferring(from: [[0, 0], [0, 1], [1, 0], [1, 1]])) + } + + static var allTests = [ + ("testSequential", testSequential) + ] +} diff --git a/Tests/DeepLearningTests/XCTestManifests.swift b/Tests/DeepLearningTests/XCTestManifests.swift index 2118f859f..768ddeccc 100644 --- a/Tests/DeepLearningTests/XCTestManifests.swift +++ b/Tests/DeepLearningTests/XCTestManifests.swift @@ -19,6 +19,7 @@ public func allTests() -> [XCTestCaseEntry] { return [ testCase(PRNGTests.allTests), testCase(TrivialModelTests.allTests), + testCase(SequentialTests.allTests), ] } #endif