From 84847c57a32cc2c7d60b52c9ae36f52ac9d3fde0 Mon Sep 17 00:00:00 2001 From: Tanmay Bakshi Date: Sat, 23 Feb 2019 17:25:16 -0500 Subject: [PATCH 1/8] Add Sigmoid Cross Entropy Loss a.k.a. "binary_crossentropy" from Keras --- Sources/DeepLearning/Loss.swift | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Sources/DeepLearning/Loss.swift b/Sources/DeepLearning/Loss.swift index 1cfaa6298..b93816eee 100644 --- a/Sources/DeepLearning/Loss.swift +++ b/Sources/DeepLearning/Loss.swift @@ -27,3 +27,12 @@ public func softmaxCrossEntropy( logits: Tensor, labels: Tensor) -> Tensor { return -(labels * logSoftmax(logits)).mean(alongAxes: 0).sum() } + +@differentiable +public func sigmoidCrossEntropy( + logits: Tensor, labels: Tensor) -> Tensor { + let loss = labels * log(logits) + + (Tensor(ones: labels.shape) - labels) * + log(Tensor(ones: logits.shape) - logits) + return -loss.mean(alongAxes: 0).sum() +} From 3573287a3a0ed458d1994e89f6dd796e2cf402ab Mon Sep 17 00:00:00 2001 From: Tanmay Bakshi Date: Sat, 23 Feb 2019 17:59:02 -0500 Subject: [PATCH 2/8] Add documentation to each loss function --- Sources/DeepLearning/Loss.swift | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Sources/DeepLearning/Loss.swift b/Sources/DeepLearning/Loss.swift index b93816eee..c2717d65a 100644 --- a/Sources/DeepLearning/Loss.swift +++ b/Sources/DeepLearning/Loss.swift @@ -16,18 +16,33 @@ import TensorFlow #endif +/// Computes the Mean Squared Error loss between logits and labels +/// +/// - Parameters +/// - logits: one-hot encoded outputs from a neural network. +/// - labels: one-hot encoded values that correspond to the correct output. @differentiable public func meanSquaredError( predicted: Tensor, expected: Tensor) -> Tensor { return (expected - predicted).squared().mean() } +/// Computes the Softmax Cross Entropy (Categorical Cross Entropy) loss between logits and labels +/// +/// - Parameters +/// - logits: one-hot encoded outputs from a neural network. +/// - labels: one-hot encoded values that correspond to the correct output. @differentiable public func softmaxCrossEntropy( logits: Tensor, labels: Tensor) -> Tensor { return -(labels * logSoftmax(logits)).mean(alongAxes: 0).sum() } +/// Computes the Sigmoid Cross Entropy (Binary Cross Entropy) loss between logits and labels +/// +/// - Parameters +/// - logits: single continuous values from 0 to 1. +/// - labels: integer values that correspond to the correct output. @differentiable public func sigmoidCrossEntropy( logits: Tensor, labels: Tensor) -> Tensor { From 2d8f21d119b8d69520a13977db50ba123834a55f Mon Sep 17 00:00:00 2001 From: Tanmay Bakshi Date: Sat, 23 Feb 2019 18:12:31 -0500 Subject: [PATCH 3/8] Update comment indentation --- Sources/DeepLearning/Loss.swift | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Sources/DeepLearning/Loss.swift b/Sources/DeepLearning/Loss.swift index c2717d65a..d5ef814a9 100644 --- a/Sources/DeepLearning/Loss.swift +++ b/Sources/DeepLearning/Loss.swift @@ -19,8 +19,8 @@ import TensorFlow /// Computes the Mean Squared Error loss between logits and labels /// /// - Parameters -/// - logits: one-hot encoded outputs from a neural network. -/// - labels: one-hot encoded values that correspond to the correct output. +/// - logits: one-hot encoded outputs from a neural network. +/// - labels: one-hot encoded values that correspond to the correct output. @differentiable public func meanSquaredError( predicted: Tensor, expected: Tensor) -> Tensor { @@ -30,8 +30,8 @@ public func meanSquaredError( /// Computes the Softmax Cross Entropy (Categorical Cross Entropy) loss between logits and labels /// /// - Parameters -/// - logits: one-hot encoded outputs from a neural network. -/// - labels: one-hot encoded values that correspond to the correct output. +/// - logits: one-hot encoded outputs from a neural network. +/// - labels: one-hot encoded values that correspond to the correct output. @differentiable public func softmaxCrossEntropy( logits: Tensor, labels: Tensor) -> Tensor { @@ -41,8 +41,8 @@ public func softmaxCrossEntropy( /// Computes the Sigmoid Cross Entropy (Binary Cross Entropy) loss between logits and labels /// /// - Parameters -/// - logits: single continuous values from 0 to 1. -/// - labels: integer values that correspond to the correct output. +/// - logits: single continuous values from 0 to 1. +/// - labels: integer values that correspond to the correct output. @differentiable public func sigmoidCrossEntropy( logits: Tensor, labels: Tensor) -> Tensor { From 7f37e492cc1956a2e3a801cc8b3103a217453812 Mon Sep 17 00:00:00 2001 From: Tanmay Bakshi Date: Sat, 23 Feb 2019 18:18:37 -0500 Subject: [PATCH 4/8] Update documentation comments --- Sources/DeepLearning/Loss.swift | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Sources/DeepLearning/Loss.swift b/Sources/DeepLearning/Loss.swift index d5ef814a9..16dadde71 100644 --- a/Sources/DeepLearning/Loss.swift +++ b/Sources/DeepLearning/Loss.swift @@ -16,7 +16,7 @@ import TensorFlow #endif -/// Computes the Mean Squared Error loss between logits and labels +/// Computes the mean squared error between logits and labels /// /// - Parameters /// - logits: one-hot encoded outputs from a neural network. @@ -27,7 +27,7 @@ public func meanSquaredError( return (expected - predicted).squared().mean() } -/// Computes the Softmax Cross Entropy (Categorical Cross Entropy) loss between logits and labels +/// Computes the softmax cross entropy (categorical cross entropy) between logits and labels /// /// - Parameters /// - logits: one-hot encoded outputs from a neural network. @@ -38,7 +38,7 @@ public func softmaxCrossEntropy( return -(labels * logSoftmax(logits)).mean(alongAxes: 0).sum() } -/// Computes the Sigmoid Cross Entropy (Binary Cross Entropy) loss between logits and labels +/// Computes the sigmoid cross entropy (binary cross entropy) between logits and labels /// /// - Parameters /// - logits: single continuous values from 0 to 1. From 87c1d3d992c360ea0665e8a2f4b1d4dddab4d494 Mon Sep 17 00:00:00 2001 From: Tanmay Bakshi Date: Sat, 23 Feb 2019 18:19:17 -0500 Subject: [PATCH 5/8] Update documentation comments --- Sources/DeepLearning/Loss.swift | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Sources/DeepLearning/Loss.swift b/Sources/DeepLearning/Loss.swift index 16dadde71..b0bcddd78 100644 --- a/Sources/DeepLearning/Loss.swift +++ b/Sources/DeepLearning/Loss.swift @@ -16,7 +16,7 @@ import TensorFlow #endif -/// Computes the mean squared error between logits and labels +/// Computes the mean squared error between logits and labels. /// /// - Parameters /// - logits: one-hot encoded outputs from a neural network. @@ -27,7 +27,7 @@ public func meanSquaredError( return (expected - predicted).squared().mean() } -/// Computes the softmax cross entropy (categorical cross entropy) between logits and labels +/// Computes the softmax cross entropy (categorical cross entropy) between logits and labels. /// /// - Parameters /// - logits: one-hot encoded outputs from a neural network. @@ -38,7 +38,7 @@ public func softmaxCrossEntropy( return -(labels * logSoftmax(logits)).mean(alongAxes: 0).sum() } -/// Computes the sigmoid cross entropy (binary cross entropy) between logits and labels +/// Computes the sigmoid cross entropy (binary cross entropy) between logits and labels. /// /// - Parameters /// - logits: single continuous values from 0 to 1. From d5106489a840a8a2af75a9afd2f8fe895f0fb4d5 Mon Sep 17 00:00:00 2001 From: Tanmay Bakshi Date: Sun, 24 Feb 2019 03:58:06 -0500 Subject: [PATCH 6/8] Reformat code --- Sources/DeepLearning/Loss.swift | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/Sources/DeepLearning/Loss.swift b/Sources/DeepLearning/Loss.swift index b0bcddd78..9d593aedc 100644 --- a/Sources/DeepLearning/Loss.swift +++ b/Sources/DeepLearning/Loss.swift @@ -18,9 +18,9 @@ import TensorFlow /// Computes the mean squared error between logits and labels. /// -/// - Parameters -/// - logits: one-hot encoded outputs from a neural network. -/// - labels: one-hot encoded values that correspond to the correct output. +/// - Parameters: +/// - logits: One-hot encoded outputs from a neural network. +/// - labels: One-hot encoded values that correspond to the correct output. @differentiable public func meanSquaredError( predicted: Tensor, expected: Tensor) -> Tensor { @@ -29,9 +29,9 @@ public func meanSquaredError( /// Computes the softmax cross entropy (categorical cross entropy) between logits and labels. /// -/// - Parameters -/// - logits: one-hot encoded outputs from a neural network. -/// - labels: one-hot encoded values that correspond to the correct output. +/// - Parameters: +/// - logits: One-hot encoded outputs from a neural network. +/// - labels: One-hot encoded values that correspond to the correct output. @differentiable public func softmaxCrossEntropy( logits: Tensor, labels: Tensor) -> Tensor { @@ -40,14 +40,15 @@ public func softmaxCrossEntropy( /// Computes the sigmoid cross entropy (binary cross entropy) between logits and labels. /// -/// - Parameters -/// - logits: single continuous values from 0 to 1. -/// - labels: integer values that correspond to the correct output. +/// - Parameters: +/// - logits: Single continuous values from `0` to `1`. +/// - labels: Integer values that correspond to the correct output. @differentiable public func sigmoidCrossEntropy( - logits: Tensor, labels: Tensor) -> Tensor { + logits: Tensor, labels: Tensor + ) -> Tensor { let loss = labels * log(logits) + - (Tensor(ones: labels.shape) - labels) * - log(Tensor(ones: logits.shape) - logits) + (Tensor(ones: labels.shape) - labels) * + log(Tensor(ones: logits.shape) - logits) return -loss.mean(alongAxes: 0).sum() } From 8b156e75f659510479c6fc30f474f7439cf98816 Mon Sep 17 00:00:00 2001 From: Richard Wei Date: Sun, 24 Feb 2019 05:06:06 -0500 Subject: [PATCH 7/8] linewrap formatting Co-Authored-By: tanmayb123 --- Sources/DeepLearning/Loss.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/DeepLearning/Loss.swift b/Sources/DeepLearning/Loss.swift index 9d593aedc..3e46c2a42 100644 --- a/Sources/DeepLearning/Loss.swift +++ b/Sources/DeepLearning/Loss.swift @@ -46,7 +46,7 @@ public func softmaxCrossEntropy( @differentiable public func sigmoidCrossEntropy( logits: Tensor, labels: Tensor - ) -> Tensor { +) -> Tensor { let loss = labels * log(logits) + (Tensor(ones: labels.shape) - labels) * log(Tensor(ones: logits.shape) - logits) From f167c525b73ccaf3b392fb63c43d4db9ec025210 Mon Sep 17 00:00:00 2001 From: Tanmay Bakshi Date: Sun, 24 Feb 2019 05:58:11 -0500 Subject: [PATCH 8/8] Remove Tensor(ones:) from SCE + formatting --- Sources/DeepLearning/Loss.swift | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Sources/DeepLearning/Loss.swift b/Sources/DeepLearning/Loss.swift index 3e46c2a42..eed80dc63 100644 --- a/Sources/DeepLearning/Loss.swift +++ b/Sources/DeepLearning/Loss.swift @@ -48,7 +48,6 @@ public func sigmoidCrossEntropy( logits: Tensor, labels: Tensor ) -> Tensor { let loss = labels * log(logits) + - (Tensor(ones: labels.shape) - labels) * - log(Tensor(ones: logits.shape) - logits) + (Tensor(1) - labels) * log(Tensor(1) - logits) return -loss.mean(alongAxes: 0).sum() }