shyrma_sparse_softmax_cross_entropy_loss_with_logits

libnd4j/include/helpers/impl/ShapeUtils.cpp

@@ -367,7 +367,8 @@ Nd4jLong* ShapeUtils<T>::evalReduceShapeInfo(const char order, std::vector<int>&
         for(int i=0; i<rank; ++i) {
             isAbsent = true;
             for(int j=0; j<size; ++j) {
-                if(i == dimensions[j]) {
+                int dim = dimensions[j] >= 0 ? dimensions[j] : dimensions[j] + rank;
+                if(i == dim) {
                     isAbsent = false;
                     break;
                 }

libnd4j/include/ops/declarable/generic/loss/softmaxCrossEntropyWithLogits.cpp

@@ -40,8 +40,6 @@ CUSTOM_OP_IMPL(softmax_cross_entropy_loss_with_logits, 2, 1, false, 0, 0) {
     REQUIRE_TRUE(labels->isSameShape(logits), 0, "SOFTMAX_CROSS_ENTROPY_LOSS_WITH_LOGITS OP: labels and logits arrays must have the same shapes, but got %s and %s correspondingly !", ShapeUtils<T>::shapeAsString(labels).c_str(), ShapeUtils<T>::shapeAsString(logits).c_str());    
     REQUIRE_TRUE(classesDim < logits->rankOf(), 0, "SOFTMAX_CROSS_ENTROPY_LOSS_WITH_LOGITS OP: class dimension must be smaller than rank of logits, but got %i and %i correspondingly !", classesDim, logits->rankOf());
 
-	T extraParams[1] = {static_cast<T>(classesDim)};
-
     NDArray<T> maxAlongDim = logits->template reduceAlongDims<simdOps::Max<T>>({classesDim}, true);
     NDArray<T> logExp = (*logits - maxAlongDim).template transform<simdOps::Exp<T>>();
     NDArray<T> logSoftMax = ( logExp / logExp.template reduceAlongDims<simdOps::Sum<T>>({classesDim}, true) ).template transform<simdOps::Log<T>>();

libnd4j/include/ops/declarable/generic/loss/sparseSoftmaxCrossEntropyWithLogits.cpp

@@ -0,0 +1,59 @@
+/*******************************************************************************
+ * Copyright (c) 2015-2018 Skymind, Inc.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com), created on 29.08.2018
+//
+
+#include <op_boilerplate.h>
+#if NOT_EXCLUDED(OP_sparse_softmax_cross_entropy_loss_with_logits)
+
+#include <ops/declarable/CustomOperations.h>
+#include<ops/declarable/helpers/losses.h>
+
+namespace nd4j {
+namespace ops  {
+
+
+//////////////////////////////////////////////////////////////////////////
+OP_IMPL(sparse_softmax_cross_entropy_loss_with_logits, 2, 1, false) {
+
+  	NDArray<T>* labels  = INPUT_VARIABLE(0);
+    NDArray<T>* logits  = INPUT_VARIABLE(1);    
+
+    NDArray<T>* output  = OUTPUT_VARIABLE(0);
+
+    const int labelsRank = labels->rankOf();
+    const int logitsRank = logits->rankOf();
+
+    // input validation    		       
+    REQUIRE_TRUE(labelsRank == logitsRank - 1, 0, "SPARSE_SOFTMAX_CROSS_ENTROPY_LOSS_WITH_LOGITS OP: input arrays should satisfy relation (labels_rank = logits_rank - 1), but got labels_rank = %i and logits_rank = %i instead !", labelsRank, logitsRank);
+
+    std::vector<Nd4jLong> labelsShape = labels->getShapeAsVector();
+    std::vector<Nd4jLong> logitsShape = logits->getShapeAsVector();
+    REQUIRE_TRUE(std::equal(logitsShape.begin(), logitsShape.end()-1, labelsShape.begin()), 0, "SPARSE_SOFTMAX_CROSS_ENTROPY_LOSS_WITH_LOGITS OP: wrong shape of labels array, its shape should be the same as logits shape with last dimension excluded, however got labels_shape = %s and logits_shape = %s instead !", ShapeUtils<T>::shapeAsString(labelsShape).c_str(), ShapeUtils<T>::shapeAsString(logitsShape).c_str());
+
+    helpers::sparseSoftmaxCrossEntropyLossWithLogits(*labels, *logits, *output);
+
+    return Status::OK();
+}
+
+
+
+}
+}
+
+#endif

libnd4j/include/ops/declarable/headers/loss.h

@@ -275,9 +275,9 @@ namespace ops {
         DECLARE_CUSTOM_OP(cosine_distance_loss, 3, 1, false, 0, 2);
         #endif
 
-          //////////////////////////////////////////////////////////////////////////
+        //////////////////////////////////////////////////////////////////////////
     /**
-       * Implementation of softmax cross-entropy loss function max(logits, 0.) - logits * labels + log(1. + exp(-abs(logits))); 
+       * Implementation of softmax cross-entropy loss function 
        * 
        * Input arrays: 
        *    0: logits - logits, type float
@@ -294,6 +294,23 @@ namespace ops {
         DECLARE_CUSTOM_OP(softmax_cross_entropy_loss_with_logits, 2, 1, false, 0, 0);  
         #endif
 
+        //////////////////////////////////////////////////////////////////////////
+    /**
+       * Implementation of sparse softmax cross-entropy loss function
+       * 
+       * Input arrays:        
+       *    0: labels - ground truth vales, expected to be within range [0, num_classes), type float.
+       *       Must have rank equal logits rank minus 1.
+       *    1: logits - logits, type float
+       *  
+       * Output array: 
+       *    0: loss values, type float. Has the same shape as labels
+       */      
+        #if NOT_EXCLUDED(OP_sparse_softmax_cross_entropy_loss_with_logits)
+        DECLARE_OP(sparse_softmax_cross_entropy_loss_with_logits, 2, 1, false);  
+        #endif
+
+
 }
 }
 

libnd4j/include/ops/declarable/helpers/cpu/losses.cpp

@@ -0,0 +1,59 @@
+/*******************************************************************************
+ * Copyright (c) 2015-2018 Skymind, Inc.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com), created on 29.08.2018
+//
+
+
+#include<ops/declarable/helpers/losses.h>
+#include <helpers/ShapeUtils.h>
+
+
+namespace nd4j 	  {
+namespace ops 	  {
+namespace helpers {
+
+
+//////////////////////////////////////////////////////////////////////////
+template <typename T>
+void sparseSoftmaxCrossEntropyLossWithLogits(const NDArray<T>& labels, const NDArray<T>& logits, NDArray<T>& output) {
+
+
+    NDArray<T> maxAlongDim = logits.template reduceAlongDims<simdOps::Max<T>>({-1}, true);
+    NDArray<T> logitsExp = (logits - maxAlongDim).template transform<simdOps::Exp<T>>();
+    NDArray<T> logSoftMax = ( logitsExp / logitsExp.template reduceAlongDims<simdOps::Sum<T>>({-1}, true) ).template transform<simdOps::Log<T>>();
+
+    const Nd4jLong labelsLen = labels.lengthOf();
+
+    std::vector<int> dimsToExclude = ShapeUtils<T>::evalDimsToExclude(logits.rankOf(), {-1});
+
+#pragma omp parallel for schedule(guided) 
+    for(Nd4jLong i = 0; i < labelsLen; ++i) {
+
+        NDArray<T> subArr = logSoftMax(i, dimsToExclude);
+        output(i) = -subArr(labels(i));
+    }
+}
+
+
+template void sparseSoftmaxCrossEntropyLossWithLogits<float>(const NDArray<float>& labels, const NDArray<float>& logits, NDArray<float>& output);
+template void sparseSoftmaxCrossEntropyLossWithLogits<float16>(const NDArray<float16>& labels, const NDArray<float16>& logits, NDArray<float16>& output);
+template void sparseSoftmaxCrossEntropyLossWithLogits<double>(const NDArray<double>& labels, const NDArray<double>& logits, NDArray<double>& output);
+
+}
+}
+}

libnd4j/include/ops/declarable/helpers/losses.h

@@ -0,0 +1,40 @@
+/*******************************************************************************
+ * Copyright (c) 2015-2018 Skymind, Inc.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author Yurii Shyrma (iuriish@yahoo.com), created on 29.08.2018
+//
+
+#ifndef LIBND4J_LOSS_H
+#define LIBND4J_LOSS_H
+
+#include <ops/declarable/helpers/helpers.h>
+
+
+namespace nd4j    {
+namespace ops     {
+namespace helpers {
+
+
+	template <typename T>
+	void sparseSoftmaxCrossEntropyLossWithLogits(const NDArray<T>& labels, const NDArray<T>& logits, NDArray<T>& output);
+
+
+}
+}
+}
+
+#endif //LIBND4J_LOSS_H

libnd4j/tests_cpu/layers_tests/DeclarableOpsTests10.cpp

@@ -422,3 +422,92 @@ TEST_F(DeclarableOpsTests10, range_test12) {
 
     delete result;
 }
+
+///////////////////////////////////////////////////////////////////
+TEST_F(DeclarableOpsTests10, sparse_softmax_cross_entropy_loss_with_logits_test1) {
+
+    NDArray<double> labels('c', {2,3},{3.,2.,1.,0.,1.,2.});
+    NDArray<double> logits('c', {2,3,4});
+    NDArray<double> expected('c', {2,3}, {1.24254, 1.34254, 1.44254, 1.54254, 1.44254, 1.34254});
+
+    logits.linspace(0.1, 0.1);
+
+    nd4j::ops::sparse_softmax_cross_entropy_loss_with_logits<double> op;
+    nd4j::ResultSet<double>* results = op.execute({&labels, &logits}, {}, {});
+
+    ASSERT_EQ(ND4J_STATUS_OK, results->status());
+
+    NDArray<double> *output = results->at(0);    
+
+    ASSERT_TRUE(expected.isSameShape(output));
+    ASSERT_TRUE(expected.equalsTo(output));
+
+    delete results;
+}
+
+///////////////////////////////////////////////////////////////////
+TEST_F(DeclarableOpsTests10, sparse_softmax_cross_entropy_loss_with_logits_test2) {
+
+    NDArray<double> labels('c', {2},{1.,0.});
+    NDArray<double> logits('c', {2,3});
+    NDArray<double> expected('c', {2}, {1.10194, 1.20194});
+
+    logits.linspace(0.1, 0.1);
+
+    nd4j::ops::sparse_softmax_cross_entropy_loss_with_logits<double> op;
+    nd4j::ResultSet<double>* results = op.execute({&labels, &logits}, {}, {});
+
+    ASSERT_EQ(ND4J_STATUS_OK, results->status());
+
+    NDArray<double> *output = results->at(0);    
+
+    ASSERT_TRUE(expected.isSameShape(output));
+    ASSERT_TRUE(expected.equalsTo(output));
+
+    delete results;
+}
+
+///////////////////////////////////////////////////////////////////
+TEST_F(DeclarableOpsTests10, sparse_softmax_cross_entropy_loss_with_logits_test3) {
+
+    NDArray<double> labels('c', {1},{0.});
+    NDArray<double> logits('c', {1,3});
+    NDArray<double> expected('c', {1}, {1.20194});
+
+    logits.linspace(0.1, 0.1);
+
+    nd4j::ops::sparse_softmax_cross_entropy_loss_with_logits<double> op;
+    nd4j::ResultSet<double>* results = op.execute({&labels, &logits}, {}, {});
+
+    ASSERT_EQ(ND4J_STATUS_OK, results->status());
+
+    NDArray<double> *output = results->at(0);    
+
+    ASSERT_TRUE(expected.isSameShape(output));
+    ASSERT_TRUE(expected.equalsTo(output));
+
+    delete results;
+}
+
+///////////////////////////////////////////////////////////////////
+TEST_F(DeclarableOpsTests10, sparse_softmax_cross_entropy_loss_with_logits_test4) {
+
+    NDArray<double> labels('c', {2},{0.,0.});
+    NDArray<double> logits('c', {2,1});
+    NDArray<double> expected('c', {2}, {0., 0.});
+
+    logits.linspace(0.1, 0.1);
+
+    nd4j::ops::sparse_softmax_cross_entropy_loss_with_logits<double> op;
+    nd4j::ResultSet<double>* results = op.execute({&labels, &logits}, {}, {});
+
+    ASSERT_EQ(ND4J_STATUS_OK, results->status());
+
+    NDArray<double> *output = results->at(0);    
+
+    ASSERT_TRUE(expected.isSameShape(output));
+    ASSERT_TRUE(expected.equalsTo(output));
+
+    delete results;
+}
+