[WIP] Benchmarking (#7241)

* inital commit * inital commit * inital commit * inital commit * inital commit * inital commit * include * parameters * next step * next step * next step * next step * next step * next step * next step * axis * Test name * Fixes * next step * next step * next step * next step * next step * next step * next step * next step * next step * More parameters, extra test * next step * Add more stuff, break build * Fixes * new line fix * concat leak fix
deeplearning4j · Mar 1, 2019 · c42d41c · c42d41c
1 parent 39bf342
commit c42d41c
Show file tree

Hide file tree

Showing 27 changed files with 1,834 additions and 4 deletions.
diff --git a/libnd4j/include/cblas.h b/libnd4j/include/cblas.h
@@ -21,7 +21,9 @@
 #ifndef NATIVEOPERATIONS_CBLAS_H
 #define NATIVEOPERATIONS_CBLAS_H
 
+#ifndef __STANDALONE_BUILD__
 #include "config.h"
+#endif
 
 #ifdef __MKL_CBLAS_H__
 // CBLAS from MKL is already included

diff --git a/libnd4j/include/graph/impl/GraphUtils.cpp b/libnd4j/include/graph/impl/GraphUtils.cpp
@@ -18,7 +18,9 @@
 // Created by GS <sgazeos@gmail.com> 3/7/2018
 //
 
+#ifndef __STANDALONE_BUILD__
 #include "config.h"
+#endif
 
 #include <graph/GraphUtils.h>
 #include <cstdlib>

diff --git a/libnd4j/include/helpers/BenchmarkHelper.h b/libnd4j/include/helpers/BenchmarkHelper.h
@@ -0,0 +1,84 @@
+/*******************************************************************************
+ * Copyright (c) 2015-2018 Skymind, Inc.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+/**
+ * @author raver119@gmail.com
+ */
+
+#ifndef LIBND4J_BENCHMARKHELPER_H
+#define LIBND4J_BENCHMARKHELPER_H
+
+
+#include <helpers/OpBenchmark.h>
+#include <helpers/benchmark/ScalarBenchmark.h>
+#include <helpers/benchmark/TransformBenchmark.h>
+#include <helpers/benchmark/ReductionBenchmark.h>
+#include <helpers/benchmark/PairwiseBenchmark.h>
+#include <ops/declarable/DeclarableOp.h>
+#include <NDArray.h>
+#include <benchmark/Parameters.h>
+#include <benchmark/PredefinedParameters.h>
+#include <benchmark/ParametersBatch.h>
+#include <benchmark/ParametersSpace.h>
+#include <benchmark/BoolParameters.h>
+#include <benchmark/IntParameters.h>
+#include <benchmark/IntPowerParameters.h>
+#include <array/ResultSet.h>
+
+namespace nd4j {
+
+    class BenchmarkHelper {
+    private:
+        unsigned int _wIterations;
+        unsigned int _rIterations;
+
+    protected:
+        void benchmarkOperation(OpBenchmark &benchmark);
+
+        void benchmarkScalarOperation(scalar::Ops op, std::string testName, double value, NDArray &x, NDArray &z);
+
+        void benchmarkDeclarableOp(nd4j::ops::DeclarableOp &op, std::string testName, Context &context);
+
+        void benchmarkGEMM(char orderA, std::initializer_list<Nd4jLong> shapeA, char orderB, std::initializer_list<Nd4jLong> shapeB, char orderC, std::initializer_list<Nd4jLong> shapeC);
+
+        void printHeader();
+    public:
+        BenchmarkHelper(unsigned int warmUpIterations = 10, unsigned int runIterations = 100);
+
+        void runOperationSuit(std::initializer_list<OpBenchmark*> benchmarks, const char *msg = nullptr);
+        void runOperationSuit(std::vector<OpBenchmark*> &benchmarks, bool postHeaders, const char *msg = nullptr);
+
+        void runOperationSuit(ScalarBenchmark *op, const std::function<void (ResultSet &, ResultSet &)>& func, const char *message = nullptr);
+        void runOperationSuit(TransformBenchmark *op, const std::function<void (ResultSet &, ResultSet &)>& func, const char *message = nullptr);
+        void runOperationSuit(ReductionBenchmark *op, const std::function<void (ResultSet &, ResultSet &)>& func, const char *message = nullptr);
+        void runOperationSuit(ReductionBenchmark *op, const std::function<void (ResultSet &, ResultSet &, ResultSet &)>& func, const char *message = nullptr);
+        void runOperationSuit(PairwiseBenchmark *op, const std::function<void (ResultSet &, ResultSet &, ResultSet &)>& func, const char *message = nullptr);
+
+
+        void runOperationSuit(TransformBenchmark *op, const std::function<void (Parameters &, ResultSet &, ResultSet &)>& func, ParametersBatch &parametersBatch, const char *message = nullptr);
+        void runOperationSuit(ScalarBenchmark *op, const std::function<void (Parameters &, ResultSet &, ResultSet &)>& func, ParametersBatch &parametersBatch, const char *message = nullptr);
+        void runOperationSuit(ReductionBenchmark *op, const std::function<void (Parameters &, ResultSet &, ResultSet &)>& func, ParametersBatch &parametersBatch, const char *message = nullptr);
+        void runOperationSuit(ReductionBenchmark *op, const std::function<void (Parameters &, ResultSet &, ResultSet &, ResultSet &)>& func, ParametersBatch &parametersBatch, const char *message = nullptr);
+        void runOperationSuit(PairwiseBenchmark *op, const std::function<void (Parameters &, ResultSet &, ResultSet &, ResultSet &)>& func, ParametersBatch &parametersBatch, const char *message = nullptr);
+
+        void runScalarSuit();
+
+        void runAllSuits();
+    };
+}
+
+
+#endif //DEV_TESTS_BENCHMARKHELPER_H
diff --git a/libnd4j/include/helpers/MKLDNNStream.h b/libnd4j/include/helpers/MKLDNNStream.h
@@ -21,7 +21,9 @@
 #ifndef LIBND4J_MKLDNNSTREAM_H
 #define LIBND4J_MKLDNNSTREAM_H
 
+#ifndef __STANDALONE_BUILD__
 #include "config.h"
+#endif
 
 #ifdef HAVE_MKLDNN
 #include <mkldnn.hpp>

diff --git a/libnd4j/include/helpers/OpBenchmark.h b/libnd4j/include/helpers/OpBenchmark.h
@@ -0,0 +1,52 @@
+//
+// Created by raver on 2/28/2019.
+//
+
+#ifndef DEV_TESTS_OPEXECUTIONER_H
+#define DEV_TESTS_OPEXECUTIONER_H
+
+#include <NativeOpExcutioner.h>
+#include <NDArray.h>
+#include <helpers/ShapeUtils.h>
+
+namespace nd4j {
+    class ND4J_EXPORT OpBenchmark {
+    protected:
+        int _opNum;
+        std::string _testName;
+        NDArray *_x = nullptr;
+        NDArray *_y = nullptr;
+        NDArray *_z = nullptr;
+        std::vector<int> _axis;
+    public:
+        OpBenchmark() = default;
+        OpBenchmark(std::string name, NDArray *x, NDArray *y, NDArray *z);
+        OpBenchmark(std::string name, NDArray *x, NDArray *z);
+        OpBenchmark(std::string name, NDArray *x, NDArray *z, std::initializer_list<int> axis);
+        OpBenchmark(std::string name, NDArray *x, NDArray *z, std::vector<int> axis);
+
+        void setOpNum(int opNum);
+        void setTestName(std::string testName);
+        void setX(NDArray *array);
+        void setY(NDArray *array);
+        void setZ(NDArray *array);
+        void setAxis(std::vector<int> axis);
+        void setAxis(std::initializer_list<int> axis);
+
+        NDArray& x();
+        int opNum();
+        std::string testName();
+
+        virtual std::string axis() = 0;
+        virtual std::string orders() = 0;
+        virtual std::string strides() = 0;
+        virtual std::string inplace() = 0;
+
+        virtual void executeOnce() = 0;
+
+        virtual OpBenchmark* clone() = 0;
+    };
+}
+
+
+#endif //DEV_TESTS_OPEXECUTIONER_H
diff --git a/libnd4j/include/helpers/ShapeUtils.h b/libnd4j/include/helpers/ShapeUtils.h
@@ -87,6 +87,7 @@ namespace nd4j {
         static std::string shapeAsString(const std::vector<Nd4jLong>& shape);
         static std::string shapeAsString(const Nd4jLong* shapeInfo);
         static std::string shapeAsString(const int rank, const Nd4jLong* shapeInfo);
+        static std::string strideAsString(const NDArray* array);
 
         // evaluate shapeInfo for diagonal array which is made using input arr elements as diagonal
         static Nd4jLong* evalDiagShapeInfo(const Nd4jLong* shapeInfo, nd4j::memory::Workspace* workspace);

diff --git a/libnd4j/include/helpers/benchmark/BasicSuit.h b/libnd4j/include/helpers/benchmark/BasicSuit.h
@@ -0,0 +1,33 @@
+/*******************************************************************************
+ * Copyright (c) 2015-2018 Skymind, Inc.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+/**
+ * @author raver119@gmail.com
+ */
+
+#ifndef DEV_TESTS_BASICSUIT_H
+#define DEV_TESTS_BASICSUIT_H
+
+namespace nd4j {
+    class BasicSuit {
+    protected:
+
+    public:
+
+    };
+}
+
+#endif //DEV_TESTS_BASICSUIT_H
diff --git a/libnd4j/include/helpers/benchmark/BoolParameters.h b/libnd4j/include/helpers/benchmark/BoolParameters.h
@@ -0,0 +1,48 @@
+/*******************************************************************************
+ * Copyright (c) 2015-2018 Skymind, Inc.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#ifndef DEV_TESTS_BOOLPARAMETERS_H
+#define DEV_TESTS_BOOLPARAMETERS_H
+
+#include <map>
+#include <vector>
+#include <string>
+#include "Parameters.h"
+#include "ParametersSpace.h"
+
+namespace nd4j {
+    class BoolParameters : public ParametersSpace {
+    protected:
+
+    public:
+        BoolParameters(std::string name) : ParametersSpace() {
+            _name = name;
+        }
+
+        std::vector<int> evaluate() override {
+            std::vector<int> result;
+            result.emplace_back(0);
+            result.emplace_back(1);
+            return result;
+        }
+    };
+}
+
+#endif //DEV_TESTS_PARAMETERSPACE_H
diff --git a/libnd4j/include/helpers/benchmark/IntParameters.h b/libnd4j/include/helpers/benchmark/IntParameters.h
@@ -0,0 +1,55 @@
+/*******************************************************************************
+ * Copyright (c) 2015-2018 Skymind, Inc.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#ifndef DEV_TESTS_INTPARAMETERS_H
+#define DEV_TESTS_INTPARAMETERS_H
+
+#include <map>
+#include <vector>
+#include <string>
+#include "Parameters.h"
+#include "ParametersSpace.h"
+
+namespace nd4j {
+    class IntParameters : public ParametersSpace {
+    protected:
+        int _start;
+        int _stop;
+        int _step;
+
+    public:
+        IntParameters(std::string name, int start, int stop, int step = 1) : ParametersSpace() {
+            _start = start;
+            _stop = stop;
+            _step = step;
+            _name = name;
+        }
+
+        std::vector<int> evaluate() override {
+            std::vector<int> result;
+            for (int e = _start; e < _stop; e += _step) {
+               result.emplace_back(e);
+            }
+            return result;
+        }
+    };
+}
+
+#endif //DEV_TESTS_PARAMETERSPACE_H
diff --git a/libnd4j/include/helpers/benchmark/IntPowerParameters.h b/libnd4j/include/helpers/benchmark/IntPowerParameters.h
@@ -0,0 +1,57 @@
+/*******************************************************************************
+ * Copyright (c) 2015-2018 Skymind, Inc.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+//
+// @author raver119@gmail.com
+//
+
+#ifndef DEV_TESTS_INTPOWERPARAMETERS_H
+#define DEV_TESTS_INTPOWERPARAMETERS_H
+
+#include <map>
+#include <vector>
+#include <string>
+#include "Parameters.h"
+#include "ParametersSpace.h"
+
+namespace nd4j {
+    class IntPowerParameters : public ParametersSpace {
+    protected:
+        int _base;
+        int _start;
+        int _stop;
+        int _step;
+
+    public:
+        IntPowerParameters(std::string name, int base, int start, int stop, int step = 1) : ParametersSpace() {
+            _base = base;
+            _start = start;
+            _stop = stop;
+            _step = step;
+            _name = name;
+        }
+
+        std::vector<int> evaluate() override {
+            std::vector<int> result;
+            for (int e = _start; e < _stop; e += _step) {
+               result.emplace_back(nd4j::math::nd4j_pow<double, double, int>(_base, e));
+            }
+            return result;
+        }
+    };
+}
+
+#endif //DEV_TESTS_PARAMETERSPACE_H