draft #5 Added google benchmark to kokkos kernel and to the CI

.github/workflows/osx.yml

@@ -109,4 +109,4 @@ jobs:
 
       - name: test
         working-directory: kokkos-kernels/build
-        run: ctest -j2 --output-on-failure --timeout 3600
+        run: ctest -j2 --output-on-failure --timeout 3600

perf_test/BenchmarkMain.cpp

@@ -0,0 +1,61 @@
+/*
+//@HEADER
+// ************************************************************************
+//
+//                        Kokkos v. 3.0
+//       Copyright (2020) National Technology & Engineering
+//               Solutions of Sandia, LLC (NTESS).
+//
+// Under the terms of Contract DE-NA0003525 with NTESS,
+// the U.S. Government retains certain rights in this software.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+//
+// 3. Neither the name of the Corporation nor the names of the
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Questions? Contact Siva Rajamanickam (srajama@sandia.gov)
+//
+// ************************************************************************
+//@HEADER
+*/
+
+#include <benchmark/benchmark.h>
+
+#include <Benchmark_Context.hpp>
+#include <Kokkos_Core.hpp>
+
+int main(int argc, char** argv) {
+  Kokkos::initialize(argc, argv);
+  benchmark::Initialize(&argc, argv);
+  benchmark::SetDefaultTimeUnit(benchmark::kSecond);
+  KokkosKernelsBenchmark::add_benchmark_context(true);
+
+  benchmark::RunSpecifiedBenchmarks();
+
+  benchmark::Shutdown();
+  Kokkos::finalize();
+  return 0;
+}

perf_test/Benchmark_Context.hpp

@@ -0,0 +1,99 @@
+/*
+//@HEADER
+// ************************************************************************
+//
+//                        Kokkos v. 3.0
+//       Copyright (2020) National Technology & Engineering
+//               Solutions of Sandia, LLC (NTESS).
+//
+// Under the terms of Contract DE-NA0003525 with NTESS,
+// the U.S. Government retains certain rights in this software.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+//
+// 3. Neither the name of the Corporation nor the names of the
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Questions? Contact Siva Rajamanickam (srajama@sandia.gov)
+//
+// ************************************************************************
+//@HEADER
+*/
+
+#ifndef KOKKOSKERNELS_PERFTEST_BENCHMARK_CONTEXT_HPP
+#define KOKKOSKENERLS_PERFTEST_BENCHMARK_CONTEXT_HPP
+
+#include <string>
+
+#include <benchmark/benchmark.h>
+
+#include <Kokkos_Core.hpp>
+
+namespace KokkosKernelsBenchmark {
+
+/// \brief Remove unwanted spaces and colon signs from input string. In case of
+/// invalid input it will return an empty string.
+std::string remove_unwanted_characters(std::string str) {
+  auto from = str.find_first_not_of(" :");
+  auto to   = str.find_last_not_of(" :");
+
+  if (from == std::string::npos || to == std::string::npos) {
+    return "";
+  }
+
+  // return extracted part of string without unwanted spaces and colon signs
+  return str.substr(from, to + 1);
+}
+
+/// \brief Extract all key:value pairs from kokkos configuration and add it to
+/// the benchmark context
+void add_kokkos_configuration(bool verbose) {
+  std::ostringstream msg;
+  Kokkos::print_configuration(msg, verbose);
+
+  // Iterate over lines returned from kokkos and extract key:value pairs
+  std::stringstream ss{msg.str()};
+  for (std::string line; std::getline(ss, line, '\n');) {
+    auto found = line.find_first_of(':');
+    if (found != std::string::npos) {
+      auto val = remove_unwanted_characters(line.substr(found + 1));
+      // Ignore line without value, for example a category name
+      if (!val.empty()) {
+        benchmark::AddCustomContext(
+            remove_unwanted_characters(line.substr(0, found)), val);
+      }
+    }
+  }
+}
+
+/// \brief Gather all context information and add it to benchmark context data
+void add_benchmark_context(bool verbose = false) {
+  // Add Kokkos configuration to benchmark context data
+  add_kokkos_configuration(verbose);
+}
+
+}  // namespace KokkosKernelsBenchmark
+
+#endif

perf_test/CMakeLists.txt

@@ -50,5 +50,102 @@ if (KokkosKernels_ENABLE_PERFTESTS)
     ADD_COMPONENT_SUBDIRECTORY(blas)
     ADD_SUBDIRECTORY(performance)
     #ADD_SUBDIRECTORY(common)
+
 endif()
 
+IF(KokkosKernels_ENABLE_BENCHMARK)
+
+    IF (KOKKOSKERNELS_HAS_TRILINOS)
+      message(FATAL_ERROR "Benchmarks are not supported when building as part of Trilinos")
+    ENDIF()
+
+    find_package(benchmark QUIET)
+
+    IF(benchmark_FOUND)
+      MESSAGE(STATUS "Using google benchmark found in ${benchmark_DIR}")
+    ELSE()
+      message(STATUS "No installed google benchmark found, fetching from GitHub")
+      include(FetchContent)
+      SET(BENCHMARK_ENABLE_TESTING OFF)
+
+      list(APPEND CMAKE_MESSAGE_INDENT "    ")
+      #Note: recent bug (google/benchmark#1441) is preventing us from using
+      # the latest benchmark release.
+      FetchContent_Declare(
+        googlebenchmark
+        URL https://github.com/google/benchmark/archive/refs/tags/v1.6.2.tar.gz
+        URL_HASH MD5=14d14849e075af116143a161bc3b927b
+      )
+      FetchContent_MakeAvailable(googlebenchmark)
+      list(POP_BACK CMAKE_MESSAGE_INDENT)
+
+      include_directories(${benchmark_SOURCE_DIR}/include)
+
+      # Suppress clang-tidy diagnostics on code that we do not have control over
+      IF(CMAKE_CXX_CLANG_TIDY)
+        SET_TARGET_PROPERTIES(benchmark PROPERTIES CXX_CLANG_TIDY "")
+      ENDIF()
+
+      target_compile_options(benchmark PRIVATE -w)
+      target_compile_options(benchmark_main PRIVATE -w)
+    ENDIF()
+
+    KOKKOSKERNELS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR})
+
+    FUNCTION(KOKKOSKERNELS_ADD_BENCHMARK NAME)
+      CMAKE_PARSE_ARGUMENTS(
+        BENCHMARK
+        ""
+        ""
+        "SOURCES"
+        ${ARGN}
+      )
+      IF(DEFINED BENCHMARK_UNPARSED_ARGUMENTS)
+        MESSAGE(
+          WARNING
+          "Unexpected arguments when adding a benchmark: "
+          ${BENCHMARK_UNPARSED_ARGUMENTS}
+        )
+      ENDIF()
+
+      SET(BENCHMARK_NAME ${PACKAGE_NAME}_${NAME})
+
+      ADD_EXECUTABLE(
+        ${BENCHMARK_NAME}
+        ${BENCHMARK_SOURCES}
+      )
+      TARGET_LINK_LIBRARIES(
+        ${BENCHMARK_NAME}
+        PRIVATE benchmark::benchmark Kokkos::kokkoskernels
+      )
+      FOREACH(SOURCE_FILE ${BENCHMARK_SOURCES})
+        SET_SOURCE_FILES_PROPERTIES(
+          ${SOURCE_FILE}
+          PROPERTIES LANGUAGE CXX
+        )
+      ENDFOREACH()
+
+      STRING(TIMESTAMP BENCHMARK_TIME "%Y-%m-%d_T%H-%M-%S" UTC)
+      SET(
+        BENCHMARK_ARGS
+        --benchmark_counters_tabular=true
+        --benchmark_out=${BENCHMARK_NAME}_${BENCHMARK_TIME}.json
+      )
+
+      ADD_TEST(
+        NAME ${BENCHMARK_NAME}
+        COMMAND ${BENCHMARK_NAME} ${BENCHMARK_ARGS}
+      )
+    ENDFUNCTION()
+
+    SET(
+      BENCHMARK_SOURCES
+      BenchmarkMain.cpp
+    )
+
+    KOKKOSKERNELS_ADD_BENCHMARK(
+      PerformanceTest_Benchmark
+      SOURCES ${BENCHMARK_SOURCES}
+    )
+
+endif()