Rebase sycl-nliber2 into develop

.jenkins

@@ -24,6 +24,36 @@ pipeline {
 
         stage('Build') {
             parallel {
+                stage('SYCL-OneAPI') {
+                    agent {
+                        dockerfile {
+                            filename 'Dockerfile.sycl'
+                            dir 'scripts/docker'
+                            additionalBuildArgs '--pull --build-arg BASE=intel/oneapi-basekit:devel-ubuntu18.04'
+                            label 'docker'
+                            args '-v /tmp/ccache.kokkos:/tmp/ccache'
+                        }
+                    }
+                    steps {
+                        sh 'ccache --zero-stats'
+                        sh '''rm -rf build && mkdir -p build && cd build && \
+                              cmake \
+                                -DCMAKE_BUILD_TYPE=Debug \
+                                -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+                                -DCMAKE_CXX_COMPILER=clang++ \
+                                -DKokkos_ENABLE_COMPILER_WARNINGS=OFF \
+                                -DKokkos_ENABLE_TESTS=ON \
+                                -DKokkos_ENABLE_SYCL=ON \
+                                -DCMAKE_CXX_STANDARD=17 \
+                              .. && \
+                              make -j8 && ctest --output-on-failure'''
+                    }
+                    post {
+                        always {
+                            sh 'ccache --show-stats'
+                        }
+                    }
+                }
                 stage('HIP-ROCm-3.8-C++14') {
                     agent {
                         dockerfile {

Makefile.kokkos

@@ -5,7 +5,7 @@ KOKKOS_VERSION_MINOR = 2
 KOKKOS_VERSION_PATCH = 0
 KOKKOS_VERSION = $(shell echo $(KOKKOS_VERSION_MAJOR)*10000+$(KOKKOS_VERSION_MINOR)*100+$(KOKKOS_VERSION_PATCH) | bc)
 
-# Options: Cuda,HIP,OpenMP,Pthread,Serial
+# Options: Cuda,HIP,OpenMP,Pthread,Serial,SYCL
 #KOKKOS_DEVICES ?= "OpenMP"
 KOKKOS_DEVICES ?= "Pthread"
 # Options: 
@@ -102,6 +102,7 @@ endif
 # Check for other Execution Spaces.
 KOKKOS_INTERNAL_USE_CUDA := $(call kokkos_has_string,$(KOKKOS_DEVICES),Cuda)
 KOKKOS_INTERNAL_USE_HIP := $(call kokkos_has_string,$(KOKKOS_DEVICES),HIP)
+KOKKOS_INTERNAL_USE_SYCL := $(call kokkos_has_string,$(KOKKOS_DEVICES),SYCL)
 KOKKOS_INTERNAL_USE_OPENMPTARGET := $(call kokkos_has_string,$(KOKKOS_DEVICES),OpenMPTarget)
 
 KOKKOS_DEVICELIST =
@@ -123,6 +124,9 @@ endif
 ifeq ($(KOKKOS_INTERNAL_USE_HIP), 1)
   KOKKOS_DEVICELIST += HIP
 endif
+ifeq ($(KOKKOS_INTERNAL_USE_SYCL), 1)
+  KOKKOS_DEVICELIST += SYCL
+endif
 ifeq ($(KOKKOS_INTERNAL_USE_OPENMPTARGET), 1)
   KOKKOS_DEVICELIST += OPENMPTARGET
   KOKKOS_INTERNAL_HAVE_CXX17_OR_NEWER := $(shell expr $(KOKKOS_INTERNAL_ENABLE_CXX17) \
@@ -161,6 +165,8 @@ KOKKOS_INTERNAL_COMPILER_CLANG       := $(call kokkos_has_string,$(KOKKOS_CXX_VE
 KOKKOS_INTERNAL_COMPILER_APPLE_CLANG := $(call kokkos_has_string,$(KOKKOS_CXX_VERSION),Apple LLVM)
 KOKKOS_INTERNAL_COMPILER_HCC         := $(call kokkos_has_string,$(KOKKOS_CXX_VERSION),HCC)
 KOKKOS_INTERNAL_COMPILER_GCC         := $(call kokkos_has_string,$(KOKKOS_CXX_VERSION),GCC)
+KOKKOS_INTERNAL_COMPILER_COMPUTEPP   := $(call kokkos_has_string,$(KOKKOS_CXX_VERSION),Codeplay)
+KOKKOS_INTERNAL_COMPILER_DPCPP       := $(call kokkos_has_string,$(KOKKOS_CXX_VERSION),DPC)
 
 # Check Host Compiler if using NVCC through nvcc_wrapper
 ifeq ($(KOKKOS_INTERNAL_COMPILER_NVCC), 1)
@@ -174,6 +180,21 @@ ifeq ($(KOKKOS_INTERNAL_COMPILER_NVCC), 1)
   endif
 endif
 
+ifeq ($(KOKKOS_INTERNAL_COMPILER_COMPUTEPP), 1)
+  KOKKOS_INTERNAL_COMPUTEPP_PATH := $(shell which compute++)
+  ifeq ($(origin COMPUTEPP_PATH), undefined)
+    COMPUTEPP_PATH = $(KOKKOS_INTERNAL_COMPUTEPP_PATH:/bin/compute++=)
+  endif
+  ifeq ($(COMPUTEPP_PATH),)
+    COMPUTEPP_PATH = $(KOKKOS_INTERNAL_COMPUTEPP_PATH:/bin/compute++=)
+  endif
+  KOKKOS_INTERNAL_COMPILER_CLANG = 1
+endif
+
+ifeq ($(KOKKOS_INTERNAL_COMPILER_DPCPP), 1)
+  KOKKOS_INTERNAL_COMPILER_CLANG = 1
+endif
+
 ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 2)
   KOKKOS_INTERNAL_COMPILER_CLANG = 1
 endif
@@ -464,6 +485,10 @@ ifeq ($(KOKKOS_INTERNAL_USE_HIP), 1)
   tmp := $(call kokkos_append_header,'$H''define KOKKOS_ENABLE_HIP')
 endif
 
+ifeq ($(KOKKOS_INTERNAL_USE_SYCL), 1)
+  tmp := $(call kokkos_append_header,'\#define KOKKOS_ENABLE_SYCL')
+endif
+
 ifeq ($(KOKKOS_INTERNAL_USE_OPENMPTARGET), 1)
   tmp := $(call kokkos_append_header,'$H''define KOKKOS_ENABLE_OPENMPTARGET')
   ifeq ($(KOKKOS_INTERNAL_COMPILER_GCC), 1)
@@ -1244,6 +1269,24 @@ ifeq ($(KOKKOS_INTERNAL_USE_HPX), 1)
   KOKKOS_TPL_LIBRARY_NAMES += hpx
 endif
 
+# Add SYCL specific flags
+ifeq ($(KOKKOS_INTERNAL_USE_SYCL), 1)
+  KOKKOS_SRC += $(wildcard $(KOKKOS_PATH)/core/src/SYCL/*.cpp)
+  KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/core/src/SYCL/*.hpp)
+  ifeq ($(KOKKOS_INTERNAL_COMPILER_COMPUTEPP), 1)
+    KOKKOS_CPPFLAGS += -I$(COMPUTEPP_PATH)/include
+    KOKKOS_CPPFLAGS += -I$(OPENCL_PATH)/include
+    KOKKOS_LDFLAGS  += -L$(COMPUTEPP_PATH)/lib
+    KOKKOS_LIBS     += -lComputeCpp -lOpenCL
+  endif
+
+  ifeq ($(KOKKOS_INTERNAL_COMPILER_DPCPP), 1)
+    KOKKOS_CPPFLAGS += -fsycl -fsycl-unnamed-lambda -DSYCL_JUST_DONT_NAME_KERNELS
+    KOKKOS_LDFLAGS += -fsycl
+    KOKKOS_LIBS  += -lOpenCL -lsycl
+  endif
+endif
+
 # Explicitly set the GCC Toolchain for Clang.
 ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1)
   KOKKOS_INTERNAL_GCC_PATH = $(shell which g++)

Makefile.targets

@@ -57,6 +57,15 @@ Kokkos_HIP_Locks.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/HIP/Kokkos_HIP
 	$(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/HIP/Kokkos_HIP_Locks.cpp
 endif
 
+ifeq ($(KOKKOS_INTERNAL_USE_SYCL), 1)
+Kokkos_SYCL.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/SYCL/Kokkos_SYCL.cpp
+	$(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/SYCL/Kokkos_SYCL.cpp
+Kokkos_SYCL_Space.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/SYCL/Kokkos_SYCL_Space.cpp
+	$(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/SYCL/Kokkos_SYCL_Space.cpp
+Kokkos_SYCL_Instance.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/SYCL/Kokkos_SYCL_Instance.cpp
+	$(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/SYCL/Kokkos_SYCL_Instance.cpp
+endif
+
 ifeq ($(KOKKOS_INTERNAL_USE_PTHREADS), 1)
 Kokkos_ThreadsExec_base.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/Threads/Kokkos_ThreadsExec_base.cpp
 	$(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/Threads/Kokkos_ThreadsExec_base.cpp

bin/sycl_cp_wrapper

@@ -0,0 +1,57 @@
+#!/bin/bash
+FLAGS=""
+dry_run=0
+host_compiler=clang++
+
+cpp_file=""
+
+while [ $# -gt 0 ]
+do
+  case $1 in
+    #show the executed command
+    --show)
+      dry_run=1
+    ;;
+    --version)
+      compute++ --version
+      exit
+    ;;
+    -ccbin)
+      shift
+      host_compiler=$1
+    ;;
+    #Source files
+    *.cpp|*.cxx|*.cc|*.C|*.c++)
+      cpp_file="$1"
+    ;;
+    #All other arguments
+    *)
+     FLAGS+=" $1"
+    ;; 
+  esac
+  shift
+done
+
+SYCLFLAGS="-sycl -intelspirmetadata -sycl-target spir64"
+
+sycl_file_full=`echo ${cpp_file} | sed 's|\.cpp|\.sycl|g'`
+sycl_file=`echo $(basename "${sycl_file_full%.*}")`.sycl
+
+link=`echo ${#cpp_file}`
+
+if [ ${dry_run} -eq 1 ]; then
+  if [ ${link} -eq 0 ]; then
+    echo "${host_compiler} ${FLAGS}"
+  else
+    echo "compute++ ${SYCLFLAGS} ${FLAGS} ${cpp_file}"
+    echo "${host_compiler} ${FLAGS} -include ${sycl_file} ${cpp_file}"
+  fi
+else
+  if [ ${link} -eq 0 ]; then
+    ${host_compiler} ${FLAGS}
+  else
+    compute++ ${SYCLFLAGS} ${FLAGS} ${cpp_file}
+    ${host_compiler} ${FLAGS} -include ${sycl_file} ${cpp_file}
+  fi
+fi
+

cmake/KokkosCore_config.h.in

@@ -21,6 +21,7 @@
 #cmakedefine KOKKOS_ENABLE_HPX
 #cmakedefine KOKKOS_ENABLE_MEMKIND
 #cmakedefine KOKKOS_ENABLE_LIBRT
+#cmakedefine KOKKOS_ENABLE_SYCL
 
 #ifndef __CUDA_ARCH__
 #cmakedefine KOKKOS_ENABLE_TM

cmake/kokkos_arch.cmake

@@ -359,6 +359,16 @@ IF (Kokkos_ENABLE_HIP)
 ENDIF()
 
 
+IF (Kokkos_ENABLE_SYCL)
+  COMPILER_SPECIFIC_FLAGS(
+    DEFAULT -fsycl
+  )
+  COMPILER_SPECIFIC_OPTIONS(
+    DEFAULT -fsycl-unnamed-lambda
+  )
+ENDIF()
+
+
 SET(CUDA_ARCH_ALREADY_SPECIFIED "")
 FUNCTION(CHECK_CUDA_ARCH ARCH FLAG)
   IF(KOKKOS_ARCH_${ARCH})
@@ -486,7 +496,7 @@ ENDIF()
 #Let's just always print things
 MESSAGE(STATUS "Execution Spaces:")
 
-FOREACH (_BACKEND CUDA OPENMPTARGET HIP)
+FOREACH (_BACKEND CUDA OPENMPTARGET HIP SYCL)
   IF(KOKKOS_ENABLE_${_BACKEND})
     IF(_DEVICE_PARALLEL)
       MESSAGE(FATAL_ERROR "Multiple device parallel execution spaces are not allowed! "

cmake/kokkos_enable_devices.cmake

@@ -128,3 +128,6 @@ KOKKOS_DEVICE_OPTION(SERIAL ${SERIAL_DEFAULT} HOST "Whether to build serial back
 KOKKOS_DEVICE_OPTION(HPX OFF HOST "Whether to build HPX backend (experimental)")
 
 KOKKOS_DEVICE_OPTION(HIP OFF DEVICE "Whether to build HIP backend")
+
+KOKKOS_DEVICE_OPTION(SYCL OFF DEVICE "Whether to build SYCL backend")
+

containers/src/Kokkos_DynRankView.hpp

@@ -1374,7 +1374,7 @@ struct DynRankSubviewTag {};
 namespace Impl {
 
 template <class SrcTraits, class... Args>
-struct ViewMapping<
+class ViewMapping<
     typename std::enable_if<
         (std::is_same<typename SrcTraits::specialize, void>::value &&
          (std::is_same<typename SrcTraits::array_layout,

core/perf_test/CMakeLists.txt

@@ -56,6 +56,46 @@ IF(Kokkos_ENABLE_HIP)
   )
 ENDIF()
 
+# FIXME_SYCL
+IF(Kokkos_ENABLE_SYCL)
+  LIST(REMOVE_ITEM SOURCES
+    PerfTestGramSchmidt.cpp
+    PerfTestHexGrad.cpp
+    PerfTest_CustomReduction.cpp
+    PerfTest_ExecSpacePartitioning.cpp
+    PerfTest_ViewCopy_a123.cpp
+    PerfTest_ViewCopy_b123.cpp
+    PerfTest_ViewCopy_c123.cpp
+    PerfTest_ViewCopy_d123.cpp
+    PerfTest_ViewCopy_a45.cpp
+    PerfTest_ViewCopy_b45.cpp
+    PerfTest_ViewCopy_c45.cpp
+    PerfTest_ViewCopy_d45.cpp
+    PerfTest_ViewCopy_a6.cpp
+    PerfTest_ViewCopy_b6.cpp
+    PerfTest_ViewCopy_c6.cpp
+    PerfTest_ViewCopy_d6.cpp
+    PerfTest_ViewCopy_a7.cpp
+    PerfTest_ViewCopy_b7.cpp
+    PerfTest_ViewCopy_c7.cpp
+    PerfTest_ViewCopy_d7.cpp
+    PerfTest_ViewCopy_a8.cpp
+    PerfTest_ViewCopy_b8.cpp
+    PerfTest_ViewCopy_c8.cpp
+    PerfTest_ViewCopy_d8.cpp
+    PerfTest_ViewFill_123.cpp
+    PerfTest_ViewFill_45.cpp
+    PerfTest_ViewFill_6.cpp
+    PerfTest_ViewFill_7.cpp
+    PerfTest_ViewFill_8.cpp
+    PerfTest_ViewResize_123.cpp
+    PerfTest_ViewResize_45.cpp
+    PerfTest_ViewResize_6.cpp
+    PerfTest_ViewResize_7.cpp
+    PerfTest_ViewResize_8.cpp
+  )
+ENDIF()
+
 IF(Kokkos_ENABLE_OPENMPTARGET)
 # FIXME OPENMPTARGET requires TeamPolicy Reductions and Custom Reduction
   LIST(REMOVE_ITEM SOURCES
@@ -83,25 +123,31 @@ IF(NOT KOKKOS_CXX_COMPILER_ID STREQUAL "MSVC")
   )
 ENDIF()
 
-KOKKOS_ADD_EXECUTABLE_AND_TEST(
-  PerformanceTest_Atomic
-  SOURCES test_atomic.cpp
-  CATEGORIES PERFORMANCE
-)
-
-IF(NOT KOKKOS_ENABLE_CUDA OR KOKKOS_ENABLE_CUDA_LAMBDA)
+# FIXME_SYCL
+IF(NOT Kokkos_ENABLE_SYCL)
   KOKKOS_ADD_EXECUTABLE_AND_TEST(
-    PerformanceTest_Atomic_MinMax
-    SOURCES test_atomic_minmax_simple.cpp
+    PerformanceTest_Atomic
+    SOURCES test_atomic.cpp
     CATEGORIES PERFORMANCE
   )
 ENDIF()
 
-KOKKOS_ADD_EXECUTABLE_AND_TEST(
-  PerformanceTest_Mempool
-  SOURCES test_mempool.cpp
-  CATEGORIES PERFORMANCE
-)
+# FIXME_SYCL
+IF(NOT KOKKOS_ENABLE_SYCL)
+  IF(NOT KOKKOS_ENABLE_CUDA OR KOKKOS_ENABLE_CUDA_LAMBDA)
+    KOKKOS_ADD_EXECUTABLE_AND_TEST(
+      PerformanceTest_Atomic_MinMax
+      SOURCES test_atomic_minmax_simple.cpp
+      CATEGORIES PERFORMANCE
+    )
+  ENDIF()
+
+  KOKKOS_ADD_EXECUTABLE_AND_TEST(
+    PerformanceTest_Mempool
+    SOURCES test_mempool.cpp
+    CATEGORIES PERFORMANCE
+  )
+ENDIF()
 
 IF(NOT Kokkos_ENABLE_OPENMPTARGET)
 # FIXME OPENMPTARGET needs tasking

core/src/CMakeLists.txt

@@ -60,6 +60,11 @@ ELSE()
   LIST(REMOVE_ITEM KOKKOS_CORE_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/impl/Kokkos_Serial_task.cpp)
 ENDIF()
 
+IF (KOKKOS_ENABLE_SYCL)
+  APPEND_GLOB(KOKKOS_CORE_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/SYCL/*.cpp)
+  APPEND_GLOB(KOKKOS_CORE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/SYCL/*.hpp)
+ENDIF()
+
 KOKKOS_ADD_LIBRARY(
   kokkoscore
   SOURCES ${KOKKOS_CORE_SRCS}

core/src/Kokkos_Atomic.hpp

@@ -90,6 +90,10 @@
 
 #define KOKKOS_ENABLE_HIP_ATOMICS
 
+#elif defined(KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_SYCL_GPU)
+
+//#define KOKKOS_ENABLE_SYCL_ATOMICS
+
 #endif
 
 #if !defined(KOKKOS_ENABLE_GNU_ATOMICS) &&    \
@@ -176,6 +180,10 @@ inline const char* atomic_query_version() {
 #include <HIP/Kokkos_HIP_Atomic.hpp>
 #endif
 
+#if defined(KOKKOS_ENABLE_SYCL)
+#include <SYCL/Kokkos_SYCL_Atomic.hpp>
+#endif
+
 #ifdef _WIN32
 #include "impl/Kokkos_Atomic_Windows.hpp"
 #endif

core/src/Kokkos_Core.hpp

@@ -79,6 +79,10 @@
 #include <Kokkos_HIP.hpp>
 #endif
 
+#if defined(KOKKOS_ENABLE_SYCL)
+#include <Kokkos_SYCL.hpp>
+#endif
+
 #include <Kokkos_AnonymousSpace.hpp>
 #include <Kokkos_Pair.hpp>
 #include <Kokkos_MemoryPool.hpp>