Merge pull request #423 from masterleinad/sycl_support

arborx · Dec 23, 2020 · 6368455 · 6368455
2 parents fe6ed46 + ed6ce4d
commit 6368455
Show file tree

Hide file tree

Showing 6 changed files with 188 additions and 8 deletions.
diff --git a/.jenkins b/.jenkins
@@ -385,6 +385,64 @@ pipeline {
                         }
                     }
                 }
+
+                stage('SYCL') {
+                    agent {
+                        dockerfile {
+                            filename "Dockerfile.sycl"
+                            dir "docker"
+                            args '-v /tmp/ccache.kokkos:/tmp/ccache'
+                            label 'NVIDIA_Tesla_V100-PCIE-32GB && nvidia-docker'
+                        }
+                    }
+                    steps {
+                        sh 'ccache --zero-stats'
+                        sh 'rm -rf build && mkdir -p build'
+                        dir('build') {
+                            sh '''
+                                cmake \
+                                    -D CMAKE_INSTALL_PREFIX=$ARBORX_DIR \
+                                    -D CMAKE_BUILD_TYPE=Release \
+                                    -D CMAKE_CXX_COMPILER=clang++ \
+                                    -D CMAKE_CXX_EXTENSIONS=OFF \
+                                    -D CMAKE_CXX_FLAGS="-Wpedantic -Wall -Wextra -Wno-unknown-cuda-version -fsycl -fsycl-targets=nvptx64-nvidia-cuda-sycldevice" \
+                                    -D CMAKE_PREFIX_PATH="$KOKKOS_DIR;$BOOST_DIR;$BENCHMARK_DIR" \
+                                    -D ARBORX_ENABLE_MPI=ON \
+                                    -D MPIEXEC_PREFLAGS="--allow-run-as-root" \
+                                    -D MPIEXEC_MAX_NUMPROCS=4 \
+                                    -D ARBORX_ENABLE_TESTS=ON \
+                                    -D ARBORX_ENABLE_EXAMPLES=ON \
+                                    -D ARBORX_ENABLE_BENCHMARKS=ON \
+                                ..
+                            '''
+                            sh 'make -j8 VERBOSE=1'
+                            sh 'ctest $CTEST_OPTIONS'
+                        }
+                    }
+                    post {
+                        always {
+                            sh 'ccache --show-stats'
+                            xunit reduceLog: false, tools:[CTest(deleteOutputFiles: true, failIfNotNew: true, pattern: 'build/Testing/**/Test.xml', skipNoTestFiles: false, stopProcessingIfError: true)]
+                        }
+                        success {
+                            sh 'cd build && make install'
+                            sh 'rm -rf test_install && mkdir -p test_install'
+                            dir('test_install') {
+                                sh 'cp -r ../examples .'
+                                sh '''
+                                    cmake \
+                                        -D CMAKE_BUILD_TYPE=Release \
+                                        -D CMAKE_CXX_COMPILER=clang++ \
+                                        -D CMAKE_CXX_EXTENSIONS=OFF \
+                                        -D CMAKE_PREFIX_PATH="$KOKKOS_DIR;$ARBORX_DIR" \
+                                    examples \
+                                '''
+                                sh 'make VERBOSE=1'
+                                sh 'make test'
+                            }
+                        }
+                    }
+                }
             }
         }
     }

diff --git a/benchmarks/bvh_driver/bvh_driver.cpp b/benchmarks/bvh_driver/bvh_driver.cpp
@@ -588,7 +588,7 @@ int main(int argc, char *argv[])
       throw std::runtime_error("OpenMPTarget backend not available!");
 #endif
 
-#ifdef KOKKOS_ENABLE_
+#ifdef KOKKOS_ENABLE_SYCL
     if (spec.backends == "all" || spec.backends == "sycl")
       register_benchmark<
           Kokkos::Experimental::SYCL,

diff --git a/docker/Dockerfile.sycl b/docker/Dockerfile.sycl
@@ -0,0 +1,116 @@
+ARG BASE=nvidia/cuda:10.2-devel
+FROM $BASE
+
+ARG NPROCS=4
+
+RUN apt-get update && apt-get install -y \
+        bc \
+        wget \
+        ccache \
+        ninja-build \
+        python3 \
+        git \
+        vim \
+        jq \
+        libopenmpi-dev \
+        && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN KEYDUMP_URL=https://cloud.cees.ornl.gov/download && \
+    KEYDUMP_FILE=keydump && \
+    wget --quiet ${KEYDUMP_URL}/${KEYDUMP_FILE} && \
+    wget --quiet ${KEYDUMP_URL}/${KEYDUMP_FILE}.sig && \
+    gpg --import ${KEYDUMP_FILE} && \
+    gpg --verify ${KEYDUMP_FILE}.sig ${KEYDUMP_FILE} && \
+    rm ${KEYDUMP_FILE}*
+
+ARG CMAKE_VERSION=3.19.0
+ENV CMAKE_DIR=/opt/cmake
+RUN CMAKE_KEY=2D2CEF1034921684 && \
+    CMAKE_URL=https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION} && \
+    CMAKE_SCRIPT=cmake-${CMAKE_VERSION}-Linux-x86_64.sh && \
+    CMAKE_SHA256=cmake-${CMAKE_VERSION}-SHA-256.txt && \
+    wget --quiet ${CMAKE_URL}/${CMAKE_SHA256} && \
+    wget --quiet ${CMAKE_URL}/${CMAKE_SHA256}.asc && \
+    wget --quiet ${CMAKE_URL}/${CMAKE_SCRIPT} && \
+    gpg --keyserver pool.sks-keyservers.net --recv-keys ${CMAKE_KEY} && \
+    gpg --verify ${CMAKE_SHA256}.asc ${CMAKE_SHA256} && \
+    grep ${CMAKE_SCRIPT} ${CMAKE_SHA256} | sha256sum --check && \
+    mkdir -p ${CMAKE_DIR} && \
+    sh ${CMAKE_SCRIPT} --skip-license --prefix=${CMAKE_DIR} && \
+    rm cmake*
+ENV PATH=${CMAKE_DIR}/bin:$PATH
+
+ENV SYCL_DIR=/opt/sycl
+RUN SYCL_VERSION=20201207 && \
+    SYCL_URL=https://github.com/intel/llvm/archive/sycl-nightly && \
+    SYCL_ARCHIVE=${SYCL_VERSION}.tar.gz && \
+    SCRATCH_DIR=/scratch && mkdir -p ${SCRATCH_DIR} && cd ${SCRATCH_DIR} && \
+    wget --quiet ${SYCL_URL}/${SYCL_ARCHIVE} && \
+    mkdir llvm && \
+    tar -xf ${SYCL_ARCHIVE} -C llvm --strip-components=1 && \
+    cd llvm && \
+    python3 buildbot/configure.py --cuda && \
+    python3 buildbot/compile.py && \
+    mkdir -p ${SYCL_DIR} && \
+    mv ${SCRATCH_DIR}/llvm/build/install/* ${SYCL_DIR} && \
+    echo "${SYCL_DIR}/lib" > /etc/ld.so.conf.d/sycl.conf && ldconfig && \
+    rm -rf ${SCRATCH_DIR}
+ENV PATH=${SYCL_DIR}/bin:$PATH
+
+# Install Boost
+ENV BOOST_DIR=/opt/boost
+RUN BOOST_VERSION=1.72.0 && \
+    BOOST_VERSION_UNDERSCORE=$(echo "$BOOST_VERSION" | sed -e "s/\./_/g") && \
+    BOOST_KEY=379CE192D401AB61 && \
+    BOOST_URL=https://dl.bintray.com/boostorg/release/${BOOST_VERSION}/source && \
+    BOOST_ARCHIVE=boost_${BOOST_VERSION_UNDERSCORE}.tar.bz2 && \
+    SCRATCH_DIR=/scratch && mkdir -p ${SCRATCH_DIR} && cd ${SCRATCH_DIR} && \
+    wget --quiet ${BOOST_URL}/${BOOST_ARCHIVE} && \
+    wget --quiet ${BOOST_URL}/${BOOST_ARCHIVE}.asc && \
+    wget --quiet ${BOOST_URL}/${BOOST_ARCHIVE}.json && \
+    wget --quiet ${BOOST_URL}/${BOOST_ARCHIVE}.json.asc && \
+    gpg --verify ${BOOST_ARCHIVE}.json.asc ${BOOST_ARCHIVE}.json && \
+    gpg --verify ${BOOST_ARCHIVE}.asc ${BOOST_ARCHIVE} && \
+    cat ${BOOST_ARCHIVE}.json | jq -r '. | .sha256 + "  " + .file' | sha256sum --check && \
+    mkdir -p boost && \
+    tar -xf ${BOOST_ARCHIVE} -C boost --strip-components=1 && \
+    cd boost && \
+    ./bootstrap.sh \
+        --prefix=${BOOST_DIR} \
+        && \
+    ./b2 -j${NPROCS} \
+        hardcode-dll-paths=true dll-path=${BOOST_DIR}/lib \
+        link=shared \
+        variant=release \
+        cxxflags=-w \
+        install \
+        && \
+    rm -rf ${SCRATCH_DIR}
+
+# Install Google Benchmark support library
+ENV BENCHMARK_DIR=/opt/benchmark
+RUN SCRATCH_DIR=/scratch && mkdir -p ${SCRATCH_DIR} && cd ${SCRATCH_DIR} && \
+    git clone https://github.com/google/benchmark.git -b v1.5.0 && \
+    cd benchmark && \
+    mkdir build && cd build && \
+    cmake -D CMAKE_BUILD_TYPE=Release -D CMAKE_INSTALL_PREFIX=${BENCHMARK_DIR} -D BENCHMARK_ENABLE_TESTING=OFF .. && \
+    make -j${NPROCS} && make install && \
+    rm -rf ${SCRATCH_DIR}
+
+# Install Kokkos, the commit is post 3.3.00 release
+ARG KOKKOS_VERSION=8c241078d0b844633cb1eae186bbb6a8a810ff5d
+ARG KOKKOS_OPTIONS="-DKokkos_ENABLE_SYCL=ON -DCMAKE_CXX_FLAGS=-Wno-unknown-cuda-version -DKokkos_ENABLE_UNSUPPORTED_ARCHS=ON -DKokkos_ARCH_VOLTA70=ON -DCMAKE_CXX_STANDARD=17"
+ENV KOKKOS_DIR=/opt/kokkos
+RUN KOKKOS_URL=https://github.com/kokkos/kokkos/archive/${KOKKOS_VERSION}.tar.gz && \
+    KOKKOS_ARCHIVE=kokkos-${KOKKOS_HASH}.tar.gz && \
+    SCRATCH_DIR=/scratch && mkdir -p ${SCRATCH_DIR} && cd ${SCRATCH_DIR} && \
+    wget --quiet ${KOKKOS_URL} --output-document=${KOKKOS_ARCHIVE} && \
+    mkdir -p kokkos && \
+    tar -xf ${KOKKOS_ARCHIVE} -C kokkos --strip-components=1 && \
+    cd kokkos && \
+    mkdir -p build && cd build && \
+    cmake -D CMAKE_BUILD_TYPE=Release -D CMAKE_INSTALL_PREFIX=${KOKKOS_DIR} -D CMAKE_CXX_COMPILER=clang++ ${KOKKOS_OPTIONS} .. && \
+    make -j${NPROCS} install && \
+    rm -rf ${SCRATCH_DIR}
diff --git a/examples/callback/example_callback.cpp b/examples/callback/example_callback.cpp
@@ -65,7 +65,7 @@ struct PrintfCallback
   KOKKOS_FUNCTION void operator()(Predicate, int primitive,
                                   OutputFunctor const &out) const
   {
-#ifndef KOKKOS_ENABLE_SYCL
+#ifndef __SYCL_DEVICE_ONLY__
     printf("Found %d from functor\n", primitive);
 #endif
     out(primitive);
@@ -102,8 +102,10 @@ int main(int argc, char *argv[])
     bvh.query(ExecutionSpace{}, FirstOctant{},
               KOKKOS_LAMBDA(auto /*predicate*/, int primitive,
                             auto /*output_functor*/) {
-#ifndef KOKKOS_ENABLE_SYCL
+#ifndef __SYCL_DEVICE_ONLY__
                 printf("Found %d from generic lambda\n", primitive);
+#else
+                (void)primitive;
 #endif
               },
               values, offsets);
@@ -120,8 +122,10 @@ int main(int argc, char *argv[])
     bvh.query(ExecutionSpace{}, NearestToOrigin{k},
               KOKKOS_LAMBDA(auto /*predicate*/, int primitive,
                             auto /*output_functor*/) {
-#ifndef KOKKOS_ENABLE_SYCL
+#ifndef __SYCL_DEVICE_ONLY__
                 printf("Found %d from generic lambda\n", primitive);
+#else
+                (void)primitive;
 #endif
               },
               values, offsets);
@@ -136,8 +140,10 @@ int main(int argc, char *argv[])
 #ifndef __NVCC__
     bvh.query(ExecutionSpace{}, FirstOctant{},
               KOKKOS_LAMBDA(auto /*predicate*/, int j) {
-#ifndef KOKKOS_ENABLE_SYCL
+#ifndef __SYCL_DEVICE_ONLY__
                 printf("%d %d %d\n", ++c(), -1, j);
+#else
+                (void)j;
 #endif
               });
 #endif

diff --git a/examples/dbscan/ArborX_DetailsDBSCANVerification.hpp b/examples/dbscan/ArborX_DetailsDBSCANVerification.hpp
@@ -52,7 +52,7 @@ bool verifyConnectedCorePointsShareIndex(ExecutionSpace const &exec_space,
 
             if (neigh_is_core_point && clusters(i) != clusters(j))
             {
-#ifndef KOKKOS_ENABLE_SYCL
+#ifndef __SYCL_DEVICE_ONLY__
               printf("Connected cores do not belong to the same cluster: "
                      "%d [%d] -> %d [%d]\n",
                      i, clusters(i), j, clusters(j));
@@ -107,7 +107,7 @@ bool verifyBoundaryPointsConnectToCorePoints(ExecutionSpace const &exec_space,
 
           if (is_boundary && !have_shared_core)
           {
-#ifndef KOKKOS_ENABLE_SYCL
+#ifndef __SYCL_DEVICE_ONLY__
             printf("Boundary point does not belong to a cluster: "
                    "%d [%d]\n",
                    i, clusters(i));

diff --git a/src/details/ArborX_DetailsTreeConstruction.hpp b/src/details/ArborX_DetailsTreeConstruction.hpp
@@ -119,7 +119,7 @@ inline void assignMortonCodes(
   using Access = AccessTraits<Primitives, PrimitivesTag>;
 
   auto const n = Access::size(primitives);
-  ARBORX_ASSERT(morton_codes.extent(0) == n);
+  ARBORX_ASSERT(static_cast<decltype(n)>(morton_codes.extent(0)) == n);
 
   using Tag = typename AccessTraitsHelper<Access>::tag;
   assignMortonCodesDispatch(Tag{}, space, primitives, morton_codes,