intel · ymeng-git · Jul 14, 2022 · Jul 12, 2022
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -27,13 +27,30 @@ message(STATUS "ENABLE_BENCHMARK: ${ENABLE_BENCHMARK}")
 option(ENABLE_DOCS "Enable Documentation" OFF)
 message(STATUS "ENABLE_DOCS: ${ENABLE_DOCS}")
 
+option(FPGA_USE_INTEL_HEXL "Use Intel HEXL" OFF)
+message(STATUS "FPGA_USE_INTEL_HEXL: ${FPGA_USE_INTEL_HEXL}")
+option(FPGA_BUILD_INTEL_HEXL "Build INTEL HEXL" OFF)
+message(STATUS "FPGA_BUILD_INTEL_HEXL: ${FPGA_BUILD_INTEL_HEXL}")
+
 if (LINUX)
     include(GNUInstallDirs)
 else()
     set(CMAKE_INSTALL_INCLUDEDIR "include")
     set(CMAKE_INSTALL_LIBDIR "lib")
 endif()
 
+if(FPGA_USE_INTEL_HEXL)
+    if(FPGA_BUILD_INTEL_HEXL)
+        message(STATUS "Intel HEXL: download ...")
+        include(cmake/intel-hexl/intel-hexl.cmake)
+    else()
+        find_package(HEXL 1.2.4)
+        if (NOT TARGET HEXL::hexl)
+            message(FATAL_ERROR "Intel HEXL: not found")
+        endif()
+    endif()
+endif()
+
 add_subdirectory(host)
 add_subdirectory(device)
 

diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt
@@ -21,6 +21,23 @@ function(bench_function kernel)
     target_link_libraries(bench_${KERNEL} PRIVATE benchmark::benchmark)
     target_link_libraries(bench_${KERNEL} PRIVATE nlohmann_json::nlohmann_json)
 
+    if(FPGA_USE_INTEL_HEXL)
+        target_compile_options(bench_${KERNEL} PRIVATE -DFPGA_USE_INTEL_HEXL)
+        if(FPGA_BUILD_INTEL_HEXL)
+            add_dependencies(bench_${KERNEL} ext_intel_hexl)
+            target_include_directories(bench_${KERNEL} PRIVATE ${CMAKE_BINARY_DIR}/hexl-install/${CMAKE_INSTALL_INCLUDEDIR})
+            target_link_directories(bench_${KERNEL} PRIVATE ${CMAKE_BINARY_DIR}/hexl-install/${CMAKE_INSTALL_LIBDIR})
+            target_link_libraries(bench_${KERNEL} PRIVATE ${CMAKE_BINARY_DIR}/hexl-install/${CMAKE_INSTALL_LIBDIR}/libhexl.so)
+        else()
+            get_target_property(
+                HEXL_INCLUDE_DIR
+                HEXL::hexl
+                INTERFACE_INCLUDE_DIRECTORIES)
+            target_include_directories(bench_${KERNEL} PRIVATE ${HEXL_INCLUDE_DIR})
+            target_link_libraries(bench_${KERNEL} PRIVATE HEXL::hexl)
+        endif()
+    endif(FPGA_USE_INTEL_HEXL)
+
     add_custom_command(
         TARGET bench_${KERNEL} POST_BUILD
         COMMAND rm -f ${KERNEL}.aocx

diff --git a/benchmark/bench_keyswitch.cpp b/benchmark/bench_keyswitch.cpp
@@ -10,7 +10,11 @@
 
 #include <benchmark/benchmark.h>
 
+#ifdef FPGA_USE_INTEL_HEXL
+#include "hexl/hexl.hpp"
+#else
 #include "hexl-fpga.h"
+#endif
 
 static uint32_t get_iter() {
     char* env = getenv("ITER");
@@ -101,8 +105,6 @@ std::vector<std::string> keyswitch::glob(const char* pattern) {
 
 void keyswitch::setup_keyswitch(const std::vector<std::string>& files) {
     for (size_t i = 0; i < files.size(); i++) {
-        std::cout << "Constructing Test Vector " << i << " from File ... "
-                  << files[i] << std::endl;
         test_vectors_.push_back(KeySwitchTestVector(files[i].c_str()));
     }
 
@@ -111,22 +113,35 @@ void keyswitch::setup_keyswitch(const std::vector<std::string>& files) {
 }
 
 void keyswitch::bench_keyswitch() {
+#ifndef FPGA_USE_INTEL_HEXL
     intel::hexl::set_worksize_KeySwitch(test_vector_size_ * n_iter);
+#endif
+
+#ifndef FPGA_USE_INTEL_HEXL
+    // HEXL-FPGA version of keyswitch
+    namespace ns = intel::hexl;
+#else
+    // HEXL CPU version of keyswitch
+    namespace ns = intel::hexl::internal;
+#endif
+
     for (size_t n = 0; n < n_iter; n++) {
         for (size_t i = 0; i < test_vector_size_; i++) {
-            intel::hexl::KeySwitch(test_vectors_[i].input.data(),
-                                   test_vectors_[i].t_target_iter_ptr.data(),
-                                   test_vectors_[0].coeff_count,
-                                   test_vectors_[0].decomp_modulus_size,
-                                   test_vectors_[0].key_modulus_size,
-                                   test_vectors_[0].rns_modulus_size,
-                                   test_vectors_[0].key_component_count,
-                                   test_vectors_[0].moduli.data(),
-                                   test_vectors_[0].key_vectors.data(),
-                                   test_vectors_[0].modswitch_factors.data());
+            ns::KeySwitch(test_vectors_[i].input.data(),
+                          test_vectors_[i].t_target_iter_ptr.data(),
+                          test_vectors_[0].coeff_count,
+                          test_vectors_[0].decomp_modulus_size,
+                          test_vectors_[0].key_modulus_size,
+                          test_vectors_[0].rns_modulus_size,
+                          test_vectors_[0].key_component_count,
+                          test_vectors_[0].moduli.data(),
+                          test_vectors_[0].key_vectors.data(),
+                          test_vectors_[0].modswitch_factors.data());
         }
     }
+#ifndef FPGA_USE_INTEL_HEXL
     intel::hexl::KeySwitchCompleted();
+#endif
 }
 
 BENCHMARK_F(keyswitch, 16384_6_7_7_2)
@@ -149,6 +164,9 @@ BENCHMARK_F(keyswitch, 16384_6_7_7_2)
 
     setup_keyswitch(files);
 
+    // warm up the FPGA kernels specially the twiddle factor dispatching kernel
+    bench_keyswitch();
+
     for (auto st : state) {
         bench_keyswitch();
     }

diff --git a/benchmark/main.cpp b/benchmark/main.cpp
@@ -3,15 +3,19 @@
 
 #include <benchmark/benchmark.h>
 
+#ifndef FPGA_USE_INTEL_HEXL
 #include "hexl-fpga.h"
+#endif
 
 int main(int argc, char** argv) {
+#ifndef FPGA_USE_INTEL_HEXL
     intel::hexl::acquire_FPGA_resources();
-
+#endif
     benchmark::Initialize(&argc, argv);
     benchmark::RunSpecifiedBenchmarks();
 
+#ifndef FPGA_USE_INTEL_HEXL
     intel::hexl::release_FPGA_resources();
-
+#endif
     return 0;
 }
diff --git a/cmake/intel-hexl/intel-hexl.cmake b/cmake/intel-hexl/intel-hexl.cmake
@@ -0,0 +1,38 @@
+# Copyright (C) 2020-2021 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+include(ExternalProject)
+
+set(INTEL_HEXL_GIT_REPO_URL https://github.com/intel/hexl.git)
+set(INTEL_HEXL_GIT_LABEL v1.2.4)
+
+ExternalProject_Add(
+    ext_intel_hexl
+    PREFIX ext_intel_hexl
+    GIT_REPOSITORY ${INTEL_HEXL_GIT_REPO_URL}
+    GIT_TAG ${INTEL_HEXL_GIT_LABEL}
+    CMAKE_ARGS
+        -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+        -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+        -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+        -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+        -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+        -DHEXL_SHARED_LIB=ON
+        -DHEXL_EXPERIMENTAL=ON
+        -DHEXL_DEBUG=OFF
+        -DHEXL_BENCHMARK=OFF
+        -DHEXL_COVERAGE=OFF
+        -DHEXL_TESTING=OFF
+        -DCMAKE_INSTALL_INCLUDEDIR=include
+        -DCMAKE_INSTALL_LIBDIR=lib
+        -DCMAKE_INSTALL_PREFIX=${CMAKE_BINARY_DIR}/hexl-install
+    EXCLUDE_FROM_ALL TRUE
+    # Skip updates
+    UPDATE_COMMAND "")
+
+ExternalProject_Get_Property(ext_intel_hexl SOURCE_DIR BINARY_DIR)
+
+add_library(libhexl INTERFACE)
+add_dependencies(libhexl ext_intel_hexl)
+target_include_directories(libhexl INTERFACE ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR})
+target_link_libraries(libhexl INTERFACE ${CMAKE_INSTALL_PREFIX}/${CAMKE_INSTALL_LIBDIR}/libhexl-fpga.so)
diff --git a/utils/perf-cmp.sh b/utils/perf-cmp.sh
@@ -0,0 +1,99 @@
+# Copyright (C) 2020-2021 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+#!/usr/bin/env bash
+
+workdir=$PWD
+results=${workdir}/results
+mkdir -p ${results}
+mkdir -p testdata
+
+setup_testdata() {
+    pushd testdata
+
+    # downloading bitstream keyswitch.aocx
+    wget https://github.com/intel/hexl-fpga/releases/download/v1.1/keyswitch.aocx
+    aocx=${PWD}/keyswitch.aocx
+    export FPGA_BITSTREAM=${aocx}
+    export FPGA_KERNEL=KEYSWITCH
+
+    # downloading test vectors testdata.zip
+    wget https://github.com/intel/hexl-fpga/releases/download/v1.1/testdata.zip
+    unzip testdata.zip
+    export KEYSWITCH_DATA_DIR=${PWD}/testdata
+
+    popd
+}
+
+build_cpu() {
+    rm -rf build
+    mkdir build
+
+    cd build
+    cmake .. \
+    -DCMAKE_INSTALL_PREFIX=./hexl-fpga-install \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DCMAKE_CXX_COMPILER=g++ \
+    -DCMAKE_C_COMPILER=gcc \
+    -DENABLE_TESTS=ON \
+    -DENABLE_BENCHMARK=ON \
+    -DFPGA_USE_INTEL_HEXL=ON \
+    -DFPGA_BUILD_INTEL_HEXL=ON
+
+    make -j
+    cd ..
+}
+
+build_fpga() {
+    rm -rf build-fpga
+    mkdir build-fpga
+
+    cd build-fpga
+    cmake .. \
+    -DCMAKE_INSTALL_PREFIX=./hexl-fpga-install \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DCMAKE_CXX_COMPILER=g++ \
+    -DCMAKE_C_COMPILER=gcc \
+    -DENABLE_TESTS=ON \
+    -DENABLE_BENCHMARK=ON \
+    -DFPGA_USE_INTEL_HEXL=OFF \
+    -DFPGA_BUILD_INTEL_HEXL=OFF
+
+    make -j
+    cd ..
+}
+
+run_cpu() {
+    iter=$1
+    export ITER=${iter}
+    ./bench_keyswitch > ${results}/cpu.iter-${iter}.log
+}
+
+run_fpga() {
+    iter=$1
+    batch=$2
+    export ITER=${iter}
+    export BATCH_SIZE_KEYSWITCH=${batch}
+    ./bench_keyswitch  > ${results}/fpga.iter-${iter}.batch-${batch}.log
+}
+
+build_cpu
+build_fpga
+
+setup_testdata
+
+pushd build/benchmark
+for i in 1 1024 4096 16384
+do
+    run_cpu $i
+done
+popd
+
+aocl program acl0 ${aocx}
+pushd build-fpga/benchmark
+batch=128
+for i in 1 1024 4096 16384
+do
+    run_fpga $i ${batch}
+done
+popd