feat: cpu instruction set detection (#570)

Author: vansangpfiev

cortex-cpp/CMakeLists.txt

@@ -59,9 +59,28 @@ endif()
 
 add_compile_definitions(CORTEX_CPP_VERSION="${CORTEX_CPP_VERSION}")
 
+if(LLAMA_CUDA)
+  add_compile_definitions(CORTEX_CUDA)
+endif()
+
+if(LLAMA_AVX512)
+  add_compile_definitions(CORTEX_AVX512)
+endif()
+
+if(LLAMA_AVX2)
+  add_compile_definitions(CORTEX_AVX2)
+endif()
+
+if(LLAMA_VULKAN)
+  add_compile_definitions(CORTEX_VULKAN)
+endif()
+
 add_subdirectory(test)
 
-add_executable(${PROJECT_NAME} main.cc)
+add_executable(${PROJECT_NAME} main.cc
+  ${CMAKE_CURRENT_SOURCE_DIR}/utils/cpuid/cpu_info.cc
+  ${CMAKE_CURRENT_SOURCE_DIR}/utils/cpuid/cpu_validation.cc
+)
 
 # ##############################################################################
 # If you include the drogon source code locally in your project, use this method

cortex-cpp/Makefile

@@ -39,12 +39,12 @@ endif
 
 pre-package:
 ifeq ($(OS),Windows_NT)
-	@powershell -Command "mkdir -p cortex-cpp\engines\cortex.llamacpp\; cp -r build\engines\cortex.llamacpp\engine.dll cortex-cpp\engines\cortex.llamacpp\;"
-	@powershell -Command "cp -r build\Release\cortex-cpp.exe .\cortex-cpp\;"
-	@powershell -Command "cp -r build-deps\_install\bin\zlib.dll .\cortex-cpp\;"
-	@powershell -Command "cp -r ..\.github\patches\windows\msvcp140.dll .\cortex-cpp\;"
-	@powershell -Command "cp -r ..\.github\patches\windows\vcruntime140_1.dll .\cortex-cpp\;"
-	@powershell -Command "cp -r ..\.github\patches\windows\vcruntime140.dll .\cortex-cpp\;"
+	@powershell -Command "mkdir -p cortex-cpp\engines\cortex.llamacpp\; cp build\engines\cortex.llamacpp\engine.dll cortex-cpp\engines\cortex.llamacpp\;"
+	@powershell -Command "cp build\Release\cortex-cpp.exe .\cortex-cpp\;"
+	@powershell -Command "cp build-deps\_install\bin\zlib.dll .\cortex-cpp\;"
+	@powershell -Command "cp ..\.github\patches\windows\msvcp140.dll .\cortex-cpp\;"
+	@powershell -Command "cp ..\.github\patches\windows\vcruntime140_1.dll .\cortex-cpp\;"
+	@powershell -Command "cp ..\.github\patches\windows\vcruntime140.dll .\cortex-cpp\;"
 else ifeq ($(shell uname -s),Linux)
 	@mkdir -p cortex-cpp/engines/cortex.llamacpp; \
 	cp build/engines/cortex.llamacpp/libengine.so cortex-cpp/engines/cortex.llamacpp/; \

cortex-cpp/controllers/server.cc

@@ -6,6 +6,8 @@
 
 #include "trantor/utils/Logger.h"
 #include "utils/cortex_utils.h"
+#include "utils/cpuid/cpu_info.h"
+#include "utils/cpuid/cpu_validation.h"
 #include "utils/logging_utils.h"
 
 using namespace inferences;
@@ -265,6 +267,15 @@ void server::LoadModel(const HttpRequestPtr& req,
     };
 
     try {
+      if (engine_type == kLlamaEngine) {
+        cortex::cpuid::CpuInfo cpu_info;
+        LOG_INFO << "CPU instruction set: " << cpu_info.to_string();
+        if (auto [res, err] = cortex::cpuid::llamacpp::IsValidInstructions();
+            !res) {
+          LOG_WARN << err;
+        }
+      }
+
       std::string abs_path =
           cortex_utils::GetCurrentPath() + get_engine_path(engine_type);
       engines_[engine_type].dl =

cortex-cpp/utils/cpuid/cpu_info.cc

@@ -0,0 +1,190 @@
+// Copyright (c) 2013 Steinwurf ApS
+// All Rights Reserved
+// Inspired by https://github.com/steinwurf/cpuid
+#include "platform.h"
+
+#include "cpu_info.h"
+#include "detail/cpu_info_impl.h"
+
+#if defined(PLATFORM_GCC_COMPATIBLE_X86)
+#include "detail/init_gcc_x86.h"
+#elif defined(PLATFORM_MSVC_X86) && !defined(PLATFORM_WINDOWS_PHONE)
+#include "detail/init_msvc_x86.h"
+#elif defined(PLATFORM_MSVC_ARM)
+#include "detail/init_msvc_arm.h"
+#elif defined(PLATFORM_CLANG_ARM) && defined(PLATFORM_IOS)
+#include "detail/init_ios_clang_arm.h"
+#elif defined(PLATFORM_GCC_COMPATIBLE_ARM) && defined(PLATFORM_LINUX)
+#include "detail/init_linux_gcc_arm.h"
+#else
+#include "detail/init_unknown.h"
+#endif
+
+namespace cortex::cpuid {
+
+CpuInfo::CpuInfo() : impl(new Impl()) {
+  init_cpuinfo(*impl);
+}
+
+CpuInfo::~CpuInfo() {}
+
+// x86 member functions
+bool CpuInfo::has_fpu() const {
+  return impl->has_fpu;
+}
+
+bool CpuInfo::has_mmx() const {
+  return impl->has_mmx;
+}
+
+bool CpuInfo::has_sse() const {
+  return impl->has_sse;
+}
+
+bool CpuInfo::has_sse2() const {
+  return impl->has_sse2;
+}
+
+bool CpuInfo::has_sse3() const {
+  return impl->has_sse3;
+}
+
+bool CpuInfo::has_ssse3() const {
+  return impl->has_ssse3;
+}
+
+bool CpuInfo::has_sse4_1() const {
+  return impl->has_sse4_1;
+}
+
+bool CpuInfo::has_sse4_2() const {
+  return impl->has_sse4_2;
+}
+
+bool CpuInfo::has_pclmulqdq() const {
+  return impl->has_pclmulqdq;
+}
+
+bool CpuInfo::has_avx() const {
+  return impl->has_avx;
+}
+
+bool CpuInfo::has_avx2() const {
+  return impl->has_avx2;
+}
+
+bool CpuInfo::has_avx512_f() const {
+  return impl->has_avx512_f;
+}
+
+bool CpuInfo::has_avx512_dq() const {
+  return impl->has_avx512_dq;
+}
+
+bool CpuInfo::has_avx512_ifma() const {
+  return impl->has_avx512_ifma;
+}
+
+bool CpuInfo::has_avx512_pf() const {
+  return impl->has_avx512_pf;
+}
+
+bool CpuInfo::has_avx512_er() const {
+  return impl->has_avx512_er;
+}
+
+bool CpuInfo::has_avx512_cd() const {
+  return impl->has_avx512_cd;
+}
+
+bool CpuInfo::has_avx512_bw() const {
+  return impl->has_avx512_bw;
+}
+
+bool CpuInfo::has_avx512_vl() const {
+  return impl->has_avx512_vl;
+}
+
+bool CpuInfo::has_avx512_vbmi() const {
+  return impl->has_avx512_vbmi;
+}
+
+bool CpuInfo::has_avx512_vbmi2() const {
+  return impl->has_avx512_vbmi2;
+}
+
+bool CpuInfo::has_avx512_vnni() const {
+  return impl->has_avx512_vnni;
+}
+
+bool CpuInfo::has_avx512_bitalg() const {
+  return impl->has_avx512_bitalg;
+}
+
+bool CpuInfo::has_avx512_vpopcntdq() const {
+  return impl->has_avx512_vpopcntdq;
+}
+
+bool CpuInfo::has_avx512_4vnniw() const {
+  return impl->has_avx512_4vnniw;
+}
+
+bool CpuInfo::has_avx512_4fmaps() const {
+  return impl->has_avx512_4fmaps;
+}
+
+bool CpuInfo::has_avx512_vp2intersect() const {
+  return impl->has_avx512_vp2intersect;
+}
+
+bool CpuInfo::has_f16c() const {
+  return impl->has_f16c;
+}
+
+bool CpuInfo::has_aes() const {
+  return impl->has_aes;
+}
+
+// ARM member functions
+bool CpuInfo::has_neon() const {
+  return impl->has_neon;
+}
+
+std::string CpuInfo::to_string() {
+  std::string s;
+  auto get = [](bool flag) -> std::string {
+    return flag ? "1" : "0";
+  };
+  s += "fpu = " + get(impl->has_fpu) + "| ";
+  s += "mmx = " + get(impl->has_mmx) + "| ";
+  s += "sse = " + get(impl->has_sse) + "| ";
+  s += "sse2 = " + get(impl->has_sse2) + "| ";
+  s += "sse3 = " + get(impl->has_sse3) + "| ";
+  s += "ssse3 = " + get(impl->has_ssse3) + "| ";
+  s += "sse4_1 = " + get(impl->has_sse4_1) + "| ";
+  s += "sse4_2 = " + get(impl->has_sse4_2) + "| ";
+  s += "pclmulqdq = " + get(impl->has_pclmulqdq) + "| ";
+  s += "avx = " + get(impl->has_avx) + "| ";
+  s += "avx2 = " + get(impl->has_avx2) + "| ";
+  s += "avx512_f = " + get(impl->has_avx512_f) + "| ";
+  s += "avx512_dq = " + get(impl->has_avx512_dq) + "| ";
+  s += "avx512_ifma = " + get(impl->has_avx512_ifma) + "| ";
+  s += "avx512_pf = " + get(impl->has_avx512_pf) + "| ";
+  s += "avx512_er = " + get(impl->has_avx512_er) + "| ";
+  s += "avx512_cd = " + get(impl->has_avx512_cd) + "| ";
+  s += "avx512_bw = " + get(impl->has_avx512_bw) + "| ";
+  s += "has_avx512_vl = " + get(impl->has_avx512_vl) + "| ";
+  s += "has_avx512_vbmi = " + get(impl->has_avx512_vbmi) + "| ";
+  s += "has_avx512_vbmi2 = " + get(impl->has_avx512_vbmi2) + "| ";
+  s += "avx512_vnni = " + get(impl->has_avx512_vnni) + "| ";
+  s += "avx512_bitalg = " + get(impl->has_avx512_bitalg) + "| ";
+  s += "avx512_vpopcntdq = " + get(impl->has_avx512_vpopcntdq) + "| ";
+  s += "avx512_4vnniw = " + get(impl->has_avx512_4vnniw) + "| ";
+  s += "avx512_4fmaps = " + get(impl->has_avx512_4fmaps) + "| ";
+  s += "avx512_vp2intersect = " + get(impl->has_avx512_vp2intersect) + "| ";
+  s += "aes = " + get(impl->has_aes) + "| ";
+  s += "f16c = " + get(impl->has_f16c) + "|";
+  return s;
+}
+
+}  // namespace cpuid

cortex-cpp/utils/cpuid/cpu_info.h

@@ -0,0 +1,131 @@
+// Copyright (c) 2013 Steinwurf ApS
+// All Rights Reserved
+// Inspired by https://github.com/steinwurf/cpuid
+#pragma once
+
+#include <memory>
+#include <string>
+
+namespace cortex::cpuid {
+/// The CpuInfo object extract information about which, if any, additional
+/// instructions are supported by the CPU.
+class CpuInfo {
+ public:
+  /// Constructor for feature detection with default values
+  CpuInfo();
+
+  /// Destructor
+  ~CpuInfo();
+
+  /// Return true if the CPU supports x87 Floating-Point Unit
+  bool has_fpu() const;
+
+  /// Return true if the CPU supports MMX
+  bool has_mmx() const;
+
+  /// Return true if the CPU supports Streaming SIMD Extensions
+  bool has_sse() const;
+
+  /// Return true if the CPU supports Streaming SIMD Extensions 2
+  bool has_sse2() const;
+
+  /// Return true if the CPU supports Streaming SIMD Extensions 3
+  bool has_sse3() const;
+
+  /// Return true if the CPU supports Supplemental Streaming SIMD Extensions 3
+  bool has_ssse3() const;
+
+  /// Return true if the CPU supports Streaming SIMD Extensions 4.1
+  bool has_sse4_1() const;
+
+  /// Return true if the CPU supports Streaming SIMD Extensions 4.2
+  bool has_sse4_2() const;
+
+  /// Return true if the CPU supports carry-less multiplication of two 64-bit
+  /// polynomials over the finite field GF(2)
+  bool has_pclmulqdq() const;
+
+  /// Return true if the CPU supports Advanced Vector Extensions
+  bool has_avx() const;
+
+  /// Return true if the CPU supports Advanced Vector Extensions 2
+  bool has_avx2() const;
+
+  /// Return true if the CPU supports AVX-512 Foundation
+  bool has_avx512_f() const;
+
+  /// Return true if the CPU supports AVX-512 Doubleword and Quadword
+  /// Instructions
+  bool has_avx512_dq() const;
+
+  /// Return true if the CPU supports AVX-512 Integer Fused Multiply Add
+  bool has_avx512_ifma() const;
+
+  /// Return true if the CPU supports AVX-512 Prefetch Instructions
+  bool has_avx512_pf() const;
+
+  /// Return true if the CPU supports AVX-512 Exponential and Reciprocal
+  /// Instructions
+  bool has_avx512_er() const;
+
+  /// Return true if the CPU supports AVX-512 Conflict Detection Instructions
+  bool has_avx512_cd() const;
+
+  /// Return true if the CPU supports AVX-512 Byte and Word Instructions
+  bool has_avx512_bw() const;
+
+  /// Return true if the CPU supports AVX-512 Vector Length Extensions
+  bool has_avx512_vl() const;
+
+  /// Return true if the CPU supports AVX-512 Vector Byte Manipulation
+  /// Instructions
+  bool has_avx512_vbmi() const;
+
+  /// Return true if the CPU supports AVX-512 Vector Byte Manipulation
+  /// Instructions 2
+  bool has_avx512_vbmi2() const;
+
+  /// Return true if the CPU supports AVX-512 Vector Neural Network
+  /// Instructions
+  bool has_avx512_vnni() const;
+
+  /// Return true if the CPU supports AVX-512 Bit Algorithms
+  bool has_avx512_bitalg() const;
+
+  /// Return true if the CPU supports Vector population count instruction
+  bool has_avx512_vpopcntdq() const;
+
+  /// Return true if the CPU supports AVX-512 Vector Neural Network
+  /// Instructions Word variable precision
+  bool has_avx512_4vnniw() const;
+
+  /// Return true if the CPU supports AVX-512 Fused Multiply Accumulation
+  /// Packed Single precision
+  bool has_avx512_4fmaps() const;
+
+  /// Return true if the CPU supports AVX-512 Vector Pair Intersection to a
+  /// Pair of Mask Registers
+  bool has_avx512_vp2intersect() const;
+
+  /// Return true if the CPU supports converting between half-precision and
+  /// standard IEEE single-precision floating-point formats
+  bool has_f16c() const;
+
+  /// Return true if the CPU supports Advanced Encryption Standard instruction
+  /// set
+  bool has_aes() const;
+
+  /// Return true if the CPU supports ARM Advanced SIMD
+  bool has_neon() const;
+
+  std::string to_string();
+
+ public:
+  /// Private implementation
+  struct Impl;
+
+ private:
+  /// Pimpl pointer
+  std::unique_ptr<Impl> impl;
+};
+}  // namespace cortex::cpuid