Merge pull request #47040 from hualongfeng/qat_batch_mode

Crypto: Add QAT batch mode Reviewed-by: Casey Bodley <cbodley@redhat.com>
ceph · Mar 11, 2023 · e64a0de · e64a0de
2 parents 2732527 + c097b24
commit e64a0de
Show file tree

Hide file tree

Showing 20 changed files with 640 additions and 391 deletions.
diff --git a/src/crypto/crypto_accel.h b/src/crypto/crypto_accel.h
@@ -17,21 +17,34 @@
 #include <cstddef>
 #include "include/Context.h"
 
+class optional_yield;
+
 class CryptoAccel;
 typedef std::shared_ptr<CryptoAccel> CryptoAccelRef;
 
 class CryptoAccel {
  public:
   CryptoAccel() {}
+  CryptoAccel(const size_t chunk_size, const size_t max_requests) {}
   virtual ~CryptoAccel() {}
 
   static const int AES_256_IVSIZE = 128/8;
   static const int AES_256_KEYSIZE = 256/8;
   virtual bool cbc_encrypt(unsigned char* out, const unsigned char* in, size_t size,
                    const unsigned char (&iv)[AES_256_IVSIZE],
-                   const unsigned char (&key)[AES_256_KEYSIZE]) = 0;
+                   const unsigned char (&key)[AES_256_KEYSIZE],
+                   optional_yield y) = 0;
   virtual bool cbc_decrypt(unsigned char* out, const unsigned char* in, size_t size,
                    const unsigned char (&iv)[AES_256_IVSIZE],
-                   const unsigned char (&key)[AES_256_KEYSIZE]) = 0;
+                   const unsigned char (&key)[AES_256_KEYSIZE],
+                   optional_yield y) = 0;
+  virtual bool cbc_encrypt_batch(unsigned char* out, const unsigned char* in, size_t size,
+                   const unsigned char iv[][AES_256_IVSIZE],
+                   const unsigned char (&key)[AES_256_KEYSIZE],
+                   optional_yield y) = 0;
+  virtual bool cbc_decrypt_batch(unsigned char* out, const unsigned char* in, size_t size,
+                   const unsigned char iv[][AES_256_IVSIZE],
+                   const unsigned char (&key)[AES_256_KEYSIZE],
+                   optional_yield y) = 0;
 };
 #endif
diff --git a/src/crypto/crypto_plugin.h b/src/crypto/crypto_plugin.h
@@ -20,6 +20,7 @@
 #include "ostream"
 
 #include "crypto/crypto_accel.h"
+#include <boost/asio/io_context.hpp>
 // -----------------------------------------------------------------------------
 
 class CryptoPlugin : public ceph::Plugin {
@@ -31,6 +32,8 @@ class CryptoPlugin : public ceph::Plugin {
   ~CryptoPlugin()
   {}
   virtual int factory(CryptoAccelRef *cs,
-                      std::ostream *ss) = 0;
+                      std::ostream *ss,
+                      const size_t chunk_size,
+                      const size_t max_requests) = 0;
 };
 #endif
diff --git a/src/crypto/isa-l/CMakeLists.txt b/src/crypto/isa-l/CMakeLists.txt
@@ -29,6 +29,9 @@ endif(HAVE_NASM_X64)
 
 add_library(ceph_crypto_isal SHARED ${isal_crypto_plugin_srcs})
 target_include_directories(ceph_crypto_isal PRIVATE ${isal_dir}/include)
+
+target_link_libraries(ceph_crypto_isal PRIVATE spawn)
+
 set_target_properties(ceph_crypto_isal PROPERTIES
   VERSION 1.0.0
   SOVERSION 1

diff --git a/src/crypto/isa-l/isal_crypto_accel.cc b/src/crypto/isa-l/isal_crypto_accel.cc
@@ -18,9 +18,10 @@
 
 bool ISALCryptoAccel::cbc_encrypt(unsigned char* out, const unsigned char* in, size_t size,
                              const unsigned char (&iv)[AES_256_IVSIZE],
-                             const unsigned char (&key)[AES_256_KEYSIZE])
+                             const unsigned char (&key)[AES_256_KEYSIZE],
+                             optional_yield y)
 {
-  if ((size % AES_256_IVSIZE) != 0) {
+  if (unlikely((size % AES_256_IVSIZE) != 0)) {
     return false;
   }
   alignas(16) struct cbc_key_data keys_blk;
@@ -31,9 +32,10 @@ bool ISALCryptoAccel::cbc_encrypt(unsigned char* out, const unsigned char* in, s
 }
 bool ISALCryptoAccel::cbc_decrypt(unsigned char* out, const unsigned char* in, size_t size,
                              const unsigned char (&iv)[AES_256_IVSIZE],
-                             const unsigned char (&key)[AES_256_KEYSIZE])
+                             const unsigned char (&key)[AES_256_KEYSIZE],
+                             optional_yield y)
 {
-  if ((size % AES_256_IVSIZE) != 0) {
+  if (unlikely((size % AES_256_IVSIZE) != 0)) {
     return false;
   }
   alignas(16) struct cbc_key_data keys_blk;

diff --git a/src/crypto/isa-l/isal_crypto_accel.h b/src/crypto/isa-l/isal_crypto_accel.h
@@ -15,6 +15,7 @@
 #ifndef ISAL_CRYPTO_ACCEL_H
 #define ISAL_CRYPTO_ACCEL_H
 #include "crypto/crypto_accel.h"
+#include "common/async/yield_context.h"
 
 class ISALCryptoAccel : public CryptoAccel {
  public:
@@ -23,9 +24,19 @@ class ISALCryptoAccel : public CryptoAccel {
 
   bool cbc_encrypt(unsigned char* out, const unsigned char* in, size_t size,
                    const unsigned char (&iv)[AES_256_IVSIZE],
-                   const unsigned char (&key)[AES_256_KEYSIZE]) override;
+                   const unsigned char (&key)[AES_256_KEYSIZE],
+                   optional_yield y) override;
   bool cbc_decrypt(unsigned char* out, const unsigned char* in, size_t size,
                    const unsigned char (&iv)[AES_256_IVSIZE],
-                   const unsigned char (&key)[AES_256_KEYSIZE]) override;
+                   const unsigned char (&key)[AES_256_KEYSIZE],
+                   optional_yield y) override;
+  bool cbc_encrypt_batch(unsigned char* out, const unsigned char* in, size_t size,
+                   const unsigned char iv[][AES_256_IVSIZE],
+                   const unsigned char (&key)[AES_256_KEYSIZE],
+                   optional_yield y) override { return false; }
+  bool cbc_decrypt_batch(unsigned char* out, const unsigned char* in, size_t size,
+                   const unsigned char iv[][AES_256_IVSIZE],
+                   const unsigned char (&key)[AES_256_KEYSIZE],
+                   optional_yield y) override { return false; }
 };
 #endif
diff --git a/src/crypto/isa-l/isal_crypto_plugin.h b/src/crypto/isa-l/isal_crypto_plugin.h
@@ -31,7 +31,9 @@ class ISALCryptoPlugin : public CryptoPlugin {
   ~ISALCryptoPlugin()
   {}
   virtual int factory(CryptoAccelRef *cs,
-                      std::ostream *ss)
+                      std::ostream *ss,
+                      const size_t chunk_size,
+                      const size_t max_requests)
   {
     if (cryptoaccel == nullptr)
     {

diff --git a/src/crypto/openssl/CMakeLists.txt b/src/crypto/openssl/CMakeLists.txt
@@ -7,7 +7,8 @@ set(openssl_crypto_plugin_srcs
 add_library(ceph_crypto_openssl SHARED ${openssl_crypto_plugin_srcs})
 target_link_libraries(ceph_crypto_openssl
     PRIVATE OpenSSL::Crypto
-    $<$<PLATFORM_ID:Windows>:ceph-common>)
+    $<$<PLATFORM_ID:Windows>:ceph-common>
+    spawn)
 target_include_directories(ceph_crypto_openssl PRIVATE ${OPENSSL_INCLUDE_DIR})
 add_dependencies(crypto_plugins ceph_crypto_openssl)
 set_target_properties(ceph_crypto_openssl PROPERTIES INSTALL_RPATH "")

diff --git a/src/crypto/openssl/openssl_crypto_accel.cc b/src/crypto/openssl/openssl_crypto_accel.cc
@@ -77,9 +77,10 @@ bool evp_transform(unsigned char* out, const unsigned char* in, size_t size,
 
 bool OpenSSLCryptoAccel::cbc_encrypt(unsigned char* out, const unsigned char* in, size_t size,
                              const unsigned char (&iv)[AES_256_IVSIZE],
-                             const unsigned char (&key)[AES_256_KEYSIZE])
+                             const unsigned char (&key)[AES_256_KEYSIZE],
+                             optional_yield y)
 {
-  if ((size % AES_256_IVSIZE) != 0) {
+  if (unlikely((size % AES_256_IVSIZE) != 0)) {
     return false;
   }
 
@@ -91,9 +92,10 @@ bool OpenSSLCryptoAccel::cbc_encrypt(unsigned char* out, const unsigned char* in
 
 bool OpenSSLCryptoAccel::cbc_decrypt(unsigned char* out, const unsigned char* in, size_t size,
                              const unsigned char (&iv)[AES_256_IVSIZE],
-                             const unsigned char (&key)[AES_256_KEYSIZE])
+                             const unsigned char (&key)[AES_256_KEYSIZE],
+                             optional_yield y)
 {
-  if ((size % AES_256_IVSIZE) != 0) {
+  if (unlikely((size % AES_256_IVSIZE) != 0)) {
     return false;
   }
 

diff --git a/src/crypto/openssl/openssl_crypto_accel.h b/src/crypto/openssl/openssl_crypto_accel.h
@@ -16,6 +16,7 @@
 #define OPENSSL_CRYPTO_ACCEL_H
 
 #include "crypto/crypto_accel.h"
+#include "common/async/yield_context.h"
 
 class OpenSSLCryptoAccel : public CryptoAccel {
  public:
@@ -24,9 +25,19 @@ class OpenSSLCryptoAccel : public CryptoAccel {
 
   bool cbc_encrypt(unsigned char* out, const unsigned char* in, size_t size,
                    const unsigned char (&iv)[AES_256_IVSIZE],
-                   const unsigned char (&key)[AES_256_KEYSIZE]) override;
+                   const unsigned char (&key)[AES_256_KEYSIZE],
+                   optional_yield y) override;
   bool cbc_decrypt(unsigned char* out, const unsigned char* in, size_t size,
                    const unsigned char (&iv)[AES_256_IVSIZE],
-                   const unsigned char (&key)[AES_256_KEYSIZE]) override;
+                   const unsigned char (&key)[AES_256_KEYSIZE],
+                   optional_yield y) override;
+  bool cbc_encrypt_batch(unsigned char* out, const unsigned char* in, size_t size,
+                   const unsigned char iv[][AES_256_IVSIZE],
+                   const unsigned char (&key)[AES_256_KEYSIZE],
+                   optional_yield y) override { return false; }
+  bool cbc_decrypt_batch(unsigned char* out, const unsigned char* in, size_t size,
+                   const unsigned char iv[][AES_256_IVSIZE],
+                   const unsigned char (&key)[AES_256_KEYSIZE],
+                   optional_yield y) override { return false; }
 };
 #endif
diff --git a/src/crypto/openssl/openssl_crypto_plugin.h b/src/crypto/openssl/openssl_crypto_plugin.h
@@ -25,7 +25,10 @@ class OpenSSLCryptoPlugin : public CryptoPlugin {
 public:
   explicit OpenSSLCryptoPlugin(CephContext* cct) : CryptoPlugin(cct)
   {}
-  int factory(CryptoAccelRef *cs, std::ostream *ss) override {
+  int factory(CryptoAccelRef *cs,
+              std::ostream *ss,
+              const size_t chunk_size,
+              const size_t max_requests) override {
     if (cryptoaccel == nullptr)
       cryptoaccel = CryptoAccelRef(new OpenSSLCryptoAccel);
 

diff --git a/src/crypto/qat/CMakeLists.txt b/src/crypto/qat/CMakeLists.txt
@@ -13,7 +13,8 @@ add_dependencies(crypto_plugins ceph_crypto_qat)
 
 target_link_libraries(ceph_crypto_qat PRIVATE
                       QatDrv::qat_s
-                      QatDrv::usdm_drv_s)
+                      QatDrv::usdm_drv_s
+                      spawn)
 
 add_dependencies(crypto_plugins ceph_crypto_qat)
 set_target_properties(ceph_crypto_qat PROPERTIES VERSION 1.0.0 SOVERSION 1)

diff --git a/src/crypto/qat/qat_crypto_accel.cc b/src/crypto/qat/qat_crypto_accel.cc
@@ -15,28 +15,33 @@
 
 #include "crypto/qat/qat_crypto_accel.h"
 
-bool QccCryptoAccel::cbc_encrypt(unsigned char* out, const unsigned char* in, size_t size,
-    const unsigned char (&iv)[AES_256_IVSIZE],
-    const unsigned char (&key)[AES_256_KEYSIZE])
-{
-  if ((size % AES_256_IVSIZE) != 0) {
+
+
+
+bool QccCryptoAccel::cbc_encrypt_batch(unsigned char* out, const unsigned char* in, size_t size,
+        const unsigned char iv[][AES_256_IVSIZE],
+        const unsigned char (&key)[AES_256_KEYSIZE],
+        optional_yield y) {
+  if (unlikely((size % AES_256_IVSIZE) != 0)) {
     return false;
   }
 
-  return qcccrypto.perform_op(out, in, size,
-      const_cast<unsigned char *>(&iv[0]),
-      const_cast<unsigned char *>(&key[0]), CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT);
+  return qcccrypto.perform_op_batch(out, in, size,
+      const_cast<unsigned char *>(&iv[0][0]),
+      const_cast<unsigned char *>(&key[0]),
+      CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT, y);
 }
 
-bool QccCryptoAccel::cbc_decrypt(unsigned char* out, const unsigned char* in, size_t size,
-    const unsigned char (&iv)[AES_256_IVSIZE],
-    const unsigned char (&key)[AES_256_KEYSIZE])
-{
-  if ((size % AES_256_IVSIZE) != 0) {
+bool QccCryptoAccel::cbc_decrypt_batch(unsigned char* out, const unsigned char* in, size_t size,
+        const unsigned char iv[][AES_256_IVSIZE],
+        const unsigned char (&key)[AES_256_KEYSIZE],
+        optional_yield y) {
+  if (unlikely((size % AES_256_IVSIZE) != 0)) {
     return false;
   }
 
-  return qcccrypto.perform_op(out, in, size,
-      const_cast<unsigned char *>(&iv[0]),
-      const_cast<unsigned char *>(&key[0]), CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT);
+  return qcccrypto.perform_op_batch(out, in, size,
+      const_cast<unsigned char *>(&iv[0][0]),
+      const_cast<unsigned char *>(&key[0]),
+      CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT, y);
 }
diff --git a/src/crypto/qat/qat_crypto_accel.h b/src/crypto/qat/qat_crypto_accel.h
@@ -18,18 +18,29 @@
 
 #include "crypto/crypto_accel.h"
 #include "crypto/qat/qcccrypto.h"
+#include "common/async/yield_context.h"
 
 class QccCryptoAccel : public CryptoAccel {
   public:
     QccCrypto qcccrypto;
-    QccCryptoAccel() { qcccrypto.init(); };
+    QccCryptoAccel(const size_t chunk_size, const size_t max_requests):qcccrypto() { qcccrypto.init(chunk_size, max_requests); };
     ~QccCryptoAccel() { qcccrypto.destroy(); };
 
     bool cbc_encrypt(unsigned char* out, const unsigned char* in, size_t size,
         const unsigned char (&iv)[AES_256_IVSIZE],
-        const unsigned char (&key)[AES_256_KEYSIZE]) override;
+        const unsigned char (&key)[AES_256_KEYSIZE],
+        optional_yield y) override { return false; }
     bool cbc_decrypt(unsigned char* out, const unsigned char* in, size_t size,
         const unsigned char (&iv)[AES_256_IVSIZE],
-        const unsigned char (&key)[AES_256_KEYSIZE]) override;
+        const unsigned char (&key)[AES_256_KEYSIZE],
+        optional_yield y) override { return false; }
+    bool cbc_encrypt_batch(unsigned char* out, const unsigned char* in, size_t size,
+        const unsigned char iv[][AES_256_IVSIZE],
+        const unsigned char (&key)[AES_256_KEYSIZE],
+        optional_yield y) override;
+    bool cbc_decrypt_batch(unsigned char* out, const unsigned char* in, size_t size,
+        const unsigned char iv[][AES_256_IVSIZE],
+        const unsigned char (&key)[AES_256_KEYSIZE],
+        optional_yield y) override;
 };
 #endif
diff --git a/src/crypto/qat/qat_crypto_plugin.h b/src/crypto/qat/qat_crypto_plugin.h
@@ -29,11 +29,11 @@ class QccCryptoPlugin : public CryptoPlugin {
   {}
   ~QccCryptoPlugin()
   {}
-  virtual int factory(CryptoAccelRef *cs, std::ostream *ss)
+  virtual int factory(CryptoAccelRef *cs, std::ostream *ss, const size_t chunk_size, const size_t max_requests)
   {
     std::lock_guard<std::mutex> l(qat_init);
     if (cryptoaccel == nullptr)
-      cryptoaccel = CryptoAccelRef(new QccCryptoAccel);
+      cryptoaccel = CryptoAccelRef(new QccCryptoAccel(chunk_size, max_requests));
 
     *cs = cryptoaccel;
     return 0;