From cbbf9397766d97eb4f92846a283dbb088fe47ae7 Mon Sep 17 00:00:00 2001 From: Changzheng Wei Date: Thu, 27 Oct 2016 14:06:17 +0800 Subject: [PATCH] Add new feature to do cipher operation for small packets using CPU instead of QAT according to threshold Add null check to small packet ctrl msg for invalid cipher names Change-Id: Ide274b49c6ae5723d2a0c279103b3ad87c60542a Signed-off-by: Steve Linsell --- .tools/Makefile.am | 1 + .tools/Makefile.in | 2 + .tools/configure | 23 ++++ .tools/configure.ac | 14 ++ README.md | 28 ++++ configure | 23 ++++ e_qat.c | 55 +++++++- e_qat.h | 33 ++++- qat_ciphers.c | 309 ++++++++++++++++++++++++++++++++++++++++---- qat_ciphers.h | 14 +- 10 files changed, 465 insertions(+), 37 deletions(-) diff --git a/.tools/Makefile.am b/.tools/Makefile.am index a332f547..0be1be0d 100644 --- a/.tools/Makefile.am +++ b/.tools/Makefile.am @@ -51,6 +51,7 @@ CFLAGS = $(cflags) $(cflags_cc_opt) $(cflags_enable_upstream_driver) \ $(cflags_enable_usdm) $(cflags_mux) $(disable_qat_rsa) \ $(disable_qat_ciphers) $(disable_qat_ecdh) $(disable_qat_ecdsa) \ $(disable_qat_dsa) $(disable_qat_dh) $(disable_qat_prf) \ + $(enable_qat_small_pkt_offload) \ $(enable_qat_debug) $(enable_qat_warnings) \ $(enable_qat_mem_debug) $(enable_qat_mem_warnings) diff --git a/.tools/Makefile.in b/.tools/Makefile.in index 1001077d..515982aa 100644 --- a/.tools/Makefile.in +++ b/.tools/Makefile.in @@ -137,6 +137,7 @@ CFLAGS = $(cflags) $(cflags_cc_opt) $(cflags_enable_upstream_driver) \ $(cflags_enable_usdm) $(cflags_mux) $(disable_qat_rsa) \ $(disable_qat_ciphers) $(disable_qat_ecdh) $(disable_qat_ecdsa) \ $(disable_qat_dsa) $(disable_qat_dh) $(disable_qat_prf) \ + $(enable_qat_small_pkt_offload) \ $(enable_qat_debug) $(enable_qat_warnings) \ $(enable_qat_mem_debug) $(enable_qat_mem_warnings) @@ -242,6 +243,7 @@ enable_qat_debug = @enable_qat_debug@ enable_qat_mem_debug = @enable_qat_mem_debug@ enable_qat_mem_warnings = @enable_qat_mem_warnings@ enable_qat_mux = @enable_qat_mux@ +enable_qat_small_pkt_offload = @enable_qat_small_pkt_offload@ enable_qat_warnings = @enable_qat_warnings@ enable_upstream_driver = @enable_upstream_driver@ enable_usdm = @enable_usdm@ diff --git a/.tools/configure b/.tools/configure index 476c7ad2..86aae8af 100755 --- a/.tools/configure +++ b/.tools/configure @@ -661,6 +661,7 @@ enable_qat_mem_warnings enable_qat_mem_debug enable_qat_warnings enable_qat_debug +enable_qat_small_pkt_offload disable_qat_prf disable_qat_dh disable_qat_dsa @@ -793,6 +794,7 @@ enable_qat_ecdsa enable_qat_dsa enable_qat_dh enable_qat_prf +enable_qat_small_pkt_offload enable_qat_debug enable_qat_warnings enable_qat_mem_debug @@ -1476,6 +1478,8 @@ Optional Features: --disable-qat_dsa Disable accelerated DSA offload --disable-qat_dh Disable accelerated DH offload --disable-qat_prf Disable accelerated PRF offload + --enable-qat_small_pkt_offload + Enable accelerated small packet CIPHER offload --enable-qat_debug Enable Engine Debug Messages --enable-qat_warnings Enable Engine Warning Messages --enable-qat_mem_debug Enable Memory Driver Debug Messages @@ -11458,6 +11462,14 @@ fi +# Check whether --enable-qat_small_pkt_offload was given. +if test "${enable_qat_small_pkt_offload+set}" = set; then : + enableval=$enable_qat_small_pkt_offload; +else + enable_qat_small_pkt_offload=unset +fi + + # Parameter parsing for enabling debug/warnings @@ -11721,6 +11733,17 @@ else $as_echo "$as_me: Not Offloading PRF to Hardware" >&6;} fi +if test "x$enable_qat_small_pkt_offload" != "xunset" +then + enable_qat_small_pkt_offload="-DOPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS" + { $as_echo "$as_me:${as_lineno-$LINENO}: Offloading small packet CIPHERS to Hardware" >&5 +$as_echo "$as_me: Offloading small packet CIPHERS to Hardware" >&6;} +else + enable_qat_small_pkt_offload="" + { $as_echo "$as_me:${as_lineno-$LINENO}: Not Offloading small packet CIPHERS to Hardware" >&5 +$as_echo "$as_me: Not Offloading small packet CIPHERS to Hardware" >&6;} +fi + if test "x$enable_qat_debug" != "xunset" then enable_qat_debug="-DQAT_DEBUG" diff --git a/.tools/configure.ac b/.tools/configure.ac index 65c41c7d..cefde76b 100644 --- a/.tools/configure.ac +++ b/.tools/configure.ac @@ -64,6 +64,11 @@ AC_ARG_ENABLE(qat_prf, , disable_qat_prf=unset) AC_SUBST(disable_qat_prf) +AC_ARG_ENABLE(qat_small_pkt_offload, + AS_HELP_STRING([--enable-qat_small_pkt_offload], + [Enable accelerated small packet CIPHER offload]), + , enable_qat_small_pkt_offload=unset) +AC_SUBST(enable_qat_small_pkt_offload) # Parameter parsing for enabling debug/warnings @@ -261,6 +266,15 @@ else AC_MSG_NOTICE([Not Offloading PRF to Hardware]) fi +if test "x$enable_qat_small_pkt_offload" != "xunset" +then + enable_qat_small_pkt_offload="-DOPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS" + AC_MSG_NOTICE([Offloading small packet CIPHERS to Hardware]) +else + enable_qat_small_pkt_offload="" + AC_MSG_NOTICE([Not Offloading small packet CIPHERS to Hardware]) +fi + if test "x$enable_qat_debug" != "xunset" then enable_qat_debug="-DQAT_DEBUG" diff --git a/README.md b/README.md index faf6dd24..86e3c8b1 100644 --- a/README.md +++ b/README.md @@ -595,6 +595,30 @@ Description: should be passed in as Param 3. Setting the value to -1 results in infinite retries. The default is 5 and the max value is 100,000. This message can be sent at any time after the engine is created. + +Message String: SET_CRYPTO_SMALL_PACKET_OFFLOAD_THRESHOLD +Param 3: 0 +Param 4: string of cipher algorithm name and threshold value +Description: + This message is used to set the threshold that determines the size + crypto packets need to be before they are offloaded to the acceleration + device. Very small crypto packets are not efficient to offload to the + accelerator as it will take longer to transfer the data to and from the + accelerator than it will to encrypt/decrypt using the main CPU. + Each EVP_CIPHER supported in Intel® Quickassist Technology + OpenSSL\* engine can set the threshold value + independently according to the algorithm name as below: + AES-128-CBC-HMAC-SHA1 + AES-256-CBC-HMAC-SHA1 + AES-128-CBC-HMAC-SHA256 + AES-256-CBC-HMAC-SHA256 + The input format should be a string like this in one line: + AES-128-CBC-HMAC-SHA1:4096,AES-256-CBC-HMAC-SHA1:8192 + Using a separator ":" between cipher name and threshold value. + Using a separator "," between different cipher configuration + The default threshold value is 2048 bytes, the minimum is 0 bytes and the + maximum is 16,384. + ``` ## Intel® Quickassist Technology OpenSSL\* Engine Build Options @@ -701,6 +725,10 @@ Optional Disable/Enable Intel® Quickassist Technology PRF offload (enabled by default) +--disable-qat_small_pkt_offload/--enable-qat_small_pkt_offload + Enable Intel® Quickassist Technology to offload small + packet cipher operations (disabled by default) + --disable-qat_debug/--enable-qat_debug Disable/Enable debug output to aid debugging. Warning: This option should never be enabled in a production environment as diff --git a/configure b/configure index a435902b..d78383e9 100755 --- a/configure +++ b/configure @@ -661,6 +661,7 @@ enable_qat_mem_warnings enable_qat_mem_debug enable_qat_warnings enable_qat_debug +enable_qat_small_pkt_offload disable_qat_prf disable_qat_dh disable_qat_dsa @@ -793,6 +794,7 @@ enable_qat_ecdsa enable_qat_dsa enable_qat_dh enable_qat_prf +enable_qat_small_pkt_offload enable_qat_debug enable_qat_warnings enable_qat_mem_debug @@ -1476,6 +1478,8 @@ Optional Features: --disable-qat_dsa Disable accelerated DSA offload --disable-qat_dh Disable accelerated DH offload --disable-qat_prf Disable accelerated PRF offload + --enable-qat_small_pkt_offload + Enable accelerated small packet CIPHER offload --enable-qat_debug Enable Engine Debug Messages --enable-qat_warnings Enable Engine Warning Messages --enable-qat_mem_debug Enable Memory Driver Debug Messages @@ -11458,6 +11462,14 @@ fi +# Check whether --enable-qat_small_pkt_offload was given. +if test "${enable_qat_small_pkt_offload+set}" = set; then : + enableval=$enable_qat_small_pkt_offload; +else + enable_qat_small_pkt_offload=unset +fi + + # Parameter parsing for enabling debug/warnings @@ -11721,6 +11733,17 @@ else $as_echo "$as_me: Not Offloading PRF to Hardware" >&6;} fi +if test "x$enable_qat_small_pkt_offload" != "xunset" +then + enable_qat_small_pkt_offload="-DOPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS" + { $as_echo "$as_me:${as_lineno-$LINENO}: Offloading small packet CIPHERS to Hardware" >&5 +$as_echo "$as_me: Offloading small packet CIPHERS to Hardware" >&6;} +else + enable_qat_small_pkt_offload="" + { $as_echo "$as_me:${as_lineno-$LINENO}: Not Offloading small packet CIPHERS to Hardware" >&5 +$as_echo "$as_me: Not Offloading small packet CIPHERS to Hardware" >&6;} +fi + if test "x$enable_qat_debug" != "xunset" then enable_qat_debug="-DQAT_DEBUG" diff --git a/e_qat.c b/e_qat.c index f46cf4ff..4d33951f 100644 --- a/e_qat.c +++ b/e_qat.c @@ -116,6 +116,8 @@ /* OpenSSL Includes */ #include #include +#include +#include /* QAT includes */ #ifdef USE_QAT_CONTIG_MEM @@ -197,6 +199,18 @@ int qat_is_event_driven() return enable_event_driven_polling; } +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS +int setQatSmallPacketThreshold(unsigned char *cipher_name, int threshold) +{ + if(threshold < 0) + threshold = 0; + else if (threshold > 16384) + threshold = 16384; + return qat_pkt_threshold_table_set_threshold(OBJ_sn2nid(cipher_name),threshold); +} + +#endif + /****************************************************************************** * function: * incr_curr_inst(void) @@ -910,6 +924,7 @@ static int qat_engine_init(ENGINE *e) #define QAT_CMD_GET_NUM_CRYPTO_INSTANCES (ENGINE_CMD_BASE + 8) #define QAT_CMD_DISABLE_EVENT_DRIVEN_POLLING_MODE (ENGINE_CMD_BASE + 9) #define QAT_CMD_SET_EPOLL_TIMEOUT (ENGINE_CMD_BASE + 10) +#define QAT_CMD_SET_CRYPTO_SMALL_PACKET_OFFLOAD_THRESHOLD (ENGINE_CMD_BASE + 11) static const ENGINE_CMD_DEFN qat_cmd_defns[] = { { @@ -967,6 +982,11 @@ static const ENGINE_CMD_DEFN qat_cmd_defns[] = { "SET_EPOLL_TIMEOUT", "Set epoll_wait timeout", ENGINE_CMD_FLAG_NUMERIC}, + { + QAT_CMD_SET_CRYPTO_SMALL_PACKET_OFFLOAD_THRESHOLD, + "SET_CRYPTO_SMALL_PACKET_OFFLOAD_THRESHOLD", + "Set QAT small packet threshold", + ENGINE_CMD_FLAG_STRING}, {0, NULL, NULL, 0} }; @@ -1122,6 +1142,30 @@ qat_engine_ctrl(ENGINE *e, int cmd, long i, void *p, void (*f) (void)) *(int *)p = numInstances; break; + case QAT_CMD_SET_CRYPTO_SMALL_PACKET_OFFLOAD_THRESHOLD: +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + if(p) { + char *token; + while((token = strsep((char **)&p, ","))) { + char *name_token = strsep(&token,":"); + char *value_token = strsep(&token,":"); + if(name_token && value_token) { + retVal = setQatSmallPacketThreshold(name_token, atoi(value_token)); + } else { + WARN("Invalid parameter!\n"); + retVal = 0; + } + } + } else { + WARN("Invalid parameter!\n"); + retVal = 0; + } +#else + WARN("QAT_CMD_SET_CRYPTO_SMALL_PACKET_OFFLOAD_THRESHOLD is not supported\n"); + retVal = 0; +#endif + break; + default: WARN("CTRL command not implemented\n"); retVal = 0; @@ -1163,7 +1207,7 @@ static int qat_engine_finish(ENGINE *e) } if (0 == enable_external_polling && !qat_is_event_driven()) { - if (icp_polling_threads[i] != NULL) { + if ((pthread_t *) icp_polling_threads[i] != 0) { pthread_join(icp_polling_threads[i], NULL); } } @@ -1173,7 +1217,7 @@ static int qat_engine_finish(ENGINE *e) } if (0 == enable_external_polling && qat_is_event_driven()) { - if (icp_polling_threads[0] != NULL) { + if ((pthread_t *) icp_polling_threads[0] != 0) { pthread_join(icp_polling_threads[0], NULL); } } @@ -1246,6 +1290,9 @@ static int qat_engine_destroy(ENGINE *e) qat_free_DH_methods(); qat_free_DSA_methods(); qat_free_RSA_methods(); +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + CRYPTO_THREAD_cleanup_local(&qat_pkt_threshold_table_key); +#endif ERR_unload_QAT_strings(); return 1; } @@ -1292,7 +1339,9 @@ static int bind_qat(ENGINE *e, const char *id) * as this function will be called by a single thread. */ qat_create_ciphers(); - +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + CRYPTO_THREAD_run_once(&qat_pkt_threshold_table_once,qat_pkt_threshold_table_make_key); +#endif DEBUG("%s: About to set mem functions\n", __func__); if (!ENGINE_set_RSA(e, qat_get_RSA_methods())) { diff --git a/e_qat.h b/e_qat.h index e735363e..6434a086 100644 --- a/e_qat.h +++ b/e_qat.h @@ -96,13 +96,8 @@ typedef struct qat_chained_ctx_t { * need to be the elements present in EVP_AES_HMAC_SHA1 defined in * crypto/evp/e_aes_cbc_hmac_sha1.c */ - AES_KEY ks; - SHA_CTX head, tail, md; + size_t payload_length; /* AAD length in decrypt case */ - union { - unsigned int tls_ver; - unsigned char tls_aad[16]; /* 13 used */ - } aux; /* QAT Session Params */ CpaInstanceHandle instanceHandle; @@ -145,6 +140,32 @@ typedef struct qat_chained_ctx_t { } qat_chained_ctx; +typedef struct qat_chained_sha1_ctx_t { + /*struct EVP_AES_HMAC_SHA1 IA_key */ + AES_KEY ks; + SHA_CTX head, tail, md; + size_t payload_length; /* AAD length in decrypt case */ + union { + unsigned int tls_ver; + unsigned char tls_aad[16]; /* 13 used */ + } aux; + + qat_chained_ctx qat_ctx; +}qat_chained_sha1_ctx; + +typedef struct qat_chained_sha256_ctx_t { + /*EVP_AES_HMAC_SHA256 IA_key*/ + AES_KEY ks; + SHA256_CTX head, tail, md; + size_t payload_length; /* AAD length in decrypt case */ + union { + unsigned int tls_ver; + unsigned char tls_aad[16]; /* 13 used */ + } aux; + + qat_chained_ctx qat_ctx; +}qat_chained_sha256_ctx; + /* qat_buffer structure for partial hash */ typedef struct qat_buffer_t { struct qat_buffer_t *next; /* next buffer in the list */ diff --git a/qat_ciphers.c b/qat_ciphers.c index 55219042..cc39c348 100644 --- a/qat_ciphers.c +++ b/qat_ciphers.c @@ -69,6 +69,7 @@ #include #include #include +#include #include #ifdef OPENSSL_ENABLE_QAT_CIPHERS @@ -108,7 +109,7 @@ const EVP_CIPHER *qat_aes_128_cbc_hmac_sha1(void) || !EVP_CIPHER_meth_set_cleanup(_hidden_aes_128_cbc_hmac_sha1, qat_aes_cbc_hmac_sha_cleanup) || !EVP_CIPHER_meth_set_impl_ctx_size(_hidden_aes_128_cbc_hmac_sha1, - sizeof(qat_chained_ctx)) + sizeof(qat_chained_sha1_ctx)) || !EVP_CIPHER_meth_set_set_asn1_params(_hidden_aes_128_cbc_hmac_sha1, EVP_CIPH_FLAG_DEFAULT_ASN1 ? NULL : @@ -146,7 +147,7 @@ const EVP_CIPHER *qat_aes_256_cbc_hmac_sha1(void) || !EVP_CIPHER_meth_set_cleanup(_hidden_aes_256_cbc_hmac_sha1, qat_aes_cbc_hmac_sha_cleanup) || !EVP_CIPHER_meth_set_impl_ctx_size(_hidden_aes_256_cbc_hmac_sha1, - sizeof(qat_chained_ctx)) + sizeof(qat_chained_sha1_ctx)) || !EVP_CIPHER_meth_set_set_asn1_params(_hidden_aes_256_cbc_hmac_sha1, EVP_CIPH_FLAG_DEFAULT_ASN1 ? NULL : @@ -184,7 +185,7 @@ const EVP_CIPHER *qat_aes_128_cbc_hmac_sha256(void) || !EVP_CIPHER_meth_set_cleanup(_hidden_aes_128_cbc_hmac_sha256, qat_aes_cbc_hmac_sha_cleanup) || !EVP_CIPHER_meth_set_impl_ctx_size(_hidden_aes_128_cbc_hmac_sha256, - sizeof(qat_chained_ctx)) + sizeof(qat_chained_sha256_ctx)) || !EVP_CIPHER_meth_set_set_asn1_params(_hidden_aes_128_cbc_hmac_sha256, EVP_CIPH_FLAG_DEFAULT_ASN1 ? NULL : @@ -222,7 +223,7 @@ const EVP_CIPHER *qat_aes_256_cbc_hmac_sha256(void) || !EVP_CIPHER_meth_set_cleanup(_hidden_aes_256_cbc_hmac_sha256, qat_aes_cbc_hmac_sha_cleanup) || !EVP_CIPHER_meth_set_impl_ctx_size(_hidden_aes_256_cbc_hmac_sha256, - sizeof(qat_chained_ctx)) + sizeof(qat_chained_sha256_ctx)) || !EVP_CIPHER_meth_set_set_asn1_params(_hidden_aes_256_cbc_hmac_sha256, EVP_CIPH_FLAG_DEFAULT_ASN1 ? NULL : @@ -288,6 +289,116 @@ void qat_free_ciphers(void) {} #endif +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS +# define CRYPTO_SMALL_PACKET_OFFLOAD_THRESHOLD_DEFAULT 2048 + +CRYPTO_ONCE qat_pkt_threshold_table_once = CRYPTO_ONCE_STATIC_INIT; +CRYPTO_THREAD_LOCAL qat_pkt_threshold_table_key; + +void qat_pkt_threshold_table_make_key(void) +{ + CRYPTO_THREAD_init_local(&qat_pkt_threshold_table_key, qat_free_pkt_threshold_table); +} + +typedef struct cipher_threshold_table_s { + int nid; + int threshold; +}PKT_THRESHOLD; + +DEFINE_LHASH_OF(PKT_THRESHOLD); + +PKT_THRESHOLD qat_pkt_threshold_table[] = { + {NID_aes_128_cbc_hmac_sha1,CRYPTO_SMALL_PACKET_OFFLOAD_THRESHOLD_DEFAULT}, + {NID_aes_256_cbc_hmac_sha1,CRYPTO_SMALL_PACKET_OFFLOAD_THRESHOLD_DEFAULT}, + {NID_aes_128_cbc_hmac_sha256,CRYPTO_SMALL_PACKET_OFFLOAD_THRESHOLD_DEFAULT}, + {NID_aes_256_cbc_hmac_sha256,CRYPTO_SMALL_PACKET_OFFLOAD_THRESHOLD_DEFAULT} +}; + +static int pkt_threshold_table_cmp(const PKT_THRESHOLD * a, const PKT_THRESHOLD * b) +{ + return (a->nid == b->nid)?0:1; +} + +static unsigned long pkt_threshold_table_hash(const PKT_THRESHOLD * a) +{ + return (unsigned long)(a->nid); +} + +LHASH_OF(PKT_THRESHOLD) *qat_create_pkt_threshold_table(void) +{ + int i; + LHASH_OF(PKT_THRESHOLD) *ret = NULL; + ret = lh_PKT_THRESHOLD_new(pkt_threshold_table_hash,pkt_threshold_table_cmp); + if(ret == NULL) { + return ret; + } + for(i = 0; i < sizeof(qat_pkt_threshold_table);i++) { + lh_PKT_THRESHOLD_insert(ret,&qat_pkt_threshold_table[i]); + } + return ret; +} + +int qat_pkt_threshold_table_set_threshold(int nid, int threshold) +{ + PKT_THRESHOLD entry,*ret; + LHASH_OF(PKT_THRESHOLD) *tbl = NULL; + if(NID_undef == nid) { + WARN("Unsupported NID\n"); + return 0; + } + if((tbl = CRYPTO_THREAD_get_local(&qat_pkt_threshold_table_key)) == NULL) { + tbl = qat_create_pkt_threshold_table(); + if(tbl != NULL) { + CRYPTO_THREAD_set_local(&qat_pkt_threshold_table_key, tbl); + } + else { + WARN("Create packet threshold table fail.\n"); + return 0; + } + } + entry.nid = nid; + ret = lh_PKT_THRESHOLD_retrieve(tbl,&entry); + if(ret == NULL) { + WARN("Threshold entry retrieve failed for the NID : %d\n",entry.nid); + return 0; + } + ret->threshold = threshold; + lh_PKT_THRESHOLD_insert(tbl, ret); + return 1; +} + +int qat_pkt_threshold_table_get_threshold(int nid) +{ + PKT_THRESHOLD entry,*ret; + LHASH_OF(PKT_THRESHOLD) *tbl = NULL; + if((tbl = CRYPTO_THREAD_get_local(&qat_pkt_threshold_table_key)) == NULL) { + tbl = qat_create_pkt_threshold_table(); + if(tbl != NULL) { + CRYPTO_THREAD_set_local(&qat_pkt_threshold_table_key, tbl); + } + else { + WARN("Create packet threshold table fail.\n"); + return 0; + } + } + entry.nid = nid; + ret = lh_PKT_THRESHOLD_retrieve(tbl,&entry); + if(ret == NULL) { + WARN("Threshold entry retrieve failed for the NID : %d\n",entry.nid); + return 0; + } + return ret->threshold; +} + +void qat_free_pkt_threshold_table(void *thread_key) +{ + LHASH_OF(PKT_THRESHOLD) *tbl = (LHASH_OF(PKT_THRESHOLD) *) thread_key; + if((tbl = CRYPTO_THREAD_get_local(&qat_pkt_threshold_table_key))) { + lh_PKT_THRESHOLD_free(tbl); + } +} + +#endif /****************************************************************************** * function: * qat_ciphers(ENGINE *e, @@ -546,7 +657,6 @@ int qat_aes_cbc_hmac_sha_init(EVP_CIPHER_CTX *ctx, const unsigned char *inkey, const unsigned char *iv, int enc) { - /* Initialise a QAT session and set the cipher keys */ qat_chained_ctx *qat_ctx = NULL; @@ -554,8 +664,48 @@ int qat_aes_cbc_hmac_sha_init(EVP_CIPHER_CTX *ctx, WARN("[%s] ctx or inkey is NULL.\n", __func__); return 0; } - - qat_ctx = qat_chained_data(ctx); +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + const EVP_CIPHER *ia_cipher = NULL; +#endif + int nid = EVP_CIPHER_CTX_nid(ctx); + switch (nid) { + case NID_aes_128_cbc_hmac_sha1: +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + ia_cipher = EVP_aes_128_cbc_hmac_sha1(); +#endif + qat_ctx = &(((qat_chained_sha1_ctx *)qat_chained_data(ctx))->qat_ctx); + break; + case NID_aes_256_cbc_hmac_sha1: +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + ia_cipher = EVP_aes_256_cbc_hmac_sha1(); +#endif + qat_ctx = &(((qat_chained_sha1_ctx *)qat_chained_data(ctx))->qat_ctx); + break; + case NID_aes_128_cbc_hmac_sha256: +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + ia_cipher = EVP_aes_128_cbc_hmac_sha256(); +#endif + qat_ctx = &(((qat_chained_sha256_ctx *)qat_chained_data(ctx))->qat_ctx); + break; + case NID_aes_256_cbc_hmac_sha256: +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + ia_cipher = EVP_aes_256_cbc_hmac_sha256(); +#endif + qat_ctx = &(((qat_chained_sha256_ctx *)qat_chained_data(ctx))->qat_ctx); + break; + default: +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + ia_cipher = NULL; +#endif + return 0; + } + if (iv) + memcpy((unsigned char *)EVP_CIPHER_CTX_original_iv(ctx), iv, EVP_CIPHER_CTX_iv_length(ctx)); + memcpy(EVP_CIPHER_CTX_iv_noconst(ctx), EVP_CIPHER_CTX_original_iv(ctx), EVP_CIPHER_CTX_iv_length(ctx)); +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + if(EVP_CIPHER_meth_get_init(ia_cipher)(ctx, inkey, iv, enc) == 0) + goto end; +#endif if (qat_ctx == NULL) { WARN("[%s] --- qat_ctx is NULL.\n", __func__); return 0; @@ -652,9 +802,51 @@ int qat_aes_cbc_hmac_sha_ctrl(EVP_CIPHER_CTX *ctx, int type, int arg, WARN("[%s] --- ctx parameter is NULL.\n", __func__); return -1; } +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + const EVP_CIPHER *ia_cipher = NULL; + size_t *payload_len_ptr = NULL; +#endif + int nid = EVP_CIPHER_CTX_nid(ctx); + switch (nid) { + case NID_aes_128_cbc_hmac_sha1: +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + ia_cipher = EVP_aes_128_cbc_hmac_sha1(); + payload_len_ptr = &(((qat_chained_sha1_ctx *)qat_chained_data(ctx))->payload_length); +#endif + evp_ctx = &(((qat_chained_sha1_ctx *)qat_chained_data(ctx))->qat_ctx); + break; + case NID_aes_256_cbc_hmac_sha1: +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + ia_cipher = EVP_aes_256_cbc_hmac_sha1(); + payload_len_ptr = &(((qat_chained_sha1_ctx *)qat_chained_data(ctx))->payload_length); +#endif + evp_ctx = &(((qat_chained_sha1_ctx *)qat_chained_data(ctx))->qat_ctx); + break; + case NID_aes_128_cbc_hmac_sha256: +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + ia_cipher = EVP_aes_128_cbc_hmac_sha256(); + payload_len_ptr = &(((qat_chained_sha256_ctx *)qat_chained_data(ctx))->payload_length); +#endif + evp_ctx = &(((qat_chained_sha256_ctx *)qat_chained_data(ctx))->qat_ctx); + break; + case NID_aes_256_cbc_hmac_sha256: +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + ia_cipher = EVP_aes_256_cbc_hmac_sha256(); + payload_len_ptr = &(((qat_chained_sha256_ctx *)qat_chained_data(ctx))->payload_length); +#endif + evp_ctx = &(((qat_chained_sha256_ctx *)qat_chained_data(ctx))->qat_ctx); + break; + default: +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + ia_cipher = NULL; +#endif + return 0; + } - evp_ctx = qat_chained_data(ctx); - +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + EVP_CIPHER_meth_get_ctrl(ia_cipher)(ctx, type, arg, ptr); +#endif + if (evp_ctx == NULL) { WARN("[%s] --- evp_ctx is NULL.\n", __func__); return -1; @@ -711,21 +903,27 @@ int qat_aes_cbc_hmac_sha_ctrl(EVP_CIPHER_CTX *ctx, int type, int arg, * the send/encrypt direction */ p = ptr; - len = - (p[arg - QAT_TLS_PAYLOADLENGTH_MSB_OFFSET] << QAT_BYTE_SHIFT | - p[arg - QAT_TLS_PAYLOADLENGTH_LSB_OFFSET]); if (arg < TLS_VIRT_HDR_SIZE) { retVal = -1; break; } +#ifdef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + len = + (p[arg - QAT_TLS_PAYLOADLENGTH_MSB_OFFSET] << QAT_BYTE_SHIFT | + p[arg - QAT_TLS_PAYLOADLENGTH_LSB_OFFSET]); +#else + len = *(unsigned int *)payload_len_ptr; +#endif + evp_ctx->tls_version = (p[arg - QAT_TLS_VERSION_MSB_OFFSET] << QAT_BYTE_SHIFT | p[arg - QAT_TLS_VERSION_LSB_OFFSET]); if (EVP_CIPHER_CTX_encrypting(ctx)) { evp_ctx->payload_length = len; +#ifdef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS if (evp_ctx->tls_version >= TLS1_1_VERSION) { len -= AES_BLOCK_SIZE; /* TODO: Why does this code reduce the len in the @@ -734,7 +932,7 @@ int qat_aes_cbc_hmac_sha_ctrl(EVP_CIPHER_CTX *ctx, int type, int arg, len >> QAT_BYTE_SHIFT; p[arg - QAT_TLS_PAYLOADLENGTH_LSB_OFFSET] = len; } - +#endif if (NULL == evp_ctx->tls_virt_hdr) { WARN("Unable to allocate memory for mac preamble in qat/n"); return -1; @@ -803,21 +1001,26 @@ int qat_aes_cbc_hmac_sha_cleanup(EVP_CIPHER_CTX *ctx) return 0; } - evp_ctx = qat_chained_data(ctx); + int nid = 0; + nid = EVP_CIPHER_CTX_nid(ctx); + switch (nid) { + case NID_aes_128_cbc_hmac_sha1: + case NID_aes_256_cbc_hmac_sha1: + evp_ctx = &(((qat_chained_sha1_ctx *)qat_chained_data(ctx))->qat_ctx); + break; + case NID_aes_128_cbc_hmac_sha256: + case NID_aes_256_cbc_hmac_sha256: + evp_ctx = &(((qat_chained_sha256_ctx *)qat_chained_data(ctx))->qat_ctx); + break; + default: + return 0; + } if (evp_ctx == NULL) { WARN("[%s] evp_ctx parameter is NULL.\n", __func__); return 0; } - if (!(evp_ctx->init)) { - /* - * It is valid to call cleanup even if the context has not been - * initialised. - */ - return retVal; - } - sessSetup = evp_ctx->session_data; if (sessSetup) { if (evp_ctx->qat_ctx) { @@ -901,7 +1104,20 @@ static int qat_aes_sha_session_init(EVP_CIPHER_CTX *ctx) return 0; } - evp_ctx = qat_chained_data(ctx); + int nid = 0; + nid = EVP_CIPHER_CTX_nid(ctx); + switch (nid) { + case NID_aes_128_cbc_hmac_sha1: + case NID_aes_256_cbc_hmac_sha1: + evp_ctx = &(((qat_chained_sha1_ctx *)qat_chained_data(ctx))->qat_ctx); + break; + case NID_aes_128_cbc_hmac_sha256: + case NID_aes_256_cbc_hmac_sha256: + evp_ctx = &(((qat_chained_sha256_ctx *)qat_chained_data(ctx))->qat_ctx); + break; + default: + return 0; + } if (evp_ctx == NULL) { WARN("[%s] --- evp_ctx is NULL.\n", __func__); @@ -954,8 +1170,6 @@ static int qat_aes_sha_session_init(EVP_CIPHER_CTX *ctx) return 0; } - evp_ctx->qat_ctx = pSessionCtx; - evp_ctx->srcBufferList.numBuffers = 2; evp_ctx->srcBufferList.pBuffers = (evp_ctx->srcFlatBuffer); evp_ctx->srcBufferList.pUserData = NULL; @@ -1020,6 +1234,7 @@ static int qat_aes_sha_session_init(EVP_CIPHER_CTX *ctx) * Create the OpData structure to remove this processing from the data * path */ + evp_ctx->qat_ctx = pSessionCtx; evp_ctx->OpData.sessionCtx = evp_ctx->qat_ctx; evp_ctx->OpData.packetType = CPA_CY_SYM_PACKET_TYPE_FULL; @@ -1080,7 +1295,49 @@ int qat_aes_cbc_hmac_sha_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, return 0; } - evp_ctx = qat_chained_data(ctx); +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + const EVP_CIPHER *ia_cipher = NULL; +#endif + + int nid = EVP_CIPHER_CTX_nid(ctx); + switch (nid) { + case NID_aes_128_cbc_hmac_sha1: +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + ia_cipher = EVP_aes_128_cbc_hmac_sha1(); +#endif + evp_ctx = &(((qat_chained_sha1_ctx *)qat_chained_data(ctx))->qat_ctx); + break; + case NID_aes_256_cbc_hmac_sha1: +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + ia_cipher = EVP_aes_256_cbc_hmac_sha1(); +#endif + evp_ctx = &(((qat_chained_sha1_ctx *)qat_chained_data(ctx))->qat_ctx); + break; + case NID_aes_128_cbc_hmac_sha256: +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + ia_cipher = EVP_aes_128_cbc_hmac_sha256(); +#endif + evp_ctx = &(((qat_chained_sha256_ctx *)qat_chained_data(ctx))->qat_ctx); + break; + case NID_aes_256_cbc_hmac_sha256: +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + ia_cipher = EVP_aes_256_cbc_hmac_sha256(); +#endif + evp_ctx = &(((qat_chained_sha256_ctx *)qat_chained_data(ctx))->qat_ctx); + break; + default: +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + ia_cipher = NULL; +#endif + return 0; + } + +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + if(len <= qat_pkt_threshold_table_get_threshold(nid)) { + retVal = EVP_CIPHER_meth_get_do_cipher(ia_cipher)(ctx, out, in, len); + return retVal; + } +#endif if (evp_ctx == NULL) { WARN("[%s] --- evp_ctx is NULL.\n", __func__); diff --git a/qat_ciphers.h b/qat_ciphers.h index c27e620e..9581034b 100644 --- a/qat_ciphers.h +++ b/qat_ciphers.h @@ -47,13 +47,15 @@ # define QAT_CIPHERS_H # include +# include # define AES_BLOCK_SIZE 16 # define AES_IV_LEN 16 # define AES_KEY_SIZE_256 32 # define AES_KEY_SIZE_128 16 -# define qat_chained_data(ctx) ((qat_chained_ctx *)EVP_CIPHER_CTX_get_cipher_data(ctx)) +# define qat_chained_data(ctx) (EVP_CIPHER_CTX_get_cipher_data(ctx)) + # define HMAC_KEY_SIZE 64 # define TLS_VIRT_HDR_SIZE 13 @@ -77,5 +79,13 @@ void qat_create_ciphers(void); void qat_free_ciphers(void); int qat_ciphers(ENGINE *e, const EVP_CIPHER **cipher, const int **nids, int nid); - +# ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS +extern CRYPTO_ONCE qat_pkt_threshold_table_once; +extern CRYPTO_THREAD_LOCAL qat_pkt_threshold_table_key; +void qat_pkt_threshold_table_make_key(void ); +LHASH_OF(PKT_THRESHOLD) *qat_create_pkt_threshold_table(void); +void qat_free_pkt_threshold_table(void *); +int qat_pkt_threshold_table_set_threshold(int nid, int threshold); +int qat_pkt_threshold_table_get_threshold(int nid); +# endif #endif /* QAT_CIPHERS_H */