diff --git a/.tools/Makefile.am b/.tools/Makefile.am index a332f547..0be1be0d 100644 --- a/.tools/Makefile.am +++ b/.tools/Makefile.am @@ -51,6 +51,7 @@ CFLAGS = $(cflags) $(cflags_cc_opt) $(cflags_enable_upstream_driver) \ $(cflags_enable_usdm) $(cflags_mux) $(disable_qat_rsa) \ $(disable_qat_ciphers) $(disable_qat_ecdh) $(disable_qat_ecdsa) \ $(disable_qat_dsa) $(disable_qat_dh) $(disable_qat_prf) \ + $(enable_qat_small_pkt_offload) \ $(enable_qat_debug) $(enable_qat_warnings) \ $(enable_qat_mem_debug) $(enable_qat_mem_warnings) diff --git a/.tools/Makefile.in b/.tools/Makefile.in index 1001077d..515982aa 100644 --- a/.tools/Makefile.in +++ b/.tools/Makefile.in @@ -137,6 +137,7 @@ CFLAGS = $(cflags) $(cflags_cc_opt) $(cflags_enable_upstream_driver) \ $(cflags_enable_usdm) $(cflags_mux) $(disable_qat_rsa) \ $(disable_qat_ciphers) $(disable_qat_ecdh) $(disable_qat_ecdsa) \ $(disable_qat_dsa) $(disable_qat_dh) $(disable_qat_prf) \ + $(enable_qat_small_pkt_offload) \ $(enable_qat_debug) $(enable_qat_warnings) \ $(enable_qat_mem_debug) $(enable_qat_mem_warnings) @@ -242,6 +243,7 @@ enable_qat_debug = @enable_qat_debug@ enable_qat_mem_debug = @enable_qat_mem_debug@ enable_qat_mem_warnings = @enable_qat_mem_warnings@ enable_qat_mux = @enable_qat_mux@ +enable_qat_small_pkt_offload = @enable_qat_small_pkt_offload@ enable_qat_warnings = @enable_qat_warnings@ enable_upstream_driver = @enable_upstream_driver@ enable_usdm = @enable_usdm@ diff --git a/.tools/configure b/.tools/configure index 476c7ad2..86aae8af 100755 --- a/.tools/configure +++ b/.tools/configure @@ -661,6 +661,7 @@ enable_qat_mem_warnings enable_qat_mem_debug enable_qat_warnings enable_qat_debug +enable_qat_small_pkt_offload disable_qat_prf disable_qat_dh disable_qat_dsa @@ -793,6 +794,7 @@ enable_qat_ecdsa enable_qat_dsa enable_qat_dh enable_qat_prf +enable_qat_small_pkt_offload enable_qat_debug enable_qat_warnings enable_qat_mem_debug @@ -1476,6 +1478,8 @@ Optional Features: --disable-qat_dsa Disable accelerated DSA offload --disable-qat_dh Disable accelerated DH offload --disable-qat_prf Disable accelerated PRF offload + --enable-qat_small_pkt_offload + Enable accelerated small packet CIPHER offload --enable-qat_debug Enable Engine Debug Messages --enable-qat_warnings Enable Engine Warning Messages --enable-qat_mem_debug Enable Memory Driver Debug Messages @@ -11458,6 +11462,14 @@ fi +# Check whether --enable-qat_small_pkt_offload was given. +if test "${enable_qat_small_pkt_offload+set}" = set; then : + enableval=$enable_qat_small_pkt_offload; +else + enable_qat_small_pkt_offload=unset +fi + + # Parameter parsing for enabling debug/warnings @@ -11721,6 +11733,17 @@ else $as_echo "$as_me: Not Offloading PRF to Hardware" >&6;} fi +if test "x$enable_qat_small_pkt_offload" != "xunset" +then + enable_qat_small_pkt_offload="-DOPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS" + { $as_echo "$as_me:${as_lineno-$LINENO}: Offloading small packet CIPHERS to Hardware" >&5 +$as_echo "$as_me: Offloading small packet CIPHERS to Hardware" >&6;} +else + enable_qat_small_pkt_offload="" + { $as_echo "$as_me:${as_lineno-$LINENO}: Not Offloading small packet CIPHERS to Hardware" >&5 +$as_echo "$as_me: Not Offloading small packet CIPHERS to Hardware" >&6;} +fi + if test "x$enable_qat_debug" != "xunset" then enable_qat_debug="-DQAT_DEBUG" diff --git a/.tools/configure.ac b/.tools/configure.ac index 65c41c7d..cefde76b 100644 --- a/.tools/configure.ac +++ b/.tools/configure.ac @@ -64,6 +64,11 @@ AC_ARG_ENABLE(qat_prf, , disable_qat_prf=unset) AC_SUBST(disable_qat_prf) +AC_ARG_ENABLE(qat_small_pkt_offload, + AS_HELP_STRING([--enable-qat_small_pkt_offload], + [Enable accelerated small packet CIPHER offload]), + , enable_qat_small_pkt_offload=unset) +AC_SUBST(enable_qat_small_pkt_offload) # Parameter parsing for enabling debug/warnings @@ -261,6 +266,15 @@ else AC_MSG_NOTICE([Not Offloading PRF to Hardware]) fi +if test "x$enable_qat_small_pkt_offload" != "xunset" +then + enable_qat_small_pkt_offload="-DOPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS" + AC_MSG_NOTICE([Offloading small packet CIPHERS to Hardware]) +else + enable_qat_small_pkt_offload="" + AC_MSG_NOTICE([Not Offloading small packet CIPHERS to Hardware]) +fi + if test "x$enable_qat_debug" != "xunset" then enable_qat_debug="-DQAT_DEBUG" diff --git a/README.md b/README.md index faf6dd24..86e3c8b1 100644 --- a/README.md +++ b/README.md @@ -595,6 +595,30 @@ Description: should be passed in as Param 3. Setting the value to -1 results in infinite retries. The default is 5 and the max value is 100,000. This message can be sent at any time after the engine is created. + +Message String: SET_CRYPTO_SMALL_PACKET_OFFLOAD_THRESHOLD +Param 3: 0 +Param 4: string of cipher algorithm name and threshold value +Description: + This message is used to set the threshold that determines the size + crypto packets need to be before they are offloaded to the acceleration + device. Very small crypto packets are not efficient to offload to the + accelerator as it will take longer to transfer the data to and from the + accelerator than it will to encrypt/decrypt using the main CPU. + Each EVP_CIPHER supported in Intel® Quickassist Technology + OpenSSL\* engine can set the threshold value + independently according to the algorithm name as below: + AES-128-CBC-HMAC-SHA1 + AES-256-CBC-HMAC-SHA1 + AES-128-CBC-HMAC-SHA256 + AES-256-CBC-HMAC-SHA256 + The input format should be a string like this in one line: + AES-128-CBC-HMAC-SHA1:4096,AES-256-CBC-HMAC-SHA1:8192 + Using a separator ":" between cipher name and threshold value. + Using a separator "," between different cipher configuration + The default threshold value is 2048 bytes, the minimum is 0 bytes and the + maximum is 16,384. + ``` ## Intel® Quickassist Technology OpenSSL\* Engine Build Options @@ -701,6 +725,10 @@ Optional Disable/Enable Intel® Quickassist Technology PRF offload (enabled by default) +--disable-qat_small_pkt_offload/--enable-qat_small_pkt_offload + Enable Intel® Quickassist Technology to offload small + packet cipher operations (disabled by default) + --disable-qat_debug/--enable-qat_debug Disable/Enable debug output to aid debugging. Warning: This option should never be enabled in a production environment as diff --git a/configure b/configure index a435902b..d78383e9 100755 --- a/configure +++ b/configure @@ -661,6 +661,7 @@ enable_qat_mem_warnings enable_qat_mem_debug enable_qat_warnings enable_qat_debug +enable_qat_small_pkt_offload disable_qat_prf disable_qat_dh disable_qat_dsa @@ -793,6 +794,7 @@ enable_qat_ecdsa enable_qat_dsa enable_qat_dh enable_qat_prf +enable_qat_small_pkt_offload enable_qat_debug enable_qat_warnings enable_qat_mem_debug @@ -1476,6 +1478,8 @@ Optional Features: --disable-qat_dsa Disable accelerated DSA offload --disable-qat_dh Disable accelerated DH offload --disable-qat_prf Disable accelerated PRF offload + --enable-qat_small_pkt_offload + Enable accelerated small packet CIPHER offload --enable-qat_debug Enable Engine Debug Messages --enable-qat_warnings Enable Engine Warning Messages --enable-qat_mem_debug Enable Memory Driver Debug Messages @@ -11458,6 +11462,14 @@ fi +# Check whether --enable-qat_small_pkt_offload was given. +if test "${enable_qat_small_pkt_offload+set}" = set; then : + enableval=$enable_qat_small_pkt_offload; +else + enable_qat_small_pkt_offload=unset +fi + + # Parameter parsing for enabling debug/warnings @@ -11721,6 +11733,17 @@ else $as_echo "$as_me: Not Offloading PRF to Hardware" >&6;} fi +if test "x$enable_qat_small_pkt_offload" != "xunset" +then + enable_qat_small_pkt_offload="-DOPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS" + { $as_echo "$as_me:${as_lineno-$LINENO}: Offloading small packet CIPHERS to Hardware" >&5 +$as_echo "$as_me: Offloading small packet CIPHERS to Hardware" >&6;} +else + enable_qat_small_pkt_offload="" + { $as_echo "$as_me:${as_lineno-$LINENO}: Not Offloading small packet CIPHERS to Hardware" >&5 +$as_echo "$as_me: Not Offloading small packet CIPHERS to Hardware" >&6;} +fi + if test "x$enable_qat_debug" != "xunset" then enable_qat_debug="-DQAT_DEBUG" diff --git a/e_qat.c b/e_qat.c index f46cf4ff..4d33951f 100644 --- a/e_qat.c +++ b/e_qat.c @@ -116,6 +116,8 @@ /* OpenSSL Includes */ #include #include +#include +#include /* QAT includes */ #ifdef USE_QAT_CONTIG_MEM @@ -197,6 +199,18 @@ int qat_is_event_driven() return enable_event_driven_polling; } +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS +int setQatSmallPacketThreshold(unsigned char *cipher_name, int threshold) +{ + if(threshold < 0) + threshold = 0; + else if (threshold > 16384) + threshold = 16384; + return qat_pkt_threshold_table_set_threshold(OBJ_sn2nid(cipher_name),threshold); +} + +#endif + /****************************************************************************** * function: * incr_curr_inst(void) @@ -910,6 +924,7 @@ static int qat_engine_init(ENGINE *e) #define QAT_CMD_GET_NUM_CRYPTO_INSTANCES (ENGINE_CMD_BASE + 8) #define QAT_CMD_DISABLE_EVENT_DRIVEN_POLLING_MODE (ENGINE_CMD_BASE + 9) #define QAT_CMD_SET_EPOLL_TIMEOUT (ENGINE_CMD_BASE + 10) +#define QAT_CMD_SET_CRYPTO_SMALL_PACKET_OFFLOAD_THRESHOLD (ENGINE_CMD_BASE + 11) static const ENGINE_CMD_DEFN qat_cmd_defns[] = { { @@ -967,6 +982,11 @@ static const ENGINE_CMD_DEFN qat_cmd_defns[] = { "SET_EPOLL_TIMEOUT", "Set epoll_wait timeout", ENGINE_CMD_FLAG_NUMERIC}, + { + QAT_CMD_SET_CRYPTO_SMALL_PACKET_OFFLOAD_THRESHOLD, + "SET_CRYPTO_SMALL_PACKET_OFFLOAD_THRESHOLD", + "Set QAT small packet threshold", + ENGINE_CMD_FLAG_STRING}, {0, NULL, NULL, 0} }; @@ -1122,6 +1142,30 @@ qat_engine_ctrl(ENGINE *e, int cmd, long i, void *p, void (*f) (void)) *(int *)p = numInstances; break; + case QAT_CMD_SET_CRYPTO_SMALL_PACKET_OFFLOAD_THRESHOLD: +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + if(p) { + char *token; + while((token = strsep((char **)&p, ","))) { + char *name_token = strsep(&token,":"); + char *value_token = strsep(&token,":"); + if(name_token && value_token) { + retVal = setQatSmallPacketThreshold(name_token, atoi(value_token)); + } else { + WARN("Invalid parameter!\n"); + retVal = 0; + } + } + } else { + WARN("Invalid parameter!\n"); + retVal = 0; + } +#else + WARN("QAT_CMD_SET_CRYPTO_SMALL_PACKET_OFFLOAD_THRESHOLD is not supported\n"); + retVal = 0; +#endif + break; + default: WARN("CTRL command not implemented\n"); retVal = 0; @@ -1163,7 +1207,7 @@ static int qat_engine_finish(ENGINE *e) } if (0 == enable_external_polling && !qat_is_event_driven()) { - if (icp_polling_threads[i] != NULL) { + if ((pthread_t *) icp_polling_threads[i] != 0) { pthread_join(icp_polling_threads[i], NULL); } } @@ -1173,7 +1217,7 @@ static int qat_engine_finish(ENGINE *e) } if (0 == enable_external_polling && qat_is_event_driven()) { - if (icp_polling_threads[0] != NULL) { + if ((pthread_t *) icp_polling_threads[0] != 0) { pthread_join(icp_polling_threads[0], NULL); } } @@ -1246,6 +1290,9 @@ static int qat_engine_destroy(ENGINE *e) qat_free_DH_methods(); qat_free_DSA_methods(); qat_free_RSA_methods(); +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + CRYPTO_THREAD_cleanup_local(&qat_pkt_threshold_table_key); +#endif ERR_unload_QAT_strings(); return 1; } @@ -1292,7 +1339,9 @@ static int bind_qat(ENGINE *e, const char *id) * as this function will be called by a single thread. */ qat_create_ciphers(); - +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + CRYPTO_THREAD_run_once(&qat_pkt_threshold_table_once,qat_pkt_threshold_table_make_key); +#endif DEBUG("%s: About to set mem functions\n", __func__); if (!ENGINE_set_RSA(e, qat_get_RSA_methods())) { diff --git a/e_qat.h b/e_qat.h index e735363e..6434a086 100644 --- a/e_qat.h +++ b/e_qat.h @@ -96,13 +96,8 @@ typedef struct qat_chained_ctx_t { * need to be the elements present in EVP_AES_HMAC_SHA1 defined in * crypto/evp/e_aes_cbc_hmac_sha1.c */ - AES_KEY ks; - SHA_CTX head, tail, md; + size_t payload_length; /* AAD length in decrypt case */ - union { - unsigned int tls_ver; - unsigned char tls_aad[16]; /* 13 used */ - } aux; /* QAT Session Params */ CpaInstanceHandle instanceHandle; @@ -145,6 +140,32 @@ typedef struct qat_chained_ctx_t { } qat_chained_ctx; +typedef struct qat_chained_sha1_ctx_t { + /*struct EVP_AES_HMAC_SHA1 IA_key */ + AES_KEY ks; + SHA_CTX head, tail, md; + size_t payload_length; /* AAD length in decrypt case */ + union { + unsigned int tls_ver; + unsigned char tls_aad[16]; /* 13 used */ + } aux; + + qat_chained_ctx qat_ctx; +}qat_chained_sha1_ctx; + +typedef struct qat_chained_sha256_ctx_t { + /*EVP_AES_HMAC_SHA256 IA_key*/ + AES_KEY ks; + SHA256_CTX head, tail, md; + size_t payload_length; /* AAD length in decrypt case */ + union { + unsigned int tls_ver; + unsigned char tls_aad[16]; /* 13 used */ + } aux; + + qat_chained_ctx qat_ctx; +}qat_chained_sha256_ctx; + /* qat_buffer structure for partial hash */ typedef struct qat_buffer_t { struct qat_buffer_t *next; /* next buffer in the list */ diff --git a/qat_ciphers.c b/qat_ciphers.c index 55219042..cc39c348 100644 --- a/qat_ciphers.c +++ b/qat_ciphers.c @@ -69,6 +69,7 @@ #include #include #include +#include #include #ifdef OPENSSL_ENABLE_QAT_CIPHERS @@ -108,7 +109,7 @@ const EVP_CIPHER *qat_aes_128_cbc_hmac_sha1(void) || !EVP_CIPHER_meth_set_cleanup(_hidden_aes_128_cbc_hmac_sha1, qat_aes_cbc_hmac_sha_cleanup) || !EVP_CIPHER_meth_set_impl_ctx_size(_hidden_aes_128_cbc_hmac_sha1, - sizeof(qat_chained_ctx)) + sizeof(qat_chained_sha1_ctx)) || !EVP_CIPHER_meth_set_set_asn1_params(_hidden_aes_128_cbc_hmac_sha1, EVP_CIPH_FLAG_DEFAULT_ASN1 ? NULL : @@ -146,7 +147,7 @@ const EVP_CIPHER *qat_aes_256_cbc_hmac_sha1(void) || !EVP_CIPHER_meth_set_cleanup(_hidden_aes_256_cbc_hmac_sha1, qat_aes_cbc_hmac_sha_cleanup) || !EVP_CIPHER_meth_set_impl_ctx_size(_hidden_aes_256_cbc_hmac_sha1, - sizeof(qat_chained_ctx)) + sizeof(qat_chained_sha1_ctx)) || !EVP_CIPHER_meth_set_set_asn1_params(_hidden_aes_256_cbc_hmac_sha1, EVP_CIPH_FLAG_DEFAULT_ASN1 ? NULL : @@ -184,7 +185,7 @@ const EVP_CIPHER *qat_aes_128_cbc_hmac_sha256(void) || !EVP_CIPHER_meth_set_cleanup(_hidden_aes_128_cbc_hmac_sha256, qat_aes_cbc_hmac_sha_cleanup) || !EVP_CIPHER_meth_set_impl_ctx_size(_hidden_aes_128_cbc_hmac_sha256, - sizeof(qat_chained_ctx)) + sizeof(qat_chained_sha256_ctx)) || !EVP_CIPHER_meth_set_set_asn1_params(_hidden_aes_128_cbc_hmac_sha256, EVP_CIPH_FLAG_DEFAULT_ASN1 ? NULL : @@ -222,7 +223,7 @@ const EVP_CIPHER *qat_aes_256_cbc_hmac_sha256(void) || !EVP_CIPHER_meth_set_cleanup(_hidden_aes_256_cbc_hmac_sha256, qat_aes_cbc_hmac_sha_cleanup) || !EVP_CIPHER_meth_set_impl_ctx_size(_hidden_aes_256_cbc_hmac_sha256, - sizeof(qat_chained_ctx)) + sizeof(qat_chained_sha256_ctx)) || !EVP_CIPHER_meth_set_set_asn1_params(_hidden_aes_256_cbc_hmac_sha256, EVP_CIPH_FLAG_DEFAULT_ASN1 ? NULL : @@ -288,6 +289,116 @@ void qat_free_ciphers(void) {} #endif +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS +# define CRYPTO_SMALL_PACKET_OFFLOAD_THRESHOLD_DEFAULT 2048 + +CRYPTO_ONCE qat_pkt_threshold_table_once = CRYPTO_ONCE_STATIC_INIT; +CRYPTO_THREAD_LOCAL qat_pkt_threshold_table_key; + +void qat_pkt_threshold_table_make_key(void) +{ + CRYPTO_THREAD_init_local(&qat_pkt_threshold_table_key, qat_free_pkt_threshold_table); +} + +typedef struct cipher_threshold_table_s { + int nid; + int threshold; +}PKT_THRESHOLD; + +DEFINE_LHASH_OF(PKT_THRESHOLD); + +PKT_THRESHOLD qat_pkt_threshold_table[] = { + {NID_aes_128_cbc_hmac_sha1,CRYPTO_SMALL_PACKET_OFFLOAD_THRESHOLD_DEFAULT}, + {NID_aes_256_cbc_hmac_sha1,CRYPTO_SMALL_PACKET_OFFLOAD_THRESHOLD_DEFAULT}, + {NID_aes_128_cbc_hmac_sha256,CRYPTO_SMALL_PACKET_OFFLOAD_THRESHOLD_DEFAULT}, + {NID_aes_256_cbc_hmac_sha256,CRYPTO_SMALL_PACKET_OFFLOAD_THRESHOLD_DEFAULT} +}; + +static int pkt_threshold_table_cmp(const PKT_THRESHOLD * a, const PKT_THRESHOLD * b) +{ + return (a->nid == b->nid)?0:1; +} + +static unsigned long pkt_threshold_table_hash(const PKT_THRESHOLD * a) +{ + return (unsigned long)(a->nid); +} + +LHASH_OF(PKT_THRESHOLD) *qat_create_pkt_threshold_table(void) +{ + int i; + LHASH_OF(PKT_THRESHOLD) *ret = NULL; + ret = lh_PKT_THRESHOLD_new(pkt_threshold_table_hash,pkt_threshold_table_cmp); + if(ret == NULL) { + return ret; + } + for(i = 0; i < sizeof(qat_pkt_threshold_table);i++) { + lh_PKT_THRESHOLD_insert(ret,&qat_pkt_threshold_table[i]); + } + return ret; +} + +int qat_pkt_threshold_table_set_threshold(int nid, int threshold) +{ + PKT_THRESHOLD entry,*ret; + LHASH_OF(PKT_THRESHOLD) *tbl = NULL; + if(NID_undef == nid) { + WARN("Unsupported NID\n"); + return 0; + } + if((tbl = CRYPTO_THREAD_get_local(&qat_pkt_threshold_table_key)) == NULL) { + tbl = qat_create_pkt_threshold_table(); + if(tbl != NULL) { + CRYPTO_THREAD_set_local(&qat_pkt_threshold_table_key, tbl); + } + else { + WARN("Create packet threshold table fail.\n"); + return 0; + } + } + entry.nid = nid; + ret = lh_PKT_THRESHOLD_retrieve(tbl,&entry); + if(ret == NULL) { + WARN("Threshold entry retrieve failed for the NID : %d\n",entry.nid); + return 0; + } + ret->threshold = threshold; + lh_PKT_THRESHOLD_insert(tbl, ret); + return 1; +} + +int qat_pkt_threshold_table_get_threshold(int nid) +{ + PKT_THRESHOLD entry,*ret; + LHASH_OF(PKT_THRESHOLD) *tbl = NULL; + if((tbl = CRYPTO_THREAD_get_local(&qat_pkt_threshold_table_key)) == NULL) { + tbl = qat_create_pkt_threshold_table(); + if(tbl != NULL) { + CRYPTO_THREAD_set_local(&qat_pkt_threshold_table_key, tbl); + } + else { + WARN("Create packet threshold table fail.\n"); + return 0; + } + } + entry.nid = nid; + ret = lh_PKT_THRESHOLD_retrieve(tbl,&entry); + if(ret == NULL) { + WARN("Threshold entry retrieve failed for the NID : %d\n",entry.nid); + return 0; + } + return ret->threshold; +} + +void qat_free_pkt_threshold_table(void *thread_key) +{ + LHASH_OF(PKT_THRESHOLD) *tbl = (LHASH_OF(PKT_THRESHOLD) *) thread_key; + if((tbl = CRYPTO_THREAD_get_local(&qat_pkt_threshold_table_key))) { + lh_PKT_THRESHOLD_free(tbl); + } +} + +#endif /****************************************************************************** * function: * qat_ciphers(ENGINE *e, @@ -546,7 +657,6 @@ int qat_aes_cbc_hmac_sha_init(EVP_CIPHER_CTX *ctx, const unsigned char *inkey, const unsigned char *iv, int enc) { - /* Initialise a QAT session and set the cipher keys */ qat_chained_ctx *qat_ctx = NULL; @@ -554,8 +664,48 @@ int qat_aes_cbc_hmac_sha_init(EVP_CIPHER_CTX *ctx, WARN("[%s] ctx or inkey is NULL.\n", __func__); return 0; } - - qat_ctx = qat_chained_data(ctx); +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + const EVP_CIPHER *ia_cipher = NULL; +#endif + int nid = EVP_CIPHER_CTX_nid(ctx); + switch (nid) { + case NID_aes_128_cbc_hmac_sha1: +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + ia_cipher = EVP_aes_128_cbc_hmac_sha1(); +#endif + qat_ctx = &(((qat_chained_sha1_ctx *)qat_chained_data(ctx))->qat_ctx); + break; + case NID_aes_256_cbc_hmac_sha1: +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + ia_cipher = EVP_aes_256_cbc_hmac_sha1(); +#endif + qat_ctx = &(((qat_chained_sha1_ctx *)qat_chained_data(ctx))->qat_ctx); + break; + case NID_aes_128_cbc_hmac_sha256: +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + ia_cipher = EVP_aes_128_cbc_hmac_sha256(); +#endif + qat_ctx = &(((qat_chained_sha256_ctx *)qat_chained_data(ctx))->qat_ctx); + break; + case NID_aes_256_cbc_hmac_sha256: +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + ia_cipher = EVP_aes_256_cbc_hmac_sha256(); +#endif + qat_ctx = &(((qat_chained_sha256_ctx *)qat_chained_data(ctx))->qat_ctx); + break; + default: +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + ia_cipher = NULL; +#endif + return 0; + } + if (iv) + memcpy((unsigned char *)EVP_CIPHER_CTX_original_iv(ctx), iv, EVP_CIPHER_CTX_iv_length(ctx)); + memcpy(EVP_CIPHER_CTX_iv_noconst(ctx), EVP_CIPHER_CTX_original_iv(ctx), EVP_CIPHER_CTX_iv_length(ctx)); +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + if(EVP_CIPHER_meth_get_init(ia_cipher)(ctx, inkey, iv, enc) == 0) + goto end; +#endif if (qat_ctx == NULL) { WARN("[%s] --- qat_ctx is NULL.\n", __func__); return 0; @@ -652,9 +802,51 @@ int qat_aes_cbc_hmac_sha_ctrl(EVP_CIPHER_CTX *ctx, int type, int arg, WARN("[%s] --- ctx parameter is NULL.\n", __func__); return -1; } +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + const EVP_CIPHER *ia_cipher = NULL; + size_t *payload_len_ptr = NULL; +#endif + int nid = EVP_CIPHER_CTX_nid(ctx); + switch (nid) { + case NID_aes_128_cbc_hmac_sha1: +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + ia_cipher = EVP_aes_128_cbc_hmac_sha1(); + payload_len_ptr = &(((qat_chained_sha1_ctx *)qat_chained_data(ctx))->payload_length); +#endif + evp_ctx = &(((qat_chained_sha1_ctx *)qat_chained_data(ctx))->qat_ctx); + break; + case NID_aes_256_cbc_hmac_sha1: +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + ia_cipher = EVP_aes_256_cbc_hmac_sha1(); + payload_len_ptr = &(((qat_chained_sha1_ctx *)qat_chained_data(ctx))->payload_length); +#endif + evp_ctx = &(((qat_chained_sha1_ctx *)qat_chained_data(ctx))->qat_ctx); + break; + case NID_aes_128_cbc_hmac_sha256: +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + ia_cipher = EVP_aes_128_cbc_hmac_sha256(); + payload_len_ptr = &(((qat_chained_sha256_ctx *)qat_chained_data(ctx))->payload_length); +#endif + evp_ctx = &(((qat_chained_sha256_ctx *)qat_chained_data(ctx))->qat_ctx); + break; + case NID_aes_256_cbc_hmac_sha256: +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + ia_cipher = EVP_aes_256_cbc_hmac_sha256(); + payload_len_ptr = &(((qat_chained_sha256_ctx *)qat_chained_data(ctx))->payload_length); +#endif + evp_ctx = &(((qat_chained_sha256_ctx *)qat_chained_data(ctx))->qat_ctx); + break; + default: +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + ia_cipher = NULL; +#endif + return 0; + } - evp_ctx = qat_chained_data(ctx); - +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + EVP_CIPHER_meth_get_ctrl(ia_cipher)(ctx, type, arg, ptr); +#endif + if (evp_ctx == NULL) { WARN("[%s] --- evp_ctx is NULL.\n", __func__); return -1; @@ -711,21 +903,27 @@ int qat_aes_cbc_hmac_sha_ctrl(EVP_CIPHER_CTX *ctx, int type, int arg, * the send/encrypt direction */ p = ptr; - len = - (p[arg - QAT_TLS_PAYLOADLENGTH_MSB_OFFSET] << QAT_BYTE_SHIFT | - p[arg - QAT_TLS_PAYLOADLENGTH_LSB_OFFSET]); if (arg < TLS_VIRT_HDR_SIZE) { retVal = -1; break; } +#ifdef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + len = + (p[arg - QAT_TLS_PAYLOADLENGTH_MSB_OFFSET] << QAT_BYTE_SHIFT | + p[arg - QAT_TLS_PAYLOADLENGTH_LSB_OFFSET]); +#else + len = *(unsigned int *)payload_len_ptr; +#endif + evp_ctx->tls_version = (p[arg - QAT_TLS_VERSION_MSB_OFFSET] << QAT_BYTE_SHIFT | p[arg - QAT_TLS_VERSION_LSB_OFFSET]); if (EVP_CIPHER_CTX_encrypting(ctx)) { evp_ctx->payload_length = len; +#ifdef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS if (evp_ctx->tls_version >= TLS1_1_VERSION) { len -= AES_BLOCK_SIZE; /* TODO: Why does this code reduce the len in the @@ -734,7 +932,7 @@ int qat_aes_cbc_hmac_sha_ctrl(EVP_CIPHER_CTX *ctx, int type, int arg, len >> QAT_BYTE_SHIFT; p[arg - QAT_TLS_PAYLOADLENGTH_LSB_OFFSET] = len; } - +#endif if (NULL == evp_ctx->tls_virt_hdr) { WARN("Unable to allocate memory for mac preamble in qat/n"); return -1; @@ -803,21 +1001,26 @@ int qat_aes_cbc_hmac_sha_cleanup(EVP_CIPHER_CTX *ctx) return 0; } - evp_ctx = qat_chained_data(ctx); + int nid = 0; + nid = EVP_CIPHER_CTX_nid(ctx); + switch (nid) { + case NID_aes_128_cbc_hmac_sha1: + case NID_aes_256_cbc_hmac_sha1: + evp_ctx = &(((qat_chained_sha1_ctx *)qat_chained_data(ctx))->qat_ctx); + break; + case NID_aes_128_cbc_hmac_sha256: + case NID_aes_256_cbc_hmac_sha256: + evp_ctx = &(((qat_chained_sha256_ctx *)qat_chained_data(ctx))->qat_ctx); + break; + default: + return 0; + } if (evp_ctx == NULL) { WARN("[%s] evp_ctx parameter is NULL.\n", __func__); return 0; } - if (!(evp_ctx->init)) { - /* - * It is valid to call cleanup even if the context has not been - * initialised. - */ - return retVal; - } - sessSetup = evp_ctx->session_data; if (sessSetup) { if (evp_ctx->qat_ctx) { @@ -901,7 +1104,20 @@ static int qat_aes_sha_session_init(EVP_CIPHER_CTX *ctx) return 0; } - evp_ctx = qat_chained_data(ctx); + int nid = 0; + nid = EVP_CIPHER_CTX_nid(ctx); + switch (nid) { + case NID_aes_128_cbc_hmac_sha1: + case NID_aes_256_cbc_hmac_sha1: + evp_ctx = &(((qat_chained_sha1_ctx *)qat_chained_data(ctx))->qat_ctx); + break; + case NID_aes_128_cbc_hmac_sha256: + case NID_aes_256_cbc_hmac_sha256: + evp_ctx = &(((qat_chained_sha256_ctx *)qat_chained_data(ctx))->qat_ctx); + break; + default: + return 0; + } if (evp_ctx == NULL) { WARN("[%s] --- evp_ctx is NULL.\n", __func__); @@ -954,8 +1170,6 @@ static int qat_aes_sha_session_init(EVP_CIPHER_CTX *ctx) return 0; } - evp_ctx->qat_ctx = pSessionCtx; - evp_ctx->srcBufferList.numBuffers = 2; evp_ctx->srcBufferList.pBuffers = (evp_ctx->srcFlatBuffer); evp_ctx->srcBufferList.pUserData = NULL; @@ -1020,6 +1234,7 @@ static int qat_aes_sha_session_init(EVP_CIPHER_CTX *ctx) * Create the OpData structure to remove this processing from the data * path */ + evp_ctx->qat_ctx = pSessionCtx; evp_ctx->OpData.sessionCtx = evp_ctx->qat_ctx; evp_ctx->OpData.packetType = CPA_CY_SYM_PACKET_TYPE_FULL; @@ -1080,7 +1295,49 @@ int qat_aes_cbc_hmac_sha_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, return 0; } - evp_ctx = qat_chained_data(ctx); +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + const EVP_CIPHER *ia_cipher = NULL; +#endif + + int nid = EVP_CIPHER_CTX_nid(ctx); + switch (nid) { + case NID_aes_128_cbc_hmac_sha1: +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + ia_cipher = EVP_aes_128_cbc_hmac_sha1(); +#endif + evp_ctx = &(((qat_chained_sha1_ctx *)qat_chained_data(ctx))->qat_ctx); + break; + case NID_aes_256_cbc_hmac_sha1: +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + ia_cipher = EVP_aes_256_cbc_hmac_sha1(); +#endif + evp_ctx = &(((qat_chained_sha1_ctx *)qat_chained_data(ctx))->qat_ctx); + break; + case NID_aes_128_cbc_hmac_sha256: +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + ia_cipher = EVP_aes_128_cbc_hmac_sha256(); +#endif + evp_ctx = &(((qat_chained_sha256_ctx *)qat_chained_data(ctx))->qat_ctx); + break; + case NID_aes_256_cbc_hmac_sha256: +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + ia_cipher = EVP_aes_256_cbc_hmac_sha256(); +#endif + evp_ctx = &(((qat_chained_sha256_ctx *)qat_chained_data(ctx))->qat_ctx); + break; + default: +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + ia_cipher = NULL; +#endif + return 0; + } + +#ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS + if(len <= qat_pkt_threshold_table_get_threshold(nid)) { + retVal = EVP_CIPHER_meth_get_do_cipher(ia_cipher)(ctx, out, in, len); + return retVal; + } +#endif if (evp_ctx == NULL) { WARN("[%s] --- evp_ctx is NULL.\n", __func__); diff --git a/qat_ciphers.h b/qat_ciphers.h index c27e620e..9581034b 100644 --- a/qat_ciphers.h +++ b/qat_ciphers.h @@ -47,13 +47,15 @@ # define QAT_CIPHERS_H # include +# include # define AES_BLOCK_SIZE 16 # define AES_IV_LEN 16 # define AES_KEY_SIZE_256 32 # define AES_KEY_SIZE_128 16 -# define qat_chained_data(ctx) ((qat_chained_ctx *)EVP_CIPHER_CTX_get_cipher_data(ctx)) +# define qat_chained_data(ctx) (EVP_CIPHER_CTX_get_cipher_data(ctx)) + # define HMAC_KEY_SIZE 64 # define TLS_VIRT_HDR_SIZE 13 @@ -77,5 +79,13 @@ void qat_create_ciphers(void); void qat_free_ciphers(void); int qat_ciphers(ENGINE *e, const EVP_CIPHER **cipher, const int **nids, int nid); - +# ifndef OPENSSL_ENABLE_QAT_SMALL_PACKET_CIPHER_OFFLOADS +extern CRYPTO_ONCE qat_pkt_threshold_table_once; +extern CRYPTO_THREAD_LOCAL qat_pkt_threshold_table_key; +void qat_pkt_threshold_table_make_key(void ); +LHASH_OF(PKT_THRESHOLD) *qat_create_pkt_threshold_table(void); +void qat_free_pkt_threshold_table(void *); +int qat_pkt_threshold_table_set_threshold(int nid, int threshold); +int qat_pkt_threshold_table_get_threshold(int nid); +# endif #endif /* QAT_CIPHERS_H */