crypto/qat: support cipher-CRC offload

This patch adds support to the QAT symmetric crypto PMD for combined
Cipher-CRC offload feature, primarily for DOCSIS, on gen2/gen3/gen4
QAT devices.

A new devarg called qat_sym_cipher_crc_enable has been
added to the PMD, which can be set on process start as follows:

-a <qat pci bdf>,qat_sym_cipher_crc_enable=1

When enabled, a capability check for the combined Cipher-CRC offload
feature is triggered to the QAT firmware during queue pair
initialization. If supported by the firmware, any subsequent runtime
DOCSIS Cipher-CRC requests handled by the QAT PMD are offloaded to the
QAT device by setting up the content descriptor and request
accordingly.

If the combined DOCSIS Cipher-CRC feature is not supported by the
firmware, the CRC continues to be calculated within the PMD, with just
the cipher portion of the request being offloaded to the QAT device.

Signed-off-by: Kevin O'Sullivan <kevin.osullivan@intel.com>
Signed-off-by: David Coyle <david.coyle@intel.com>
Acked-by: Kai Ji <kai.ji@intel.com>

doc/guides/cryptodevs/qat.rst

@@ -294,6 +294,29 @@ by comma. When the same parameter is used more than once first occurrence of the
 is used.
 Maximum threshold that can be set is 32.
 
+
+Running QAT PMD with Cipher-CRC offload feature
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Support has been added to the QAT symmetric crypto PMD for combined Cipher-CRC offload,
+primarily for the Crypto-CRC DOCSIS security protocol, on GEN2/GEN3/GEN4 QAT devices.
+
+The following devarg enables a Cipher-CRC offload capability check to determine
+if the feature is supported on the QAT device.
+
+- qat_sym_cipher_crc_enable
+
+When enabled, a capability check for the combined Cipher-CRC offload feature is triggered
+to the QAT firmware during queue pair initialization. If supported by the firmware,
+any subsequent runtime Crypto-CRC DOCSIS security protocol requests handled by the QAT PMD
+are offloaded to the QAT device by setting up the content descriptor and request accordingly.
+If not supported, the CRC is calculated by the QAT PMD using the NET CRC API.
+
+To use this feature the user must set the devarg on process start as a device additional devarg::
+
+ -a 03:01.1,qat_sym_cipher_crc_enable=1
+
+
 Running QAT PMD with Intel IPSEC MB library for symmetric precomputes function
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

doc/guides/rel_notes/release_23_07.rst

@@ -108,6 +108,10 @@ New Features
   for new capability registers, large passthrough BAR and some
   performance enhancements for UPT.
 
+* **Updated Intel QuickAssist Technology (QAT) crypto driver.**
+
+  * Added support for combined Cipher-CRC offload for DOCSIS for QAT GENs 2,3 and 4.
+
 
 Removed Items
 -------------

drivers/common/qat/qat_device.c

@@ -149,7 +149,13 @@ qat_dev_parse_cmd(const char *str, struct qat_dev_cmd_param
 			} else {
 				memcpy(value_str, arg2, iter);
 				value = strtol(value_str, NULL, 10);
-				if (value > MAX_QP_THRESHOLD_SIZE) {
+				if (strcmp(param,
+					 SYM_CIPHER_CRC_ENABLE_NAME) == 0) {
+					if (value < 0 || value > 1) {
+						QAT_LOG(DEBUG, "The value for qat_sym_cipher_crc_enable should be set to 0 or 1, setting to 0");
+						value = 0;
+					}
+				} else if (value > MAX_QP_THRESHOLD_SIZE) {
 					QAT_LOG(DEBUG, "Exceeded max size of"
 						" threshold, setting to %d",
 						MAX_QP_THRESHOLD_SIZE);
@@ -369,6 +375,7 @@ static int qat_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 			{ SYM_ENQ_THRESHOLD_NAME, 0 },
 			{ ASYM_ENQ_THRESHOLD_NAME, 0 },
 			{ COMP_ENQ_THRESHOLD_NAME, 0 },
+			{ SYM_CIPHER_CRC_ENABLE_NAME, 0 },
 			[QAT_CMD_SLICE_MAP_POS] = { QAT_CMD_SLICE_MAP, 0},
 			{ NULL, 0 },
 	};

drivers/common/qat/qat_device.h

@@ -21,8 +21,9 @@
 #define SYM_ENQ_THRESHOLD_NAME "qat_sym_enq_threshold"
 #define ASYM_ENQ_THRESHOLD_NAME "qat_asym_enq_threshold"
 #define COMP_ENQ_THRESHOLD_NAME "qat_comp_enq_threshold"
+#define SYM_CIPHER_CRC_ENABLE_NAME "qat_sym_cipher_crc_enable"
 #define QAT_CMD_SLICE_MAP "qat_cmd_slice_disable"
-#define QAT_CMD_SLICE_MAP_POS	4
+#define QAT_CMD_SLICE_MAP_POS	5
 #define MAX_QP_THRESHOLD_SIZE	32
 
 /**

drivers/common/qat/qat_qp.c

@@ -11,6 +11,9 @@
 #include <bus_pci_driver.h>
 #include <rte_atomic.h>
 #include <rte_prefetch.h>
+#ifdef RTE_LIB_SECURITY
+#include <rte_ether.h>
+#endif
 
 #include "qat_logs.h"
 #include "qat_device.h"
@@ -24,6 +27,44 @@
 #define ADF_MAX_DESC				4096
 #define ADF_MIN_DESC				128
 
+#ifdef BUILD_QAT_SYM
+/* Cipher-CRC capability check test parameters */
+static const uint8_t cipher_crc_cap_check_iv[] = {
+	0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11,
+	0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11
+};
+
+static const uint8_t cipher_crc_cap_check_key[] = {
+	0x00, 0x00, 0x00, 0x00, 0xAA, 0xBB, 0xCC, 0xDD,
+	0xEE, 0xFF, 0x00, 0x11, 0x22, 0x33, 0x44, 0x55
+};
+
+static const uint8_t cipher_crc_cap_check_plaintext[] = {
+	/* Outer protocol header */
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	/* Ethernet frame */
+	0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x06, 0x05,
+	0x04, 0x03, 0x02, 0x01, 0x08, 0x00, 0xAA, 0xAA,
+	0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA,
+	/* CRC */
+	0xFF, 0xFF, 0xFF, 0xFF
+};
+
+static const uint8_t cipher_crc_cap_check_ciphertext[] = {
+	/* Outer protocol header */
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	/* Ethernet frame */
+	0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x06, 0x05,
+	0x04, 0x03, 0x02, 0x01, 0xD6, 0xE2, 0x70, 0x5C,
+	0xE6, 0x4D, 0xCC, 0x8C, 0x47, 0xB7, 0x09, 0xD6,
+	/* CRC */
+	0x54, 0x85, 0xF8, 0x32
+};
+
+static const uint8_t cipher_crc_cap_check_cipher_offset = 18;
+static const uint8_t cipher_crc_cap_check_crc_offset = 6;
+#endif
+
 struct qat_qp_hw_spec_funcs*
 	qat_qp_hw_spec[QAT_N_GENS];
 
@@ -957,6 +998,142 @@ qat_cq_get_fw_version(struct qat_qp *qp)
 	return -EINVAL;
 }
 
+#ifdef BUILD_QAT_SYM
+/* Sends an LA bulk req message to determine if a QAT device supports Cipher-CRC
+ * offload. This assumes that there are no inflight messages, i.e. assumes
+ * there's space  on the qp, one message is sent and only one response
+ * collected. The status bit of the response and returned data are checked.
+ * Returns:
+ *     1 if status bit indicates success and returned data matches expected
+ *     data (i.e. Cipher-CRC supported)
+ *     0 if status bit indicates error or returned data does not match expected
+ *     data (i.e. Cipher-CRC not supported)
+ *     Negative error code in case of error
+ */
+int
+qat_cq_get_fw_cipher_crc_cap(struct qat_qp *qp)
+{
+	struct qat_queue *queue = &(qp->tx_q);
+	uint8_t *base_addr = (uint8_t *)queue->base_addr;
+	struct icp_qat_fw_la_bulk_req cipher_crc_cap_msg = {{0}};
+	struct icp_qat_fw_comn_resp response = {{0}};
+	struct icp_qat_fw_la_cipher_req_params *cipher_param;
+	struct icp_qat_fw_la_auth_req_params *auth_param;
+	struct qat_sym_session *session;
+	phys_addr_t phy_src_addr;
+	uint64_t *src_data_addr;
+	int ret;
+
+	session = rte_zmalloc(NULL, sizeof(struct qat_sym_session), 0);
+	if (session == NULL)
+		return -EINVAL;
+
+	/* Verify the session physical address is known */
+	rte_iova_t session_paddr = rte_mem_virt2iova(session);
+	if (session_paddr == 0 || session_paddr == RTE_BAD_IOVA) {
+		QAT_LOG(ERR, "Session physical address unknown.");
+		return -EINVAL;
+	}
+
+	/* Prepare the LA bulk request */
+	ret = qat_cipher_crc_cap_msg_sess_prepare(session,
+					session_paddr,
+					cipher_crc_cap_check_key,
+					sizeof(cipher_crc_cap_check_key),
+					qp->qat_dev_gen);
+	if (ret < 0) {
+		rte_free(session);
+		/* Returning 0 here to allow qp setup to continue, but
+		 * indicate that Cipher-CRC offload is not supported on the
+		 * device
+		 */
+		return 0;
+	}
+
+	cipher_crc_cap_msg = session->fw_req;
+
+	src_data_addr = rte_zmalloc(NULL,
+					sizeof(cipher_crc_cap_check_plaintext),
+					0);
+	if (src_data_addr == NULL) {
+		rte_free(session);
+		return -EINVAL;
+	}
+
+	rte_memcpy(src_data_addr,
+			cipher_crc_cap_check_plaintext,
+			sizeof(cipher_crc_cap_check_plaintext));
+
+	phy_src_addr = rte_mem_virt2iova(src_data_addr);
+	if (phy_src_addr == 0 || phy_src_addr == RTE_BAD_IOVA) {
+		QAT_LOG(ERR, "Source physical address unknown.");
+		return -EINVAL;
+	}
+
+	cipher_crc_cap_msg.comn_mid.src_data_addr = phy_src_addr;
+	cipher_crc_cap_msg.comn_mid.src_length =
+					sizeof(cipher_crc_cap_check_plaintext);
+	cipher_crc_cap_msg.comn_mid.dest_data_addr = phy_src_addr;
+	cipher_crc_cap_msg.comn_mid.dst_length =
+					sizeof(cipher_crc_cap_check_plaintext);
+
+	cipher_param = (void *)&cipher_crc_cap_msg.serv_specif_rqpars;
+	auth_param = (void *)((uint8_t *)cipher_param +
+			ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
+
+	rte_memcpy(cipher_param->u.cipher_IV_array,
+			cipher_crc_cap_check_iv,
+			sizeof(cipher_crc_cap_check_iv));
+
+	cipher_param->cipher_offset = cipher_crc_cap_check_cipher_offset;
+	cipher_param->cipher_length =
+			sizeof(cipher_crc_cap_check_plaintext) -
+			cipher_crc_cap_check_cipher_offset;
+	auth_param->auth_off = cipher_crc_cap_check_crc_offset;
+	auth_param->auth_len = sizeof(cipher_crc_cap_check_plaintext) -
+				cipher_crc_cap_check_crc_offset -
+				RTE_ETHER_CRC_LEN;
+
+	ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(
+			cipher_crc_cap_msg.comn_hdr.serv_specif_flags,
+			ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
+
+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
+	QAT_DP_HEXDUMP_LOG(DEBUG, "LA Bulk request", &cipher_crc_cap_msg,
+			sizeof(cipher_crc_cap_msg));
+#endif
+
+	/* Send the cipher_crc_cap_msg request */
+	memcpy(base_addr + queue->tail,
+	       &cipher_crc_cap_msg,
+	       sizeof(cipher_crc_cap_msg));
+	queue->tail = adf_modulo(queue->tail + queue->msg_size,
+			queue->modulo_mask);
+	txq_write_tail(qp->qat_dev_gen, qp, queue);
+
+	/* Check for response and verify data is same as ciphertext */
+	if (qat_cq_dequeue_response(qp, &response)) {
+#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
+		QAT_DP_HEXDUMP_LOG(DEBUG, "LA response:", &response,
+				sizeof(response));
+#endif
+
+		if (memcmp(src_data_addr,
+				cipher_crc_cap_check_ciphertext,
+				sizeof(cipher_crc_cap_check_ciphertext)) != 0)
+			ret = 0; /* Cipher-CRC offload not supported */
+		else
+			ret = 1;
+	} else {
+		ret = -EINVAL;
+	}
+
+	rte_free(src_data_addr);
+	rte_free(session);
+	return ret;
+}
+#endif
+
 __rte_weak int
 qat_comp_process_response(void **op __rte_unused, uint8_t *resp __rte_unused,
 			  void *op_cookie __rte_unused,

drivers/common/qat/qat_qp.h

@@ -153,6 +153,11 @@ qat_qp_get_hw_data(struct qat_pci_device *qat_dev,
 int
 qat_cq_get_fw_version(struct qat_qp *qp);
 
+#ifdef BUILD_QAT_SYM
+int
+qat_cq_get_fw_cipher_crc_cap(struct qat_qp *qp);
+#endif
+
 /* Needed for weak function*/
 int
 qat_comp_process_response(void **op __rte_unused, uint8_t *resp __rte_unused,

drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c

@@ -163,7 +163,7 @@ qat_sym_crypto_qp_setup_gen2(struct rte_cryptodev *dev, uint16_t qp_id,
 		QAT_LOG(DEBUG, "unknown QAT firmware version");
 
 	/* set capabilities based on the fw version */
-	qat_sym_private->internal_capabilities = QAT_SYM_CAP_VALID |
+	qat_sym_private->internal_capabilities |= QAT_SYM_CAP_VALID |
 			((ret >= MIXED_CRYPTO_MIN_FW_VER) ?
 					QAT_SYM_CAP_MIXED_CRYPTO : 0);
 	return 0;

drivers/crypto/qat/dev/qat_crypto_pmd_gens.h

@@ -399,8 +399,13 @@ qat_sym_convert_op_to_vec_chain(struct rte_crypto_op *op,
 		cipher_ofs = op->sym->cipher.data.offset >> 3;
 		break;
 	case 0:
-		cipher_len = op->sym->cipher.data.length;
-		cipher_ofs = op->sym->cipher.data.offset;
+		if (ctx->bpi_ctx) {
+			cipher_len = qat_bpicipher_preprocess(ctx, op);
+			cipher_ofs = op->sym->cipher.data.offset;
+		} else {
+			cipher_len = op->sym->cipher.data.length;
+			cipher_ofs = op->sym->cipher.data.offset;
+		}
 		break;
 	default:
 		QAT_DP_LOG(ERR,
@@ -428,8 +433,10 @@ qat_sym_convert_op_to_vec_chain(struct rte_crypto_op *op,
 
 	max_len = RTE_MAX(cipher_ofs + cipher_len, auth_ofs + auth_len);
 
-	/* digest in buffer check. Needed only for wireless algos */
-	if (ret == 1) {
+	/* digest in buffer check. Needed only for wireless algos
+	 * or combined cipher-crc operations
+	 */
+	if (ret == 1 || ctx->bpi_ctx) {
 		/* Handle digest-encrypted cases, i.e.
 		 * auth-gen-then-cipher-encrypt and
 		 * cipher-decrypt-then-auth-verify
@@ -456,8 +463,9 @@ qat_sym_convert_op_to_vec_chain(struct rte_crypto_op *op,
 					auth_len;
 
 		/* Then check if digest-encrypted conditions are met */
-		if ((auth_ofs + auth_len < cipher_ofs + cipher_len) &&
-				(digest->iova == auth_end_iova))
+		if (((auth_ofs + auth_len < cipher_ofs + cipher_len) &&
+				(digest->iova == auth_end_iova)) ||
+				ctx->bpi_ctx)
 			max_len = RTE_MAX(max_len, auth_ofs + auth_len +
 					ctx->digest_length);
 	}
@@ -692,9 +700,9 @@ enqueue_one_chain_job_gen1(struct qat_sym_session *ctx,
 			auth_param->auth_len;
 
 	/* Then check if digest-encrypted conditions are met */
-	if ((auth_param->auth_off + auth_param->auth_len <
+	if (((auth_param->auth_off + auth_param->auth_len <
 		cipher_param->cipher_offset + cipher_param->cipher_length) &&
-			(digest->iova == auth_iova_end)) {
+			(digest->iova == auth_iova_end)) || ctx->bpi_ctx) {
 		/* Handle partial digest encryption */
 		if (cipher_param->cipher_offset + cipher_param->cipher_length <
 			auth_param->auth_off + auth_param->auth_len +

drivers/crypto/qat/dev/qat_sym_pmd_gen1.c

@@ -1205,6 +1205,10 @@ qat_sym_crypto_set_session_gen1(void *cryptodev __rte_unused, void *session)
 	} else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER) {
 		/* do_auth = 0; do_cipher = 1; */
 		build_request = qat_sym_build_op_cipher_gen1;
+	} else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER_CRC) {
+		/* do_auth = 1; do_cipher = 1; */
+		build_request = qat_sym_build_op_chain_gen1;
+		handle_mixed = 1;
 	}
 
 	if (build_request)

drivers/crypto/qat/qat_crypto.c

@@ -172,5 +172,25 @@ qat_cryptodev_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 			qat_asym_init_op_cookie(qp->op_cookies[i]);
 	}
 
-	return ret;
+	if (qat_private->cipher_crc_offload_enable) {
+		ret = qat_cq_get_fw_cipher_crc_cap(qp);
+		if (ret < 0) {
+			qat_cryptodev_qp_release(dev, qp_id);
+			return ret;
+		}
+
+		if (ret != 0)
+			QAT_LOG(DEBUG, "Cipher CRC supported on QAT device");
+		else
+			QAT_LOG(DEBUG, "Cipher CRC not supported on QAT device");
+
+		/* Only send the cipher crc offload capability message once */
+		qat_private->cipher_crc_offload_enable = 0;
+		/* Set cipher crc offload indicator */
+		if (ret)
+			qat_private->internal_capabilities |=
+						QAT_SYM_CAP_CIPHER_CRC;
+	}
+
+	return 0;
 }

drivers/crypto/qat/qat_crypto.h

@@ -36,6 +36,7 @@ struct qat_cryptodev_private {
 	/* Shared memzone for storing capabilities */
 	uint16_t min_enq_burst_threshold;
 	uint32_t internal_capabilities; /* see flags QAT_SYM_CAP_xxx */
+	bool cipher_crc_offload_enable;
 	enum qat_service_type service_type;
 };