Skip to content
Permalink
Browse files

Code improvement and bug fixes for QAT support

1. Support QAT when ZFS is root file-system:
   When ZFS module is loaded before QAT started, the QAT can
   be started again in post-process, e.g.:
   echo 0 > /sys/module/zfs/parameters/zfs_qat_compress_disable
   echo 0 > /sys/module/zfs/parameters/zfs_qat_encrypt_disable
   echo 0 > /sys/module/zfs/parameters/zfs_qat_checksum_disable
2. Verify alder checksum of the de-compress result
3. Allocate Digest, IV and AAD buffer in physical contiguous
   memory by QAT_PHYS_CONTIG_ALLOC.

Signed-off-by: Weigang Li <weigang.li@intel.com>
Signed-off-by: Chengfeix Zhu <chengfeix.zhu@intel.com>
  • Loading branch information...
cfzhu committed Apr 11, 2019
1 parent 83472fa commit 7c533ddcc3e555af9018116126b8e074216c96a8
Showing with 143 additions and 38 deletions.
  1. +14 −13 module/zfs/qat.c
  2. +5 −2 module/zfs/qat.h
  3. +37 −2 module/zfs/qat_compress.c
  4. +87 −21 module/zfs/qat_crypt.c
@@ -66,18 +66,6 @@ qat_mem_free_contig(void **pp_mem_addr)
int
qat_init(void)
{
int ret;

ret = qat_dc_init();
if (ret != 0)
return (ret);

ret = qat_crypt_init();
if (ret != 0) {
qat_dc_fini();
return (ret);
}

qat_ksp = kstat_create("zfs", 0, "qat", "misc",
KSTAT_TYPE_NAMED, sizeof (qat_stats) / sizeof (kstat_named_t),
KSTAT_FLAG_VIRTUAL);
@@ -86,6 +74,19 @@ qat_init(void)
kstat_install(qat_ksp);
}

/*
* Just set the disable flag when qat init failed, qat can be
* turned on again in post-process after zfs module is loaded, e.g.:
* echo 0 > /sys/module/zfs/parameters/zfs_qat_compress_disable
*/
if (qat_dc_init() != 0)
zfs_qat_compress_disable = 1;

if (qat_cy_init() != 0) {
zfs_qat_checksum_disable = 1;
zfs_qat_encrypt_disable = 1;
}

return (0);
}

@@ -97,7 +98,7 @@ qat_fini(void)
qat_ksp = NULL;
}

qat_crypt_fini();
qat_cy_fini();
qat_dc_fini();
}

@@ -147,6 +147,9 @@ typedef struct qat_stats {
QAT_STAT_INCR(stat, 1)

extern qat_stats_t qat_stats;
extern int zfs_qat_compress_disable;
extern int zfs_qat_checksum_disable;
extern int zfs_qat_encrypt_disable;

/* inlined for performance */
static inline struct page *
@@ -167,8 +170,8 @@ void qat_mem_free_contig(void **pp_mem_addr);

extern int qat_dc_init(void);
extern void qat_dc_fini(void);
extern int qat_crypt_init(void);
extern void qat_crypt_fini(void);
extern int qat_cy_init(void);
extern void qat_cy_fini(void);
extern int qat_init(void);
extern void qat_fini(void);

@@ -24,7 +24,9 @@
#include <linux/vmalloc.h>
#include <linux/pagemap.h>
#include <linux/completion.h>
#include <linux/mod_compat.h>
#include <sys/zfs_context.h>
#include <sys/byteorder.h>
#include <sys/zio.h>
#include "qat.h"

@@ -111,6 +113,9 @@ qat_dc_init(void)
Cpa32U buff_meta_size = 0;
CpaDcSessionSetupData sd = {0};

if (qat_dc_init_done)
return (0);

status = cpaDcGetNumInstances(&num_inst);
if (status != CPA_STATUS_SUCCESS)
return (-1);
@@ -252,6 +257,7 @@ qat_compress_impl(qat_compress_dir_t dir, char *src, int src_len,
Cpa32U num_add_buf = (add_len >> PAGE_SHIFT) + 2;
Cpa32U bytes_left;
Cpa32U dst_pages = 0;
Cpa32U adler32 = 0;
char *data;
struct page *page;
struct page **in_pages = NULL;
@@ -468,6 +474,12 @@ qat_compress_impl(qat_compress_dir_t dir, char *src, int src_len,
goto fail;
}

/* verify adler checksum */
adler32 = *(Cpa32U *)(src + dc_results.consumed + ZLIB_HEAD_SZ);
if (adler32 != BSWAP_32(dc_results.checksum)) {
status = CPA_STATUS_FAIL;
goto fail;
}
*c_len = dc_results.produced;
QAT_STAT_INCR(decomp_total_out_bytes, *c_len);
}
@@ -534,7 +546,30 @@ qat_compress(qat_compress_dir_t dir, char *src, int src_len,
return (ret);
}

module_param(zfs_qat_compress_disable, int, 0644);
MODULE_PARM_DESC(zfs_qat_compress_disable, "Disable QAT compression");
static int
param_set_qat_compress(const char *val, struct kernel_param *kp)
{
int ret;
int *pvalue = kp->arg;
ret = param_set_int(val, kp);
if (ret)
return (ret);
/*
* zfs_qat_compress_disable = 0: enable qat compress
* try to initialize qat instance if it has not been done
*/
if (*pvalue == 0 && !qat_dc_init_done) {
ret = qat_dc_init();
if (ret != 0) {
zfs_qat_compress_disable = 1;
return (ret);
}
}
return (ret);
}

module_param_call(zfs_qat_compress_disable, param_set_qat_compress,
param_get_int, &zfs_qat_compress_disable, 0644);
MODULE_PARM_DESC(zfs_qat_compress_disable, "Enable/Disable QAT compression");

#endif
@@ -31,6 +31,7 @@
#include <linux/vmalloc.h>
#include <linux/pagemap.h>
#include <linux/completion.h>
#include <linux/mod_compat.h>
#include <sys/zfs_context.h>
#include <sys/zio_crypt.h>
#include "lac/cpa_cy_im.h"
@@ -50,7 +51,7 @@
static Cpa32U inst_num = 0;
static Cpa16U num_inst = 0;
static CpaInstanceHandle cy_inst_handles[QAT_CRYPT_MAX_INSTANCES];
static boolean_t qat_crypt_init_done = B_FALSE;
static boolean_t qat_cy_init_done = B_FALSE;
int zfs_qat_encrypt_disable = 0;
int zfs_qat_checksum_disable = 0;

@@ -76,7 +77,7 @@ boolean_t
qat_crypt_use_accel(size_t s_len)
{
return (!zfs_qat_encrypt_disable &&
qat_crypt_init_done &&
qat_cy_init_done &&
s_len >= QAT_MIN_BUF_SIZE &&
s_len <= QAT_MAX_BUF_SIZE);
}
@@ -85,26 +86,29 @@ boolean_t
qat_checksum_use_accel(size_t s_len)
{
return (!zfs_qat_checksum_disable &&
qat_crypt_init_done &&
qat_cy_init_done &&
s_len >= QAT_MIN_BUF_SIZE &&
s_len <= QAT_MAX_BUF_SIZE);
}

void
qat_crypt_clean(void)
qat_cy_clean(void)
{
for (Cpa16U i = 0; i < num_inst; i++)
cpaCyStopInstance(cy_inst_handles[i]);

num_inst = 0;
qat_crypt_init_done = B_FALSE;
qat_cy_init_done = B_FALSE;
}

int
qat_crypt_init(void)
qat_cy_init(void)
{
CpaStatus status = CPA_STATUS_FAIL;

if (qat_cy_init_done)
return (0);

status = cpaCyGetNumInstances(&num_inst);
if (status != CPA_STATUS_SUCCESS)
return (-1);
@@ -131,21 +135,21 @@ qat_crypt_init(void)
goto error;
}

qat_crypt_init_done = B_TRUE;
qat_cy_init_done = B_TRUE;
return (0);

error:
qat_crypt_clean();
qat_cy_clean();
return (-1);
}

void
qat_crypt_fini(void)
qat_cy_fini(void)
{
if (!qat_crypt_init_done)
if (!qat_cy_init_done)
return;

qat_crypt_clean();
qat_cy_clean();
}

static CpaStatus
@@ -351,6 +355,21 @@ qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf,
nr_bufs * sizeof (CpaFlatBuffer));
if (status != CPA_STATUS_SUCCESS)
goto fail;
status = QAT_PHYS_CONTIG_ALLOC(&op_data.pDigestResult,
ZIO_DATA_MAC_LEN);
if (status != CPA_STATUS_SUCCESS)
goto fail;
status = QAT_PHYS_CONTIG_ALLOC(&op_data.pIv,
ZIO_DATA_IV_LEN);
if (status != CPA_STATUS_SUCCESS)
goto fail;
if (aad_len > 0) {
status = QAT_PHYS_CONTIG_ALLOC(&op_data.pAdditionalAuthData,
aad_len);
if (status != CPA_STATUS_SUCCESS)
goto fail;
bcopy(aad_buf, op_data.pAdditionalAuthData, aad_len);
}

bytes_left = enc_len;
data = src_buf;
@@ -389,18 +408,13 @@ qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf,

op_data.sessionCtx = cy_session_ctx;
op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
op_data.pIv = NULL; /* set this later as the J0 block */
op_data.ivLenInBytes = 0;
op_data.cryptoStartSrcOffsetInBytes = 0;
op_data.messageLenToCipherInBytes = 0;
op_data.hashStartSrcOffsetInBytes = 0;
op_data.messageLenToHashInBytes = 0;
op_data.pDigestResult = 0;
op_data.messageLenToCipherInBytes = enc_len;
op_data.ivLenInBytes = ZIO_DATA_IV_LEN;
op_data.pDigestResult = digest_buf;
op_data.pAdditionalAuthData = aad_buf;
op_data.pIv = iv_buf;
bcopy(iv_buf, op_data.pIv, ZIO_DATA_IV_LEN);

cb.verify_result = CPA_FALSE;
init_completion(&cb.complete);
@@ -420,6 +434,8 @@ qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf,
goto fail;
}

/* save digest result to digest_buf */
bcopy(op_data.pDigestResult, digest_buf, ZIO_DATA_MAC_LEN);
if (dir == QAT_ENCRYPT)
QAT_STAT_INCR(encrypt_total_out_bytes, enc_len);
else
@@ -435,6 +451,10 @@ qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf,
kunmap(out_pages[i]);

cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx);
if (aad_len > 0)
QAT_PHYS_CONTIG_FREE(op_data.pAdditionalAuthData);
QAT_PHYS_CONTIG_FREE(op_data.pIv);
QAT_PHYS_CONTIG_FREE(op_data.pDigestResult);
QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData);
QAT_PHYS_CONTIG_FREE(dst_buffer_list.pPrivateMetaData);
QAT_PHYS_CONTIG_FREE(cy_session_ctx);
@@ -557,10 +577,56 @@ qat_checksum(uint64_t cksum, uint8_t *buf, uint64_t size, zio_cksum_t *zcp)
return (status);
}

module_param(zfs_qat_encrypt_disable, int, 0644);
MODULE_PARM_DESC(zfs_qat_encrypt_disable, "Disable QAT encryption");
static int
param_set_qat_encrypt(const char *val, struct kernel_param *kp)
{
int ret;
int *pvalue = kp->arg;
ret = param_set_int(val, kp);
if (ret)
return (ret);
/*
* zfs_qat_encrypt_disable = 0: enable qat encrypt
* try to initialize qat instance if it has not been done
*/
if (*pvalue == 0 && !qat_cy_init_done) {
ret = qat_cy_init();
if (ret != 0) {
zfs_qat_encrypt_disable = 1;
return (ret);
}
}
return (ret);
}

static int
param_set_qat_checksum(const char *val, struct kernel_param *kp)
{
int ret;
int *pvalue = kp->arg;
ret = param_set_int(val, kp);
if (ret)
return (ret);
/*
* set_checksum_param_ops = 0: enable qat checksum
* try to initialize qat instance if it has not been done
*/
if (*pvalue == 0 && !qat_cy_init_done) {
ret = qat_cy_init();
if (ret != 0) {
zfs_qat_checksum_disable = 1;
return (ret);
}
}
return (ret);
}

module_param_call(zfs_qat_encrypt_disable, param_set_qat_encrypt,
param_get_int, &zfs_qat_encrypt_disable, 0644);
MODULE_PARM_DESC(zfs_qat_encrypt_disable, "Enable/Disable QAT encryption");

module_param(zfs_qat_checksum_disable, int, 0644);
MODULE_PARM_DESC(zfs_qat_checksum_disable, "Disable QAT checksumming");
module_param_call(zfs_qat_checksum_disable, param_set_qat_checksum,
param_get_int, &zfs_qat_checksum_disable, 0644);
MODULE_PARM_DESC(zfs_qat_checksum_disable, "Enable/Disable QAT checksumming");

#endif

0 comments on commit 7c533dd

Please sign in to comment.
You can’t perform that action at this time.