diff --git a/app/test-crypto-perf/cperf.h b/app/test-crypto-perf/cperf.h index 293ba9406e..c9f7f81749 100644 --- a/app/test-crypto-perf/cperf.h +++ b/app/test-crypto-perf/cperf.h @@ -41,7 +41,10 @@ struct cperf_options; struct cperf_test_vector; struct cperf_op_fns; -typedef void *(*cperf_constructor_t)(uint8_t dev_id, uint16_t qp_id, +typedef void *(*cperf_constructor_t)( + struct rte_mempool *sess_mp, + uint8_t dev_id, + uint16_t qp_id, const struct cperf_options *options, const struct cperf_test_vector *t_vec, const struct cperf_op_fns *op_fns); diff --git a/app/test-crypto-perf/cperf_ops.c b/app/test-crypto-perf/cperf_ops.c index d7182784a3..a5bf09be94 100644 --- a/app/test-crypto-perf/cperf_ops.c +++ b/app/test-crypto-perf/cperf_ops.c @@ -367,7 +367,8 @@ cperf_set_ops_aead(struct rte_crypto_op **ops, } static struct rte_cryptodev_sym_session * -cperf_create_session(uint8_t dev_id, +cperf_create_session(struct rte_mempool *sess_mp, + uint8_t dev_id, const struct cperf_options *options, const struct cperf_test_vector *test_vector, uint16_t iv_offset) @@ -377,6 +378,7 @@ cperf_create_session(uint8_t dev_id, struct rte_crypto_sym_xform aead_xform; struct rte_cryptodev_sym_session *sess = NULL; + sess = rte_cryptodev_sym_session_create(sess_mp); /* * cipher only */ @@ -401,7 +403,8 @@ cperf_create_session(uint8_t dev_id, cipher_xform.cipher.iv.length = 0; } /* create crypto session */ - sess = rte_cryptodev_sym_session_create(dev_id, &cipher_xform); + rte_cryptodev_sym_session_init(dev_id, sess, &cipher_xform, + sess_mp); /* * auth only */ @@ -427,7 +430,8 @@ cperf_create_session(uint8_t dev_id, auth_xform.auth.iv.length = 0; } /* create crypto session */ - sess = rte_cryptodev_sym_session_create(dev_id, &auth_xform); + rte_cryptodev_sym_session_init(dev_id, sess, &auth_xform, + sess_mp); /* * cipher and auth */ @@ -483,13 +487,13 @@ cperf_create_session(uint8_t dev_id, if (options->op_type == CPERF_CIPHER_THEN_AUTH) { cipher_xform.next = &auth_xform; /* create crypto session */ - sess = rte_cryptodev_sym_session_create(dev_id, - &cipher_xform); + rte_cryptodev_sym_session_init(dev_id, + sess, &cipher_xform, sess_mp); } else { /* auth then cipher */ auth_xform.next = &cipher_xform; /* create crypto session */ - sess = rte_cryptodev_sym_session_create(dev_id, - &auth_xform); + rte_cryptodev_sym_session_init(dev_id, + sess, &auth_xform, sess_mp); } } else { /* options->op_type == CPERF_AEAD */ aead_xform.type = RTE_CRYPTO_SYM_XFORM_AEAD; @@ -509,7 +513,8 @@ cperf_create_session(uint8_t dev_id, options->aead_aad_sz; /* Create crypto session */ - sess = rte_cryptodev_sym_session_create(dev_id, &aead_xform); + rte_cryptodev_sym_session_init(dev_id, + sess, &aead_xform, sess_mp); } return sess; diff --git a/app/test-crypto-perf/cperf_ops.h b/app/test-crypto-perf/cperf_ops.h index bb83cd5b26..1f8fa9370c 100644 --- a/app/test-crypto-perf/cperf_ops.h +++ b/app/test-crypto-perf/cperf_ops.h @@ -41,6 +41,7 @@ typedef struct rte_cryptodev_sym_session *(*cperf_sessions_create_t)( + struct rte_mempool *sess_mp, uint8_t dev_id, const struct cperf_options *options, const struct cperf_test_vector *test_vector, uint16_t iv_offset); diff --git a/app/test-crypto-perf/cperf_test_latency.c b/app/test-crypto-perf/cperf_test_latency.c index 8841d396a0..1a42602fe7 100644 --- a/app/test-crypto-perf/cperf_test_latency.c +++ b/app/test-crypto-perf/cperf_test_latency.c @@ -79,8 +79,10 @@ cperf_latency_test_free(struct cperf_latency_ctx *ctx, uint32_t mbuf_nb) uint32_t i; if (ctx) { - if (ctx->sess) - rte_cryptodev_sym_session_free(ctx->dev_id, ctx->sess); + if (ctx->sess) { + rte_cryptodev_sym_session_clear(ctx->dev_id, ctx->sess); + rte_cryptodev_sym_session_free(ctx->sess); + } if (ctx->mbufs_in) { for (i = 0; i < mbuf_nb; i++) @@ -191,7 +193,8 @@ cperf_mbuf_create(struct rte_mempool *mempool, } void * -cperf_latency_test_constructor(uint8_t dev_id, uint16_t qp_id, +cperf_latency_test_constructor(struct rte_mempool *sess_mp, + uint8_t dev_id, uint16_t qp_id, const struct cperf_options *options, const struct cperf_test_vector *test_vector, const struct cperf_op_fns *op_fns) @@ -216,7 +219,8 @@ cperf_latency_test_constructor(uint8_t dev_id, uint16_t qp_id, sizeof(struct rte_crypto_sym_op) + sizeof(struct cperf_op_result *); - ctx->sess = op_fns->sess_create(dev_id, options, test_vector, iv_offset); + ctx->sess = op_fns->sess_create(sess_mp, dev_id, options, test_vector, + iv_offset); if (ctx->sess == NULL) goto err; diff --git a/app/test-crypto-perf/cperf_test_latency.h b/app/test-crypto-perf/cperf_test_latency.h index 6a2cf610da..1bbedb4ea0 100644 --- a/app/test-crypto-perf/cperf_test_latency.h +++ b/app/test-crypto-perf/cperf_test_latency.h @@ -43,7 +43,10 @@ #include "cperf_test_vectors.h" void * -cperf_latency_test_constructor(uint8_t dev_id, uint16_t qp_id, +cperf_latency_test_constructor( + struct rte_mempool *sess_mp, + uint8_t dev_id, + uint16_t qp_id, const struct cperf_options *options, const struct cperf_test_vector *test_vector, const struct cperf_op_fns *ops_fn); diff --git a/app/test-crypto-perf/cperf_test_throughput.c b/app/test-crypto-perf/cperf_test_throughput.c index 87fac0fb70..30250446cd 100644 --- a/app/test-crypto-perf/cperf_test_throughput.c +++ b/app/test-crypto-perf/cperf_test_throughput.c @@ -64,8 +64,10 @@ cperf_throughput_test_free(struct cperf_throughput_ctx *ctx, uint32_t mbuf_nb) uint32_t i; if (ctx) { - if (ctx->sess) - rte_cryptodev_sym_session_free(ctx->dev_id, ctx->sess); + if (ctx->sess) { + rte_cryptodev_sym_session_clear(ctx->dev_id, ctx->sess); + rte_cryptodev_sym_session_free(ctx->sess); + } if (ctx->mbufs_in) { for (i = 0; i < mbuf_nb; i++) @@ -175,7 +177,8 @@ cperf_mbuf_create(struct rte_mempool *mempool, } void * -cperf_throughput_test_constructor(uint8_t dev_id, uint16_t qp_id, +cperf_throughput_test_constructor(struct rte_mempool *sess_mp, + uint8_t dev_id, uint16_t qp_id, const struct cperf_options *options, const struct cperf_test_vector *test_vector, const struct cperf_op_fns *op_fns) @@ -199,7 +202,8 @@ cperf_throughput_test_constructor(uint8_t dev_id, uint16_t qp_id, uint16_t iv_offset = sizeof(struct rte_crypto_op) + sizeof(struct rte_crypto_sym_op); - ctx->sess = op_fns->sess_create(dev_id, options, test_vector, iv_offset); + ctx->sess = op_fns->sess_create(sess_mp, dev_id, options, test_vector, + iv_offset); if (ctx->sess == NULL) goto err; diff --git a/app/test-crypto-perf/cperf_test_throughput.h b/app/test-crypto-perf/cperf_test_throughput.h index f1b5766cd3..987d0c31e9 100644 --- a/app/test-crypto-perf/cperf_test_throughput.h +++ b/app/test-crypto-perf/cperf_test_throughput.h @@ -44,7 +44,10 @@ void * -cperf_throughput_test_constructor(uint8_t dev_id, uint16_t qp_id, +cperf_throughput_test_constructor( + struct rte_mempool *sess_mp, + uint8_t dev_id, + uint16_t qp_id, const struct cperf_options *options, const struct cperf_test_vector *test_vector, const struct cperf_op_fns *ops_fn); diff --git a/app/test-crypto-perf/cperf_test_verify.c b/app/test-crypto-perf/cperf_test_verify.c index 3ee056009d..9c37954f74 100644 --- a/app/test-crypto-perf/cperf_test_verify.c +++ b/app/test-crypto-perf/cperf_test_verify.c @@ -68,8 +68,10 @@ cperf_verify_test_free(struct cperf_verify_ctx *ctx, uint32_t mbuf_nb) uint32_t i; if (ctx) { - if (ctx->sess) - rte_cryptodev_sym_session_free(ctx->dev_id, ctx->sess); + if (ctx->sess) { + rte_cryptodev_sym_session_clear(ctx->dev_id, ctx->sess); + rte_cryptodev_sym_session_free(ctx->sess); + } if (ctx->mbufs_in) { for (i = 0; i < mbuf_nb; i++) @@ -179,7 +181,8 @@ cperf_mbuf_create(struct rte_mempool *mempool, } void * -cperf_verify_test_constructor(uint8_t dev_id, uint16_t qp_id, +cperf_verify_test_constructor(struct rte_mempool *sess_mp, + uint8_t dev_id, uint16_t qp_id, const struct cperf_options *options, const struct cperf_test_vector *test_vector, const struct cperf_op_fns *op_fns) @@ -203,7 +206,8 @@ cperf_verify_test_constructor(uint8_t dev_id, uint16_t qp_id, uint16_t iv_offset = sizeof(struct rte_crypto_op) + sizeof(struct rte_crypto_sym_op); - ctx->sess = op_fns->sess_create(dev_id, options, test_vector, iv_offset); + ctx->sess = op_fns->sess_create(sess_mp, dev_id, options, test_vector, + iv_offset); if (ctx->sess == NULL) goto err; diff --git a/app/test-crypto-perf/cperf_test_verify.h b/app/test-crypto-perf/cperf_test_verify.h index 3fa78ee6ae..e67b48d34d 100644 --- a/app/test-crypto-perf/cperf_test_verify.h +++ b/app/test-crypto-perf/cperf_test_verify.h @@ -44,7 +44,10 @@ void * -cperf_verify_test_constructor(uint8_t dev_id, uint16_t qp_id, +cperf_verify_test_constructor( + struct rte_mempool *sess_mp, + uint8_t dev_id, + uint16_t qp_id, const struct cperf_options *options, const struct cperf_test_vector *test_vector, const struct cperf_op_fns *ops_fn); diff --git a/app/test-crypto-perf/main.c b/app/test-crypto-perf/main.c index cdd3cc83f0..e625c7418d 100644 --- a/app/test-crypto-perf/main.c +++ b/app/test-crypto-perf/main.c @@ -107,13 +107,11 @@ cperf_initialize_cryptodev(struct cperf_options *opts, uint8_t *enabled_cdevs, uint32_t max_sess_size = 0, sess_size; for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) { - sess_size = sizeof(struct rte_cryptodev_sym_session) + - rte_cryptodev_get_private_session_size(cdev_id); + sess_size = rte_cryptodev_get_private_session_size(cdev_id); if (sess_size > max_sess_size) max_sess_size = sess_size; } - for (i = 0; i < enabled_cdev_count && i < RTE_CRYPTO_MAX_DEVS; i++) { cdev_id = enabled_cdevs[i]; @@ -475,7 +473,10 @@ main(int argc, char **argv) cdev_id = enabled_cdevs[i]; - ctx[cdev_id] = cperf_testmap[opts.test].constructor(cdev_id, 0, + uint8_t socket_id = rte_cryptodev_socket_id(cdev_id); + + ctx[cdev_id] = cperf_testmap[opts.test].constructor( + session_pool_socket[socket_id], cdev_id, 0, &opts, t_vec, &op_fns); if (ctx[cdev_id] == NULL) { RTE_LOG(ERR, USER1, "Test run constructor failed\n"); diff --git a/doc/guides/rel_notes/deprecation.rst b/doc/guides/rel_notes/deprecation.rst index d8dca78ba0..68a1ccfd11 100644 --- a/doc/guides/rel_notes/deprecation.rst +++ b/doc/guides/rel_notes/deprecation.rst @@ -54,17 +54,6 @@ Deprecation Notices Target release for removal of the legacy API will be defined once most PMDs have switched to rte_flow. -* cryptodev: API changes are planned for 17.08 for the sessions management - to make it agnostic to the underlying devices, removing coupling with - crypto PMDs, so a single session can be used on multiple devices. - - An API of followed functions will be changed to allow operate on multiple - devices with one session: - - - ``rte_cryptodev_sym_session_create`` - - ``rte_cryptodev_sym_session_free`` - - ``rte_cryptodev_sym_session_pool_create`` - * librte_table: The ``key_mask`` parameter will be added to all the hash tables that currently do not have it, as well as to the hash compute function prototype. The non-"do-sig" versions of the hash tables will be removed diff --git a/doc/guides/rel_notes/release_17_08.rst b/doc/guides/rel_notes/release_17_08.rst index 0842edd920..de72b1c957 100644 --- a/doc/guides/rel_notes/release_17_08.rst +++ b/doc/guides/rel_notes/release_17_08.rst @@ -116,6 +116,8 @@ New Features * ``dev_id`` field has been removed. * ``driver_id`` field has been removed. * Mempool pointer ``mp`` has been removed. + * Replaced ``private`` marker with array of pointers to private data sessions + ``sess_private_data``. * **Updated cryptodev library.** @@ -123,6 +125,10 @@ New Features necessary to use a combination of cipher and authentication structures anymore. * Added helper functions for crypto device driver identification. + * Added support for multi-device sessions, so a single session can be + used in multiple drivers. + * Added functions to initialize and free individual driver private data + with a same session. * **Updated dpaa2_sec crypto PMD.** @@ -231,6 +237,9 @@ API Changes * ``rte_cryptodev_queue_pair_attach_sym_session()`` and ``rte_cryptodev_queue_pair_dettach_sym_session()`` functions require the new parameter ``device id``. + * Modified parameters of ``rte_cryptodev_sym_session_create()``, to accept + ``mempool``, instead of ``device id`` and ``rte_crypto_sym_xform``. + * Remove ``device id`` parameter from ``rte_cryptodev_sym_session_free()``. ABI Changes diff --git a/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c b/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c index d226c2348a..91c3801c2b 100644 --- a/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c +++ b/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c @@ -155,22 +155,37 @@ aesni_gcm_get_session(struct aesni_gcm_qp *qp, struct rte_crypto_op *op) struct rte_crypto_sym_op *sym_op = op->sym; if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { - sess = (struct aesni_gcm_session *)sym_op->session->_private; + if (likely(sym_op->session != NULL)) + sess = (struct aesni_gcm_session *) + get_session_private_data( + sym_op->session, + cryptodev_driver_id); } else { void *_sess; + void *_sess_private_data = NULL; - if (rte_mempool_get(qp->sess_mp, &_sess)) - return sess; + if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) + return NULL; - sess = (struct aesni_gcm_session *) - ((struct rte_cryptodev_sym_session *)_sess)->_private; + if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) + return NULL; - if (unlikely(aesni_gcm_set_session_parameters(qp->ops, sess, - sym_op->xform) != 0)) { + sess = (struct aesni_gcm_session *)_sess_private_data; + + if (unlikely(aesni_gcm_set_session_parameters(qp->ops, + sess, sym_op->xform) != 0)) { rte_mempool_put(qp->sess_mp, _sess); + rte_mempool_put(qp->sess_mp, _sess_private_data); sess = NULL; } + sym_op->session = (struct rte_cryptodev_sym_session *)_sess; + set_session_private_data(sym_op->session, cryptodev_driver_id, + _sess_private_data); } + + if (unlikely(sess == NULL)) + op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; + return sess; } @@ -370,13 +385,11 @@ process_gcm_crypto_op(struct aesni_gcm_qp *qp, struct rte_crypto_op *op, * - Returns NULL on invalid job */ static void -post_process_gcm_crypto_op(struct rte_crypto_op *op) +post_process_gcm_crypto_op(struct rte_crypto_op *op, + struct aesni_gcm_session *session) { struct rte_mbuf *m = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src; - struct aesni_gcm_session *session = - (struct aesni_gcm_session *)op->sym->session->_private; - op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; /* Verify digest if required */ @@ -411,6 +424,7 @@ post_process_gcm_crypto_op(struct rte_crypto_op *op) * Process a completed GCM request * * @param qp Queue Pair to process + * @param op Crypto operation * @param job JOB_AES_HMAC job * * @return @@ -418,12 +432,17 @@ post_process_gcm_crypto_op(struct rte_crypto_op *op) */ static void handle_completed_gcm_crypto_op(struct aesni_gcm_qp *qp, - struct rte_crypto_op *op) + struct rte_crypto_op *op, + struct aesni_gcm_session *sess) { - post_process_gcm_crypto_op(op); + post_process_gcm_crypto_op(op, sess); /* Free session if a session-less crypto op */ if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { + memset(sess, 0, sizeof(struct aesni_gcm_session)); + memset(op->sym->session, 0, + rte_cryptodev_get_header_session_size()); + rte_mempool_put(qp->sess_mp, sess); rte_mempool_put(qp->sess_mp, op->sym->session); op->sym->session = NULL; } @@ -458,7 +477,7 @@ aesni_gcm_pmd_dequeue_burst(void *queue_pair, break; } - handle_completed_gcm_crypto_op(qp, ops[i]); + handle_completed_gcm_crypto_op(qp, ops[i], sess); } qp->qp_stats.dequeued_count += i; diff --git a/drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c b/drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c index 85c7485aac..21052cda40 100644 --- a/drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c +++ b/drivers/crypto/aesni_gcm/aesni_gcm_pmd_ops.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. + * Copyright(c) 2016 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -302,32 +302,55 @@ aesni_gcm_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) } /** Configure a aesni gcm session from a crypto xform chain */ -static void * -aesni_gcm_pmd_session_configure(struct rte_cryptodev *dev, - struct rte_crypto_sym_xform *xform, void *sess) +static int +aesni_gcm_pmd_session_configure(struct rte_cryptodev *dev __rte_unused, + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) { + void *sess_private_data; struct aesni_gcm_private *internals = dev->data->dev_private; if (unlikely(sess == NULL)) { GCM_LOG_ERR("invalid session struct"); - return NULL; + return -1; } + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -1; + } if (aesni_gcm_set_session_parameters(gcm_ops[internals->vector_mode], - sess, xform) != 0) { + sess_private_data, xform) != 0) { GCM_LOG_ERR("failed configure session parameters"); - return NULL; + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return -1; } - return sess; + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -aesni_gcm_pmd_session_clear(struct rte_cryptodev *dev __rte_unused, void *sess) +aesni_gcm_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { - if (sess) - memset(sess, 0, sizeof(struct aesni_gcm_session)); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + + /* Zero out the whole structure */ + if (sess_priv) { + memset(sess_priv, 0, sizeof(struct aesni_gcm_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } } struct rte_cryptodev_ops aesni_gcm_pmd_ops = { diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c index 699779a3ab..3aaa070e2c 100644 --- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c +++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c @@ -354,25 +354,38 @@ get_session(struct aesni_mb_qp *qp, struct rte_crypto_op *op) { struct aesni_mb_session *sess = NULL; - if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) - sess = (struct aesni_mb_session *)op->sym->session->_private; - else { + if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { + if (likely(op->sym->session != NULL)) + sess = (struct aesni_mb_session *) + get_session_private_data( + op->sym->session, + cryptodev_driver_id); + } else { void *_sess = NULL; + void *_sess_private_data = NULL; if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) return NULL; - sess = (struct aesni_mb_session *) - ((struct rte_cryptodev_sym_session *)_sess)->_private; + if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) + return NULL; + + sess = (struct aesni_mb_session *)_sess_private_data; if (unlikely(aesni_mb_set_session_parameters(qp->op_fns, sess, op->sym->xform) != 0)) { rte_mempool_put(qp->sess_mp, _sess); + rte_mempool_put(qp->sess_mp, _sess_private_data); sess = NULL; } op->sym->session = (struct rte_cryptodev_sym_session *)_sess; + set_session_private_data(op->sym->session, cryptodev_driver_id, + _sess_private_data); } + if (unlikely(sess == NULL)) + op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; + return sess; } @@ -512,19 +525,21 @@ verify_digest(JOB_AES_HMAC *job, struct rte_crypto_op *op) { /** * Process a completed job and return rte_mbuf which job processed * + * @param qp Queue Pair to process * @param job JOB_AES_HMAC job to process * * @return - * - Returns processed mbuf which is trimmed of output digest used in - * verification of supplied digest in the case of a HASH_CIPHER operation + * - Returns processed crypto operation which mbuf is trimmed of output digest + * used in verification of supplied digest. * - Returns NULL on invalid job */ static inline struct rte_crypto_op * post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job) { struct rte_crypto_op *op = (struct rte_crypto_op *)job->user_data; - - struct aesni_mb_session *sess; + struct aesni_mb_session *sess = get_session_private_data( + op->sym->session, + cryptodev_driver_id); if (unlikely(op->status == RTE_CRYPTO_OP_STATUS_ENQUEUED)) { switch (job->status) { @@ -532,9 +547,6 @@ post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job) op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; if (job->hash_alg != NULL_HASH) { - sess = (struct aesni_mb_session *) - op->sym->session->_private; - if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) verify_digest(job, op); @@ -547,6 +559,10 @@ post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job) /* Free session if a session-less crypto op */ if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { + memset(sess, 0, sizeof(struct aesni_mb_session)); + memset(op->sym->session, 0, + rte_cryptodev_get_header_session_size()); + rte_mempool_put(qp->sess_mp, sess); rte_mempool_put(qp->sess_mp, op->sym->session); op->sym->session = NULL; } diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c index b5abb6ce2e..ae74ae394b 100644 --- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c +++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd_ops.c @@ -479,36 +479,56 @@ aesni_mb_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) } /** Configure a aesni multi-buffer session from a crypto xform chain */ -static void * +static int aesni_mb_pmd_session_configure(struct rte_cryptodev *dev, - struct rte_crypto_sym_xform *xform, void *sess) + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) { + void *sess_private_data; struct aesni_mb_private *internals = dev->data->dev_private; if (unlikely(sess == NULL)) { MB_LOG_ERR("invalid session struct"); - return NULL; + return -1; + } + + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -1; } if (aesni_mb_set_session_parameters(&job_ops[internals->vector_mode], - sess, xform) != 0) { + sess_private_data, xform) != 0) { MB_LOG_ERR("failed configure session parameters"); - return NULL; + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return -1; } - return sess; + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -aesni_mb_pmd_session_clear(struct rte_cryptodev *dev __rte_unused, void *sess) +aesni_mb_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { - /* - * Current just resetting the whole data structure, need to investigate - * whether a more selective reset of key would be more performant - */ - if (sess) - memset(sess, 0, sizeof(struct aesni_mb_session)); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + + /* Zero out the whole structure */ + if (sess_priv) { + memset(sess_priv, 0, sizeof(struct aesni_mb_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } } struct rte_cryptodev_ops aesni_mb_pmd_ops = { diff --git a/drivers/crypto/armv8/rte_armv8_pmd.c b/drivers/crypto/armv8/rte_armv8_pmd.c index 590803a26c..eeeaab7a79 100644 --- a/drivers/crypto/armv8/rte_armv8_pmd.c +++ b/drivers/crypto/armv8/rte_armv8_pmd.c @@ -557,24 +557,32 @@ get_session(struct armv8_crypto_qp *qp, struct rte_crypto_op *op) /* get existing session */ if (likely(op->sym->session != NULL)) { sess = (struct armv8_crypto_session *) - op->sym->session->_private; + get_session_private_data( + op->sym->session, + cryptodev_driver_id); } } else { /* provide internal session */ void *_sess = NULL; + void *_sess_private_data = NULL; - if (!rte_mempool_get(qp->sess_mp, (void **)&_sess)) { - sess = (struct armv8_crypto_session *) - ((struct rte_cryptodev_sym_session *)_sess) - ->_private; - - if (unlikely(armv8_crypto_set_session_parameters( - sess, op->sym->xform) != 0)) { - rte_mempool_put(qp->sess_mp, _sess); - sess = NULL; - } else - op->sym->session = _sess; + if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) + return NULL; + + if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) + return NULL; + + sess = (struct armv8_crypto_session *)_sess_private_data; + + if (unlikely(armv8_crypto_set_session_parameters(sess, + op->sym->xform) != 0)) { + rte_mempool_put(qp->sess_mp, _sess); + rte_mempool_put(qp->sess_mp, _sess_private_data); + sess = NULL; } + op->sym->session = (struct rte_cryptodev_sym_session *)_sess; + set_session_private_data(op->sym->session, cryptodev_driver_id, + _sess_private_data); } if (unlikely(sess == NULL)) @@ -704,6 +712,9 @@ process_op(const struct armv8_crypto_qp *qp, struct rte_crypto_op *op, /* Free session if a session-less crypto op */ if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { memset(sess, 0, sizeof(struct armv8_crypto_session)); + memset(op->sym->session, 0, + rte_cryptodev_get_header_session_size()); + rte_mempool_put(qp->sess_mp, sess); rte_mempool_put(qp->sess_mp, op->sym->session); op->sym->session = NULL; } diff --git a/drivers/crypto/armv8/rte_armv8_pmd_ops.c b/drivers/crypto/armv8/rte_armv8_pmd_ops.c index 1bce2ea2b7..e14b68c955 100644 --- a/drivers/crypto/armv8/rte_armv8_pmd_ops.c +++ b/drivers/crypto/armv8/rte_armv8_pmd_ops.c @@ -318,33 +318,54 @@ armv8_crypto_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) } /** Configure the session from a crypto xform chain */ -static void * -armv8_crypto_pmd_session_configure(struct rte_cryptodev *dev __rte_unused, - struct rte_crypto_sym_xform *xform, void *sess) +static int +armv8_crypto_pmd_session_configure(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) { + void *sess_private_data; + if (unlikely(sess == NULL)) { ARMV8_CRYPTO_LOG_ERR("invalid session struct"); - return NULL; + return -1; + } + + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -1; } - if (armv8_crypto_set_session_parameters( - sess, xform) != 0) { + if (armv8_crypto_set_session_parameters(sess_private_data, xform) != 0) { ARMV8_CRYPTO_LOG_ERR("failed configure session parameters"); - return NULL; + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return -1; } - return sess; + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -armv8_crypto_pmd_session_clear(struct rte_cryptodev *dev __rte_unused, - void *sess) +armv8_crypto_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); /* Zero out the whole structure */ - if (sess) - memset(sess, 0, sizeof(struct armv8_crypto_session)); + if (sess_priv) { + memset(sess_priv, 0, sizeof(struct armv8_crypto_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } } struct rte_cryptodev_ops armv8_crypto_pmd_ops = { diff --git a/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c b/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c index 6bfa812506..b8ac0f1a9b 100644 --- a/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c +++ b/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c @@ -619,7 +619,9 @@ dpaa2_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, /*Clear the unused FD fields before sending*/ memset(&fd_arr[loop], 0, sizeof(struct qbman_fd)); sess = (dpaa2_sec_session *) - (*ops)->sym->session->_private; + get_session_private_data( + (*ops)->sym->session, + cryptodev_driver_id); mb_pool = (*ops)->sym->m_src->pool; bpid = mempool_to_bpid(mb_pool); ret = build_sec_fd(sess, *ops, &fd_arr[loop], bpid); @@ -1498,8 +1500,8 @@ dpaa2_sec_aead_chain_init(struct rte_cryptodev *dev, return -1; } -static void * -dpaa2_sec_session_configure(struct rte_cryptodev *dev, +static int +dpaa2_sec_set_session_parameters(struct rte_cryptodev *dev, struct rte_crypto_sym_xform *xform, void *sess) { dpaa2_sec_session *session = sess; @@ -1508,7 +1510,7 @@ dpaa2_sec_session_configure(struct rte_cryptodev *dev, if (unlikely(sess == NULL)) { RTE_LOG(ERR, PMD, "invalid session struct"); - return NULL; + return -1; } /* Default IV length = 0 */ @@ -1544,24 +1546,59 @@ dpaa2_sec_session_configure(struct rte_cryptodev *dev, } else { RTE_LOG(ERR, PMD, "Invalid crypto type"); - return NULL; + return -1; + } + + return 0; +} + +static int +dpaa2_sec_session_configure(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) +{ + void *sess_private_data; + + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -1; } - return session; + if (dpaa2_sec_set_session_parameters(dev, xform, sess_private_data) != 0) { + PMD_DRV_LOG(ERR, "DPAA2 PMD: failed to configure " + "session parameters"); + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return -1; + } + + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -dpaa2_sec_session_clear(struct rte_cryptodev *dev __rte_unused, void *sess) +dpaa2_sec_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { PMD_INIT_FUNC_TRACE(); - dpaa2_sec_session *s = (dpaa2_sec_session *)sess; + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + dpaa2_sec_session *s = (dpaa2_sec_session *)sess_priv; - if (s) { + if (sess_priv) { rte_free(s->ctxt); rte_free(s->cipher_key.data); rte_free(s->auth_key.data); memset(sess, 0, sizeof(dpaa2_sec_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); } } diff --git a/drivers/crypto/kasumi/rte_kasumi_pmd.c b/drivers/crypto/kasumi/rte_kasumi_pmd.c index 35afa99c2f..cff40fb55c 100644 --- a/drivers/crypto/kasumi/rte_kasumi_pmd.c +++ b/drivers/crypto/kasumi/rte_kasumi_pmd.c @@ -163,23 +163,40 @@ kasumi_set_session_parameters(struct kasumi_session *sess, static struct kasumi_session * kasumi_get_session(struct kasumi_qp *qp, struct rte_crypto_op *op) { - struct kasumi_session *sess; + struct kasumi_session *sess = NULL; if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { - sess = (struct kasumi_session *)op->sym->session->_private; - } else { - struct rte_cryptodev_sym_session *c_sess = NULL; + if (likely(op->sym->session != NULL)) + sess = (struct kasumi_session *) + get_session_private_data( + op->sym->session, + cryptodev_driver_id); + } else { + void *_sess = NULL; + void *_sess_private_data = NULL; + + if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) + return NULL; - if (rte_mempool_get(qp->sess_mp, (void **)&c_sess)) + if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) return NULL; - sess = (struct kasumi_session *)c_sess->_private; + sess = (struct kasumi_session *)_sess_private_data; if (unlikely(kasumi_set_session_parameters(sess, - op->sym->xform) != 0)) - return NULL; + op->sym->xform) != 0)) { + rte_mempool_put(qp->sess_mp, _sess); + rte_mempool_put(qp->sess_mp, _sess_private_data); + sess = NULL; + } + op->sym->session = (struct rte_cryptodev_sym_session *)_sess; + set_session_private_data(op->sym->session, cryptodev_driver_id, + _sess_private_data); } + if (unlikely(sess == NULL)) + op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; + return sess; } @@ -354,6 +371,10 @@ process_ops(struct rte_crypto_op **ops, struct kasumi_session *session, ops[i]->status = RTE_CRYPTO_OP_STATUS_SUCCESS; /* Free session if a session-less crypto op. */ if (ops[i]->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { + memset(session, 0, sizeof(struct kasumi_session)); + memset(ops[i]->sym->session, 0, + rte_cryptodev_get_header_session_size()); + rte_mempool_put(qp->sess_mp, session); rte_mempool_put(qp->sess_mp, ops[i]->sym->session); ops[i]->sym->session = NULL; } @@ -406,7 +427,8 @@ process_op_bit(struct rte_crypto_op *op, struct kasumi_session *session, /* Free session if a session-less crypto op. */ if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { - rte_mempool_put(qp->sess_mp, op->sym->session); + memset(op->sym->session, 0, sizeof(struct kasumi_session)); + rte_cryptodev_sym_session_free(op->sym->session); op->sym->session = NULL; } diff --git a/drivers/crypto/kasumi/rte_kasumi_pmd_ops.c b/drivers/crypto/kasumi/rte_kasumi_pmd_ops.c index 5cb00247f1..e7bbc2941d 100644 --- a/drivers/crypto/kasumi/rte_kasumi_pmd_ops.c +++ b/drivers/crypto/kasumi/rte_kasumi_pmd_ops.c @@ -292,33 +292,54 @@ kasumi_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) } /** Configure a KASUMI session from a crypto xform chain */ -static void * +static int kasumi_pmd_session_configure(struct rte_cryptodev *dev __rte_unused, - struct rte_crypto_sym_xform *xform, void *sess) + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) { + void *sess_private_data; + if (unlikely(sess == NULL)) { KASUMI_LOG_ERR("invalid session struct"); - return NULL; + return -1; + } + + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -1; } - if (kasumi_set_session_parameters(sess, xform) != 0) { + if (kasumi_set_session_parameters(sess_private_data, xform) != 0) { KASUMI_LOG_ERR("failed configure session parameters"); - return NULL; + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return -1; } - return sess; + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -kasumi_pmd_session_clear(struct rte_cryptodev *dev __rte_unused, void *sess) +kasumi_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { - /* - * Current just resetting the whole data structure, need to investigate - * whether a more selective reset of key would be more performant - */ - if (sess) - memset(sess, 0, sizeof(struct kasumi_session)); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + + /* Zero out the whole structure */ + if (sess_priv) { + memset(sess_priv, 0, sizeof(struct kasumi_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } } struct rte_cryptodev_ops kasumi_pmd_ops = { diff --git a/drivers/crypto/null/null_crypto_pmd.c b/drivers/crypto/null/null_crypto_pmd.c index 27cb9a2f9c..7f7dee6425 100644 --- a/drivers/crypto/null/null_crypto_pmd.c +++ b/drivers/crypto/null/null_crypto_pmd.c @@ -84,6 +84,14 @@ process_op(const struct null_crypto_qp *qp, struct rte_crypto_op *op, /* set status as successful by default */ op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; + /* Free session if a session-less crypto op. */ + if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { + memset(op->sym->session, 0, + sizeof(struct null_crypto_session)); + rte_cryptodev_sym_session_free(op->sym->session); + op->sym->session = NULL; + } + /* * if crypto session and operation are valid just enqueue the packet * in the processed ring @@ -94,24 +102,35 @@ process_op(const struct null_crypto_qp *qp, struct rte_crypto_op *op, static struct null_crypto_session * get_session(struct null_crypto_qp *qp, struct rte_crypto_op *op) { - struct null_crypto_session *sess; + struct null_crypto_session *sess = NULL; struct rte_crypto_sym_op *sym_op = op->sym; if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { - if (unlikely(sym_op->session == NULL)) + if (likely(sym_op->session != NULL)) + sess = (struct null_crypto_session *) + get_session_private_data( + sym_op->session, cryptodev_driver_id); + } else { + void *_sess = NULL; + void *_sess_private_data = NULL; + + if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) return NULL; - sess = (struct null_crypto_session *)sym_op->session->_private; - } else { - struct rte_cryptodev_sym_session *c_sess = NULL; - - if (rte_mempool_get(qp->sess_mp, (void **)&c_sess)) + if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) return NULL; - sess = (struct null_crypto_session *)c_sess->_private; - - if (null_crypto_set_session_parameters(sess, sym_op->xform) != 0) - return NULL; + sess = (struct null_crypto_session *)_sess_private_data; + + if (unlikely(null_crypto_set_session_parameters(sess, + sym_op->xform) != 0)) { + rte_mempool_put(qp->sess_mp, _sess); + rte_mempool_put(qp->sess_mp, _sess_private_data); + sess = NULL; + } + sym_op->session = (struct rte_cryptodev_sym_session *)_sess; + set_session_private_data(sym_op->session, cryptodev_driver_id, + _sess_private_data); } return sess; diff --git a/drivers/crypto/null/null_crypto_pmd_ops.c b/drivers/crypto/null/null_crypto_pmd_ops.c index a3f2e4c949..d57644d46b 100644 --- a/drivers/crypto/null/null_crypto_pmd_ops.c +++ b/drivers/crypto/null/null_crypto_pmd_ops.c @@ -299,33 +299,54 @@ null_crypto_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) } /** Configure a null crypto session from a crypto xform chain */ -static void * +static int null_crypto_pmd_session_configure(struct rte_cryptodev *dev __rte_unused, - struct rte_crypto_sym_xform *xform, void *sess) + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mp) { - int retval; + void *sess_private_data; if (unlikely(sess == NULL)) { NULL_CRYPTO_LOG_ERR("invalid session struct"); - return NULL; + return -1; + } + + if (rte_mempool_get(mp, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -1; } - retval = null_crypto_set_session_parameters( - (struct null_crypto_session *)sess, xform); - if (retval != 0) { + + if (null_crypto_set_session_parameters(sess_private_data, xform) != 0) { NULL_CRYPTO_LOG_ERR("failed configure session parameters"); - return NULL; + + /* Return session to mempool */ + rte_mempool_put(mp, sess_private_data); + return -1; } - return sess; + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -null_crypto_pmd_session_clear(struct rte_cryptodev *dev __rte_unused, - void *sess) +null_crypto_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { - if (sess) - memset(sess, 0, sizeof(struct null_crypto_session)); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + + /* Zero out the whole structure */ + if (sess_priv) { + memset(sess_priv, 0, sizeof(struct null_crypto_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } } struct rte_cryptodev_ops pmd_ops = { diff --git a/drivers/crypto/openssl/rte_openssl_pmd.c b/drivers/crypto/openssl/rte_openssl_pmd.c index 6f5937d26b..d943d72bbc 100644 --- a/drivers/crypto/openssl/rte_openssl_pmd.c +++ b/drivers/crypto/openssl/rte_openssl_pmd.c @@ -560,23 +560,31 @@ get_session(struct openssl_qp *qp, struct rte_crypto_op *op) /* get existing session */ if (likely(op->sym->session != NULL)) sess = (struct openssl_session *) - op->sym->session->_private; - } else { + get_session_private_data( + op->sym->session, + cryptodev_driver_id); + } else { /* provide internal session */ void *_sess = NULL; + void *_sess_private_data = NULL; - if (!rte_mempool_get(qp->sess_mp, (void **)&_sess)) { - sess = (struct openssl_session *) - ((struct rte_cryptodev_sym_session *)_sess) - ->_private; - - if (unlikely(openssl_set_session_parameters( - sess, op->sym->xform) != 0)) { - rte_mempool_put(qp->sess_mp, _sess); - sess = NULL; - } else - op->sym->session = _sess; + if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) + return NULL; + + if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) + return NULL; + + sess = (struct openssl_session *)_sess_private_data; + + if (unlikely(openssl_set_session_parameters(sess, + op->sym->xform) != 0)) { + rte_mempool_put(qp->sess_mp, _sess); + rte_mempool_put(qp->sess_mp, _sess_private_data); + sess = NULL; } + op->sym->session = (struct rte_cryptodev_sym_session *)_sess; + set_session_private_data(op->sym->session, cryptodev_driver_id, + _sess_private_data); } if (sess == NULL) @@ -1318,6 +1326,9 @@ process_op(const struct openssl_qp *qp, struct rte_crypto_op *op, if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { openssl_reset_session(sess); memset(sess, 0, sizeof(struct openssl_session)); + memset(op->sym->session, 0, + rte_cryptodev_get_header_session_size()); + rte_mempool_put(qp->sess_mp, sess); rte_mempool_put(qp->sess_mp, op->sym->session); op->sym->session = NULL; } diff --git a/drivers/crypto/openssl/rte_openssl_pmd_ops.c b/drivers/crypto/openssl/rte_openssl_pmd_ops.c index 6093b615ed..1f8a011f67 100644 --- a/drivers/crypto/openssl/rte_openssl_pmd_ops.c +++ b/drivers/crypto/openssl/rte_openssl_pmd_ops.c @@ -668,36 +668,56 @@ openssl_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) } /** Configure the session from a crypto xform chain */ -static void * +static int openssl_pmd_session_configure(struct rte_cryptodev *dev __rte_unused, - struct rte_crypto_sym_xform *xform, void *sess) + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) { + void *sess_private_data; + if (unlikely(sess == NULL)) { OPENSSL_LOG_ERR("invalid session struct"); - return NULL; + return -1; + } + + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -1; } if (openssl_set_session_parameters( - sess, xform) != 0) { + sess_private_data, xform) != 0) { OPENSSL_LOG_ERR("failed configure session parameters"); - return NULL; + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return -1; } - return sess; + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -openssl_pmd_session_clear(struct rte_cryptodev *dev __rte_unused, void *sess) +openssl_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { - /* - * Current just resetting the whole data structure, need to investigate - * whether a more selective reset of key would be more performant - */ - if (sess) { - openssl_reset_session(sess); - memset(sess, 0, sizeof(struct openssl_session)); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + + /* Zero out the whole structure */ + if (sess_priv) { + openssl_reset_session(sess_priv); + memset(sess_priv, 0, sizeof(struct openssl_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); } } diff --git a/drivers/crypto/qat/qat_crypto.c b/drivers/crypto/qat/qat_crypto.c index d0638a253d..3e621c5c41 100644 --- a/drivers/crypto/qat/qat_crypto.c +++ b/drivers/crypto/qat/qat_crypto.c @@ -216,23 +216,23 @@ static inline int qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, struct qat_crypto_op_cookie *qat_op_cookie); -void qat_crypto_sym_clear_session(struct rte_cryptodev *dev, - void *session) +void +qat_crypto_sym_clear_session(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { - struct qat_session *sess = session; - phys_addr_t cd_paddr; - PMD_INIT_FUNC_TRACE(); - if (sess) { - if (sess->bpi_ctx) { - bpi_cipher_ctx_free(sess->bpi_ctx); - sess->bpi_ctx = NULL; - } - cd_paddr = sess->cd_paddr; - memset(sess, 0, qat_crypto_sym_get_session_private_size(dev)); - sess->cd_paddr = cd_paddr; - } else - PMD_DRV_LOG(ERR, "NULL session"); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + struct qat_session *s = (struct qat_session *)sess_priv; + + if (sess_priv) { + if (s->bpi_ctx) + bpi_cipher_ctx_free(s->bpi_ctx); + memset(s, 0, qat_crypto_sym_get_session_private_size(dev)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } } static int @@ -450,9 +450,37 @@ qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, return NULL; } - -void * +int qat_crypto_sym_configure_session(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) +{ + void *sess_private_data; + + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -1; + } + + if (qat_crypto_set_session_parameters(dev, xform, sess_private_data) != 0) { + PMD_DRV_LOG(ERR, "Crypto QAT PMD: failed to configure " + "session parameters"); + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return -1; + } + + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; +} + +int +qat_crypto_set_session_parameters(struct rte_cryptodev *dev, struct rte_crypto_sym_xform *xform, void *session_private) { struct qat_session *session = session_private; @@ -460,6 +488,10 @@ qat_crypto_sym_configure_session(struct rte_cryptodev *dev, int qat_cmd_id; PMD_INIT_FUNC_TRACE(); + /* Set context descriptor physical address */ + session->cd_paddr = rte_mempool_virt2phy(NULL, session) + + offsetof(struct qat_session, cd); + /* Get requested QAT command id */ qat_cmd_id = qat_get_cmd_id(xform); if (qat_cmd_id < 0 || qat_cmd_id >= ICP_QAT_FW_LA_CMD_DELIMITER) { @@ -514,10 +546,10 @@ qat_crypto_sym_configure_session(struct rte_cryptodev *dev, goto error_out; } - return session; + return 0; error_out: - return NULL; + return -1; } struct qat_session * @@ -946,7 +978,10 @@ qat_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops, rx_op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; } else { struct qat_session *sess = (struct qat_session *) - (rx_op->sym->session->_private); + get_session_private_data( + rx_op->sym->session, + cryptodev_qat_driver_id); + if (sess->bpi_ctx) qat_bpicipher_postprocess(sess, rx_op); rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; @@ -1072,7 +1107,14 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, return -EINVAL; } - ctx = (struct qat_session *)op->sym->session->_private; + ctx = (struct qat_session *)get_session_private_data( + op->sym->session, cryptodev_qat_driver_id); + + if (unlikely(ctx == NULL)) { + PMD_DRV_LOG(ERR, "Session was not created for this device"); + return -EINVAL; + } + qat_req = (struct icp_qat_fw_la_bulk_req *)out_msg; rte_mov128((uint8_t *)qat_req, (const uint8_t *)&(ctx->fw_req)); qat_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)op; @@ -1371,17 +1413,6 @@ static inline uint32_t adf_modulo(uint32_t data, uint32_t shift) return data - mult; } -void qat_crypto_sym_session_init(struct rte_mempool *mp, void *sym_sess) -{ - struct rte_cryptodev_sym_session *sess = sym_sess; - struct qat_session *s = (void *)sess->_private; - - PMD_INIT_FUNC_TRACE(); - s->cd_paddr = rte_mempool_virt2phy(mp, sess) + - offsetof(struct qat_session, cd) + - offsetof(struct rte_cryptodev_sym_session, _private); -} - int qat_dev_config(__rte_unused struct rte_cryptodev *dev, __rte_unused struct rte_cryptodev_config *config) { diff --git a/drivers/crypto/qat/qat_crypto.h b/drivers/crypto/qat/qat_crypto.h index 1258b6d452..d9d8887337 100644 --- a/drivers/crypto/qat/qat_crypto.h +++ b/drivers/crypto/qat/qat_crypto.h @@ -114,11 +114,15 @@ qat_pmd_session_mempool_create(struct rte_cryptodev *dev, extern unsigned qat_crypto_sym_get_session_private_size(struct rte_cryptodev *dev); -extern void -qat_crypto_sym_session_init(struct rte_mempool *mempool, void *priv_sess); - -extern void * +extern int qat_crypto_sym_configure_session(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool); + + +int +qat_crypto_set_session_parameters(struct rte_cryptodev *dev, struct rte_crypto_sym_xform *xform, void *session_private); struct qat_session * @@ -136,8 +140,8 @@ qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, extern void -qat_crypto_sym_clear_session(struct rte_cryptodev *dev, void *session); - +qat_crypto_sym_clear_session(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *session); extern uint16_t qat_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops, diff --git a/drivers/crypto/qat/rte_qat_cryptodev.c b/drivers/crypto/qat/rte_qat_cryptodev.c index 1c5ff77541..9a710e6006 100644 --- a/drivers/crypto/qat/rte_qat_cryptodev.c +++ b/drivers/crypto/qat/rte_qat_cryptodev.c @@ -73,7 +73,6 @@ static struct rte_cryptodev_ops crypto_qat_ops = { /* Crypto related operations */ .session_get_size = qat_crypto_sym_get_session_private_size, .session_configure = qat_crypto_sym_configure_session, - .session_initialize = qat_crypto_sym_session_init, .session_clear = qat_crypto_sym_clear_session }; diff --git a/drivers/crypto/scheduler/scheduler_failover.c b/drivers/crypto/scheduler/scheduler_failover.c index 162a29bb67..2aa13f8e77 100644 --- a/drivers/crypto/scheduler/scheduler_failover.c +++ b/drivers/crypto/scheduler/scheduler_failover.c @@ -49,57 +49,18 @@ struct fo_scheduler_qp_ctx { }; static __rte_always_inline uint16_t -failover_slave_enqueue(struct scheduler_slave *slave, uint8_t slave_idx, +failover_slave_enqueue(struct scheduler_slave *slave, struct rte_crypto_op **ops, uint16_t nb_ops) { uint16_t i, processed_ops; - struct rte_cryptodev_sym_session *sessions[nb_ops]; - struct scheduler_session *sess0, *sess1, *sess2, *sess3; for (i = 0; i < nb_ops && i < 4; i++) rte_prefetch0(ops[i]->sym->session); - for (i = 0; (i < (nb_ops - 8)) && (nb_ops > 8); i += 4) { - rte_prefetch0(ops[i + 4]->sym->session); - rte_prefetch0(ops[i + 5]->sym->session); - rte_prefetch0(ops[i + 6]->sym->session); - rte_prefetch0(ops[i + 7]->sym->session); - - sess0 = (struct scheduler_session *) - ops[i]->sym->session->_private; - sess1 = (struct scheduler_session *) - ops[i+1]->sym->session->_private; - sess2 = (struct scheduler_session *) - ops[i+2]->sym->session->_private; - sess3 = (struct scheduler_session *) - ops[i+3]->sym->session->_private; - - sessions[i] = ops[i]->sym->session; - sessions[i + 1] = ops[i + 1]->sym->session; - sessions[i + 2] = ops[i + 2]->sym->session; - sessions[i + 3] = ops[i + 3]->sym->session; - - ops[i]->sym->session = sess0->sessions[slave_idx]; - ops[i + 1]->sym->session = sess1->sessions[slave_idx]; - ops[i + 2]->sym->session = sess2->sessions[slave_idx]; - ops[i + 3]->sym->session = sess3->sessions[slave_idx]; - } - - for (; i < nb_ops; i++) { - sess0 = (struct scheduler_session *) - ops[i]->sym->session->_private; - sessions[i] = ops[i]->sym->session; - ops[i]->sym->session = sess0->sessions[slave_idx]; - } - processed_ops = rte_cryptodev_enqueue_burst(slave->dev_id, slave->qp_id, ops, nb_ops); slave->nb_inflight_cops += processed_ops; - if (unlikely(processed_ops < nb_ops)) - for (i = processed_ops; i < nb_ops; i++) - ops[i]->sym->session = sessions[i]; - return processed_ops; } @@ -114,11 +75,11 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) return 0; enqueued_ops = failover_slave_enqueue(&qp_ctx->primary_slave, - PRIMARY_SLAVE_IDX, ops, nb_ops); + ops, nb_ops); if (enqueued_ops < nb_ops) enqueued_ops += failover_slave_enqueue(&qp_ctx->secondary_slave, - SECONDARY_SLAVE_IDX, &ops[enqueued_ops], + &ops[enqueued_ops], nb_ops - enqueued_ops); return enqueued_ops; diff --git a/drivers/crypto/scheduler/scheduler_multicore.c b/drivers/crypto/scheduler/scheduler_multicore.c index 8757552f7c..a568be4f11 100644 --- a/drivers/crypto/scheduler/scheduler_multicore.c +++ b/drivers/crypto/scheduler/scheduler_multicore.c @@ -168,7 +168,6 @@ mc_scheduler_worker(struct rte_cryptodev *dev) struct scheduler_slave *slave; struct rte_crypto_op *enq_ops[MC_SCHED_BUFFER_SIZE]; struct rte_crypto_op *deq_ops[MC_SCHED_BUFFER_SIZE]; - struct scheduler_session *sess0, *sess1, *sess2, *sess3; uint16_t processed_ops; uint16_t left_op = 0; uint16_t left_op_idx = 0; @@ -202,44 +201,6 @@ mc_scheduler_worker(struct rte_cryptodev *dev) uint16_t nb_deq_ops = rte_ring_dequeue_burst(enq_ring, (void *)enq_ops, MC_SCHED_BUFFER_SIZE, NULL); if (nb_deq_ops) { - uint16_t i; - - for (i = 0; i < nb_deq_ops && i < 4; i++) - rte_prefetch0(enq_ops[i]->sym->session); - - for (i = 0; (i < (nb_deq_ops - 8)) - && (nb_deq_ops > 8); i += 4) { - sess0 = (struct scheduler_session *) - enq_ops[i]->sym->session->_private; - sess1 = (struct scheduler_session *) - enq_ops[i+1]->sym->session->_private; - sess2 = (struct scheduler_session *) - enq_ops[i+2]->sym->session->_private; - sess3 = (struct scheduler_session *) - enq_ops[i+3]->sym->session->_private; - - enq_ops[i]->sym->session = - sess0->sessions[worker_idx]; - enq_ops[i + 1]->sym->session = - sess1->sessions[worker_idx]; - enq_ops[i + 2]->sym->session = - sess2->sessions[worker_idx]; - enq_ops[i + 3]->sym->session = - sess3->sessions[worker_idx]; - - rte_prefetch0(enq_ops[i + 4]->sym->session); - rte_prefetch0(enq_ops[i + 5]->sym->session); - rte_prefetch0(enq_ops[i + 6]->sym->session); - rte_prefetch0(enq_ops[i + 7]->sym->session); - } - - for (; i < nb_deq_ops; i++) { - sess0 = (struct scheduler_session *) - enq_ops[i]->sym->session->_private; - enq_ops[i]->sym->session = - sess0->sessions[worker_idx]; - } - processed_ops = rte_cryptodev_enqueue_burst(slave->dev_id, slave->qp_id, enq_ops, nb_deq_ops); diff --git a/drivers/crypto/scheduler/scheduler_pkt_size_distr.c b/drivers/crypto/scheduler/scheduler_pkt_size_distr.c index 6b628dfa60..1dd1bc3219 100644 --- a/drivers/crypto/scheduler/scheduler_pkt_size_distr.c +++ b/drivers/crypto/scheduler/scheduler_pkt_size_distr.c @@ -67,7 +67,6 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) struct scheduler_qp_ctx *qp_ctx = qp; struct psd_scheduler_qp_ctx *psd_qp_ctx = qp_ctx->private_qp_ctx; struct rte_crypto_op *sched_ops[NB_PKT_SIZE_SLAVES][nb_ops]; - struct scheduler_session *sess; uint32_t in_flight_ops[NB_PKT_SIZE_SLAVES] = { psd_qp_ctx->primary_slave.nb_inflight_cops, psd_qp_ctx->secondary_slave.nb_inflight_cops @@ -97,8 +96,6 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) rte_prefetch0(ops[i + 7]->sym); rte_prefetch0(ops[i + 7]->sym->session); - sess = (struct scheduler_session *) - ops[i]->sym->session->_private; /* job_len is initialized as cipher data length, once * it is 0, equals to auth data length */ @@ -118,11 +115,8 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) } sched_ops[p_enq_op->slave_idx][p_enq_op->pos] = ops[i]; - ops[i]->sym->session = sess->sessions[p_enq_op->slave_idx]; p_enq_op->pos++; - sess = (struct scheduler_session *) - ops[i+1]->sym->session->_private; job_len = ops[i+1]->sym->cipher.data.length; job_len += (ops[i+1]->sym->cipher.data.length == 0) * ops[i+1]->sym->auth.data.length; @@ -135,11 +129,8 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) } sched_ops[p_enq_op->slave_idx][p_enq_op->pos] = ops[i+1]; - ops[i+1]->sym->session = sess->sessions[p_enq_op->slave_idx]; p_enq_op->pos++; - sess = (struct scheduler_session *) - ops[i+2]->sym->session->_private; job_len = ops[i+2]->sym->cipher.data.length; job_len += (ops[i+2]->sym->cipher.data.length == 0) * ops[i+2]->sym->auth.data.length; @@ -152,12 +143,8 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) } sched_ops[p_enq_op->slave_idx][p_enq_op->pos] = ops[i+2]; - ops[i+2]->sym->session = sess->sessions[p_enq_op->slave_idx]; p_enq_op->pos++; - sess = (struct scheduler_session *) - ops[i+3]->sym->session->_private; - job_len = ops[i+3]->sym->cipher.data.length; job_len += (ops[i+3]->sym->cipher.data.length == 0) * ops[i+3]->sym->auth.data.length; @@ -170,14 +157,10 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) } sched_ops[p_enq_op->slave_idx][p_enq_op->pos] = ops[i+3]; - ops[i+3]->sym->session = sess->sessions[p_enq_op->slave_idx]; p_enq_op->pos++; } for (; i < nb_ops; i++) { - sess = (struct scheduler_session *) - ops[i]->sym->session->_private; - job_len = ops[i]->sym->cipher.data.length; job_len += (ops[i]->sym->cipher.data.length == 0) * ops[i]->sym->auth.data.length; @@ -190,7 +173,6 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) } sched_ops[p_enq_op->slave_idx][p_enq_op->pos] = ops[i]; - ops[i]->sym->session = sess->sessions[p_enq_op->slave_idx]; p_enq_op->pos++; } diff --git a/drivers/crypto/scheduler/scheduler_pmd_ops.c b/drivers/crypto/scheduler/scheduler_pmd_ops.c index b9d89737c3..c450f6a0a2 100644 --- a/drivers/crypto/scheduler/scheduler_pmd_ops.c +++ b/drivers/crypto/scheduler/scheduler_pmd_ops.c @@ -85,8 +85,10 @@ scheduler_attach_init_slave(struct rte_cryptodev *dev) /** Configure device */ static int scheduler_pmd_config(struct rte_cryptodev *dev, - struct rte_cryptodev_config *config __rte_unused) + struct rte_cryptodev_config *config) { + struct scheduler_ctx *sched_ctx = dev->data->dev_private; + uint32_t i; int ret; /* although scheduler_attach_init_slave presents multiple times, @@ -96,6 +98,15 @@ scheduler_pmd_config(struct rte_cryptodev *dev, if (ret < 0) return ret; + for (i = 0; i < sched_ctx->nb_slaves; i++) { + uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id; + + ret = rte_cryptodev_configure(slave_dev_id, config, + dev->data->session_pool); + if (ret < 0) + break; + } + return ret; } @@ -474,37 +485,39 @@ scheduler_pmd_qp_count(struct rte_cryptodev *dev) static uint32_t scheduler_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) { - return sizeof(struct scheduler_session); + struct scheduler_ctx *sched_ctx = dev->data->dev_private; + uint8_t i = 0; + uint32_t max_priv_sess_size = 0; + + /* Check what is the maximum private session size for all slaves */ + for (i = 0; i < sched_ctx->nb_slaves; i++) { + uint8_t slave_dev_id = sched_ctx->slaves[i].dev_id; + struct rte_cryptodev *dev = &rte_cryptodevs[slave_dev_id]; + uint32_t priv_sess_size = (*dev->dev_ops->session_get_size)(dev); + + if (max_priv_sess_size < priv_sess_size) + max_priv_sess_size = priv_sess_size; + } + + return max_priv_sess_size; } static int -config_slave_sess(struct scheduler_ctx *sched_ctx, - struct rte_crypto_sym_xform *xform, - struct scheduler_session *sess, - uint32_t create) +scheduler_pmd_session_configure(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) { + struct scheduler_ctx *sched_ctx = dev->data->dev_private; uint32_t i; for (i = 0; i < sched_ctx->nb_slaves; i++) { struct scheduler_slave *slave = &sched_ctx->slaves[i]; - if (sess->sessions[i]) { - if (create) - continue; - /* !create */ - sess->sessions[i] = rte_cryptodev_sym_session_free( - slave->dev_id, sess->sessions[i]); - } else { - if (!create) - continue; - /* create */ - sess->sessions[i] = - rte_cryptodev_sym_session_create( - slave->dev_id, xform); - if (!sess->sessions[i]) { - config_slave_sess(sched_ctx, NULL, sess, 0); - return -1; - } + if (rte_cryptodev_sym_session_init(slave->dev_id, sess, + xform, mempool) < 0) { + CS_LOG_ERR("unabled to config sym session"); + return -1; } } @@ -514,27 +527,17 @@ config_slave_sess(struct scheduler_ctx *sched_ctx, /** Clear the memory of session so it doesn't leave key material behind */ static void scheduler_pmd_session_clear(struct rte_cryptodev *dev, - void *sess) + struct rte_cryptodev_sym_session *sess) { struct scheduler_ctx *sched_ctx = dev->data->dev_private; + uint32_t i; - config_slave_sess(sched_ctx, NULL, sess, 0); - - memset(sess, 0, sizeof(struct scheduler_session)); -} - -static void * -scheduler_pmd_session_configure(struct rte_cryptodev *dev, - struct rte_crypto_sym_xform *xform, void *sess) -{ - struct scheduler_ctx *sched_ctx = dev->data->dev_private; + /* Clear private data of slaves */ + for (i = 0; i < sched_ctx->nb_slaves; i++) { + struct scheduler_slave *slave = &sched_ctx->slaves[i]; - if (config_slave_sess(sched_ctx, xform, sess, 1) < 0) { - CS_LOG_ERR("unabled to config sym session"); - return NULL; + rte_cryptodev_sym_session_clear(slave->dev_id, sess); } - - return sess; } struct rte_cryptodev_ops scheduler_pmd_ops = { diff --git a/drivers/crypto/scheduler/scheduler_pmd_private.h b/drivers/crypto/scheduler/scheduler_pmd_private.h index b38567cd17..a786d3a4d9 100644 --- a/drivers/crypto/scheduler/scheduler_pmd_private.h +++ b/drivers/crypto/scheduler/scheduler_pmd_private.h @@ -105,10 +105,6 @@ struct scheduler_qp_ctx { uint32_t seqn; } __rte_cache_aligned; -struct scheduler_session { - struct rte_cryptodev_sym_session *sessions[ - RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES]; -}; extern uint8_t cryptodev_driver_id; diff --git a/drivers/crypto/scheduler/scheduler_roundrobin.c b/drivers/crypto/scheduler/scheduler_roundrobin.c index 0116276462..4a847281ac 100644 --- a/drivers/crypto/scheduler/scheduler_roundrobin.c +++ b/drivers/crypto/scheduler/scheduler_roundrobin.c @@ -52,8 +52,6 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) uint32_t slave_idx = rr_qp_ctx->last_enq_slave_idx; struct scheduler_slave *slave = &rr_qp_ctx->slaves[slave_idx]; uint16_t i, processed_ops; - struct rte_cryptodev_sym_session *sessions[nb_ops]; - struct scheduler_session *sess0, *sess1, *sess2, *sess3; if (unlikely(nb_ops == 0)) return 0; @@ -61,39 +59,6 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) for (i = 0; i < nb_ops && i < 4; i++) rte_prefetch0(ops[i]->sym->session); - for (i = 0; (i < (nb_ops - 8)) && (nb_ops > 8); i += 4) { - sess0 = (struct scheduler_session *) - ops[i]->sym->session->_private; - sess1 = (struct scheduler_session *) - ops[i+1]->sym->session->_private; - sess2 = (struct scheduler_session *) - ops[i+2]->sym->session->_private; - sess3 = (struct scheduler_session *) - ops[i+3]->sym->session->_private; - - sessions[i] = ops[i]->sym->session; - sessions[i + 1] = ops[i + 1]->sym->session; - sessions[i + 2] = ops[i + 2]->sym->session; - sessions[i + 3] = ops[i + 3]->sym->session; - - ops[i]->sym->session = sess0->sessions[slave_idx]; - ops[i + 1]->sym->session = sess1->sessions[slave_idx]; - ops[i + 2]->sym->session = sess2->sessions[slave_idx]; - ops[i + 3]->sym->session = sess3->sessions[slave_idx]; - - rte_prefetch0(ops[i + 4]->sym->session); - rte_prefetch0(ops[i + 5]->sym->session); - rte_prefetch0(ops[i + 6]->sym->session); - rte_prefetch0(ops[i + 7]->sym->session); - } - - for (; i < nb_ops; i++) { - sess0 = (struct scheduler_session *) - ops[i]->sym->session->_private; - sessions[i] = ops[i]->sym->session; - ops[i]->sym->session = sess0->sessions[slave_idx]; - } - processed_ops = rte_cryptodev_enqueue_burst(slave->dev_id, slave->qp_id, ops, nb_ops); @@ -102,12 +67,6 @@ schedule_enqueue(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) rr_qp_ctx->last_enq_slave_idx += 1; rr_qp_ctx->last_enq_slave_idx %= rr_qp_ctx->nb_slaves; - /* recover session if enqueue is failed */ - if (unlikely(processed_ops < nb_ops)) { - for (i = processed_ops; i < nb_ops; i++) - ops[i]->sym->session = sessions[i]; - } - return processed_ops; } diff --git a/drivers/crypto/snow3g/rte_snow3g_pmd.c b/drivers/crypto/snow3g/rte_snow3g_pmd.c index f28b8d6817..107e1b4247 100644 --- a/drivers/crypto/snow3g/rte_snow3g_pmd.c +++ b/drivers/crypto/snow3g/rte_snow3g_pmd.c @@ -163,23 +163,41 @@ snow3g_set_session_parameters(struct snow3g_session *sess, static struct snow3g_session * snow3g_get_session(struct snow3g_qp *qp, struct rte_crypto_op *op) { - struct snow3g_session *sess; + struct snow3g_session *sess = NULL; if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { - sess = (struct snow3g_session *)op->sym->session->_private; - } else { - struct rte_cryptodev_sym_session *c_sess = NULL; + if (likely(op->sym->session != NULL)) + sess = (struct snow3g_session *) + get_session_private_data( + op->sym->session, + cryptodev_driver_id); + } else { + void *_sess = NULL; + void *_sess_private_data = NULL; + + if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) + return NULL; - if (rte_mempool_get(qp->sess_mp, (void **)&c_sess)) + if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) return NULL; - sess = (struct snow3g_session *)c_sess->_private; + sess = (struct snow3g_session *)_sess_private_data; if (unlikely(snow3g_set_session_parameters(sess, - op->sym->xform) != 0)) - return NULL; + op->sym->xform) != 0)) { + rte_mempool_put(qp->sess_mp, _sess); + rte_mempool_put(qp->sess_mp, _sess_private_data); + sess = NULL; + } + op->sym->session = (struct rte_cryptodev_sym_session *)_sess; + set_session_private_data(op->sym->session, cryptodev_driver_id, + _sess_private_data); } + if (unlikely(sess == NULL)) + op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; + + return sess; } @@ -355,6 +373,10 @@ process_ops(struct rte_crypto_op **ops, struct snow3g_session *session, ops[i]->status = RTE_CRYPTO_OP_STATUS_SUCCESS; /* Free session if a session-less crypto op. */ if (ops[i]->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { + memset(session, 0, sizeof(struct snow3g_session)); + memset(ops[i]->sym->session, 0, + rte_cryptodev_get_header_session_size()); + rte_mempool_put(qp->sess_mp, session); rte_mempool_put(qp->sess_mp, ops[i]->sym->session); ops[i]->sym->session = NULL; } @@ -407,7 +429,8 @@ process_op_bit(struct rte_crypto_op *op, struct snow3g_session *session, /* Free session if a session-less crypto op. */ if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { - rte_mempool_put(qp->sess_mp, op->sym->session); + memset(op->sym->session, 0, sizeof(struct snow3g_session)); + rte_cryptodev_sym_session_free(op->sym->session); op->sym->session = NULL; } diff --git a/drivers/crypto/snow3g/rte_snow3g_pmd_ops.c b/drivers/crypto/snow3g/rte_snow3g_pmd_ops.c index e3fa0fa764..3accba5c35 100644 --- a/drivers/crypto/snow3g/rte_snow3g_pmd_ops.c +++ b/drivers/crypto/snow3g/rte_snow3g_pmd_ops.c @@ -290,33 +290,54 @@ snow3g_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) } /** Configure a SNOW 3G session from a crypto xform chain */ -static void * +static int snow3g_pmd_session_configure(struct rte_cryptodev *dev __rte_unused, - struct rte_crypto_sym_xform *xform, void *sess) + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) { + void *sess_private_data; + if (unlikely(sess == NULL)) { SNOW3G_LOG_ERR("invalid session struct"); - return NULL; + return -1; + } + + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -1; } - if (snow3g_set_session_parameters(sess, xform) != 0) { + if (snow3g_set_session_parameters(sess_private_data, xform) != 0) { SNOW3G_LOG_ERR("failed configure session parameters"); - return NULL; + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return -1; } - return sess; + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -snow3g_pmd_session_clear(struct rte_cryptodev *dev __rte_unused, void *sess) +snow3g_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { - /* - * Current just resetting the whole data structure, need to investigate - * whether a more selective reset of key would be more performant - */ - if (sess) - memset(sess, 0, sizeof(struct snow3g_session)); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + + /* Zero out the whole structure */ + if (sess_priv) { + memset(sess_priv, 0, sizeof(struct snow3g_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } } struct rte_cryptodev_ops snow3g_pmd_ops = { diff --git a/drivers/crypto/zuc/rte_zuc_pmd.c b/drivers/crypto/zuc/rte_zuc_pmd.c index 951f1f455a..664c58ff6d 100644 --- a/drivers/crypto/zuc/rte_zuc_pmd.c +++ b/drivers/crypto/zuc/rte_zuc_pmd.c @@ -162,23 +162,40 @@ zuc_set_session_parameters(struct zuc_session *sess, static struct zuc_session * zuc_get_session(struct zuc_qp *qp, struct rte_crypto_op *op) { - struct zuc_session *sess; + struct zuc_session *sess = NULL; if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { - sess = (struct zuc_session *)op->sym->session->_private; - } else { - struct rte_cryptodev_sym_session *c_sess = NULL; + if (likely(op->sym->session != NULL)) + sess = (struct zuc_session *)get_session_private_data( + op->sym->session, + cryptodev_driver_id); + } else { + void *_sess = NULL; + void *_sess_private_data = NULL; + + if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) + return NULL; - if (rte_mempool_get(qp->sess_mp, (void **)&c_sess)) + if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) return NULL; - sess = (struct zuc_session *)c_sess->_private; + sess = (struct zuc_session *)_sess_private_data; if (unlikely(zuc_set_session_parameters(sess, - op->sym->xform) != 0)) - return NULL; + op->sym->xform) != 0)) { + rte_mempool_put(qp->sess_mp, _sess); + rte_mempool_put(qp->sess_mp, _sess_private_data); + sess = NULL; + } + op->sym->session = (struct rte_cryptodev_sym_session *)_sess; + set_session_private_data(op->sym->session, cryptodev_driver_id, + _sess_private_data); } + if (unlikely(sess == NULL)) + op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; + + return sess; } @@ -337,6 +354,10 @@ process_ops(struct rte_crypto_op **ops, struct zuc_session *session, ops[i]->status = RTE_CRYPTO_OP_STATUS_SUCCESS; /* Free session if a session-less crypto op. */ if (ops[i]->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { + memset(session, 0, sizeof(struct zuc_session)); + memset(ops[i]->sym->session, 0, + rte_cryptodev_get_header_session_size()); + rte_mempool_put(qp->sess_mp, session); rte_mempool_put(qp->sess_mp, ops[i]->sym->session); ops[i]->sym->session = NULL; } diff --git a/drivers/crypto/zuc/rte_zuc_pmd_ops.c b/drivers/crypto/zuc/rte_zuc_pmd_ops.c index abfa0e26d9..adef3432b8 100644 --- a/drivers/crypto/zuc/rte_zuc_pmd_ops.c +++ b/drivers/crypto/zuc/rte_zuc_pmd_ops.c @@ -290,33 +290,54 @@ zuc_pmd_session_get_size(struct rte_cryptodev *dev __rte_unused) } /** Configure a ZUC session from a crypto xform chain */ -static void * +static int zuc_pmd_session_configure(struct rte_cryptodev *dev __rte_unused, - struct rte_crypto_sym_xform *xform, void *sess) + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) { + void *sess_private_data; + if (unlikely(sess == NULL)) { ZUC_LOG_ERR("invalid session struct"); - return NULL; + return -1; + } + + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -1; } - if (zuc_set_session_parameters(sess, xform) != 0) { + if (zuc_set_session_parameters(sess_private_data, xform) != 0) { ZUC_LOG_ERR("failed configure session parameters"); - return NULL; + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return -1; } - return sess; + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; } /** Clear the memory of session so it doesn't leave key material behind */ static void -zuc_pmd_session_clear(struct rte_cryptodev *dev __rte_unused, void *sess) +zuc_pmd_session_clear(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { - /* - * Current just resetting the whole data structure, need to investigate - * whether a more selective reset of key would be more performant - */ - if (sess) - memset(sess, 0, sizeof(struct zuc_session)); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + + /* Zero out the whole structure */ + if (sess_priv) { + memset(sess_priv, 0, sizeof(struct zuc_session)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } } struct rte_cryptodev_ops zuc_pmd_ops = { diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-secgw/ipsec-secgw.c index 4d6c7ceb67..708eadd199 100644 --- a/examples/ipsec-secgw/ipsec-secgw.c +++ b/examples/ipsec-secgw/ipsec-secgw.c @@ -1242,8 +1242,7 @@ cryptodevs_init(void) uint32_t max_sess_sz = 0, sess_sz; for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) { - sess_sz = sizeof(struct rte_cryptodev_sym_session) + - rte_cryptodev_get_private_session_size(cdev_id); + sess_sz = rte_cryptodev_get_private_session_size(cdev_id); if (sess_sz > max_sess_sz) max_sess_sz = sess_sz; } diff --git a/examples/ipsec-secgw/ipsec.c b/examples/ipsec-secgw/ipsec.c index f09dce91bf..0afb9d6764 100644 --- a/examples/ipsec-secgw/ipsec.c +++ b/examples/ipsec-secgw/ipsec.c @@ -45,7 +45,7 @@ #include "esp.h" static inline int -create_session(struct ipsec_ctx *ipsec_ctx __rte_unused, struct ipsec_sa *sa) +create_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa) { struct rte_cryptodev_info cdev_info; unsigned long cdev_id_qp = 0; @@ -72,7 +72,10 @@ create_session(struct ipsec_ctx *ipsec_ctx __rte_unused, struct ipsec_sa *sa) ipsec_ctx->tbl[cdev_id_qp].qp); sa->crypto_session = rte_cryptodev_sym_session_create( - ipsec_ctx->tbl[cdev_id_qp].id, sa->xforms); + ipsec_ctx->session_pool); + rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id, + sa->crypto_session, sa->xforms, + ipsec_ctx->session_pool); rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id, &cdev_info); if (cdev_info.sym.max_nb_sessions_per_qp > 0) { diff --git a/examples/l2fwd-crypto/main.c b/examples/l2fwd-crypto/main.c index 9b24e51aaf..cba29ce19b 100644 --- a/examples/l2fwd-crypto/main.c +++ b/examples/l2fwd-crypto/main.c @@ -649,6 +649,9 @@ static struct rte_cryptodev_sym_session * initialize_crypto_session(struct l2fwd_crypto_options *options, uint8_t cdev_id) { struct rte_crypto_sym_xform *first_xform; + struct rte_cryptodev_sym_session *session; + uint8_t socket_id = rte_cryptodev_socket_id(cdev_id); + struct rte_mempool *sess_mp = session_pool_socket[socket_id]; if (options->xform_chain == L2FWD_CRYPTO_AEAD) { first_xform = &options->aead_xform; @@ -664,7 +667,16 @@ initialize_crypto_session(struct l2fwd_crypto_options *options, uint8_t cdev_id) first_xform = &options->auth_xform; } - return rte_cryptodev_sym_session_create(cdev_id, first_xform); + session = rte_cryptodev_sym_session_create(sess_mp); + + if (session == NULL) + return NULL; + + if (rte_cryptodev_sym_session_init(cdev_id, session, + first_xform, sess_mp) < 0) + return NULL; + + return session; } static void @@ -1935,8 +1947,7 @@ initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports, } for (cdev_id = 0; cdev_id < cdev_count; cdev_id++) { - sess_sz = sizeof(struct rte_cryptodev_sym_session) + - rte_cryptodev_get_private_session_size(cdev_id); + sess_sz = rte_cryptodev_get_private_session_size(cdev_id); if (sess_sz > max_sess_sz) max_sess_sz = sess_sz; } diff --git a/lib/librte_cryptodev/rte_cryptodev.c b/lib/librte_cryptodev/rte_cryptodev.c index dfced857ab..373c05baed 100644 --- a/lib/librte_cryptodev/rte_cryptodev.c +++ b/lib/librte_cryptodev/rte_cryptodev.c @@ -69,6 +69,8 @@ #include "rte_cryptodev.h" #include "rte_cryptodev_pmd.h" +static uint8_t nb_drivers; + struct rte_cryptodev rte_crypto_devices[RTE_CRYPTO_MAX_DEVS]; struct rte_cryptodev *rte_cryptodevs = &rte_crypto_devices[0]; @@ -1080,53 +1082,47 @@ rte_cryptodev_pmd_callback_process(struct rte_cryptodev *dev, } -static void -rte_cryptodev_sym_session_init(struct rte_mempool *mp, - const struct rte_cryptodev *dev, - struct rte_cryptodev_sym_session *sess) +int +rte_cryptodev_sym_session_init(uint8_t dev_id, + struct rte_cryptodev_sym_session *sess, + struct rte_crypto_sym_xform *xforms, + struct rte_mempool *mp) { - memset(sess, 0, mp->elt_size); + struct rte_cryptodev *dev; + uint8_t index; - if (dev->dev_ops->session_initialize) - (*dev->dev_ops->session_initialize)(mp, sess); -} + dev = rte_cryptodev_pmd_get_dev(dev_id); + if (sess == NULL || xforms == NULL || dev == NULL) + return -1; -struct rte_cryptodev_sym_session * -rte_cryptodev_sym_session_create(uint8_t dev_id, - struct rte_crypto_sym_xform *xform) -{ - struct rte_cryptodev *dev; - struct rte_cryptodev_sym_session *sess; - void *_sess; + index = dev->driver_id; - if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) { - CDEV_LOG_ERR("Invalid dev_id=%d", dev_id); - return NULL; + if (sess->sess_private_data[index] == NULL) { + if (dev->dev_ops->session_configure(dev, xforms, sess, mp) < 0) { + CDEV_LOG_ERR( + "dev_id %d failed to configure session details", + dev_id); + return -1; + } } - dev = &rte_crypto_devices[dev_id]; + return 0; +} + +struct rte_cryptodev_sym_session * +rte_cryptodev_sym_session_create(struct rte_mempool *mp) +{ + struct rte_cryptodev_sym_session *sess; /* Allocate a session structure from the session pool */ - if (rte_mempool_get(dev->data->session_pool, &_sess)) { - CDEV_LOG_ERR("Couldn't get object from session mempool"); + if (rte_mempool_get(mp, (void *)&sess)) { + CDEV_LOG_ERR("couldn't get object from session mempool"); return NULL; } - sess = _sess; - - rte_cryptodev_sym_session_init(dev->data->session_pool, dev, - sess); - RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->session_configure, NULL); - if (dev->dev_ops->session_configure(dev, xform, sess->_private) == - NULL) { - CDEV_LOG_ERR("dev_id %d failed to configure session details", - dev_id); - - /* Return session to mempool */ - rte_mempool_put(dev->data->session_pool, _sess); - return NULL; - } + /* Clear device session pointer */ + memset(sess, 0, (sizeof(void *) * nb_drivers)); return sess; } @@ -1146,7 +1142,10 @@ rte_cryptodev_queue_pair_attach_sym_session(uint8_t dev_id, uint16_t qp_id, /* The API is optional, not returning error if driver do not suuport */ RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->qp_attach_session, 0); - if (dev->dev_ops->qp_attach_session(dev, qp_id, sess->_private)) { + + void *sess_priv = get_session_private_data(sess, dev->driver_id); + + if (dev->dev_ops->qp_attach_session(dev, qp_id, sess_priv)) { CDEV_LOG_ERR("dev_id %d failed to attach qp: %d with session", dev_id, qp_id); return -EPERM; @@ -1170,7 +1169,10 @@ rte_cryptodev_queue_pair_detach_sym_session(uint8_t dev_id, uint16_t qp_id, /* The API is optional, not returning error if driver do not suuport */ RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->qp_detach_session, 0); - if (dev->dev_ops->qp_detach_session(dev, qp_id, sess->_private)) { + + void *sess_priv = get_session_private_data(sess, dev->driver_id); + + if (dev->dev_ops->qp_detach_session(dev, qp_id, sess_priv)) { CDEV_LOG_ERR("dev_id %d failed to detach qp: %d from session", dev_id, qp_id); return -EPERM; @@ -1178,34 +1180,62 @@ rte_cryptodev_queue_pair_detach_sym_session(uint8_t dev_id, uint16_t qp_id, return 0; } -struct rte_cryptodev_sym_session * -rte_cryptodev_sym_session_free(uint8_t dev_id, + +int +rte_cryptodev_sym_session_clear(uint8_t dev_id, struct rte_cryptodev_sym_session *sess) { struct rte_cryptodev *dev; - if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) { - CDEV_LOG_ERR("Invalid dev_id=%d", dev_id); - return sess; - } + dev = rte_cryptodev_pmd_get_dev(dev_id); - dev = &rte_crypto_devices[dev_id]; + if (dev == NULL || sess == NULL) + return -EINVAL; + + dev->dev_ops->session_clear(dev, sess); - /* Let device implementation clear session material */ - RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->session_clear, sess); - dev->dev_ops->session_clear(dev, (void *)sess->_private); + return 0; +} + +int +rte_cryptodev_sym_session_free(struct rte_cryptodev_sym_session *sess) +{ + uint8_t i; + void *sess_priv; + struct rte_mempool *sess_mp; + + if (sess == NULL) + return -EINVAL; + + /* Check that all device private data has been freed */ + for (i = 0; i < nb_drivers; i++) { + sess_priv = get_session_private_data(sess, i); + if (sess_priv != NULL) + return -EBUSY; + } /* Return session to mempool */ - struct rte_mempool *mp = rte_mempool_from_obj(sess); - rte_mempool_put(mp, (void *)sess); + sess_mp = rte_mempool_from_obj(sess); + rte_mempool_put(sess_mp, sess); - return NULL; + return 0; +} + +unsigned int +rte_cryptodev_get_header_session_size(void) +{ + /* + * Header contains pointers to the private data + * of all registered drivers + */ + return (sizeof(void *) * nb_drivers); } unsigned int rte_cryptodev_get_private_session_size(uint8_t dev_id) { struct rte_cryptodev *dev; + unsigned int header_size = sizeof(void *) * nb_drivers; unsigned int priv_sess_size; if (!rte_cryptodev_pmd_is_valid_dev(dev_id)) @@ -1218,6 +1248,14 @@ rte_cryptodev_get_private_session_size(uint8_t dev_id) priv_sess_size = (*dev->dev_ops->session_get_size)(dev); + /* + * If size is less than session header size, + * return the latter, as this guarantees that + * sessionless operations will work + */ + if (priv_sess_size < header_size) + return header_size; + return priv_sess_size; } @@ -1333,8 +1371,6 @@ struct cryptodev_driver { uint8_t id; }; -static uint8_t nb_drivers; - int rte_cryptodev_driver_id_get(const char *name) { diff --git a/lib/librte_cryptodev/rte_cryptodev.h b/lib/librte_cryptodev/rte_cryptodev.h index 508c672e0b..3ba3efb7e8 100644 --- a/lib/librte_cryptodev/rte_cryptodev.h +++ b/lib/librte_cryptodev/rte_cryptodev.h @@ -883,50 +883,80 @@ rte_cryptodev_enqueue_burst(uint8_t dev_id, uint16_t qp_id, /** Cryptodev symmetric crypto session */ struct rte_cryptodev_sym_session { RTE_STD_C11 - __extension__ char _private[0]; + __extension__ void *sess_private_data[0]; /**< Private session material */ }; /** - * Initialise a session for symmetric cryptographic operations. + * Create symmetric crypto session header (generic with no private data) * - * This function is used by the client to initialize immutable - * parameters of symmetric cryptographic operation. - * To perform the operation the rte_cryptodev_enqueue_burst function is - * used. Each mbuf should contain a reference to the session - * pointer returned from this function contained within it's crypto_op if a - * session-based operation is being provisioned. Memory to contain the session - * information is allocated from within mempool managed by the cryptodev. + * @param mempool Symmetric session mempool to allocate session + * objects from + * @return + * - On success return pointer to sym-session + * - On failure returns NULL + */ +struct rte_cryptodev_sym_session * +rte_cryptodev_sym_session_create(struct rte_mempool *mempool); + +/** + * Frees symmetric crypto session header, after checking that all + * the device private data has been freed, returning it + * to its original mempool. * - * The rte_cryptodev_session_free must be called to free allocated - * memory when the session is no longer required. + * @param sess Session header to be freed. * - * @param dev_id The device identifier. - * @param xform Crypto transform chain. + * @return + * - 0 if successful. + * - -EINVAL if session is NULL. + * - -EBUSY if not all device private data has been freed. + */ +int +rte_cryptodev_sym_session_free(struct rte_cryptodev_sym_session *sess); +/** + * Fill out private data for the device id, based on its device type. + * + * @param dev_id ID of device that we want the session to be used on + * @param sess Session where the private data will be attached to + * @param xforms Symmetric crypto transform operations to apply on flow + * processed with this session + * @param mempool Mempool where the private data is allocated. * * @return - * Pointer to the created session or NULL + * - On success, zero. + * - On failure, a negative value. */ -extern struct rte_cryptodev_sym_session * -rte_cryptodev_sym_session_create(uint8_t dev_id, - struct rte_crypto_sym_xform *xform); +int +rte_cryptodev_sym_session_init(uint8_t dev_id, + struct rte_cryptodev_sym_session *sess, + struct rte_crypto_sym_xform *xforms, + struct rte_mempool *mempool); /** - * Free the memory associated with a previously allocated session. + * Frees private data for the device id, based on its device type, + * returning it to its mempool. * - * @param dev_id The device identifier. - * @param session Session pointer previously allocated by - * *rte_cryptodev_sym_session_create*. + * @param dev_id ID of device that uses the session. + * @param sess Session containing the reference to the private data * * @return - * NULL on successful freeing of session. - * Session pointer on failure to free session. + * - 0 if successful. + * - -EINVAL if device is invalid or session is NULL. */ -extern struct rte_cryptodev_sym_session * -rte_cryptodev_sym_session_free(uint8_t dev_id, - struct rte_cryptodev_sym_session *session); +int +rte_cryptodev_sym_session_clear(uint8_t dev_id, + struct rte_cryptodev_sym_session *sess); + +/** + * Get the size of the header session, for all registered drivers. + * + * @return + * Size of the header session. + */ +unsigned int +rte_cryptodev_get_header_session_size(void); /** * Get the size of the private session data for a device. diff --git a/lib/librte_cryptodev/rte_cryptodev_pmd.h b/lib/librte_cryptodev/rte_cryptodev_pmd.h index 5911b838b7..2896171e45 100644 --- a/lib/librte_cryptodev/rte_cryptodev_pmd.h +++ b/lib/librte_cryptodev/rte_cryptodev_pmd.h @@ -285,20 +285,25 @@ typedef void (*cryptodev_sym_initialize_session_t)(struct rte_mempool *mempool, * @param dev Crypto device pointer * @param xform Single or chain of crypto xforms * @param priv_sess Pointer to cryptodev's private session structure + * @param mp Mempool where the private session is allocated * * @return - * - Returns private session structure on success. - * - Returns NULL on failure. + * - Returns 0 if private session structure have been created successfully. + * - Returns -1 on failure. */ -typedef void * (*cryptodev_sym_configure_session_t)(struct rte_cryptodev *dev, - struct rte_crypto_sym_xform *xform, void *session_private); +typedef int (*cryptodev_sym_configure_session_t)(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *session, + struct rte_mempool *mp); /** - * Free Crypto session. - * @param session Cryptodev session structure to free + * Free driver private session data. + * + * @param dev Crypto device pointer + * @param sess Cryptodev session structure */ typedef void (*cryptodev_sym_free_session_t)(struct rte_cryptodev *dev, - void *session_private); + struct rte_cryptodev_sym_session *sess); /** * Optional API for drivers to attach sessions with queue pair. @@ -413,6 +418,19 @@ void rte_cryptodev_pmd_callback_process(struct rte_cryptodev *dev, int rte_cryptodev_pmd_create_dev_name(char *name, const char *dev_name_prefix); +static inline void * +get_session_private_data(const struct rte_cryptodev_sym_session *sess, + uint8_t driver_id) { + return sess->sess_private_data[driver_id]; +} + +static inline void +set_session_private_data(struct rte_cryptodev_sym_session *sess, + uint8_t driver_id, void *private_data) +{ + sess->sess_private_data[driver_id] = private_data; +} + #ifdef __cplusplus } #endif diff --git a/lib/librte_cryptodev/rte_cryptodev_version.map b/lib/librte_cryptodev/rte_cryptodev_version.map index d817421824..e9ba88ac5b 100644 --- a/lib/librte_cryptodev/rte_cryptodev_version.map +++ b/lib/librte_cryptodev/rte_cryptodev_version.map @@ -66,10 +66,13 @@ DPDK_17.08 { rte_cryptodev_driver_id_get; rte_cryptodev_driver_name_get; rte_cryptodev_get_aead_algo_enum; + rte_cryptodev_get_header_session_size; rte_cryptodev_get_private_session_size; rte_cryptodev_pci_generic_probe; rte_cryptodev_pci_generic_remove; rte_cryptodev_sym_capability_check_aead; + rte_cryptodev_sym_session_init; + rte_cryptodev_sym_session_clear; rte_cryptodev_vdev_parse_init_params; rte_cryptodev_vdev_pmd_init; rte_crypto_aead_algorithm_strings; diff --git a/test/test/test_cryptodev.c b/test/test/test_cryptodev.c index e5d6c07eba..745f2611ac 100644 --- a/test/test/test_cryptodev.c +++ b/test/test/test_cryptodev.c @@ -68,7 +68,6 @@ struct crypto_testsuite_params { struct rte_mempool *large_mbuf_pool; struct rte_mempool *op_mpool; struct rte_mempool *session_mpool; - struct rte_mempool *slave_session_mpool; struct rte_cryptodev_config conf; struct rte_cryptodev_qp_conf qp_conf; @@ -387,12 +386,15 @@ testsuite_setup(void) ts_params->conf.nb_queue_pairs = info.max_nb_queue_pairs; ts_params->conf.socket_id = SOCKET_ID_ANY; - unsigned int session_size = sizeof(struct rte_cryptodev_sym_session) + - rte_cryptodev_get_private_session_size(dev_id); + unsigned int session_size = rte_cryptodev_get_private_session_size(dev_id); + /* + * Create mempool with maximum number of sessions * 2, + * to include the session headers + */ ts_params->session_mpool = rte_mempool_create( "test_sess_mp", - info.sym.max_nb_sessions, + info.sym.max_nb_sessions * 2, session_size, 0, 0, NULL, NULL, NULL, NULL, SOCKET_ID_ANY, @@ -439,11 +441,6 @@ testsuite_teardown(void) rte_mempool_free(ts_params->session_mpool); ts_params->session_mpool = NULL; } - - if (ts_params->slave_session_mpool != NULL) { - rte_mempool_free(ts_params->slave_session_mpool); - ts_params->slave_session_mpool = NULL; - } } static int @@ -494,8 +491,9 @@ ut_teardown(void) /* free crypto session structure */ if (ut_params->sess) { - rte_cryptodev_sym_session_free(ts_params->valid_devs[0], + rte_cryptodev_sym_session_clear(ts_params->valid_devs[0], ut_params->sess); + rte_cryptodev_sym_session_free(ut_params->sess); ut_params->sess = NULL; } @@ -1280,10 +1278,13 @@ test_AES_CBC_HMAC_SHA1_encrypt_digest(void) ut_params->auth_xform.auth.key.data = hmac_sha1_key; ut_params->auth_xform.auth.digest_length = DIGEST_BYTE_LENGTH_SHA1; - /* Create crypto session*/ ut_params->sess = rte_cryptodev_sym_session_create( - ts_params->valid_devs[0], - &ut_params->cipher_xform); + ts_params->session_mpool); + + /* Create crypto session*/ + rte_cryptodev_sym_session_init(ts_params->valid_devs[0], + ut_params->sess, &ut_params->cipher_xform, + ts_params->session_mpool); TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); /* Generate crypto op data structure */ @@ -1496,7 +1497,9 @@ test_AES_cipheronly_mb_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_AESNI_MB_PMD)), BLKCIPHER_AES_CIPHERONLY_TYPE); @@ -1513,7 +1516,9 @@ test_AES_docsis_mb_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_AESNI_MB_PMD)), BLKCIPHER_AES_DOCSIS_TYPE); @@ -1530,7 +1535,9 @@ test_AES_docsis_qat_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_QAT_SYM_PMD)), BLKCIPHER_AES_DOCSIS_TYPE); @@ -1547,7 +1554,9 @@ test_DES_docsis_qat_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_QAT_SYM_PMD)), BLKCIPHER_DES_DOCSIS_TYPE); @@ -1564,7 +1573,9 @@ test_authonly_mb_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_AESNI_MB_PMD)), BLKCIPHER_AUTHONLY_TYPE); @@ -1581,7 +1592,9 @@ test_AES_chain_mb_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_AESNI_MB_PMD)), BLKCIPHER_AES_CHAIN_TYPE); @@ -1600,7 +1613,9 @@ test_AES_cipheronly_scheduler_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_SCHEDULER_PMD)), BLKCIPHER_AES_CIPHERONLY_TYPE); @@ -1617,7 +1632,9 @@ test_AES_chain_scheduler_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_SCHEDULER_PMD)), BLKCIPHER_AES_CHAIN_TYPE); @@ -1634,7 +1651,9 @@ test_authonly_scheduler_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_SCHEDULER_PMD)), BLKCIPHER_AUTHONLY_TYPE); @@ -1653,7 +1672,9 @@ test_AES_chain_openssl_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD)), BLKCIPHER_AES_CHAIN_TYPE); @@ -1670,7 +1691,9 @@ test_AES_cipheronly_openssl_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD)), BLKCIPHER_AES_CIPHERONLY_TYPE); @@ -1687,7 +1710,9 @@ test_AES_chain_qat_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_QAT_SYM_PMD)), BLKCIPHER_AES_CHAIN_TYPE); @@ -1704,7 +1729,9 @@ test_AES_cipheronly_qat_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_QAT_SYM_PMD)), BLKCIPHER_AES_CIPHERONLY_TYPE); @@ -1721,7 +1748,9 @@ test_AES_chain_dpaa2_sec_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_DPAA2_SEC_PMD)), BLKCIPHER_AES_CHAIN_TYPE); @@ -1738,7 +1767,9 @@ test_AES_cipheronly_dpaa2_sec_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_DPAA2_SEC_PMD)), BLKCIPHER_AES_CIPHERONLY_TYPE); @@ -1755,7 +1786,9 @@ test_authonly_dpaa2_sec_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(RTE_CRYPTODEV_DPAA2_SEC_PMD)), BLKCIPHER_AUTHONLY_TYPE); @@ -1772,7 +1805,9 @@ test_authonly_openssl_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD)), BLKCIPHER_AUTHONLY_TYPE); @@ -1789,7 +1824,9 @@ test_AES_chain_armv8_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_ARMV8_PMD)), BLKCIPHER_AES_CHAIN_TYPE); @@ -1809,6 +1846,7 @@ create_wireless_algo_hash_session(uint8_t dev_id, { uint8_t hash_key[key_len]; + struct crypto_testsuite_params *ts_params = &testsuite_params; struct crypto_unittest_params *ut_params = &unittest_params; memcpy(hash_key, key, key_len); @@ -1826,8 +1864,11 @@ create_wireless_algo_hash_session(uint8_t dev_id, ut_params->auth_xform.auth.digest_length = auth_len; ut_params->auth_xform.auth.iv.offset = IV_OFFSET; ut_params->auth_xform.auth.iv.length = iv_len; - ut_params->sess = rte_cryptodev_sym_session_create(dev_id, - &ut_params->auth_xform); + ut_params->sess = rte_cryptodev_sym_session_create( + ts_params->session_mpool); + + rte_cryptodev_sym_session_init(dev_id, ut_params->sess, + &ut_params->auth_xform, ts_params->session_mpool); TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); return 0; } @@ -1841,6 +1882,7 @@ create_wireless_algo_cipher_session(uint8_t dev_id, { uint8_t cipher_key[key_len]; + struct crypto_testsuite_params *ts_params = &testsuite_params; struct crypto_unittest_params *ut_params = &unittest_params; memcpy(cipher_key, key, key_len); @@ -1859,9 +1901,11 @@ create_wireless_algo_cipher_session(uint8_t dev_id, TEST_HEXDUMP(stdout, "key:", key, key_len); /* Create Crypto session */ - ut_params->sess = rte_cryptodev_sym_session_create(dev_id, - &ut_params-> - cipher_xform); + ut_params->sess = rte_cryptodev_sym_session_create( + ts_params->session_mpool); + + rte_cryptodev_sym_session_init(dev_id, ut_params->sess, + &ut_params->cipher_xform, ts_params->session_mpool); TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); return 0; } @@ -1940,6 +1984,7 @@ create_wireless_algo_cipher_auth_session(uint8_t dev_id, { uint8_t cipher_auth_key[key_len]; + struct crypto_testsuite_params *ts_params = &testsuite_params; struct crypto_unittest_params *ut_params = &unittest_params; memcpy(cipher_auth_key, key, key_len); @@ -1972,8 +2017,11 @@ create_wireless_algo_cipher_auth_session(uint8_t dev_id, TEST_HEXDUMP(stdout, "key:", key, key_len); /* Create Crypto session*/ - ut_params->sess = rte_cryptodev_sym_session_create(dev_id, - &ut_params->cipher_xform); + ut_params->sess = rte_cryptodev_sym_session_create( + ts_params->session_mpool); + + rte_cryptodev_sym_session_init(dev_id, ut_params->sess, + &ut_params->cipher_xform, ts_params->session_mpool); TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); return 0; @@ -1990,6 +2038,7 @@ create_wireless_cipher_auth_session(uint8_t dev_id, const uint8_t key_len = tdata->key.len; uint8_t cipher_auth_key[key_len]; + struct crypto_testsuite_params *ts_params = &testsuite_params; struct crypto_unittest_params *ut_params = &unittest_params; const uint8_t *key = tdata->key.data; const uint8_t auth_len = tdata->digest.len; @@ -2027,8 +2076,11 @@ create_wireless_cipher_auth_session(uint8_t dev_id, TEST_HEXDUMP(stdout, "key:", key, key_len); /* Create Crypto session*/ - ut_params->sess = rte_cryptodev_sym_session_create(dev_id, - &ut_params->cipher_xform); + ut_params->sess = rte_cryptodev_sym_session_create( + ts_params->session_mpool); + + rte_cryptodev_sym_session_init(dev_id, ut_params->sess, + &ut_params->cipher_xform, ts_params->session_mpool); TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); return 0; @@ -2056,6 +2108,7 @@ create_wireless_algo_auth_cipher_session(uint8_t dev_id, { uint8_t auth_cipher_key[key_len]; + struct crypto_testsuite_params *ts_params = &testsuite_params; struct crypto_unittest_params *ut_params = &unittest_params; memcpy(auth_cipher_key, key, key_len); @@ -2085,8 +2138,11 @@ create_wireless_algo_auth_cipher_session(uint8_t dev_id, TEST_HEXDUMP(stdout, "key:", key, key_len); /* Create Crypto session*/ - ut_params->sess = rte_cryptodev_sym_session_create(dev_id, - &ut_params->auth_xform); + ut_params->sess = rte_cryptodev_sym_session_create( + ts_params->session_mpool); + + rte_cryptodev_sym_session_init(dev_id, ut_params->sess, + &ut_params->auth_xform, ts_params->session_mpool); TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); @@ -4536,7 +4592,9 @@ test_3DES_chain_qat_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_QAT_SYM_PMD)), BLKCIPHER_3DES_CHAIN_TYPE); @@ -4553,7 +4611,9 @@ test_DES_cipheronly_qat_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_QAT_SYM_PMD)), BLKCIPHER_DES_CIPHERONLY_TYPE); @@ -4570,7 +4630,9 @@ test_DES_docsis_openssl_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD)), BLKCIPHER_DES_DOCSIS_TYPE); @@ -4587,7 +4649,9 @@ test_3DES_chain_dpaa2_sec_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_DPAA2_SEC_PMD)), BLKCIPHER_3DES_CHAIN_TYPE); @@ -4604,7 +4668,9 @@ test_3DES_cipheronly_dpaa2_sec_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_DPAA2_SEC_PMD)), BLKCIPHER_3DES_CIPHERONLY_TYPE); @@ -4621,7 +4687,9 @@ test_3DES_cipheronly_qat_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_QAT_SYM_PMD)), BLKCIPHER_3DES_CIPHERONLY_TYPE); @@ -4638,7 +4706,9 @@ test_3DES_chain_openssl_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD)), BLKCIPHER_3DES_CHAIN_TYPE); @@ -4655,7 +4725,9 @@ test_3DES_cipheronly_openssl_all(void) int status; status = test_blockcipher_all_tests(ts_params->mbuf_pool, - ts_params->op_mpool, ts_params->valid_devs[0], + ts_params->op_mpool, + ts_params->session_mpool, + ts_params->valid_devs[0], rte_cryptodev_driver_id_get( RTE_STR(CRYPTODEV_NAME_OPENSSL_PMD)), BLKCIPHER_3DES_CIPHERONLY_TYPE); @@ -4675,6 +4747,7 @@ create_gcm_session(uint8_t dev_id, enum rte_crypto_aead_operation op, { uint8_t aead_key[key_len]; + struct crypto_testsuite_params *ts_params = &testsuite_params; struct crypto_unittest_params *ut_params = &unittest_params; memcpy(aead_key, key, key_len); @@ -4694,8 +4767,11 @@ create_gcm_session(uint8_t dev_id, enum rte_crypto_aead_operation op, TEST_HEXDUMP(stdout, "key:", key, key_len); /* Create Crypto session*/ - ut_params->sess = rte_cryptodev_sym_session_create(dev_id, - &ut_params->aead_xform); + ut_params->sess = rte_cryptodev_sym_session_create( + ts_params->session_mpool); + + rte_cryptodev_sym_session_init(dev_id, ut_params->sess, + &ut_params->aead_xform, ts_params->session_mpool); TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); @@ -5656,7 +5732,11 @@ static int MD5_HMAC_create_session(struct crypto_testsuite_params *ts_params, ut_params->auth_xform.auth.key.data = key; ut_params->sess = rte_cryptodev_sym_session_create( - ts_params->valid_devs[0], &ut_params->auth_xform); + ts_params->session_mpool); + + rte_cryptodev_sym_session_init(ts_params->valid_devs[0], + ut_params->sess, &ut_params->auth_xform, + ts_params->session_mpool); if (ut_params->sess == NULL) return TEST_FAILED; @@ -5831,9 +5911,13 @@ test_multi_session(void) /* Create multiple crypto sessions*/ for (i = 0; i < dev_info.sym.max_nb_sessions; i++) { + sessions[i] = rte_cryptodev_sym_session_create( - ts_params->valid_devs[0], - &ut_params->auth_xform); + ts_params->session_mpool); + + rte_cryptodev_sym_session_init(ts_params->valid_devs[0], + sessions[i], &ut_params->auth_xform, + ts_params->session_mpool); TEST_ASSERT_NOT_NULL(sessions[i], "Session creation failed at session number %u", i); @@ -5869,14 +5953,17 @@ test_multi_session(void) } /* Next session create should fail */ - sessions[i] = rte_cryptodev_sym_session_create(ts_params->valid_devs[0], - &ut_params->auth_xform); + rte_cryptodev_sym_session_init(ts_params->valid_devs[0], + sessions[i], &ut_params->auth_xform, + ts_params->session_mpool); TEST_ASSERT_NULL(sessions[i], "Session creation succeeded unexpectedly!"); - for (i = 0; i < dev_info.sym.max_nb_sessions; i++) - rte_cryptodev_sym_session_free(ts_params->valid_devs[0], + for (i = 0; i < dev_info.sym.max_nb_sessions; i++) { + rte_cryptodev_sym_session_clear(ts_params->valid_devs[0], sessions[i]); + rte_cryptodev_sym_session_free(sessions[i]); + } rte_free(sessions); @@ -5934,6 +6021,9 @@ test_multi_session_random_usage(void) * dev_info.sym.max_nb_sessions) + 1, 0); for (i = 0; i < MB_SESSION_NUMBER; i++) { + sessions[i] = rte_cryptodev_sym_session_create( + ts_params->session_mpool); + rte_memcpy(&ut_paramz[i].ut_params, &testsuite_params, sizeof(struct crypto_unittest_params)); @@ -5942,9 +6032,11 @@ test_multi_session_random_usage(void) ut_paramz[i].cipher_key, ut_paramz[i].hmac_key); /* Create multiple crypto sessions*/ - sessions[i] = rte_cryptodev_sym_session_create( + rte_cryptodev_sym_session_init( ts_params->valid_devs[0], - &ut_paramz[i].ut_params.auth_xform); + sessions[i], + &ut_paramz[i].ut_params.auth_xform, + ts_params->session_mpool); TEST_ASSERT_NOT_NULL(sessions[i], "Session creation failed at session number %u", @@ -5987,9 +6079,11 @@ test_multi_session_random_usage(void) } } - for (i = 0; i < MB_SESSION_NUMBER; i++) - rte_cryptodev_sym_session_free(ts_params->valid_devs[0], + for (i = 0; i < MB_SESSION_NUMBER; i++) { + rte_cryptodev_sym_session_clear(ts_params->valid_devs[0], sessions[i]); + rte_cryptodev_sym_session_free(sessions[i]); + } rte_free(sessions); @@ -6013,9 +6107,14 @@ test_null_cipher_only_operation(void) ut_params->cipher_xform.cipher.algo = RTE_CRYPTO_CIPHER_NULL; ut_params->cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT; - /* Create Crypto session*/ ut_params->sess = rte_cryptodev_sym_session_create( - ts_params->valid_devs[0], &ut_params->cipher_xform); + ts_params->session_mpool); + + /* Create Crypto session*/ + rte_cryptodev_sym_session_init(ts_params->valid_devs[0], + ut_params->sess, + &ut_params->cipher_xform, + ts_params->session_mpool); TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); /* Generate Crypto op data structure */ @@ -6070,9 +6169,13 @@ test_null_auth_only_operation(void) ut_params->auth_xform.auth.algo = RTE_CRYPTO_AUTH_NULL; ut_params->auth_xform.auth.op = RTE_CRYPTO_AUTH_OP_GENERATE; - /* Create Crypto session*/ ut_params->sess = rte_cryptodev_sym_session_create( - ts_params->valid_devs[0], &ut_params->auth_xform); + ts_params->session_mpool); + + /* Create Crypto session*/ + rte_cryptodev_sym_session_init(ts_params->valid_devs[0], + ut_params->sess, &ut_params->auth_xform, + ts_params->session_mpool); TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); /* Generate Crypto op data structure */ @@ -6126,9 +6229,13 @@ test_null_cipher_auth_operation(void) ut_params->auth_xform.auth.algo = RTE_CRYPTO_AUTH_NULL; ut_params->auth_xform.auth.op = RTE_CRYPTO_AUTH_OP_GENERATE; - /* Create Crypto session*/ ut_params->sess = rte_cryptodev_sym_session_create( - ts_params->valid_devs[0], &ut_params->cipher_xform); + ts_params->session_mpool); + + /* Create Crypto session*/ + rte_cryptodev_sym_session_init(ts_params->valid_devs[0], + ut_params->sess, &ut_params->cipher_xform, + ts_params->session_mpool); TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); /* Generate Crypto op data structure */ @@ -6192,9 +6299,13 @@ test_null_auth_cipher_operation(void) ut_params->auth_xform.auth.algo = RTE_CRYPTO_AUTH_NULL; ut_params->auth_xform.auth.op = RTE_CRYPTO_AUTH_OP_GENERATE; - /* Create Crypto session*/ ut_params->sess = rte_cryptodev_sym_session_create( - ts_params->valid_devs[0], &ut_params->cipher_xform); + ts_params->session_mpool); + + /* Create Crypto session*/ + rte_cryptodev_sym_session_init(ts_params->valid_devs[0], + ut_params->sess, &ut_params->cipher_xform, + ts_params->session_mpool); TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); /* Generate Crypto op data structure */ @@ -6240,6 +6351,7 @@ test_null_invalid_operation(void) { struct crypto_testsuite_params *ts_params = &testsuite_params; struct crypto_unittest_params *ut_params = &unittest_params; + int ret; /* Setup Cipher Parameters */ ut_params->cipher_xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER; @@ -6248,10 +6360,14 @@ test_null_invalid_operation(void) ut_params->cipher_xform.cipher.algo = RTE_CRYPTO_CIPHER_AES_CBC; ut_params->cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT; - /* Create Crypto session*/ ut_params->sess = rte_cryptodev_sym_session_create( - ts_params->valid_devs[0], &ut_params->cipher_xform); - TEST_ASSERT_NULL(ut_params->sess, + ts_params->session_mpool); + + /* Create Crypto session*/ + ret = rte_cryptodev_sym_session_init(ts_params->valid_devs[0], + ut_params->sess, &ut_params->cipher_xform, + ts_params->session_mpool); + TEST_ASSERT(ret == -1, "Session creation succeeded unexpectedly"); @@ -6262,10 +6378,14 @@ test_null_invalid_operation(void) ut_params->auth_xform.auth.algo = RTE_CRYPTO_AUTH_SHA1_HMAC; ut_params->auth_xform.auth.op = RTE_CRYPTO_AUTH_OP_GENERATE; - /* Create Crypto session*/ ut_params->sess = rte_cryptodev_sym_session_create( - ts_params->valid_devs[0], &ut_params->auth_xform); - TEST_ASSERT_NULL(ut_params->sess, + ts_params->session_mpool); + + /* Create Crypto session*/ + ret = rte_cryptodev_sym_session_init(ts_params->valid_devs[0], + ut_params->sess, &ut_params->auth_xform, + ts_params->session_mpool); + TEST_ASSERT(ret == -1, "Session creation succeeded unexpectedly"); return TEST_SUCCESS; @@ -6299,9 +6419,13 @@ test_null_burst_operation(void) ut_params->auth_xform.auth.algo = RTE_CRYPTO_AUTH_NULL; ut_params->auth_xform.auth.op = RTE_CRYPTO_AUTH_OP_GENERATE; - /* Create Crypto session*/ ut_params->sess = rte_cryptodev_sym_session_create( - ts_params->valid_devs[0], &ut_params->cipher_xform); + ts_params->session_mpool); + + /* Create Crypto session*/ + rte_cryptodev_sym_session_init(ts_params->valid_devs[0], + ut_params->sess, &ut_params->cipher_xform, + ts_params->session_mpool); TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); TEST_ASSERT_EQUAL(rte_crypto_op_bulk_alloc(ts_params->op_mpool, @@ -6414,6 +6538,7 @@ static int create_gmac_session(uint8_t dev_id, { uint8_t auth_key[tdata->key.len]; + struct crypto_testsuite_params *ts_params = &testsuite_params; struct crypto_unittest_params *ut_params = &unittest_params; memcpy(auth_key, tdata->key.data, tdata->key.len); @@ -6430,8 +6555,12 @@ static int create_gmac_session(uint8_t dev_id, ut_params->auth_xform.auth.iv.length = tdata->iv.len; - ut_params->sess = rte_cryptodev_sym_session_create(dev_id, - &ut_params->auth_xform); + ut_params->sess = rte_cryptodev_sym_session_create( + ts_params->session_mpool); + + rte_cryptodev_sym_session_init(dev_id, ut_params->sess, + &ut_params->auth_xform, + ts_params->session_mpool); TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); @@ -6791,6 +6920,7 @@ create_auth_session(struct crypto_unittest_params *ut_params, const struct test_crypto_vector *reference, enum rte_crypto_auth_operation auth_op) { + struct crypto_testsuite_params *ts_params = &testsuite_params; uint8_t auth_key[reference->auth_key.len + 1]; memcpy(auth_key, reference->auth_key.data, reference->auth_key.len); @@ -6805,8 +6935,12 @@ create_auth_session(struct crypto_unittest_params *ut_params, ut_params->auth_xform.auth.digest_length = reference->digest.len; /* Create Crypto session*/ - ut_params->sess = rte_cryptodev_sym_session_create(dev_id, - &ut_params->auth_xform); + ut_params->sess = rte_cryptodev_sym_session_create( + ts_params->session_mpool); + + rte_cryptodev_sym_session_init(dev_id, ut_params->sess, + &ut_params->auth_xform, + ts_params->session_mpool); TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); @@ -6820,6 +6954,7 @@ create_auth_cipher_session(struct crypto_unittest_params *ut_params, enum rte_crypto_auth_operation auth_op, enum rte_crypto_cipher_operation cipher_op) { + struct crypto_testsuite_params *ts_params = &testsuite_params; uint8_t cipher_key[reference->cipher_key.len + 1]; uint8_t auth_key[reference->auth_key.len + 1]; @@ -6853,8 +6988,12 @@ create_auth_cipher_session(struct crypto_unittest_params *ut_params, } /* Create Crypto session*/ - ut_params->sess = rte_cryptodev_sym_session_create(dev_id, - &ut_params->auth_xform); + ut_params->sess = rte_cryptodev_sym_session_create( + ts_params->session_mpool); + + rte_cryptodev_sym_session_init(dev_id, ut_params->sess, + &ut_params->auth_xform, + ts_params->session_mpool); TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); @@ -7710,30 +7849,32 @@ test_scheduler_attach_slave_op(void) continue; /* - * Create a separate mempool for the slaves, as they need different - * session size and then configure them to store the pointer - * to this mempool + * Create the session mempool again, since now there are new devices + * to use the mempool. */ - unsigned int session_size = sizeof(struct rte_cryptodev_sym_session) + - rte_cryptodev_get_private_session_size(i); + if (ts_params->session_mpool) { + rte_mempool_free(ts_params->session_mpool); + ts_params->session_mpool = NULL; + } + unsigned int session_size = rte_cryptodev_get_private_session_size(i); - if (ts_params->slave_session_mpool == NULL) { - ts_params->slave_session_mpool = rte_mempool_create( - "test_slave_sess_mp", - info.sym.max_nb_sessions, - session_size, - 0, 0, NULL, NULL, NULL, NULL, - SOCKET_ID_ANY, 0); + /* + * Create mempool with maximum number of sessions * 2, + * to include the session headers + */ + if (ts_params->session_mpool == NULL) { + ts_params->session_mpool = rte_mempool_create( + "test_sess_mp", + info.sym.max_nb_sessions * 2, + session_size, + 0, 0, NULL, NULL, NULL, + NULL, SOCKET_ID_ANY, + 0); - TEST_ASSERT_NOT_NULL(ts_params->slave_session_mpool, + TEST_ASSERT_NOT_NULL(ts_params->session_mpool, "session mempool allocation failed"); } - TEST_ASSERT_SUCCESS(rte_cryptodev_configure(i, - &ts_params->conf, ts_params->slave_session_mpool), - "Failed to configure cryptodev %u with %u qps", - i, ts_params->conf.nb_queue_pairs); - ret = rte_cryptodev_scheduler_slave_attach(sched_id, (uint8_t)i); diff --git a/test/test/test_cryptodev_blockcipher.c b/test/test/test_cryptodev_blockcipher.c index 0b512c2e9d..e822c96c62 100644 --- a/test/test/test_cryptodev_blockcipher.c +++ b/test/test/test_cryptodev_blockcipher.c @@ -53,6 +53,7 @@ static int test_blockcipher_one_case(const struct blockcipher_test_case *t, struct rte_mempool *mbuf_pool, struct rte_mempool *op_mpool, + struct rte_mempool *sess_mpool, uint8_t dev_id, int driver_id, char *test_msg) @@ -65,8 +66,8 @@ test_blockcipher_one_case(const struct blockcipher_test_case *t, struct rte_crypto_sym_xform *init_xform = NULL; struct rte_crypto_sym_op *sym_op = NULL; struct rte_crypto_op *op = NULL; - struct rte_cryptodev_sym_session *sess = NULL; struct rte_cryptodev_info dev_info; + struct rte_cryptodev_sym_session *sess = NULL; int status = TEST_SUCCESS; const struct blockcipher_test_data *tdata = t->test_data; @@ -340,8 +341,10 @@ test_blockcipher_one_case(const struct blockcipher_test_case *t, /* create session for sessioned op */ if (!(t->feature_mask & BLOCKCIPHER_TEST_FEATURE_SESSIONLESS)) { - sess = rte_cryptodev_sym_session_create(dev_id, - init_xform); + sess = rte_cryptodev_sym_session_create(sess_mpool); + + rte_cryptodev_sym_session_init(dev_id, sess, init_xform, + sess_mpool); if (!sess) { snprintf(test_msg, BLOCKCIPHER_TEST_MSG_LEN, "line %u " "FAILED: %s", __LINE__, @@ -561,8 +564,10 @@ test_blockcipher_one_case(const struct blockcipher_test_case *t, error_exit: if (!(t->feature_mask & BLOCKCIPHER_TEST_FEATURE_SESSIONLESS)) { - if (sess) - rte_cryptodev_sym_session_free(dev_id, sess); + if (sess) { + rte_cryptodev_sym_session_clear(dev_id, sess); + rte_cryptodev_sym_session_free(sess); + } if (cipher_xform) rte_free(cipher_xform); if (auth_xform) @@ -584,6 +589,7 @@ test_blockcipher_one_case(const struct blockcipher_test_case *t, int test_blockcipher_all_tests(struct rte_mempool *mbuf_pool, struct rte_mempool *op_mpool, + struct rte_mempool *sess_mpool, uint8_t dev_id, int driver_id, enum blockcipher_test_type test_type) @@ -675,7 +681,7 @@ test_blockcipher_all_tests(struct rte_mempool *mbuf_pool, continue; status = test_blockcipher_one_case(tc, mbuf_pool, op_mpool, - dev_id, driver_id, test_msg); + sess_mpool, dev_id, driver_id, test_msg); printf(" %u) TestCase %s %s\n", test_index ++, tc->test_descr, test_msg); diff --git a/test/test/test_cryptodev_blockcipher.h b/test/test/test_cryptodev_blockcipher.h index 22fb4201e3..22b8d20612 100644 --- a/test/test/test_cryptodev_blockcipher.h +++ b/test/test/test_cryptodev_blockcipher.h @@ -125,6 +125,7 @@ struct blockcipher_test_data { int test_blockcipher_all_tests(struct rte_mempool *mbuf_pool, struct rte_mempool *op_mpool, + struct rte_mempool *sess_mpool, uint8_t dev_id, int driver_id, enum blockcipher_test_type test_type); diff --git a/test/test/test_cryptodev_perf.c b/test/test/test_cryptodev_perf.c index 9caba87605..e21f5e7af7 100644 --- a/test/test/test_cryptodev_perf.c +++ b/test/test/test_cryptodev_perf.c @@ -108,6 +108,8 @@ struct symmetric_session_attrs { uint32_t digest_len; }; +static struct rte_cryptodev_sym_session *test_crypto_session; + #define ALIGN_POW2_ROUNDUP(num, align) \ (((num) + (align) - 1) & ~((align) - 1)) @@ -156,18 +158,18 @@ struct crypto_unittest_params { uint8_t *digest; }; -static struct rte_cryptodev_sym_session * +static int test_perf_create_snow3g_session(uint8_t dev_id, enum chain_mode chain, enum rte_crypto_cipher_algorithm cipher_algo, unsigned int cipher_key_len, enum rte_crypto_auth_algorithm auth_algo); -static struct rte_cryptodev_sym_session * +static int test_perf_create_openssl_session(uint8_t dev_id, enum chain_mode chain, enum rte_crypto_cipher_algorithm cipher_algo, unsigned int cipher_key_len, enum rte_crypto_auth_algorithm auth_algo, enum rte_crypto_aead_algorithm aead_algo); -static struct rte_cryptodev_sym_session * +static int test_perf_create_armv8_session(uint8_t dev_id, enum chain_mode chain, enum rte_crypto_cipher_algorithm cipher_algo, unsigned int cipher_key_len, @@ -487,9 +489,11 @@ ut_teardown(void) unsigned i; /* free crypto session structure */ - if (ut_params->sess) - rte_cryptodev_sym_session_free(ts_params->dev_id, + if (ut_params->sess) { + rte_cryptodev_sym_session_clear(ts_params->dev_id, ut_params->sess); + rte_cryptodev_sym_session_free(ut_params->sess); + } /* free crypto operation structure */ if (ut_params->op) @@ -1969,10 +1973,13 @@ test_perf_crypto_qp_vary_burst_size(uint16_t dev_num) ut_params->auth_xform.auth.digest_length = DIGEST_BYTE_LENGTH_SHA256; /* Create Crypto session*/ - ut_params->sess = rte_cryptodev_sym_session_create(ts_params->dev_id, - &ut_params->cipher_xform); - TEST_ASSERT_NOT_NULL(ut_params->sess, "Session creation failed"); + test_crypto_session = rte_cryptodev_sym_session_create(ts_params->sess_mp); + + rte_cryptodev_sym_session_init(ts_params->dev_id, test_crypto_session, + &ut_params->cipher_xform, ts_params->sess_mp); + + TEST_ASSERT_NOT_NULL(test_crypto_session, "Session creation failed"); /* Generate Crypto op data structure(s) */ for (i = 0; i < num_to_submit ; i++) { @@ -1994,7 +2001,7 @@ test_perf_crypto_qp_vary_burst_size(uint16_t dev_num) rte_crypto_op_alloc(ts_params->op_mpool, RTE_CRYPTO_OP_TYPE_SYMMETRIC); - rte_crypto_op_attach_sym_session(op, ut_params->sess); + rte_crypto_op_attach_sym_session(op, test_crypto_session); op->sym->auth.digest.data = ut_params->digest; op->sym->auth.digest.phys_addr = rte_pktmbuf_mtophys_offset(m, @@ -2105,9 +2112,12 @@ test_perf_snow3G_optimise_cyclecount(struct perf_test_params *pparams) } /* Create Crypto session*/ - sess = test_perf_create_snow3g_session(ts_params->dev_id, + if (test_perf_create_snow3g_session(ts_params->dev_id, pparams->chain, pparams->cipher_algo, - pparams->key_length, pparams->auth_algo); + pparams->key_length, pparams->auth_algo) == 0) + sess = test_crypto_session; + else + sess = NULL; TEST_ASSERT_NOT_NULL(sess, "Session creation failed"); /* Generate Crypto op data structure(s)*/ @@ -2211,7 +2221,10 @@ test_perf_snow3G_optimise_cyclecount(struct perf_test_params *pparams) rte_pktmbuf_free(c_ops[i]->sym->m_src); rte_crypto_op_free(c_ops[i]); } - rte_cryptodev_sym_session_free(ts_params->dev_id, sess); + + rte_cryptodev_sym_session_clear(ts_params->dev_id, + sess); + rte_cryptodev_sym_session_free(sess); return TEST_SUCCESS; } @@ -2290,10 +2303,13 @@ test_perf_openssl_optimise_cyclecount(struct perf_test_params *pparams) } /* Create Crypto session*/ - sess = test_perf_create_openssl_session(ts_params->dev_id, + if (test_perf_create_openssl_session(ts_params->dev_id, pparams->chain, pparams->cipher_algo, pparams->key_length, pparams->auth_algo, - pparams->aead_algo); + pparams->aead_algo) == 0) + sess = test_crypto_session; + else + sess = NULL; TEST_ASSERT_NOT_NULL(sess, "Session creation failed"); /* Generate Crypto op data structure(s)*/ @@ -2425,7 +2441,9 @@ test_perf_openssl_optimise_cyclecount(struct perf_test_params *pparams) rte_pktmbuf_free(c_ops[i]->sym->m_src); rte_crypto_op_free(c_ops[i]); } - rte_cryptodev_sym_session_free(ts_params->dev_id, sess); + + rte_cryptodev_sym_session_clear(ts_params->dev_id, sess); + rte_cryptodev_sym_session_free(sess); return TEST_SUCCESS; } @@ -2452,10 +2470,12 @@ test_perf_armv8_optimise_cyclecount(struct perf_test_params *pparams) } /* Create Crypto session*/ - sess = test_perf_create_armv8_session(ts_params->dev_id, + if (test_perf_create_armv8_session(ts_params->dev_id, pparams->chain, pparams->cipher_algo, - pparams->key_length, pparams->auth_algo); - TEST_ASSERT_NOT_NULL(sess, "Session creation failed"); + pparams->key_length, pparams->auth_algo) == 0) + sess = test_crypto_session; + else + sess = NULL; /* Generate Crypto op data structure(s)*/ for (i = 0; i < num_to_submit ; i++) { @@ -2672,12 +2692,13 @@ static uint8_t snow3g_hash_key[] = { 0x1E, 0x26, 0x98, 0xD2, 0xE2, 0x2A, 0xD5, 0x7E }; -static struct rte_cryptodev_sym_session * +static int test_perf_create_aes_sha_session(uint8_t dev_id, enum chain_mode chain, enum rte_crypto_cipher_algorithm cipher_algo, unsigned cipher_key_len, enum rte_crypto_auth_algorithm auth_algo) { + struct crypto_testsuite_params *ts_params = &testsuite_params; struct rte_crypto_sym_xform cipher_xform = { 0 }; struct rte_crypto_sym_xform auth_xform = { 0 }; @@ -2701,33 +2722,42 @@ test_perf_create_aes_sha_session(uint8_t dev_id, enum chain_mode chain, auth_xform.auth.digest_length = get_auth_digest_length(auth_algo); } + + test_crypto_session = rte_cryptodev_sym_session_create(ts_params->sess_mp); switch (chain) { case CIPHER_HASH: cipher_xform.next = &auth_xform; auth_xform.next = NULL; /* Create Crypto session*/ - return rte_cryptodev_sym_session_create(dev_id, &cipher_xform); + return rte_cryptodev_sym_session_init(dev_id, + test_crypto_session, &cipher_xform, + ts_params->sess_mp); case HASH_CIPHER: auth_xform.next = &cipher_xform; cipher_xform.next = NULL; /* Create Crypto session*/ - return rte_cryptodev_sym_session_create(dev_id, &auth_xform); + return rte_cryptodev_sym_session_init(dev_id, + test_crypto_session, &auth_xform, + ts_params->sess_mp); case CIPHER_ONLY: cipher_xform.next = NULL; /* Create Crypto session*/ - return rte_cryptodev_sym_session_create(dev_id, &cipher_xform); + return rte_cryptodev_sym_session_init(dev_id, + test_crypto_session, &cipher_xform, + ts_params->sess_mp); default: - return NULL; + return -1; } } #define SNOW3G_CIPHER_IV_LENGTH 16 -static struct rte_cryptodev_sym_session * +static int test_perf_create_snow3g_session(uint8_t dev_id, enum chain_mode chain, enum rte_crypto_cipher_algorithm cipher_algo, unsigned cipher_key_len, enum rte_crypto_auth_algorithm auth_algo) { + struct crypto_testsuite_params *ts_params = &testsuite_params; struct rte_crypto_sym_xform cipher_xform = {0}; struct rte_crypto_sym_xform auth_xform = {0}; @@ -2755,37 +2785,47 @@ test_perf_create_snow3g_session(uint8_t dev_id, enum chain_mode chain, auth_xform.auth.iv.offset = IV_OFFSET + SNOW3G_CIPHER_IV_LENGTH; auth_xform.auth.iv.length = SNOW3G_CIPHER_IV_LENGTH; + test_crypto_session = rte_cryptodev_sym_session_create(ts_params->sess_mp); switch (chain) { case CIPHER_HASH: cipher_xform.next = &auth_xform; auth_xform.next = NULL; /* Create Crypto session*/ - return rte_cryptodev_sym_session_create(dev_id, &cipher_xform); + return rte_cryptodev_sym_session_init(dev_id, + test_crypto_session, &cipher_xform, + ts_params->sess_mp); case HASH_CIPHER: auth_xform.next = &cipher_xform; cipher_xform.next = NULL; /* Create Crypto session*/ - return rte_cryptodev_sym_session_create(dev_id, &auth_xform); + return rte_cryptodev_sym_session_init(dev_id, + test_crypto_session, &auth_xform, + ts_params->sess_mp); case CIPHER_ONLY: cipher_xform.next = NULL; /* Create Crypto session*/ - return rte_cryptodev_sym_session_create(dev_id, &cipher_xform); + return rte_cryptodev_sym_session_init(dev_id, + test_crypto_session, &cipher_xform, + ts_params->sess_mp); case HASH_ONLY: auth_xform.next = NULL; /* Create Crypto session */ - return rte_cryptodev_sym_session_create(dev_id, &auth_xform); + return rte_cryptodev_sym_session_init(dev_id, + test_crypto_session, &auth_xform, + ts_params->sess_mp); default: - return NULL; + return -1; } } -static struct rte_cryptodev_sym_session * +static int test_perf_create_openssl_session(uint8_t dev_id, enum chain_mode chain, enum rte_crypto_cipher_algorithm cipher_algo, unsigned int key_len, enum rte_crypto_auth_algorithm auth_algo, enum rte_crypto_aead_algorithm aead_algo) { + struct crypto_testsuite_params *ts_params = &testsuite_params; struct rte_crypto_sym_xform cipher_xform = { 0 }; struct rte_crypto_sym_xform auth_xform = { 0 }; struct rte_crypto_sym_xform aead_xform = { 0 }; @@ -2809,7 +2849,7 @@ test_perf_create_openssl_session(uint8_t dev_id, enum chain_mode chain, cipher_xform.cipher.iv.length = AES_CIPHER_IV_LENGTH; break; default: - return NULL; + return -1; } cipher_xform.cipher.key.length = key_len; @@ -2824,7 +2864,7 @@ test_perf_create_openssl_session(uint8_t dev_id, enum chain_mode chain, auth_xform.auth.key.data = hmac_sha_key; break; default: - return NULL; + return -1; } auth_xform.auth.key.length = get_auth_key_max_length(auth_algo); @@ -2844,37 +2884,45 @@ test_perf_create_openssl_session(uint8_t dev_id, enum chain_mode chain, aead_xform.aead.digest_length = get_aead_digest_length(aead_algo); break; default: - return NULL; + return -1; } aead_xform.aead.key.length = key_len; } + test_crypto_session = rte_cryptodev_sym_session_create(ts_params->sess_mp); switch (chain) { case CIPHER_HASH: cipher_xform.next = &auth_xform; auth_xform.next = NULL; /* Create Crypto session*/ - return rte_cryptodev_sym_session_create(dev_id, &cipher_xform); + return rte_cryptodev_sym_session_init(dev_id, + test_crypto_session, &cipher_xform, + ts_params->sess_mp); case HASH_CIPHER: auth_xform.next = &cipher_xform; cipher_xform.next = NULL; /* Create Crypto session*/ - return rte_cryptodev_sym_session_create(dev_id, &auth_xform); + return rte_cryptodev_sym_session_init(dev_id, + test_crypto_session, &auth_xform, + ts_params->sess_mp); case AEAD: /* Create Crypto session*/ - return rte_cryptodev_sym_session_create(dev_id, &aead_xform); + return rte_cryptodev_sym_session_init(dev_id, + test_crypto_session, &aead_xform, + ts_params->sess_mp); default: - return NULL; + return -1; } } -static struct rte_cryptodev_sym_session * +static int test_perf_create_armv8_session(uint8_t dev_id, enum chain_mode chain, enum rte_crypto_cipher_algorithm cipher_algo, unsigned int cipher_key_len, enum rte_crypto_auth_algorithm auth_algo) { + struct crypto_testsuite_params *ts_params = &testsuite_params; struct rte_crypto_sym_xform cipher_xform = { 0 }; struct rte_crypto_sym_xform auth_xform = { 0 }; @@ -2887,7 +2935,7 @@ test_perf_create_armv8_session(uint8_t dev_id, enum chain_mode chain, cipher_xform.cipher.key.data = aes_cbc_128_key; break; default: - return NULL; + return -1; } cipher_xform.cipher.key.length = cipher_key_len; @@ -2901,6 +2949,8 @@ test_perf_create_armv8_session(uint8_t dev_id, enum chain_mode chain, auth_xform.auth.digest_length = get_auth_digest_length(auth_algo); + rte_cryptodev_sym_session_create(ts_params->sess_mp); + switch (chain) { case CIPHER_HASH: cipher_xform.next = &auth_xform; @@ -2908,16 +2958,20 @@ test_perf_create_armv8_session(uint8_t dev_id, enum chain_mode chain, /* Encrypt and hash the result */ cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT; /* Create Crypto session*/ - return rte_cryptodev_sym_session_create(dev_id, &cipher_xform); + return rte_cryptodev_sym_session_init(dev_id, + test_crypto_session, &cipher_xform, + ts_params->sess_mp); case HASH_CIPHER: auth_xform.next = &cipher_xform; cipher_xform.next = NULL; /* Hash encrypted message and decrypt */ cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_DECRYPT; /* Create Crypto session*/ - return rte_cryptodev_sym_session_create(dev_id, &auth_xform); + return rte_cryptodev_sym_session_init(dev_id, + test_crypto_session, &auth_xform, + ts_params->sess_mp); default: - return NULL; + return -1; } } @@ -3167,9 +3221,12 @@ test_perf_aes_sha(uint8_t dev_id, uint16_t queue_id, } /* Create Crypto session*/ - sess = test_perf_create_aes_sha_session(ts_params->dev_id, + if (test_perf_create_aes_sha_session(ts_params->dev_id, pparams->chain, pparams->cipher_algo, - pparams->key_length, pparams->auth_algo); + pparams->key_length, pparams->auth_algo) == 0) + sess = test_crypto_session; + else + sess = NULL; TEST_ASSERT_NOT_NULL(sess, "Session creation failed"); /* Generate a burst of crypto operations */ @@ -3264,7 +3321,9 @@ test_perf_aes_sha(uint8_t dev_id, uint16_t queue_id, for (i = 0; i < pparams->burst_size * NUM_MBUF_SETS; i++) rte_pktmbuf_free(mbufs[i]); - rte_cryptodev_sym_session_free(dev_id, sess); + + rte_cryptodev_sym_session_clear(ts_params->dev_id, sess); + rte_cryptodev_sym_session_free(sess); printf("\n"); return TEST_SUCCESS; @@ -3300,9 +3359,12 @@ test_perf_snow3g(uint8_t dev_id, uint16_t queue_id, } /* Create Crypto session*/ - sess = test_perf_create_snow3g_session(ts_params->dev_id, + if (test_perf_create_snow3g_session(ts_params->dev_id, pparams->chain, pparams->cipher_algo, - pparams->key_length, pparams->auth_algo); + pparams->key_length, pparams->auth_algo) == 0) + sess = test_crypto_session; + else + sess = NULL; TEST_ASSERT_NOT_NULL(sess, "Session creation failed"); /* Generate a burst of crypto operations */ @@ -3429,7 +3491,9 @@ test_perf_snow3g(uint8_t dev_id, uint16_t queue_id, for (i = 0; i < pparams->burst_size * NUM_MBUF_SETS; i++) rte_pktmbuf_free(mbufs[i]); - rte_cryptodev_sym_session_free(dev_id, sess); + + rte_cryptodev_sym_session_clear(ts_params->dev_id, sess); + rte_cryptodev_sym_session_free(sess); printf("\n"); return TEST_SUCCESS; @@ -3486,10 +3550,13 @@ test_perf_openssl(uint8_t dev_id, uint16_t queue_id, } /* Create Crypto session*/ - sess = test_perf_create_openssl_session(ts_params->dev_id, + if (test_perf_create_openssl_session(ts_params->dev_id, pparams->chain, pparams->cipher_algo, pparams->key_length, pparams->auth_algo, - pparams->aead_algo); + pparams->aead_algo) == 0) + sess = test_crypto_session; + else + sess = NULL; TEST_ASSERT_NOT_NULL(sess, "Session creation failed"); /* Generate a burst of crypto operations */ @@ -3582,7 +3649,9 @@ test_perf_openssl(uint8_t dev_id, uint16_t queue_id, for (i = 0; i < pparams->burst_size * NUM_MBUF_SETS; i++) rte_pktmbuf_free(mbufs[i]); - rte_cryptodev_sym_session_free(dev_id, sess); + + rte_cryptodev_sym_session_clear(ts_params->dev_id, sess); + rte_cryptodev_sym_session_free(sess); printf("\n"); return TEST_SUCCESS; @@ -3617,9 +3686,12 @@ test_perf_armv8(uint8_t dev_id, uint16_t queue_id, } /* Create Crypto session*/ - sess = test_perf_create_armv8_session(ts_params->dev_id, + if (test_perf_create_armv8_session(ts_params->dev_id, pparams->chain, pparams->cipher_algo, - pparams->key_length, pparams->auth_algo); + pparams->key_length, pparams->auth_algo) == 0) + sess = test_crypto_session; + else + sess = NULL; TEST_ASSERT_NOT_NULL(sess, "Session creation failed"); /* Generate a burst of crypto operations */ @@ -4177,7 +4249,7 @@ test_perf_aes_cbc_vary_burst_size(void) static struct rte_cryptodev_sym_session * test_perf_create_session(uint8_t dev_id, struct perf_test_params *pparams) { - static struct rte_cryptodev_sym_session *sess; + struct crypto_testsuite_params *ts_params = &testsuite_params; struct rte_crypto_sym_xform aead_xform = { 0 }; uint8_t aead_key[pparams->session_attrs->key_aead_len]; @@ -4197,9 +4269,12 @@ test_perf_create_session(uint8_t dev_id, struct perf_test_params *pparams) aead_xform.aead.add_auth_data_length = pparams->session_attrs->aad_len; aead_xform.aead.digest_length = pparams->session_attrs->digest_len; - sess = rte_cryptodev_sym_session_create(dev_id, &aead_xform); + test_crypto_session = rte_cryptodev_sym_session_create(ts_params->sess_mp); + + rte_cryptodev_sym_session_init(dev_id, test_crypto_session, + &aead_xform, ts_params->sess_mp); - return sess; + return test_crypto_session; } static inline struct rte_crypto_op * @@ -4418,7 +4493,9 @@ perf_AES_GCM(uint8_t dev_id, uint16_t queue_id, for (i = 0; i < burst; i++) rte_pktmbuf_free(mbufs[i]); - rte_cryptodev_sym_session_free(dev_id, sess); + + rte_cryptodev_sym_session_clear(ts_params->dev_id, sess); + rte_cryptodev_sym_session_free(sess); return 0; }