From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 91ED6A0547; Tue, 26 Oct 2021 18:46:05 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id BEB224113A; Tue, 26 Oct 2021 18:45:54 +0200 (CEST) Received: from mga17.intel.com (mga17.intel.com [192.55.52.151]) by mails.dpdk.org (Postfix) with ESMTP id 91BCD410D5 for ; Tue, 26 Oct 2021 18:45:40 +0200 (CEST) X-IronPort-AV: E=McAfee;i="6200,9189,10149"; a="210742893" X-IronPort-AV: E=Sophos;i="5.87,184,1631602800"; d="scan'208";a="210742893" Received: from fmsmga004.fm.intel.com ([10.253.24.48]) by fmsmga107.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 26 Oct 2021 09:45:20 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.87,184,1631602800"; d="scan'208";a="554771466" Received: from silpixa00400272.ir.intel.com (HELO silpixa00400272.ger.corp.intel.com) ([10.237.223.111]) by fmsmga004.fm.intel.com with ESMTP; 26 Oct 2021 09:45:19 -0700 From: Kai Ji To: dev@dpdk.org Cc: Fan Zhang , Arek Kusztal , Kai Ji Date: Tue, 26 Oct 2021 17:45:00 +0100 Message-Id: <20211026164502.18016-8-kai.ji@intel.com> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20211026164502.18016-1-kai.ji@intel.com> References: <20211022170354.13503-1-roy.fan.zhang@intel.com> <20211026164502.18016-1-kai.ji@intel.com> Subject: [dpdk-dev] [dpdk-dev v5 7/9] crypto/qat: unified device private data structure X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" From: Fan Zhang This patch unifies the QAT symmetric and asymmetric device private data structures and functions. Signed-off-by: Arek Kusztal Signed-off-by: Fan Zhang Signed-off-by: Kai Ji --- drivers/common/qat/meson.build | 2 +- drivers/common/qat/qat_common.c | 15 ++ drivers/common/qat/qat_common.h | 3 + drivers/common/qat/qat_device.h | 7 +- drivers/crypto/qat/qat_asym_pmd.c | 216 ++++------------------- drivers/crypto/qat/qat_asym_pmd.h | 29 +--- drivers/crypto/qat/qat_crypto.c | 172 ++++++++++++++++++ drivers/crypto/qat/qat_crypto.h | 78 +++++++++ drivers/crypto/qat/qat_sym_pmd.c | 250 +++++---------------------- drivers/crypto/qat/qat_sym_pmd.h | 21 +-- drivers/crypto/qat/qat_sym_session.c | 15 +- 11 files changed, 361 insertions(+), 447 deletions(-) create mode 100644 drivers/crypto/qat/qat_crypto.c create mode 100644 drivers/crypto/qat/qat_crypto.h diff --git a/drivers/common/qat/meson.build b/drivers/common/qat/meson.build index 8a1c6d64e8..29fd0168ea 100644 --- a/drivers/common/qat/meson.build +++ b/drivers/common/qat/meson.build @@ -71,7 +71,7 @@ endif if qat_crypto foreach f: ['qat_sym_pmd.c', 'qat_sym.c', 'qat_sym_session.c', - 'qat_sym_hw_dp.c', 'qat_asym_pmd.c', 'qat_asym.c'] + 'qat_sym_hw_dp.c', 'qat_asym_pmd.c', 'qat_asym.c', 'qat_crypto.c'] sources += files(join_paths(qat_crypto_relpath, f)) endforeach deps += ['security'] diff --git a/drivers/common/qat/qat_common.c b/drivers/common/qat/qat_common.c index 5343a1451e..59e7e02622 100644 --- a/drivers/common/qat/qat_common.c +++ b/drivers/common/qat/qat_common.c @@ -6,6 +6,21 @@ #include "qat_device.h" #include "qat_logs.h" +const char * +qat_service_get_str(enum qat_service_type type) +{ + switch (type) { + case QAT_SERVICE_SYMMETRIC: + return "sym"; + case QAT_SERVICE_ASYMMETRIC: + return "asym"; + case QAT_SERVICE_COMPRESSION: + return "comp"; + default: + return "invalid"; + } +} + int qat_sgl_fill_array(struct rte_mbuf *buf, int64_t offset, void *list_in, uint32_t data_len, diff --git a/drivers/common/qat/qat_common.h b/drivers/common/qat/qat_common.h index a7632e31f8..9411a79301 100644 --- a/drivers/common/qat/qat_common.h +++ b/drivers/common/qat/qat_common.h @@ -91,4 +91,7 @@ void qat_stats_reset(struct qat_pci_device *dev, enum qat_service_type service); +const char * +qat_service_get_str(enum qat_service_type type); + #endif /* _QAT_COMMON_H_ */ diff --git a/drivers/common/qat/qat_device.h b/drivers/common/qat/qat_device.h index e7c7e9af95..85fae7b7c7 100644 --- a/drivers/common/qat/qat_device.h +++ b/drivers/common/qat/qat_device.h @@ -76,8 +76,7 @@ struct qat_device_info { extern struct qat_device_info qat_pci_devs[]; -struct qat_sym_dev_private; -struct qat_asym_dev_private; +struct qat_cryptodev_private; struct qat_comp_dev_private; /* @@ -106,14 +105,14 @@ struct qat_pci_device { /**< links to qps set up for each service, index same as on API */ /* Data relating to symmetric crypto service */ - struct qat_sym_dev_private *sym_dev; + struct qat_cryptodev_private *sym_dev; /**< link back to cryptodev private data */ int qat_sym_driver_id; /**< Symmetric driver id used by this device */ /* Data relating to asymmetric crypto service */ - struct qat_asym_dev_private *asym_dev; + struct qat_cryptodev_private *asym_dev; /**< link back to cryptodev private data */ int qat_asym_driver_id; diff --git a/drivers/crypto/qat/qat_asym_pmd.c b/drivers/crypto/qat/qat_asym_pmd.c index 0944d27a4d..042f39ddcc 100644 --- a/drivers/crypto/qat/qat_asym_pmd.c +++ b/drivers/crypto/qat/qat_asym_pmd.c @@ -6,6 +6,7 @@ #include "qat_logs.h" +#include "qat_crypto.h" #include "qat_asym.h" #include "qat_asym_pmd.h" #include "qat_sym_capabilities.h" @@ -18,190 +19,45 @@ static const struct rte_cryptodev_capabilities qat_gen1_asym_capabilities[] = { RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST() }; -static int qat_asym_qp_release(struct rte_cryptodev *dev, - uint16_t queue_pair_id); - -static int qat_asym_dev_config(__rte_unused struct rte_cryptodev *dev, - __rte_unused struct rte_cryptodev_config *config) -{ - return 0; -} - -static int qat_asym_dev_start(__rte_unused struct rte_cryptodev *dev) -{ - return 0; -} - -static void qat_asym_dev_stop(__rte_unused struct rte_cryptodev *dev) -{ - -} - -static int qat_asym_dev_close(struct rte_cryptodev *dev) -{ - int i, ret; - - for (i = 0; i < dev->data->nb_queue_pairs; i++) { - ret = qat_asym_qp_release(dev, i); - if (ret < 0) - return ret; - } - - return 0; -} - -static void qat_asym_dev_info_get(struct rte_cryptodev *dev, - struct rte_cryptodev_info *info) -{ - struct qat_asym_dev_private *internals = dev->data->dev_private; - struct qat_pci_device *qat_dev = internals->qat_dev; - - if (info != NULL) { - info->max_nb_queue_pairs = qat_qps_per_service(qat_dev, - QAT_SERVICE_ASYMMETRIC); - info->feature_flags = dev->feature_flags; - info->capabilities = internals->qat_dev_capabilities; - info->driver_id = qat_asym_driver_id; - /* No limit of number of sessions */ - info->sym.max_nb_sessions = 0; - } -} - -static void qat_asym_stats_get(struct rte_cryptodev *dev, - struct rte_cryptodev_stats *stats) -{ - struct qat_common_stats qat_stats = {0}; - struct qat_asym_dev_private *qat_priv; - - if (stats == NULL || dev == NULL) { - QAT_LOG(ERR, "invalid ptr: stats %p, dev %p", stats, dev); - return; - } - qat_priv = dev->data->dev_private; - - qat_stats_get(qat_priv->qat_dev, &qat_stats, QAT_SERVICE_ASYMMETRIC); - stats->enqueued_count = qat_stats.enqueued_count; - stats->dequeued_count = qat_stats.dequeued_count; - stats->enqueue_err_count = qat_stats.enqueue_err_count; - stats->dequeue_err_count = qat_stats.dequeue_err_count; -} - -static void qat_asym_stats_reset(struct rte_cryptodev *dev) +void +qat_asym_init_op_cookie(void *op_cookie) { - struct qat_asym_dev_private *qat_priv; + int j; + struct qat_asym_op_cookie *cookie = op_cookie; - if (dev == NULL) { - QAT_LOG(ERR, "invalid asymmetric cryptodev ptr %p", dev); - return; - } - qat_priv = dev->data->dev_private; + cookie->input_addr = rte_mempool_virt2iova(cookie) + + offsetof(struct qat_asym_op_cookie, + input_params_ptrs); - qat_stats_reset(qat_priv->qat_dev, QAT_SERVICE_ASYMMETRIC); -} - -static int qat_asym_qp_release(struct rte_cryptodev *dev, - uint16_t queue_pair_id) -{ - struct qat_asym_dev_private *qat_private = dev->data->dev_private; - enum qat_device_gen qat_dev_gen = qat_private->qat_dev->qat_dev_gen; - - QAT_LOG(DEBUG, "Release asym qp %u on device %d", - queue_pair_id, dev->data->dev_id); - - qat_private->qat_dev->qps_in_use[QAT_SERVICE_ASYMMETRIC][queue_pair_id] - = NULL; - - return qat_qp_release(qat_dev_gen, (struct qat_qp **) - &(dev->data->queue_pairs[queue_pair_id])); -} + cookie->output_addr = rte_mempool_virt2iova(cookie) + + offsetof(struct qat_asym_op_cookie, + output_params_ptrs); -static int qat_asym_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, - const struct rte_cryptodev_qp_conf *qp_conf, - int socket_id) -{ - struct qat_qp_config qat_qp_conf; - struct qat_qp *qp; - int ret = 0; - uint32_t i; - - struct qat_qp **qp_addr = - (struct qat_qp **)&(dev->data->queue_pairs[qp_id]); - struct qat_asym_dev_private *qat_private = dev->data->dev_private; - struct qat_pci_device *qat_dev = qat_private->qat_dev; - const struct qat_qp_hw_data *asym_hw_qps = - qat_gen_config[qat_private->qat_dev->qat_dev_gen] - .qp_hw_data[QAT_SERVICE_ASYMMETRIC]; - const struct qat_qp_hw_data *qp_hw_data = asym_hw_qps + qp_id; - - /* If qp is already in use free ring memory and qp metadata. */ - if (*qp_addr != NULL) { - ret = qat_asym_qp_release(dev, qp_id); - if (ret < 0) - return ret; - } - if (qp_id >= qat_qps_per_service(qat_dev, QAT_SERVICE_ASYMMETRIC)) { - QAT_LOG(ERR, "qp_id %u invalid for this device", qp_id); - return -EINVAL; - } - - qat_qp_conf.hw = qp_hw_data; - qat_qp_conf.cookie_size = sizeof(struct qat_asym_op_cookie); - qat_qp_conf.nb_descriptors = qp_conf->nb_descriptors; - qat_qp_conf.socket_id = socket_id; - qat_qp_conf.service_str = "asym"; - - ret = qat_qp_setup(qat_private->qat_dev, qp_addr, qp_id, &qat_qp_conf); - if (ret != 0) - return ret; - - /* store a link to the qp in the qat_pci_device */ - qat_private->qat_dev->qps_in_use[QAT_SERVICE_ASYMMETRIC][qp_id] - = *qp_addr; - - qp = (struct qat_qp *)*qp_addr; - qp->min_enq_burst_threshold = qat_private->min_enq_burst_threshold; - - for (i = 0; i < qp->nb_descriptors; i++) { - int j; - - struct qat_asym_op_cookie __rte_unused *cookie = - qp->op_cookies[i]; - cookie->input_addr = rte_mempool_virt2iova(cookie) + + for (j = 0; j < 8; j++) { + cookie->input_params_ptrs[j] = + rte_mempool_virt2iova(cookie) + offsetof(struct qat_asym_op_cookie, - input_params_ptrs); - - cookie->output_addr = rte_mempool_virt2iova(cookie) + + input_array[j]); + cookie->output_params_ptrs[j] = + rte_mempool_virt2iova(cookie) + offsetof(struct qat_asym_op_cookie, - output_params_ptrs); - - for (j = 0; j < 8; j++) { - cookie->input_params_ptrs[j] = - rte_mempool_virt2iova(cookie) + - offsetof(struct qat_asym_op_cookie, - input_array[j]); - cookie->output_params_ptrs[j] = - rte_mempool_virt2iova(cookie) + - offsetof(struct qat_asym_op_cookie, - output_array[j]); - } + output_array[j]); } - - return ret; } -struct rte_cryptodev_ops crypto_qat_ops = { +static struct rte_cryptodev_ops crypto_qat_ops = { /* Device related operations */ - .dev_configure = qat_asym_dev_config, - .dev_start = qat_asym_dev_start, - .dev_stop = qat_asym_dev_stop, - .dev_close = qat_asym_dev_close, - .dev_infos_get = qat_asym_dev_info_get, + .dev_configure = qat_cryptodev_config, + .dev_start = qat_cryptodev_start, + .dev_stop = qat_cryptodev_stop, + .dev_close = qat_cryptodev_close, + .dev_infos_get = qat_cryptodev_info_get, - .stats_get = qat_asym_stats_get, - .stats_reset = qat_asym_stats_reset, - .queue_pair_setup = qat_asym_qp_setup, - .queue_pair_release = qat_asym_qp_release, + .stats_get = qat_cryptodev_stats_get, + .stats_reset = qat_cryptodev_stats_reset, + .queue_pair_setup = qat_cryptodev_qp_setup, + .queue_pair_release = qat_cryptodev_qp_release, /* Crypto related operations */ .asym_session_get_size = qat_asym_session_get_private_size, @@ -241,15 +97,14 @@ qat_asym_dev_create(struct qat_pci_device *qat_pci_dev, struct qat_device_info *qat_dev_instance = &qat_pci_devs[qat_pci_dev->qat_dev_id]; struct rte_cryptodev_pmd_init_params init_params = { - .name = "", - .socket_id = - qat_dev_instance->pci_dev->device.numa_node, - .private_data_size = sizeof(struct qat_asym_dev_private) + .name = "", + .socket_id = qat_dev_instance->pci_dev->device.numa_node, + .private_data_size = sizeof(struct qat_cryptodev_private) }; char name[RTE_CRYPTODEV_NAME_MAX_LEN]; char capa_memz_name[RTE_CRYPTODEV_NAME_MAX_LEN]; struct rte_cryptodev *cryptodev; - struct qat_asym_dev_private *internals; + struct qat_cryptodev_private *internals; if (qat_pci_dev->qat_dev_gen == QAT_GEN4) { QAT_LOG(ERR, "Asymmetric crypto PMD not supported on QAT 4xxx"); @@ -310,8 +165,9 @@ qat_asym_dev_create(struct qat_pci_device *qat_pci_dev, internals = cryptodev->data->dev_private; internals->qat_dev = qat_pci_dev; - internals->asym_dev_id = cryptodev->data->dev_id; + internals->dev_id = cryptodev->data->dev_id; internals->qat_dev_capabilities = qat_gen1_asym_capabilities; + internals->service_type = QAT_SERVICE_ASYMMETRIC; internals->capa_mz = rte_memzone_lookup(capa_memz_name); if (internals->capa_mz == NULL) { @@ -347,7 +203,7 @@ qat_asym_dev_create(struct qat_pci_device *qat_pci_dev, rte_cryptodev_pmd_probing_finish(cryptodev); QAT_LOG(DEBUG, "Created QAT ASYM device %s as cryptodev instance %d", - cryptodev->data->name, internals->asym_dev_id); + cryptodev->data->name, internals->dev_id); return 0; } @@ -365,7 +221,7 @@ qat_asym_dev_destroy(struct qat_pci_device *qat_pci_dev) /* free crypto device */ cryptodev = rte_cryptodev_pmd_get_dev( - qat_pci_dev->asym_dev->asym_dev_id); + qat_pci_dev->asym_dev->dev_id); rte_cryptodev_pmd_destroy(cryptodev); qat_pci_devs[qat_pci_dev->qat_dev_id].asym_rte_dev.name = NULL; qat_pci_dev->asym_dev = NULL; diff --git a/drivers/crypto/qat/qat_asym_pmd.h b/drivers/crypto/qat/qat_asym_pmd.h index 3b5abddec8..c493796511 100644 --- a/drivers/crypto/qat/qat_asym_pmd.h +++ b/drivers/crypto/qat/qat_asym_pmd.h @@ -15,21 +15,8 @@ extern uint8_t qat_asym_driver_id; -/** private data structure for a QAT device. - * This QAT device is a device offering only asymmetric crypto service, - * there can be one of these on each qat_pci_device (VF). - */ -struct qat_asym_dev_private { - struct qat_pci_device *qat_dev; - /**< The qat pci device hosting the service */ - uint8_t asym_dev_id; - /**< Device instance for this rte_cryptodev */ - const struct rte_cryptodev_capabilities *qat_dev_capabilities; - /* QAT device asymmetric crypto capabilities */ - const struct rte_memzone *capa_mz; - /* Shared memzone for storing capabilities */ - uint16_t min_enq_burst_threshold; -}; +void +qat_asym_init_op_cookie(void *op_cookie); uint16_t qat_asym_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops, @@ -39,16 +26,4 @@ uint16_t qat_asym_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops); -int qat_asym_session_configure(struct rte_cryptodev *dev, - struct rte_crypto_asym_xform *xform, - struct rte_cryptodev_asym_session *sess, - struct rte_mempool *mempool); - -int -qat_asym_dev_create(struct qat_pci_device *qat_pci_dev, - struct qat_dev_cmd_param *qat_dev_cmd_param); - -int -qat_asym_dev_destroy(struct qat_pci_device *qat_pci_dev); - #endif /* _QAT_ASYM_PMD_H_ */ diff --git a/drivers/crypto/qat/qat_crypto.c b/drivers/crypto/qat/qat_crypto.c new file mode 100644 index 0000000000..01d2439b93 --- /dev/null +++ b/drivers/crypto/qat/qat_crypto.c @@ -0,0 +1,172 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + +#include "qat_device.h" +#include "qat_qp.h" +#include "qat_crypto.h" +#include "qat_sym.h" +#include "qat_asym.h" + +int +qat_cryptodev_config(__rte_unused struct rte_cryptodev *dev, + __rte_unused struct rte_cryptodev_config *config) +{ + return 0; +} + +int +qat_cryptodev_start(__rte_unused struct rte_cryptodev *dev) +{ + return 0; +} + +void +qat_cryptodev_stop(__rte_unused struct rte_cryptodev *dev) +{ +} + +int +qat_cryptodev_close(struct rte_cryptodev *dev) +{ + int i, ret; + + for (i = 0; i < dev->data->nb_queue_pairs; i++) { + ret = dev->dev_ops->queue_pair_release(dev, i); + if (ret < 0) + return ret; + } + + return 0; +} + +void +qat_cryptodev_info_get(struct rte_cryptodev *dev, + struct rte_cryptodev_info *info) +{ + struct qat_cryptodev_private *qat_private = dev->data->dev_private; + struct qat_pci_device *qat_dev = qat_private->qat_dev; + enum qat_service_type service_type = qat_private->service_type; + + if (info != NULL) { + info->max_nb_queue_pairs = + qat_qps_per_service(qat_dev, service_type); + info->feature_flags = dev->feature_flags; + info->capabilities = qat_private->qat_dev_capabilities; + info->driver_id = qat_sym_driver_id; + /* No limit of number of sessions */ + info->sym.max_nb_sessions = 0; + } +} + +void +qat_cryptodev_stats_get(struct rte_cryptodev *dev, + struct rte_cryptodev_stats *stats) +{ + struct qat_common_stats qat_stats = {0}; + struct qat_cryptodev_private *qat_priv; + + if (stats == NULL || dev == NULL) { + QAT_LOG(ERR, "invalid ptr: stats %p, dev %p", stats, dev); + return; + } + qat_priv = dev->data->dev_private; + + qat_stats_get(qat_priv->qat_dev, &qat_stats, qat_priv->service_type); + stats->enqueued_count = qat_stats.enqueued_count; + stats->dequeued_count = qat_stats.dequeued_count; + stats->enqueue_err_count = qat_stats.enqueue_err_count; + stats->dequeue_err_count = qat_stats.dequeue_err_count; +} + +void +qat_cryptodev_stats_reset(struct rte_cryptodev *dev) +{ + struct qat_cryptodev_private *qat_priv; + + if (dev == NULL) { + QAT_LOG(ERR, "invalid cryptodev ptr %p", dev); + return; + } + qat_priv = dev->data->dev_private; + + qat_stats_reset(qat_priv->qat_dev, qat_priv->service_type); + +} + +int +qat_cryptodev_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id) +{ + struct qat_cryptodev_private *qat_private = dev->data->dev_private; + struct qat_pci_device *qat_dev = qat_private->qat_dev; + enum qat_device_gen qat_dev_gen = qat_dev->qat_dev_gen; + enum qat_service_type service_type = qat_private->service_type; + + QAT_LOG(DEBUG, "Release %s qp %u on device %d", + qat_service_get_str(service_type), + queue_pair_id, dev->data->dev_id); + + qat_private->qat_dev->qps_in_use[service_type][queue_pair_id] = NULL; + + return qat_qp_release(qat_dev_gen, (struct qat_qp **) + &(dev->data->queue_pairs[queue_pair_id])); +} + +int +qat_cryptodev_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, + const struct rte_cryptodev_qp_conf *qp_conf, int socket_id) +{ + struct qat_qp **qp_addr = + (struct qat_qp **)&(dev->data->queue_pairs[qp_id]); + struct qat_cryptodev_private *qat_private = dev->data->dev_private; + struct qat_pci_device *qat_dev = qat_private->qat_dev; + enum qat_service_type service_type = qat_private->service_type; + struct qat_qp_config qat_qp_conf = {0}; + struct qat_qp *qp; + int ret = 0; + uint32_t i; + + /* If qp is already in use free ring memory and qp metadata. */ + if (*qp_addr != NULL) { + ret = dev->dev_ops->queue_pair_release(dev, qp_id); + if (ret < 0) + return -EBUSY; + } + if (qp_id >= qat_qps_per_service(qat_dev, service_type)) { + QAT_LOG(ERR, "qp_id %u invalid for this device", qp_id); + return -EINVAL; + } + + qat_qp_conf.hw = qat_qp_get_hw_data(qat_dev, service_type, + qp_id); + if (qat_qp_conf.hw == NULL) { + QAT_LOG(ERR, "qp_id %u invalid for this device", qp_id); + return -EINVAL; + } + + qat_qp_conf.cookie_size = service_type == QAT_SERVICE_SYMMETRIC ? + sizeof(struct qat_sym_op_cookie) : + sizeof(struct qat_asym_op_cookie); + qat_qp_conf.nb_descriptors = qp_conf->nb_descriptors; + qat_qp_conf.socket_id = socket_id; + qat_qp_conf.service_str = qat_service_get_str(service_type); + + ret = qat_qp_setup(qat_dev, qp_addr, qp_id, &qat_qp_conf); + if (ret != 0) + return ret; + + /* store a link to the qp in the qat_pci_device */ + qat_dev->qps_in_use[service_type][qp_id] = *qp_addr; + + qp = (struct qat_qp *)*qp_addr; + qp->min_enq_burst_threshold = qat_private->min_enq_burst_threshold; + + for (i = 0; i < qp->nb_descriptors; i++) { + if (service_type == QAT_SERVICE_SYMMETRIC) + qat_sym_init_op_cookie(qp->op_cookies[i]); + else + qat_asym_init_op_cookie(qp->op_cookies[i]); + } + + return ret; +} diff --git a/drivers/crypto/qat/qat_crypto.h b/drivers/crypto/qat/qat_crypto.h new file mode 100644 index 0000000000..3803fef19d --- /dev/null +++ b/drivers/crypto/qat/qat_crypto.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Intel Corporation + */ + + #ifndef _QAT_CRYPTO_H_ + #define _QAT_CRYPTO_H_ + +#include +#ifdef RTE_LIB_SECURITY +#include +#endif + +#include "qat_device.h" + +extern uint8_t qat_sym_driver_id; +extern uint8_t qat_asym_driver_id; + +/** helper macro to set cryptodev capability range **/ +#define CAP_RNG(n, l, r, i) .n = {.min = l, .max = r, .increment = i} + +#define CAP_RNG_ZERO(n) .n = {.min = 0, .max = 0, .increment = 0} +/** helper macro to set cryptodev capability value **/ +#define CAP_SET(n, v) .n = v + +/** private data structure for a QAT device. + * there can be one of these on each qat_pci_device (VF). + */ +struct qat_cryptodev_private { + struct qat_pci_device *qat_dev; + /**< The qat pci device hosting the service */ + uint8_t dev_id; + /**< Device instance for this rte_cryptodev */ + const struct rte_cryptodev_capabilities *qat_dev_capabilities; + /* QAT device symmetric crypto capabilities */ + const struct rte_memzone *capa_mz; + /* Shared memzone for storing capabilities */ + uint16_t min_enq_burst_threshold; + uint32_t internal_capabilities; /* see flags QAT_SYM_CAP_xxx */ + enum qat_service_type service_type; +}; + +struct qat_capabilities_info { + struct rte_cryptodev_capabilities *data; + uint64_t size; +}; + +int +qat_cryptodev_config(struct rte_cryptodev *dev, + struct rte_cryptodev_config *config); + +int +qat_cryptodev_start(struct rte_cryptodev *dev); + +void +qat_cryptodev_stop(struct rte_cryptodev *dev); + +int +qat_cryptodev_close(struct rte_cryptodev *dev); + +void +qat_cryptodev_info_get(struct rte_cryptodev *dev, + struct rte_cryptodev_info *info); + +void +qat_cryptodev_stats_get(struct rte_cryptodev *dev, + struct rte_cryptodev_stats *stats); + +void +qat_cryptodev_stats_reset(struct rte_cryptodev *dev); + +int +qat_cryptodev_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, + const struct rte_cryptodev_qp_conf *qp_conf, int socket_id); + +int +qat_cryptodev_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id); + +#endif diff --git a/drivers/crypto/qat/qat_sym_pmd.c b/drivers/crypto/qat/qat_sym_pmd.c index 5b8ee4bee6..dec877cfab 100644 --- a/drivers/crypto/qat/qat_sym_pmd.c +++ b/drivers/crypto/qat/qat_sym_pmd.c @@ -13,6 +13,7 @@ #endif #include "qat_logs.h" +#include "qat_crypto.h" #include "qat_sym.h" #include "qat_sym_session.h" #include "qat_sym_pmd.h" @@ -59,213 +60,19 @@ static const struct rte_security_capability qat_security_capabilities[] = { }; #endif -static int qat_sym_qp_release(struct rte_cryptodev *dev, - uint16_t queue_pair_id); - -static int qat_sym_dev_config(__rte_unused struct rte_cryptodev *dev, - __rte_unused struct rte_cryptodev_config *config) -{ - return 0; -} - -static int qat_sym_dev_start(__rte_unused struct rte_cryptodev *dev) -{ - return 0; -} - -static void qat_sym_dev_stop(__rte_unused struct rte_cryptodev *dev) -{ - return; -} - -static int qat_sym_dev_close(struct rte_cryptodev *dev) -{ - int i, ret; - - for (i = 0; i < dev->data->nb_queue_pairs; i++) { - ret = qat_sym_qp_release(dev, i); - if (ret < 0) - return ret; - } - - return 0; -} - -static void qat_sym_dev_info_get(struct rte_cryptodev *dev, - struct rte_cryptodev_info *info) -{ - struct qat_sym_dev_private *internals = dev->data->dev_private; - struct qat_pci_device *qat_dev = internals->qat_dev; - - if (info != NULL) { - info->max_nb_queue_pairs = - qat_qps_per_service(qat_dev, QAT_SERVICE_SYMMETRIC); - info->feature_flags = dev->feature_flags; - info->capabilities = internals->qat_dev_capabilities; - info->driver_id = qat_sym_driver_id; - /* No limit of number of sessions */ - info->sym.max_nb_sessions = 0; - } -} - -static void qat_sym_stats_get(struct rte_cryptodev *dev, - struct rte_cryptodev_stats *stats) -{ - struct qat_common_stats qat_stats = {0}; - struct qat_sym_dev_private *qat_priv; - - if (stats == NULL || dev == NULL) { - QAT_LOG(ERR, "invalid ptr: stats %p, dev %p", stats, dev); - return; - } - qat_priv = dev->data->dev_private; - - qat_stats_get(qat_priv->qat_dev, &qat_stats, QAT_SERVICE_SYMMETRIC); - stats->enqueued_count = qat_stats.enqueued_count; - stats->dequeued_count = qat_stats.dequeued_count; - stats->enqueue_err_count = qat_stats.enqueue_err_count; - stats->dequeue_err_count = qat_stats.dequeue_err_count; -} - -static void qat_sym_stats_reset(struct rte_cryptodev *dev) -{ - struct qat_sym_dev_private *qat_priv; - - if (dev == NULL) { - QAT_LOG(ERR, "invalid cryptodev ptr %p", dev); - return; - } - qat_priv = dev->data->dev_private; - - qat_stats_reset(qat_priv->qat_dev, QAT_SERVICE_SYMMETRIC); - -} - -static int qat_sym_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id) -{ - struct qat_sym_dev_private *qat_private = dev->data->dev_private; - enum qat_device_gen qat_dev_gen = qat_private->qat_dev->qat_dev_gen; - - QAT_LOG(DEBUG, "Release sym qp %u on device %d", - queue_pair_id, dev->data->dev_id); - - qat_private->qat_dev->qps_in_use[QAT_SERVICE_SYMMETRIC][queue_pair_id] - = NULL; - - return qat_qp_release(qat_dev_gen, (struct qat_qp **) - &(dev->data->queue_pairs[queue_pair_id])); -} - -static int qat_sym_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, - const struct rte_cryptodev_qp_conf *qp_conf, - int socket_id) -{ - struct qat_qp *qp; - int ret = 0; - uint32_t i; - struct qat_qp_config qat_qp_conf; - struct qat_qp **qp_addr = - (struct qat_qp **)&(dev->data->queue_pairs[qp_id]); - struct qat_sym_dev_private *qat_private = dev->data->dev_private; - struct qat_pci_device *qat_dev = qat_private->qat_dev; - - /* If qp is already in use free ring memory and qp metadata. */ - if (*qp_addr != NULL) { - ret = qat_sym_qp_release(dev, qp_id); - if (ret < 0) - return ret; - } - if (qp_id >= qat_qps_per_service(qat_dev, QAT_SERVICE_SYMMETRIC)) { - QAT_LOG(ERR, "qp_id %u invalid for this device", qp_id); - return -EINVAL; - } - - qat_qp_conf.hw = qat_qp_get_hw_data(qat_dev, QAT_SERVICE_SYMMETRIC, - qp_id); - if (qat_qp_conf.hw == NULL) { - QAT_LOG(ERR, "qp_id %u invalid for this device", qp_id); - return -EINVAL; - } - - qat_qp_conf.cookie_size = sizeof(struct qat_sym_op_cookie); - qat_qp_conf.nb_descriptors = qp_conf->nb_descriptors; - qat_qp_conf.socket_id = socket_id; - qat_qp_conf.service_str = "sym"; - - ret = qat_qp_setup(qat_private->qat_dev, qp_addr, qp_id, &qat_qp_conf); - if (ret != 0) - return ret; - - /* store a link to the qp in the qat_pci_device */ - qat_private->qat_dev->qps_in_use[QAT_SERVICE_SYMMETRIC][qp_id] - = *qp_addr; - - qp = (struct qat_qp *)*qp_addr; - qp->min_enq_burst_threshold = qat_private->min_enq_burst_threshold; - - for (i = 0; i < qp->nb_descriptors; i++) { - - struct qat_sym_op_cookie *cookie = - qp->op_cookies[i]; - - cookie->qat_sgl_src_phys_addr = - rte_mempool_virt2iova(cookie) + - offsetof(struct qat_sym_op_cookie, - qat_sgl_src); - - cookie->qat_sgl_dst_phys_addr = - rte_mempool_virt2iova(cookie) + - offsetof(struct qat_sym_op_cookie, - qat_sgl_dst); - - cookie->opt.spc_gmac.cd_phys_addr = - rte_mempool_virt2iova(cookie) + - offsetof(struct qat_sym_op_cookie, - opt.spc_gmac.cd_cipher); - - } - - /* Get fw version from QAT (GEN2), skip if we've got it already */ - if (qp->qat_dev_gen == QAT_GEN2 && !(qat_private->internal_capabilities - & QAT_SYM_CAP_VALID)) { - ret = qat_cq_get_fw_version(qp); - - if (ret < 0) { - qat_sym_qp_release(dev, qp_id); - return ret; - } - - if (ret != 0) - QAT_LOG(DEBUG, "QAT firmware version: %d.%d.%d", - (ret >> 24) & 0xff, - (ret >> 16) & 0xff, - (ret >> 8) & 0xff); - else - QAT_LOG(DEBUG, "unknown QAT firmware version"); - - /* set capabilities based on the fw version */ - qat_private->internal_capabilities = QAT_SYM_CAP_VALID | - ((ret >= MIXED_CRYPTO_MIN_FW_VER) ? - QAT_SYM_CAP_MIXED_CRYPTO : 0); - ret = 0; - } - - return ret; -} - static struct rte_cryptodev_ops crypto_qat_ops = { /* Device related operations */ - .dev_configure = qat_sym_dev_config, - .dev_start = qat_sym_dev_start, - .dev_stop = qat_sym_dev_stop, - .dev_close = qat_sym_dev_close, - .dev_infos_get = qat_sym_dev_info_get, + .dev_configure = qat_cryptodev_config, + .dev_start = qat_cryptodev_start, + .dev_stop = qat_cryptodev_stop, + .dev_close = qat_cryptodev_close, + .dev_infos_get = qat_cryptodev_info_get, - .stats_get = qat_sym_stats_get, - .stats_reset = qat_sym_stats_reset, - .queue_pair_setup = qat_sym_qp_setup, - .queue_pair_release = qat_sym_qp_release, + .stats_get = qat_cryptodev_stats_get, + .stats_reset = qat_cryptodev_stats_reset, + .queue_pair_setup = qat_cryptodev_qp_setup, + .queue_pair_release = qat_cryptodev_qp_release, /* Crypto related operations */ .sym_session_get_size = qat_sym_session_get_private_size, @@ -295,6 +102,27 @@ static struct rte_security_ops security_qat_ops = { }; #endif +void +qat_sym_init_op_cookie(void *op_cookie) +{ + struct qat_sym_op_cookie *cookie = op_cookie; + + cookie->qat_sgl_src_phys_addr = + rte_mempool_virt2iova(cookie) + + offsetof(struct qat_sym_op_cookie, + qat_sgl_src); + + cookie->qat_sgl_dst_phys_addr = + rte_mempool_virt2iova(cookie) + + offsetof(struct qat_sym_op_cookie, + qat_sgl_dst); + + cookie->opt.spc_gmac.cd_phys_addr = + rte_mempool_virt2iova(cookie) + + offsetof(struct qat_sym_op_cookie, + opt.spc_gmac.cd_cipher); +} + static uint16_t qat_sym_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops) @@ -330,15 +158,14 @@ qat_sym_dev_create(struct qat_pci_device *qat_pci_dev, &qat_pci_devs[qat_pci_dev->qat_dev_id]; struct rte_cryptodev_pmd_init_params init_params = { - .name = "", - .socket_id = - qat_dev_instance->pci_dev->device.numa_node, - .private_data_size = sizeof(struct qat_sym_dev_private) + .name = "", + .socket_id = qat_dev_instance->pci_dev->device.numa_node, + .private_data_size = sizeof(struct qat_cryptodev_private) }; char name[RTE_CRYPTODEV_NAME_MAX_LEN]; char capa_memz_name[RTE_CRYPTODEV_NAME_MAX_LEN]; struct rte_cryptodev *cryptodev; - struct qat_sym_dev_private *internals; + struct qat_cryptodev_private *internals; const struct rte_cryptodev_capabilities *capabilities; uint64_t capa_size; @@ -424,8 +251,9 @@ qat_sym_dev_create(struct qat_pci_device *qat_pci_dev, internals = cryptodev->data->dev_private; internals->qat_dev = qat_pci_dev; + internals->service_type = QAT_SERVICE_SYMMETRIC; - internals->sym_dev_id = cryptodev->data->dev_id; + internals->dev_id = cryptodev->data->dev_id; switch (qat_pci_dev->qat_dev_gen) { case QAT_GEN1: capabilities = qat_gen1_sym_capabilities; @@ -480,7 +308,7 @@ qat_sym_dev_create(struct qat_pci_device *qat_pci_dev, qat_pci_dev->sym_dev = internals; QAT_LOG(DEBUG, "Created QAT SYM device %s as cryptodev instance %d", - cryptodev->data->name, internals->sym_dev_id); + cryptodev->data->name, internals->dev_id); rte_cryptodev_pmd_probing_finish(cryptodev); @@ -511,7 +339,7 @@ qat_sym_dev_destroy(struct qat_pci_device *qat_pci_dev) rte_memzone_free(qat_pci_dev->sym_dev->capa_mz); /* free crypto device */ - cryptodev = rte_cryptodev_pmd_get_dev(qat_pci_dev->sym_dev->sym_dev_id); + cryptodev = rte_cryptodev_pmd_get_dev(qat_pci_dev->sym_dev->dev_id); #ifdef RTE_LIB_SECURITY rte_free(cryptodev->security_ctx); cryptodev->security_ctx = NULL; diff --git a/drivers/crypto/qat/qat_sym_pmd.h b/drivers/crypto/qat/qat_sym_pmd.h index e0992cbe27..d49b732ca0 100644 --- a/drivers/crypto/qat/qat_sym_pmd.h +++ b/drivers/crypto/qat/qat_sym_pmd.h @@ -14,6 +14,7 @@ #endif #include "qat_sym_capabilities.h" +#include "qat_crypto.h" #include "qat_device.h" /** Intel(R) QAT Symmetric Crypto PMD driver name */ @@ -25,23 +26,6 @@ extern uint8_t qat_sym_driver_id; -/** private data structure for a QAT device. - * This QAT device is a device offering only symmetric crypto service, - * there can be one of these on each qat_pci_device (VF). - */ -struct qat_sym_dev_private { - struct qat_pci_device *qat_dev; - /**< The qat pci device hosting the service */ - uint8_t sym_dev_id; - /**< Device instance for this rte_cryptodev */ - const struct rte_cryptodev_capabilities *qat_dev_capabilities; - /* QAT device symmetric crypto capabilities */ - const struct rte_memzone *capa_mz; - /* Shared memzone for storing capabilities */ - uint16_t min_enq_burst_threshold; - uint32_t internal_capabilities; /* see flags QAT_SYM_CAP_xxx */ -}; - int qat_sym_dev_create(struct qat_pci_device *qat_pci_dev, struct qat_dev_cmd_param *qat_dev_cmd_param); @@ -49,5 +33,8 @@ qat_sym_dev_create(struct qat_pci_device *qat_pci_dev, int qat_sym_dev_destroy(struct qat_pci_device *qat_pci_dev); +void +qat_sym_init_op_cookie(void *op_cookie); + #endif #endif /* _QAT_SYM_PMD_H_ */ diff --git a/drivers/crypto/qat/qat_sym_session.c b/drivers/crypto/qat/qat_sym_session.c index 3f2f6736fc..8ca475ca8b 100644 --- a/drivers/crypto/qat/qat_sym_session.c +++ b/drivers/crypto/qat/qat_sym_session.c @@ -131,7 +131,7 @@ bpi_cipher_ctx_init(enum rte_crypto_cipher_algorithm cryptodev_algo, static int qat_is_cipher_alg_supported(enum rte_crypto_cipher_algorithm algo, - struct qat_sym_dev_private *internals) + struct qat_cryptodev_private *internals) { int i = 0; const struct rte_cryptodev_capabilities *capability; @@ -152,7 +152,7 @@ qat_is_cipher_alg_supported(enum rte_crypto_cipher_algorithm algo, static int qat_is_auth_alg_supported(enum rte_crypto_auth_algorithm algo, - struct qat_sym_dev_private *internals) + struct qat_cryptodev_private *internals) { int i = 0; const struct rte_cryptodev_capabilities *capability; @@ -267,7 +267,7 @@ qat_sym_session_configure_cipher(struct rte_cryptodev *dev, struct rte_crypto_sym_xform *xform, struct qat_sym_session *session) { - struct qat_sym_dev_private *internals = dev->data->dev_private; + struct qat_cryptodev_private *internals = dev->data->dev_private; struct rte_crypto_cipher_xform *cipher_xform = NULL; enum qat_device_gen qat_dev_gen = internals->qat_dev->qat_dev_gen; @@ -532,7 +532,8 @@ static void qat_sym_session_handle_mixed(const struct rte_cryptodev *dev, struct qat_sym_session *session) { - const struct qat_sym_dev_private *qat_private = dev->data->dev_private; + const struct qat_cryptodev_private *qat_private = + dev->data->dev_private; enum qat_device_gen min_dev_gen = (qat_private->internal_capabilities & QAT_SYM_CAP_MIXED_CRYPTO) ? QAT_GEN2 : QAT_GEN3; @@ -564,7 +565,7 @@ qat_sym_session_set_parameters(struct rte_cryptodev *dev, struct rte_crypto_sym_xform *xform, void *session_private) { struct qat_sym_session *session = session_private; - struct qat_sym_dev_private *internals = dev->data->dev_private; + struct qat_cryptodev_private *internals = dev->data->dev_private; enum qat_device_gen qat_dev_gen = internals->qat_dev->qat_dev_gen; int ret; int qat_cmd_id; @@ -707,7 +708,7 @@ qat_sym_session_configure_auth(struct rte_cryptodev *dev, struct qat_sym_session *session) { struct rte_crypto_auth_xform *auth_xform = qat_get_auth_xform(xform); - struct qat_sym_dev_private *internals = dev->data->dev_private; + struct qat_cryptodev_private *internals = dev->data->dev_private; const uint8_t *key_data = auth_xform->key.data; uint8_t key_length = auth_xform->key.length; enum qat_device_gen qat_dev_gen = @@ -875,7 +876,7 @@ qat_sym_session_configure_aead(struct rte_cryptodev *dev, { struct rte_crypto_aead_xform *aead_xform = &xform->aead; enum rte_crypto_auth_operation crypto_operation; - struct qat_sym_dev_private *internals = + struct qat_cryptodev_private *internals = dev->data->dev_private; enum qat_device_gen qat_dev_gen = internals->qat_dev->qat_dev_gen; -- 2.17.1