From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id D40E6A00C5; Mon, 6 Jul 2020 19:38:06 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id A7A7F1DB89; Mon, 6 Jul 2020 19:38:06 +0200 (CEST) Received: from mga01.intel.com (mga01.intel.com [192.55.52.88]) by dpdk.org (Postfix) with ESMTP id 2D0611DB56 for ; Mon, 6 Jul 2020 19:38:04 +0200 (CEST) IronPort-SDR: vodkJ3/NZeHa7YQ8ohn14OKBuLXlboHqRx4b0ov+HZsyGzsgpHfSPybkpBdI4h4NsuzeyWhXGc AOpz0Q1wBhZQ== X-IronPort-AV: E=McAfee;i="6000,8403,9673"; a="165531341" X-IronPort-AV: E=Sophos;i="5.75,320,1589266800"; d="scan'208";a="165531341" X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga002.jf.intel.com ([10.7.209.21]) by fmsmga101.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 06 Jul 2020 10:38:02 -0700 IronPort-SDR: EdHuELkW6IKvI0zGiXx4FLnyMpQQbh6t6B4ChsLHg0OsKzSuY7/R6NLP7Abk3u4d+fokKAeuiV gqjYg1e+766g== X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.75,320,1589266800"; d="scan'208";a="297077932" Received: from unknown (HELO akusztax-MOBL.ger.corp.intel.com) ([10.104.113.62]) by orsmga002.jf.intel.com with ESMTP; 06 Jul 2020 10:37:54 -0700 From: Arek Kusztal To: dev@dpdk.org Cc: akhil.goyal@nxp.com, fiona.trahe@intel.com, Arek Kusztal Date: Mon, 6 Jul 2020 19:35:35 +0200 Message-Id: <20200706173535.14812-1-arkadiuszx.kusztal@intel.com> X-Mailer: git-send-email 2.19.1.windows.1 MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Subject: [dpdk-dev] [PATCH v4] crypto/qat: add handling of multi process X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" This patch improves handling of multi process applications using Intel QuickAssist Technology PMD. Signed-off-by: Arek Kusztal --- v4: - fixed checkpatch issues doc/guides/rel_notes/release_20_08.rst | 4 ++ drivers/common/qat/qat_device.c | 111 +++++++++++++++++++++++---------- drivers/common/qat/qat_device.h | 69 +++++++++++--------- drivers/common/qat/qat_qp.c | 26 ++++++-- drivers/compress/qat/qat_comp_pmd.c | 49 ++++++++++++--- drivers/compress/qat/qat_comp_pmd.h | 2 + drivers/crypto/qat/qat_asym.c | 4 +- drivers/crypto/qat/qat_asym_pmd.c | 72 +++++++++++++++++---- drivers/crypto/qat/qat_asym_pmd.h | 4 +- drivers/crypto/qat/qat_sym.c | 2 +- drivers/crypto/qat/qat_sym.h | 2 +- drivers/crypto/qat/qat_sym_pmd.c | 105 +++++++++++++++++++++++++------ drivers/crypto/qat/qat_sym_pmd.h | 4 +- 13 files changed, 337 insertions(+), 117 deletions(-) diff --git a/doc/guides/rel_notes/release_20_08.rst b/doc/guides/rel_notes/release_20_08.rst index 159da34..ec73ece 100644 --- a/doc/guides/rel_notes/release_20_08.rst +++ b/doc/guides/rel_notes/release_20_08.rst @@ -134,6 +134,10 @@ New Features See the :doc:`../sample_app_ug/l2_forward_real_virtual` for more details of this parameter usage. +* **Updated the Intel QuickAssist Technology crypto and compression PMDs.** + + Implemented improvements in handling multi process applications + using QAT symmetric/asymmetric crypto and compression services. Removed Items ------------- diff --git a/drivers/common/qat/qat_device.c b/drivers/common/qat/qat_device.c index 2b41d9a..a6ab29f 100644 --- a/drivers/common/qat/qat_device.c +++ b/drivers/common/qat/qat_device.c @@ -32,8 +32,8 @@ struct qat_gen_hw_data qat_gen_config[] = { }, }; - -static struct qat_pci_device qat_pci_devices[RTE_PMD_QAT_MAX_PCI_DEVICES]; +/* per-process array of device data */ +struct qat_device_info qat_pci_devs[RTE_PMD_QAT_MAX_PCI_DEVICES]; static int qat_nb_pci_devices; /* @@ -60,26 +60,20 @@ static const struct rte_pci_id pci_id_qat_map[] = { }; static struct qat_pci_device * -qat_pci_get_dev(uint8_t dev_id) -{ - return &qat_pci_devices[dev_id]; -} - -static struct qat_pci_device * qat_pci_get_named_dev(const char *name) { - struct qat_pci_device *dev; unsigned int i; if (name == NULL) return NULL; for (i = 0; i < RTE_PMD_QAT_MAX_PCI_DEVICES; i++) { - dev = &qat_pci_devices[i]; - - if ((dev->attached == QAT_ATTACHED) && - (strcmp(dev->name, name) == 0)) - return dev; + if (qat_pci_devs[i].mz && + (strcmp(((struct qat_pci_device *) + qat_pci_devs[i].mz->addr)->name, name) + == 0)) + return (struct qat_pci_device *) + qat_pci_devs[i].mz->addr; } return NULL; @@ -88,13 +82,14 @@ qat_pci_get_named_dev(const char *name) static uint8_t qat_pci_find_free_device_index(void) { - uint8_t dev_id; + uint8_t dev_id; - for (dev_id = 0; dev_id < RTE_PMD_QAT_MAX_PCI_DEVICES; dev_id++) { - if (qat_pci_devices[dev_id].attached == QAT_DETACHED) - break; - } - return dev_id; + for (dev_id = 0; dev_id < RTE_PMD_QAT_MAX_PCI_DEVICES; + dev_id++) { + if (qat_pci_devs[dev_id].mz == NULL) + break; + } + return dev_id; } struct qat_pci_device * @@ -169,12 +164,31 @@ qat_pci_device_allocate(struct rte_pci_device *pci_dev, struct qat_dev_cmd_param *qat_dev_cmd_param) { struct qat_pci_device *qat_dev; - uint8_t qat_dev_id; + uint8_t qat_dev_id = 0; char name[QAT_DEV_NAME_MAX_LEN]; struct rte_devargs *devargs = pci_dev->device.devargs; rte_pci_device_name(&pci_dev->addr, name, sizeof(name)); snprintf(name+strlen(name), QAT_DEV_NAME_MAX_LEN-strlen(name), "_qat"); + + if (rte_eal_process_type() == RTE_PROC_SECONDARY) { + const struct rte_memzone *mz = rte_memzone_lookup(name); + + if (mz == NULL) { + QAT_LOG(ERR, + "Secondary can't find %s mz, did primary create device?", + name); + return NULL; + } + qat_dev = mz->addr; + qat_pci_devs[qat_dev->qat_dev_id].mz = mz; + qat_pci_devs[qat_dev->qat_dev_id].pci_dev = pci_dev; + qat_nb_pci_devices++; + QAT_LOG(DEBUG, "QAT device %d found, name %s, total QATs %d", + qat_dev->qat_dev_id, qat_dev->name, qat_nb_pci_devices); + return qat_dev; + } + if (qat_pci_get_named_dev(name) != NULL) { QAT_LOG(ERR, "QAT device with name %s already allocated!", name); @@ -187,12 +201,22 @@ qat_pci_device_allocate(struct rte_pci_device *pci_dev, return NULL; } - qat_dev = qat_pci_get_dev(qat_dev_id); + qat_pci_devs[qat_dev_id].mz = rte_memzone_reserve(name, + sizeof(struct qat_pci_device), + rte_socket_id(), 0); + + if (qat_pci_devs[qat_dev_id].mz == NULL) { + QAT_LOG(ERR, "Error when allocating memzone for QAT_%d", + qat_dev_id); + return NULL; + } + + qat_dev = qat_pci_devs[qat_dev_id].mz->addr; memset(qat_dev, 0, sizeof(*qat_dev)); strlcpy(qat_dev->name, name, QAT_DEV_NAME_MAX_LEN); qat_dev->qat_dev_id = qat_dev_id; - qat_dev->pci_dev = pci_dev; - switch (qat_dev->pci_dev->id.device_id) { + qat_pci_devs[qat_dev_id].pci_dev = pci_dev; + switch (pci_dev->id.device_id) { case 0x0443: qat_dev->qat_dev_gen = QAT_GEN1; break; @@ -206,6 +230,7 @@ qat_pci_device_allocate(struct rte_pci_device *pci_dev, break; default: QAT_LOG(ERR, "Invalid dev_id, can't determine generation"); + rte_memzone_free(qat_pci_devs[qat_dev->qat_dev_id].mz); return NULL; } @@ -213,22 +238,20 @@ qat_pci_device_allocate(struct rte_pci_device *pci_dev, qat_dev_parse_cmd(devargs->drv_str, qat_dev_cmd_param); rte_spinlock_init(&qat_dev->arb_csr_lock); - - qat_dev->attached = QAT_ATTACHED; - qat_nb_pci_devices++; - QAT_LOG(DEBUG, "QAT device %d allocated, name %s, total QATs %d", + QAT_LOG(DEBUG, "QAT device %d found, name %s, total QATs %d", qat_dev->qat_dev_id, qat_dev->name, qat_nb_pci_devices); return qat_dev; } -int +static int qat_pci_device_release(struct rte_pci_device *pci_dev) { struct qat_pci_device *qat_dev; char name[QAT_DEV_NAME_MAX_LEN]; + int busy = 0; if (pci_dev == NULL) return -EINVAL; @@ -238,15 +261,35 @@ qat_pci_device_release(struct rte_pci_device *pci_dev) qat_dev = qat_pci_get_named_dev(name); if (qat_dev != NULL) { + struct qat_device_info *inst = + &qat_pci_devs[qat_dev->qat_dev_id]; /* Check that there are no service devs still on pci device */ - if (qat_dev->sym_dev != NULL) - return -EBUSY; - qat_dev->attached = QAT_DETACHED; + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + if (qat_dev->sym_dev != NULL) { + QAT_LOG(DEBUG, "QAT sym device %s is busy", + name); + busy = 1; + } + if (qat_dev->asym_dev != NULL) { + QAT_LOG(DEBUG, "QAT asym device %s is busy", + name); + busy = 1; + } + if (qat_dev->comp_dev != NULL) { + QAT_LOG(DEBUG, "QAT comp device %s is busy", + name); + busy = 1; + } + if (busy) + return -EBUSY; + rte_memzone_free(inst->mz); + } + memset(inst, 0, sizeof(struct qat_device_info)); qat_nb_pci_devices--; + QAT_LOG(DEBUG, "QAT device %s released, total QATs %d", + name, qat_nb_pci_devices); } - QAT_LOG(DEBUG, "QAT device %s released, total QATs %d", - name, qat_nb_pci_devices); return 0; } diff --git a/drivers/common/qat/qat_device.h b/drivers/common/qat/qat_device.h index 09a4c55..271968c 100644 --- a/drivers/common/qat/qat_device.h +++ b/drivers/common/qat/qat_device.h @@ -32,6 +32,37 @@ enum qat_comp_num_im_buffers { QAT_NUM_INTERM_BUFS_GEN3 = 20 }; +struct qat_device_info { + const struct rte_memzone *mz; + /**< mz to store the qat_pci_device so it can be + * shared across processes + */ + struct rte_pci_device *pci_dev; + struct rte_device sym_rte_dev; + /**< This represents the crypto sym subset of this pci device. + * Register with this rather than with the one in + * pci_dev so that its driver can have a crypto-specific name + */ + + struct rte_device asym_rte_dev; + /**< This represents the crypto asym subset of this pci device. + * Register with this rather than with the one in + * pci_dev so that its driver can have a crypto-specific name + */ + + struct rte_device comp_rte_dev; + /**< This represents the compression subset of this pci device. + * Register with this rather than with the one in + * pci_dev so that its driver can have a compression-specific name + */ +}; + +extern struct qat_device_info qat_pci_devs[]; + +struct qat_sym_dev_private; +struct qat_asym_dev_private; +struct qat_comp_dev_private; + /* * This struct holds all the data about a QAT pci device * including data about all services it supports. @@ -39,27 +70,20 @@ enum qat_comp_num_im_buffers { * - hw_data * - config data * - runtime data + * Note: as this data can be shared in a multi-process scenario, + * any pointers in it must also point to shared memory. */ -struct qat_sym_dev_private; -struct qat_asym_dev_private; -struct qat_comp_dev_private; - struct qat_pci_device { /* Data used by all services */ char name[QAT_DEV_NAME_MAX_LEN]; /**< Name of qat pci device */ uint8_t qat_dev_id; - /**< Device instance for this qat pci device */ - struct rte_pci_device *pci_dev; - /**< PCI information. */ + /**< Id of device instance for this qat pci device */ enum qat_device_gen qat_dev_gen; /**< QAT device generation */ rte_spinlock_t arb_csr_lock; /**< lock to protect accesses to the arbiter CSR */ - __extension__ - uint8_t attached : 1; - /**< Flag indicating the device is attached */ struct qat_qp *qps_in_use[QAT_MAX_SERVICES][ADF_MAX_QPS_ON_ANY_SERVICE]; /**< links to qps set up for each service, index same as on API */ @@ -67,32 +91,18 @@ struct qat_pci_device { /* Data relating to symmetric crypto service */ struct qat_sym_dev_private *sym_dev; /**< link back to cryptodev private data */ - struct rte_device sym_rte_dev; - /**< This represents the crypto sym subset of this pci device. - * Register with this rather than with the one in - * pci_dev so that its driver can have a crypto-specific name - */ + int qat_sym_driver_id; + /**< Symmetric driver id used by this device */ /* Data relating to asymmetric crypto service */ struct qat_asym_dev_private *asym_dev; /**< link back to cryptodev private data */ - struct rte_device asym_rte_dev; - /**< This represents the crypto asym subset of this pci device. - * Register with this rather than with the one in - * pci_dev so that its driver can have a crypto-specific name - */ + int qat_asym_driver_id; + /**< Symmetric driver id used by this device */ /* Data relating to compression service */ struct qat_comp_dev_private *comp_dev; /**< link back to compressdev private data */ - struct rte_device comp_rte_dev; - /**< This represents the compression subset of this pci device. - * Register with this rather than with the one in - * pci_dev so that its driver can have a compression-specific name - */ - - /* Data relating to asymmetric crypto service */ - }; struct qat_gen_hw_data { @@ -107,9 +117,6 @@ struct qat_pci_device * qat_pci_device_allocate(struct rte_pci_device *pci_dev, struct qat_dev_cmd_param *qat_dev_cmd_param); -int -qat_pci_device_release(struct rte_pci_device *pci_dev); - struct qat_pci_device * qat_get_qat_dev_from_pci_dev(struct rte_pci_device *pci_dev); diff --git a/drivers/common/qat/qat_qp.c b/drivers/common/qat/qat_qp.c index 8e6dd04..aacd4ab 100644 --- a/drivers/common/qat/qat_qp.c +++ b/drivers/common/qat/qat_qp.c @@ -193,7 +193,8 @@ int qat_qp_setup(struct qat_pci_device *qat_dev, { struct qat_qp *qp; - struct rte_pci_device *pci_dev = qat_dev->pci_dev; + struct rte_pci_device *pci_dev = + qat_pci_devs[qat_dev->qat_dev_id].pci_dev; char op_cookie_pool_name[RTE_RING_NAMESIZE]; uint32_t i; @@ -274,7 +275,7 @@ int qat_qp_setup(struct qat_pci_device *qat_dev, qp->nb_descriptors, qat_qp_conf->cookie_size, 64, 0, NULL, NULL, NULL, NULL, - qat_dev->pci_dev->device.numa_node, + pci_dev->device.numa_node, 0); if (!qp->op_cookie_pool) { QAT_LOG(ERR, "QAT PMD Cannot create" @@ -379,7 +380,8 @@ qat_queue_create(struct qat_pci_device *qat_dev, struct qat_queue *queue, uint64_t queue_base; void *io_addr; const struct rte_memzone *qp_mz; - struct rte_pci_device *pci_dev = qat_dev->pci_dev; + struct rte_pci_device *pci_dev = + qat_pci_devs[qat_dev->qat_dev_id].pci_dev; int ret = 0; uint16_t desc_size = (dir == ADF_RING_DIR_TX ? qp_conf->hw->tx_msg_size : qp_conf->hw->rx_msg_size); @@ -403,7 +405,7 @@ qat_queue_create(struct qat_pci_device *qat_dev, struct qat_queue *queue, qp_conf->service_str, "qp_mem", queue->hw_bundle_number, queue->hw_queue_number); qp_mz = queue_dma_zone_reserve(queue->memz_name, queue_size_bytes, - qat_dev->pci_dev->device.numa_node); + pci_dev->device.numa_node); if (qp_mz == NULL) { QAT_LOG(ERR, "Failed to allocate ring memzone"); return -ENOMEM; @@ -627,9 +629,23 @@ qat_enqueue_op_burst(void *qp, void **ops, uint16_t nb_ops) while (nb_ops_sent != nb_ops_possible) { - ret = tmp_qp->build_request(*ops, base_addr + tail, + if (tmp_qp->service_type == QAT_SERVICE_SYMMETRIC) { +#ifdef BUILD_QAT_SYM + ret = qat_sym_build_request(*ops, base_addr + tail, tmp_qp->op_cookies[tail >> queue->trailz], tmp_qp->qat_dev_gen); +#endif + } else if (tmp_qp->service_type == QAT_SERVICE_COMPRESSION) { + ret = qat_comp_build_request(*ops, base_addr + tail, + tmp_qp->op_cookies[tail >> queue->trailz], + tmp_qp->qat_dev_gen); + } else if (tmp_qp->service_type == QAT_SERVICE_ASYMMETRIC) { +#ifdef BUILD_QAT_ASYM + ret = qat_asym_build_request(*ops, base_addr + tail, + tmp_qp->op_cookies[tail >> queue->trailz], + tmp_qp->qat_dev_gen); +#endif + } if (ret != 0) { tmp_qp->stats.enqueue_err_count++; /* This message cannot be enqueued */ diff --git a/drivers/compress/qat/qat_comp_pmd.c b/drivers/compress/qat/qat_comp_pmd.c index fe62de5..4a5baf3 100644 --- a/drivers/compress/qat/qat_comp_pmd.c +++ b/drivers/compress/qat/qat_comp_pmd.c @@ -655,6 +655,8 @@ qat_comp_dev_create(struct qat_pci_device *qat_pci_dev, struct qat_dev_cmd_param *qat_dev_cmd_param) { int i = 0; + struct qat_device_info *qat_dev_instance = + &qat_pci_devs[qat_pci_dev->qat_dev_id]; if (qat_pci_dev->qat_dev_gen == QAT_GEN3) { QAT_LOG(ERR, "Compression PMD not supported on QAT P5xxx"); return 0; @@ -662,24 +664,27 @@ qat_comp_dev_create(struct qat_pci_device *qat_pci_dev, struct rte_compressdev_pmd_init_params init_params = { .name = "", - .socket_id = qat_pci_dev->pci_dev->device.numa_node, + .socket_id = qat_dev_instance->pci_dev->device.numa_node, }; char name[RTE_COMPRESSDEV_NAME_MAX_LEN]; + char capa_memz_name[RTE_COMPRESSDEV_NAME_MAX_LEN]; struct rte_compressdev *compressdev; struct qat_comp_dev_private *comp_dev; + const struct rte_compressdev_capabilities *capabilities; + uint64_t capa_size; snprintf(name, RTE_COMPRESSDEV_NAME_MAX_LEN, "%s_%s", qat_pci_dev->name, "comp"); QAT_LOG(DEBUG, "Creating QAT COMP device %s", name); /* Populate subset device to use in compressdev device creation */ - qat_pci_dev->comp_rte_dev.driver = &compdev_qat_driver; - qat_pci_dev->comp_rte_dev.numa_node = - qat_pci_dev->pci_dev->device.numa_node; - qat_pci_dev->comp_rte_dev.devargs = NULL; + qat_dev_instance->comp_rte_dev.driver = &compdev_qat_driver; + qat_dev_instance->comp_rte_dev.numa_node = + qat_dev_instance->pci_dev->device.numa_node; + qat_dev_instance->comp_rte_dev.devargs = NULL; compressdev = rte_compressdev_pmd_create(name, - &(qat_pci_dev->comp_rte_dev), + &(qat_dev_instance->comp_rte_dev), sizeof(struct qat_comp_dev_private), &init_params); @@ -694,25 +699,48 @@ qat_comp_dev_create(struct qat_pci_device *qat_pci_dev, compressdev->feature_flags = RTE_COMPDEV_FF_HW_ACCELERATED; + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return 0; + + snprintf(capa_memz_name, RTE_COMPRESSDEV_NAME_MAX_LEN, + "QAT_COMP_CAPA_GEN_%d", + qat_pci_dev->qat_dev_gen); comp_dev = compressdev->data->dev_private; comp_dev->qat_dev = qat_pci_dev; comp_dev->compressdev = compressdev; - qat_pci_dev->comp_dev = comp_dev; switch (qat_pci_dev->qat_dev_gen) { case QAT_GEN1: case QAT_GEN2: case QAT_GEN3: - comp_dev->qat_dev_capabilities = qat_comp_gen_capabilities; + capabilities = qat_comp_gen_capabilities; + capa_size = sizeof(qat_comp_gen_capabilities); break; default: - comp_dev->qat_dev_capabilities = qat_comp_gen_capabilities; + capabilities = qat_comp_gen_capabilities; + capa_size = sizeof(qat_comp_gen_capabilities); QAT_LOG(DEBUG, "QAT gen %d capabilities unknown, default to GEN1", qat_pci_dev->qat_dev_gen); break; } + comp_dev->capa_mz = rte_memzone_reserve(capa_memz_name, + capa_size, + rte_socket_id(), 0); + if (comp_dev->capa_mz == NULL) { + QAT_LOG(DEBUG, + "Error allocating memzone for capabilities, destroying PMD for %s", + name); + memset(&qat_dev_instance->comp_rte_dev, 0, + sizeof(qat_dev_instance->comp_rte_dev)); + rte_compressdev_pmd_destroy(compressdev); + return -EFAULT; + } + + memcpy(comp_dev->capa_mz->addr, capabilities, capa_size); + comp_dev->qat_dev_capabilities = comp_dev->capa_mz->addr; + while (1) { if (qat_dev_cmd_param[i].name == NULL) break; @@ -721,6 +749,7 @@ qat_comp_dev_create(struct qat_pci_device *qat_pci_dev, qat_dev_cmd_param[i].val; i++; } + qat_pci_dev->comp_dev = comp_dev; QAT_LOG(DEBUG, "Created QAT COMP device %s as compressdev instance %d", @@ -740,6 +769,8 @@ qat_comp_dev_destroy(struct qat_pci_device *qat_pci_dev) if (comp_dev == NULL) return 0; + if (rte_eal_process_type() == RTE_PROC_PRIMARY) + rte_memzone_free(qat_pci_dev->comp_dev->capa_mz); /* clean up any resources used by the device */ qat_comp_dev_close(comp_dev->compressdev); diff --git a/drivers/compress/qat/qat_comp_pmd.h b/drivers/compress/qat/qat_comp_pmd.h index 5c7fa9f..ed27120 100644 --- a/drivers/compress/qat/qat_comp_pmd.h +++ b/drivers/compress/qat/qat_comp_pmd.h @@ -32,6 +32,8 @@ struct qat_comp_dev_private { /**< The device's pool for qat_comp_xforms */ struct rte_mempool *streampool; /**< The device's pool for qat_comp_streams */ + const struct rte_memzone *capa_mz; + /* Shared memzone for storing capabilities */ uint16_t min_enq_burst_threshold; }; diff --git a/drivers/crypto/qat/qat_asym.c b/drivers/crypto/qat/qat_asym.c index ae0dd79..8597381 100644 --- a/drivers/crypto/qat/qat_asym.c +++ b/drivers/crypto/qat/qat_asym.c @@ -475,7 +475,7 @@ qat_asym_build_request(void *in_op, if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { ctx = (struct qat_asym_session *) get_asym_session_private_data( - op->asym->session, cryptodev_qat_asym_driver_id); + op->asym->session, qat_asym_driver_id); if (unlikely(ctx == NULL)) { QAT_LOG(ERR, "Session has not been created for this device"); goto error; @@ -693,7 +693,7 @@ qat_asym_process_response(void **op, uint8_t *resp, if (rx_op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { ctx = (struct qat_asym_session *)get_asym_session_private_data( - rx_op->asym->session, cryptodev_qat_asym_driver_id); + rx_op->asym->session, qat_asym_driver_id); qat_asym_collect_response(rx_op, cookie, ctx->xform); } else if (rx_op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { qat_asym_collect_response(rx_op, cookie, rx_op->asym->xform); diff --git a/drivers/crypto/qat/qat_asym_pmd.c b/drivers/crypto/qat/qat_asym_pmd.c index af3cc25..9b1676b 100644 --- a/drivers/crypto/qat/qat_asym_pmd.c +++ b/drivers/crypto/qat/qat_asym_pmd.c @@ -11,7 +11,7 @@ #include "qat_sym_capabilities.h" #include "qat_asym_capabilities.h" -uint8_t cryptodev_qat_asym_driver_id; +uint8_t qat_asym_driver_id; static const struct rte_cryptodev_capabilities qat_gen1_asym_capabilities[] = { QAT_BASE_GEN1_ASYM_CAPABILITIES, @@ -63,7 +63,7 @@ static void qat_asym_dev_info_get(struct rte_cryptodev *dev, QAT_SERVICE_ASYMMETRIC); info->feature_flags = dev->feature_flags; info->capabilities = internals->qat_dev_capabilities; - info->driver_id = cryptodev_qat_asym_driver_id; + info->driver_id = qat_asym_driver_id; /* No limit of number of sessions */ info->sym.max_nb_sessions = 0; } @@ -239,33 +239,50 @@ qat_asym_dev_create(struct qat_pci_device *qat_pci_dev, struct qat_dev_cmd_param *qat_dev_cmd_param) { int i = 0; + struct qat_device_info *qat_dev_instance = + &qat_pci_devs[qat_pci_dev->qat_dev_id]; struct rte_cryptodev_pmd_init_params init_params = { .name = "", - .socket_id = qat_pci_dev->pci_dev->device.numa_node, + .socket_id = + qat_dev_instance->pci_dev->device.numa_node, .private_data_size = sizeof(struct qat_asym_dev_private) }; char name[RTE_CRYPTODEV_NAME_MAX_LEN]; + char capa_memz_name[RTE_CRYPTODEV_NAME_MAX_LEN]; struct rte_cryptodev *cryptodev; struct qat_asym_dev_private *internals; + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + qat_pci_dev->qat_asym_driver_id = + qat_asym_driver_id; + } else if (rte_eal_process_type() == RTE_PROC_SECONDARY) { + if (qat_pci_dev->qat_asym_driver_id != + qat_asym_driver_id) { + QAT_LOG(ERR, + "Device %s have different driver id than corresponding device in primary process", + name); + return -(EFAULT); + } + } + snprintf(name, RTE_CRYPTODEV_NAME_MAX_LEN, "%s_%s", qat_pci_dev->name, "asym"); QAT_LOG(DEBUG, "Creating QAT ASYM device %s\n", name); /* Populate subset device to use in cryptodev device creation */ - qat_pci_dev->asym_rte_dev.driver = &cryptodev_qat_asym_driver; - qat_pci_dev->asym_rte_dev.numa_node = - qat_pci_dev->pci_dev->device.numa_node; - qat_pci_dev->asym_rte_dev.devargs = NULL; + qat_dev_instance->asym_rte_dev.driver = &cryptodev_qat_asym_driver; + qat_dev_instance->asym_rte_dev.numa_node = + qat_dev_instance->pci_dev->device.numa_node; + qat_dev_instance->asym_rte_dev.devargs = NULL; cryptodev = rte_cryptodev_pmd_create(name, - &(qat_pci_dev->asym_rte_dev), &init_params); + &(qat_dev_instance->asym_rte_dev), &init_params); if (cryptodev == NULL) return -ENODEV; - qat_pci_dev->asym_rte_dev.name = cryptodev->data->name; - cryptodev->driver_id = cryptodev_qat_asym_driver_id; + qat_dev_instance->asym_rte_dev.name = cryptodev->data->name; + cryptodev->driver_id = qat_asym_driver_id; cryptodev->dev_ops = &crypto_qat_ops; cryptodev->enqueue_burst = qat_asym_pmd_enqueue_op_burst; @@ -276,13 +293,39 @@ qat_asym_dev_create(struct qat_pci_device *qat_pci_dev, RTE_CRYPTODEV_FF_ASYM_SESSIONLESS | RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_EXP | RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_QT; + + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return 0; + + snprintf(capa_memz_name, RTE_CRYPTODEV_NAME_MAX_LEN, + "QAT_ASYM_CAPA_GEN_%d", + qat_pci_dev->qat_dev_gen); internals = cryptodev->data->dev_private; internals->qat_dev = qat_pci_dev; - qat_pci_dev->asym_dev = internals; internals->asym_dev_id = cryptodev->data->dev_id; internals->qat_dev_capabilities = qat_gen1_asym_capabilities; + internals->capa_mz = rte_memzone_lookup(capa_memz_name); + if (internals->capa_mz == NULL) { + internals->capa_mz = rte_memzone_reserve(capa_memz_name, + sizeof(qat_gen1_asym_capabilities), + rte_socket_id(), 0); + } + if (internals->capa_mz == NULL) { + QAT_LOG(DEBUG, + "Error allocating memzone for capabilities, destroying PMD for %s", + name); + rte_cryptodev_pmd_destroy(cryptodev); + memset(&qat_dev_instance->asym_rte_dev, 0, + sizeof(qat_dev_instance->asym_rte_dev)); + return -EFAULT; + } + + memcpy(internals->capa_mz->addr, qat_gen1_asym_capabilities, + sizeof(qat_gen1_asym_capabilities)); + internals->qat_dev_capabilities = internals->capa_mz->addr; + while (1) { if (qat_dev_cmd_param[i].name == NULL) break; @@ -292,6 +335,7 @@ qat_asym_dev_create(struct qat_pci_device *qat_pci_dev, i++; } + qat_pci_dev->asym_dev = internals; QAT_LOG(DEBUG, "Created QAT ASYM device %s as cryptodev instance %d", cryptodev->data->name, internals->asym_dev_id); return 0; @@ -307,11 +351,13 @@ qat_asym_dev_destroy(struct qat_pci_device *qat_pci_dev) if (qat_pci_dev->asym_dev == NULL) return 0; + if (rte_eal_process_type() == RTE_PROC_PRIMARY) + rte_memzone_free(qat_pci_dev->asym_dev->capa_mz); /* free crypto device */ cryptodev = rte_cryptodev_pmd_get_dev( qat_pci_dev->asym_dev->asym_dev_id); rte_cryptodev_pmd_destroy(cryptodev); - qat_pci_dev->asym_rte_dev.name = NULL; + qat_pci_devs[qat_pci_dev->qat_dev_id].asym_rte_dev.name = NULL; qat_pci_dev->asym_dev = NULL; return 0; @@ -320,4 +366,4 @@ qat_asym_dev_destroy(struct qat_pci_device *qat_pci_dev) static struct cryptodev_driver qat_crypto_drv; RTE_PMD_REGISTER_CRYPTO_DRIVER(qat_crypto_drv, cryptodev_qat_asym_driver, - cryptodev_qat_asym_driver_id); + qat_asym_driver_id); diff --git a/drivers/crypto/qat/qat_asym_pmd.h b/drivers/crypto/qat/qat_asym_pmd.h index 0535bc6..3b5abdd 100644 --- a/drivers/crypto/qat/qat_asym_pmd.h +++ b/drivers/crypto/qat/qat_asym_pmd.h @@ -13,7 +13,7 @@ #define CRYPTODEV_NAME_QAT_ASYM_PMD crypto_qat_asym -extern uint8_t cryptodev_qat_asym_driver_id; +extern uint8_t qat_asym_driver_id; /** private data structure for a QAT device. * This QAT device is a device offering only asymmetric crypto service, @@ -26,6 +26,8 @@ struct qat_asym_dev_private { /**< Device instance for this rte_cryptodev */ const struct rte_cryptodev_capabilities *qat_dev_capabilities; /* QAT device asymmetric crypto capabilities */ + const struct rte_memzone *capa_mz; + /* Shared memzone for storing capabilities */ uint16_t min_enq_burst_threshold; }; diff --git a/drivers/crypto/qat/qat_sym.c b/drivers/crypto/qat/qat_sym.c index 55251d1..6d568ab 100644 --- a/drivers/crypto/qat/qat_sym.c +++ b/drivers/crypto/qat/qat_sym.c @@ -206,7 +206,7 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg, return -EINVAL; } else if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { ctx = (struct qat_sym_session *)get_sym_session_private_data( - op->sym->session, cryptodev_qat_driver_id); + op->sym->session, qat_sym_driver_id); #ifdef RTE_LIBRTE_SECURITY } else { ctx = (struct qat_sym_session *)get_sec_session_private_data( diff --git a/drivers/crypto/qat/qat_sym.h b/drivers/crypto/qat/qat_sym.h index 02e7239..dbca74e 100644 --- a/drivers/crypto/qat/qat_sym.h +++ b/drivers/crypto/qat/qat_sym.h @@ -197,7 +197,7 @@ qat_sym_process_response(void **op, uint8_t *resp) sess = (struct qat_sym_session *) get_sym_session_private_data( rx_op->sym->session, - cryptodev_qat_driver_id); + qat_sym_driver_id); } rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; diff --git a/drivers/crypto/qat/qat_sym_pmd.c b/drivers/crypto/qat/qat_sym_pmd.c index 1561752..021669c 100644 --- a/drivers/crypto/qat/qat_sym_pmd.c +++ b/drivers/crypto/qat/qat_sym_pmd.c @@ -19,7 +19,7 @@ #define MIXED_CRYPTO_MIN_FW_VER 0x04090000 -uint8_t cryptodev_qat_driver_id; +uint8_t qat_sym_driver_id; static const struct rte_cryptodev_capabilities qat_gen1_sym_capabilities[] = { QAT_BASE_GEN1_SYM_CAPABILITIES, @@ -32,6 +32,12 @@ static const struct rte_cryptodev_capabilities qat_gen2_sym_capabilities[] = { RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST() }; +static const struct rte_cryptodev_capabilities qat_gen3_sym_capabilities[] = { + QAT_BASE_GEN1_SYM_CAPABILITIES, + QAT_EXTRA_GEN2_SYM_CAPABILITIES, + RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST() +}; + #ifdef RTE_LIBRTE_SECURITY static const struct rte_cryptodev_capabilities qat_security_sym_capabilities[] = { @@ -92,7 +98,7 @@ static void qat_sym_dev_info_get(struct rte_cryptodev *dev, qat_qps_per_service(sym_hw_qps, QAT_SERVICE_SYMMETRIC); info->feature_flags = dev->feature_flags; info->capabilities = internals->qat_dev_capabilities; - info->driver_id = cryptodev_qat_driver_id; + info->driver_id = qat_sym_driver_id; /* No limit of number of sessions */ info->sym.max_nb_sessions = 0; } @@ -304,14 +310,40 @@ qat_sym_dev_create(struct qat_pci_device *qat_pci_dev, struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused) { int i = 0; + struct qat_device_info *qat_dev_instance = + &qat_pci_devs[qat_pci_dev->qat_dev_id]; + struct rte_cryptodev_pmd_init_params init_params = { .name = "", - .socket_id = qat_pci_dev->pci_dev->device.numa_node, + .socket_id = + qat_dev_instance->pci_dev->device.numa_node, .private_data_size = sizeof(struct qat_sym_dev_private) }; char name[RTE_CRYPTODEV_NAME_MAX_LEN]; + char capa_memz_name[RTE_CRYPTODEV_NAME_MAX_LEN]; struct rte_cryptodev *cryptodev; struct qat_sym_dev_private *internals; + const struct rte_cryptodev_capabilities *capabilities; + uint64_t capa_size; + /* + * All processes must use same driver id so they can share sessions. + * Store driver_id so we can validate that all processes have the same + * value, typically they have, but could differ if binaries built + * separately. + */ + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + qat_pci_dev->qat_sym_driver_id = + qat_sym_driver_id; + } else if (rte_eal_process_type() == RTE_PROC_SECONDARY) { + if (qat_pci_dev->qat_sym_driver_id != + qat_sym_driver_id) { + QAT_LOG(ERR, + "Device %s have different driver id than corresponding device in primary process", + name); + return -(EFAULT); + } + } + #ifdef RTE_LIBRTE_SECURITY struct rte_security_ctx *security_instance; #endif @@ -321,19 +353,19 @@ qat_sym_dev_create(struct qat_pci_device *qat_pci_dev, QAT_LOG(DEBUG, "Creating QAT SYM device %s", name); /* Populate subset device to use in cryptodev device creation */ - qat_pci_dev->sym_rte_dev.driver = &cryptodev_qat_sym_driver; - qat_pci_dev->sym_rte_dev.numa_node = - qat_pci_dev->pci_dev->device.numa_node; - qat_pci_dev->sym_rte_dev.devargs = NULL; + qat_dev_instance->sym_rte_dev.driver = &cryptodev_qat_sym_driver; + qat_dev_instance->sym_rte_dev.numa_node = + qat_dev_instance->pci_dev->device.numa_node; + qat_dev_instance->sym_rte_dev.devargs = NULL; cryptodev = rte_cryptodev_pmd_create(name, - &(qat_pci_dev->sym_rte_dev), &init_params); + &(qat_dev_instance->sym_rte_dev), &init_params); if (cryptodev == NULL) return -ENODEV; - qat_pci_dev->sym_rte_dev.name = cryptodev->data->name; - cryptodev->driver_id = cryptodev_qat_driver_id; + qat_dev_instance->sym_rte_dev.name = cryptodev->data->name; + cryptodev->driver_id = qat_sym_driver_id; cryptodev->dev_ops = &crypto_qat_ops; cryptodev->enqueue_burst = qat_sym_pmd_enqueue_op_burst; @@ -350,6 +382,13 @@ qat_sym_dev_create(struct qat_pci_device *qat_pci_dev, RTE_CRYPTODEV_FF_DIGEST_ENCRYPTED | RTE_CRYPTODEV_FF_SECURITY; + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return 0; + + snprintf(capa_memz_name, RTE_CRYPTODEV_NAME_MAX_LEN, + "QAT_SYM_CAPA_GEN_%d", + qat_pci_dev->qat_dev_gen); + #ifdef RTE_LIBRTE_SECURITY security_instance = rte_malloc("qat_sec", sizeof(struct rte_security_ctx), @@ -368,24 +407,48 @@ qat_sym_dev_create(struct qat_pci_device *qat_pci_dev, internals = cryptodev->data->dev_private; internals->qat_dev = qat_pci_dev; - qat_pci_dev->sym_dev = internals; internals->sym_dev_id = cryptodev->data->dev_id; switch (qat_pci_dev->qat_dev_gen) { case QAT_GEN1: - internals->qat_dev_capabilities = qat_gen1_sym_capabilities; + capabilities = qat_gen1_sym_capabilities; + capa_size = sizeof(qat_gen1_sym_capabilities); break; case QAT_GEN2: + capabilities = qat_gen2_sym_capabilities; + capa_size = sizeof(qat_gen2_sym_capabilities); + break; case QAT_GEN3: - internals->qat_dev_capabilities = qat_gen2_sym_capabilities; + capabilities = qat_gen3_sym_capabilities; + capa_size = sizeof(qat_gen3_sym_capabilities); break; default: - internals->qat_dev_capabilities = qat_gen2_sym_capabilities; QAT_LOG(DEBUG, - "QAT gen %d capabilities unknown, default to GEN2", - qat_pci_dev->qat_dev_gen); - break; + "QAT gen %d capabilities unknown", + qat_pci_dev->qat_dev_gen); + rte_cryptodev_pmd_destroy(cryptodev); + memset(&qat_dev_instance->sym_rte_dev, 0, + sizeof(qat_dev_instance->sym_rte_dev)); + return -(EINVAL); + } + internals->capa_mz = rte_memzone_lookup(capa_memz_name); + if (internals->capa_mz == NULL) { + internals->capa_mz = rte_memzone_reserve(capa_memz_name, + capa_size, + rte_socket_id(), 0); } + if (internals->capa_mz == NULL) { + QAT_LOG(DEBUG, + "Error allocating memzone for capabilities, destroying PMD for %s", + name); + rte_cryptodev_pmd_destroy(cryptodev); + memset(&qat_dev_instance->sym_rte_dev, 0, + sizeof(qat_dev_instance->sym_rte_dev)); + return -EFAULT; + } + + memcpy(internals->capa_mz->addr, capabilities, capa_size); + internals->qat_dev_capabilities = internals->capa_mz->addr; while (1) { if (qat_dev_cmd_param[i].name == NULL) @@ -396,8 +459,10 @@ qat_sym_dev_create(struct qat_pci_device *qat_pci_dev, i++; } + qat_pci_dev->sym_dev = internals; QAT_LOG(DEBUG, "Created QAT SYM device %s as cryptodev instance %d", cryptodev->data->name, internals->sym_dev_id); + return 0; } @@ -410,6 +475,8 @@ qat_sym_dev_destroy(struct qat_pci_device *qat_pci_dev) return -ENODEV; if (qat_pci_dev->sym_dev == NULL) return 0; + if (rte_eal_process_type() == RTE_PROC_PRIMARY) + rte_memzone_free(qat_pci_dev->sym_dev->capa_mz); /* free crypto device */ cryptodev = rte_cryptodev_pmd_get_dev(qat_pci_dev->sym_dev->sym_dev_id); @@ -417,7 +484,7 @@ qat_sym_dev_destroy(struct qat_pci_device *qat_pci_dev) rte_free(cryptodev->security_ctx); #endif rte_cryptodev_pmd_destroy(cryptodev); - qat_pci_dev->sym_rte_dev.name = NULL; + qat_pci_devs[qat_pci_dev->qat_dev_id].sym_rte_dev.name = NULL; qat_pci_dev->sym_dev = NULL; return 0; @@ -426,4 +493,4 @@ qat_sym_dev_destroy(struct qat_pci_device *qat_pci_dev) static struct cryptodev_driver qat_crypto_drv; RTE_PMD_REGISTER_CRYPTO_DRIVER(qat_crypto_drv, cryptodev_qat_sym_driver, - cryptodev_qat_driver_id); + qat_sym_driver_id); diff --git a/drivers/crypto/qat/qat_sym_pmd.h b/drivers/crypto/qat/qat_sym_pmd.h index c625fef..f32a77e 100644 --- a/drivers/crypto/qat/qat_sym_pmd.h +++ b/drivers/crypto/qat/qat_sym_pmd.h @@ -23,7 +23,7 @@ #define QAT_SYM_CAP_MIXED_CRYPTO (1 << 0) #define QAT_SYM_CAP_VALID (1 << 31) -extern uint8_t cryptodev_qat_driver_id; +extern uint8_t qat_sym_driver_id; /** private data structure for a QAT device. * This QAT device is a device offering only symmetric crypto service, @@ -36,6 +36,8 @@ struct qat_sym_dev_private { /**< Device instance for this rte_cryptodev */ const struct rte_cryptodev_capabilities *qat_dev_capabilities; /* QAT device symmetric crypto capabilities */ + const struct rte_memzone *capa_mz; + /* Shared memzone for storing capabilities */ uint16_t min_enq_burst_threshold; uint32_t internal_capabilities; /* see flags QAT_SYM_CAP_xxx */ }; -- 2.1.0