* [PATCH v2 1/9] crypto/ionic: introduce AMD Pensando ionic crypto driver
2024-04-30 20:21 ` [PATCH v2 0/9] " Andrew Boyer
@ 2024-04-30 20:21 ` Andrew Boyer
2024-04-30 20:21 ` [PATCH v2 2/9] crypto/ionic: add the firmware interface definition file Andrew Boyer
` (7 subsequent siblings)
8 siblings, 0 replies; 19+ messages in thread
From: Andrew Boyer @ 2024-04-30 20:21 UTC (permalink / raw)
To: dev; +Cc: Andrew Boyer
Introduce a new crypto PMD for AMD Pensando hardware accelerators. It
allows applications running directly on the AMD Pensando DSC to offload
cryptographic operations to hardware cryptographic blocks.
Add support for the cryptodevs to the common ionic library.
Add the driver skeleton.
Add a skeleton features list and guide.
Hook the new PMD up to the build.
Update MAINTAINERS.
Signed-off-by: Andrew Boyer <andrew.boyer@amd.com>
---
MAINTAINERS | 7 ++
doc/guides/cryptodevs/features/ionic.ini | 32 +++++
doc/guides/cryptodevs/index.rst | 1 +
doc/guides/cryptodevs/ionic.rst | 28 +++++
drivers/common/ionic/ionic_common.h | 2 +
drivers/common/ionic/ionic_common_uio.c | 48 +++++++-
drivers/common/ionic/version.map | 1 +
drivers/crypto/ionic/ionic_crypto.h | 92 ++++++++++++++
drivers/crypto/ionic/ionic_crypto_main.c | 148 +++++++++++++++++++++++
drivers/crypto/ionic/ionic_crypto_vdev.c | 91 ++++++++++++++
drivers/crypto/ionic/meson.build | 13 ++
drivers/crypto/meson.build | 1 +
12 files changed, 463 insertions(+), 1 deletion(-)
create mode 100644 doc/guides/cryptodevs/features/ionic.ini
create mode 100644 doc/guides/cryptodevs/ionic.rst
create mode 100644 drivers/crypto/ionic/ionic_crypto.h
create mode 100644 drivers/crypto/ionic/ionic_crypto_main.c
create mode 100644 drivers/crypto/ionic/ionic_crypto_vdev.c
create mode 100644 drivers/crypto/ionic/meson.build
diff --git a/MAINTAINERS b/MAINTAINERS
index 7abb3aee49..7cf999371c 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1075,6 +1075,13 @@ F: drivers/crypto/ccp/
F: doc/guides/cryptodevs/ccp.rst
F: doc/guides/cryptodevs/features/ccp.ini
+AMD Pensando ionic crypto
+M: Andrew Boyer <andrew.boyer@amd.com>
+F: drivers/crypto/ionic/
+F: drivers/common/ionic/
+F: doc/guides/cryptodevs/ionic.rst
+F: doc/guides/cryptodevs/features/ionic.ini
+
ARMv8 Crypto
M: Ruifeng Wang <ruifeng.wang@arm.com>
F: drivers/crypto/armv8/
diff --git a/doc/guides/cryptodevs/features/ionic.ini b/doc/guides/cryptodevs/features/ionic.ini
new file mode 100644
index 0000000000..62b7e9e8f2
--- /dev/null
+++ b/doc/guides/cryptodevs/features/ionic.ini
@@ -0,0 +1,32 @@
+;
+; Supported features of the 'ionic' crypto driver.
+;
+; Refer to default.ini for the full list of available PMD features.
+;
+[Features]
+
+;
+; Supported crypto algorithms of 'ionic' crypto driver.
+;
+[Cipher]
+
+;
+; Supported authentication algorithms of 'ionic' crypto driver.
+;
+[Auth]
+
+;
+; Supported AEAD algorithms of 'ionic' crypto driver.
+;
+[AEAD]
+
+;
+; Supported Asymmetric algorithms of the 'ionic' crypto driver.
+;
+[Asymmetric]
+
+;
+; Supported Operating systems of the 'ionic' crypto driver.
+;
+[OS]
+Linux = Y
diff --git a/doc/guides/cryptodevs/index.rst b/doc/guides/cryptodevs/index.rst
index cb4ce227e9..1e57a9fe86 100644
--- a/doc/guides/cryptodevs/index.rst
+++ b/doc/guides/cryptodevs/index.rst
@@ -20,6 +20,7 @@ Crypto Device Drivers
cnxk
dpaa2_sec
dpaa_sec
+ ionic
kasumi
octeontx
openssl
diff --git a/doc/guides/cryptodevs/ionic.rst b/doc/guides/cryptodevs/ionic.rst
new file mode 100644
index 0000000000..c9173deb2f
--- /dev/null
+++ b/doc/guides/cryptodevs/ionic.rst
@@ -0,0 +1,28 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+ Copyright 2021-2024 Advanced Micro Devices, Inc.
+
+IONIC Crypto Driver
+===================
+
+The ionic crypto driver provides support for offloading cryptographic operations
+to hardware cryptographic blocks on AMD Pensando server adapters.
+It currently supports the below models:
+
+- DSC-25 dual-port 25G Distributed Services Card `(pdf) <https://pensandoio.secure.force.com/DownloadFile?id=a0L4T000004IKurUAG>`__
+- DSC-100 dual-port 100G Distributed Services Card `(pdf) <https://pensandoio.secure.force.com/DownloadFile?id=a0L4T000004IKuwUAG>`__
+- DSC-200 dual-port 200G Distributed Services Card `(pdf) <https://www.amd.com/system/files/documents/pensando-dsc-200-product-brief.pdf>`__
+
+Please visit the AMD Pensando web site at https://www.amd.com/en/accelerators/pensando for more information.
+
+Device Support
+--------------
+
+The ionic crypto PMD currently supports running directly on the device's embedded
+processors. It does not yet support host-side access via PCI.
+For help running the PMD, please contact AMD Pensando support.
+
+Runtime Configuration
+---------------------
+
+None
+
diff --git a/drivers/common/ionic/ionic_common.h b/drivers/common/ionic/ionic_common.h
index eb4850e24c..c4a15fdf2b 100644
--- a/drivers/common/ionic/ionic_common.h
+++ b/drivers/common/ionic/ionic_common.h
@@ -32,6 +32,8 @@ struct ionic_dev_bar {
__rte_internal
void ionic_uio_scan_mnet_devices(void);
+__rte_internal
+void ionic_uio_scan_mcrypt_devices(void);
__rte_internal
void ionic_uio_get_rsrc(const char *name, int idx, struct ionic_dev_bar *bar);
diff --git a/drivers/common/ionic/ionic_common_uio.c b/drivers/common/ionic/ionic_common_uio.c
index e5c73faf96..c647b22eaf 100644
--- a/drivers/common/ionic/ionic_common_uio.c
+++ b/drivers/common/ionic/ionic_common_uio.c
@@ -23,10 +23,12 @@
#define IONIC_MDEV_UNK "mdev_unknown"
#define IONIC_MNIC "cpu_mnic"
+#define IONIC_MCRYPT "cpu_mcrypt"
#define IONIC_MAX_NAME_LEN 20
#define IONIC_MAX_MNETS 5
-#define IONIC_MAX_DEVICES (IONIC_MAX_MNETS)
+#define IONIC_MAX_MCPTS 1
+#define IONIC_MAX_DEVICES (IONIC_MAX_MNETS + IONIC_MAX_MCPTS)
#define IONIC_MAX_U16_IDX 0xFFFF
#define IONIC_UIO_MAX_TRIES 32
@@ -49,6 +51,7 @@ struct ionic_map_tbl ionic_mdev_map[IONIC_MAX_DEVICES] = {
{ "net_ionic2", 2, IONIC_MAX_U16_IDX, IONIC_MDEV_UNK },
{ "net_ionic3", 3, IONIC_MAX_U16_IDX, IONIC_MDEV_UNK },
{ "net_ionic4", 4, IONIC_MAX_U16_IDX, IONIC_MDEV_UNK },
+ { "crypto_ionic0", 5, IONIC_MAX_U16_IDX, IONIC_MDEV_UNK },
};
struct uio_name {
@@ -143,6 +146,49 @@ ionic_uio_scan_mnet_devices(void)
}
}
+void
+ionic_uio_scan_mcrypt_devices(void)
+{
+ struct ionic_map_tbl *map;
+ char devname[IONIC_MAX_NAME_LEN];
+ struct uio_name name_cache[IONIC_MAX_DEVICES];
+ bool done;
+ int mdev_idx = 0;
+ int uio_idx;
+ int i;
+ static bool scan_done;
+
+ if (scan_done)
+ return;
+
+ scan_done = true;
+
+ uio_fill_name_cache(name_cache, IONIC_MCRYPT);
+
+ for (i = IONIC_MAX_MNETS; i < IONIC_MAX_DEVICES; i++) {
+ done = false;
+
+ while (!done) {
+ if (mdev_idx > IONIC_MAX_MDEV_SCAN)
+ break;
+
+ /* Look for a matching mcrypt */
+ snprintf(devname, IONIC_MAX_NAME_LEN,
+ IONIC_MCRYPT "%d", mdev_idx);
+ uio_idx = uio_get_idx_for_devname(name_cache, devname);
+ if (uio_idx >= 0) {
+ map = &ionic_mdev_map[i];
+ map->uio_idx = (uint16_t)uio_idx;
+ strlcpy(map->mdev_name, devname,
+ IONIC_MAX_NAME_LEN);
+ done = true;
+ }
+
+ mdev_idx++;
+ }
+ }
+}
+
static int
uio_get_multi_dev_uionum(const char *name)
{
diff --git a/drivers/common/ionic/version.map b/drivers/common/ionic/version.map
index 484330c437..db532d4ffc 100644
--- a/drivers/common/ionic/version.map
+++ b/drivers/common/ionic/version.map
@@ -2,6 +2,7 @@ INTERNAL {
global:
ionic_uio_scan_mnet_devices;
+ ionic_uio_scan_mcrypt_devices;
ionic_uio_get_rsrc;
ionic_uio_rel_rsrc;
diff --git a/drivers/crypto/ionic/ionic_crypto.h b/drivers/crypto/ionic/ionic_crypto.h
new file mode 100644
index 0000000000..86750f0cbd
--- /dev/null
+++ b/drivers/crypto/ionic/ionic_crypto.h
@@ -0,0 +1,92 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021-2024 Advanced Micro Devices, Inc.
+ */
+
+#ifndef _IONIC_CRYPTO_H_
+#define _IONIC_CRYPTO_H_
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <inttypes.h>
+
+#include <rte_common.h>
+#include <rte_dev.h>
+#include <rte_cryptodev.h>
+#include <cryptodev_pmd.h>
+#include <rte_log.h>
+
+#include "ionic_common.h"
+#include "ionic_regs.h"
+
+/* Devargs */
+/* NONE */
+
+extern int iocpt_logtype;
+#define RTE_LOGTYPE_IOCPT iocpt_logtype
+
+#define IOCPT_PRINT(level, ...) \
+ RTE_LOG_LINE_PREFIX(level, IOCPT, "%s(): ", __func__, __VA_ARGS__)
+
+#define IOCPT_PRINT_CALL() IOCPT_PRINT(DEBUG, " >>")
+
+struct iocpt_dev_bars {
+ struct ionic_dev_bar bar[IONIC_BARS_MAX];
+ uint32_t num_bars;
+};
+
+#define IOCPT_DEV_F_INITED BIT(0)
+#define IOCPT_DEV_F_UP BIT(1)
+#define IOCPT_DEV_F_FW_RESET BIT(2)
+
+/* Combined dev / LIF object */
+struct iocpt_dev {
+ const char *name;
+ struct iocpt_dev_bars bars;
+
+ const struct iocpt_dev_intf *intf;
+ void *bus_dev;
+ struct rte_cryptodev *crypto_dev;
+
+ uint32_t max_qps;
+ uint32_t max_sessions;
+ uint16_t state;
+ uint8_t driver_id;
+ uint8_t socket_id;
+
+ uint64_t features;
+ uint32_t hw_features;
+};
+
+struct iocpt_dev_intf {
+ int (*setup_bars)(struct iocpt_dev *dev);
+ void (*unmap_bars)(struct iocpt_dev *dev);
+};
+
+static inline int
+iocpt_setup_bars(struct iocpt_dev *dev)
+{
+ if (dev->intf->setup_bars == NULL)
+ return -EINVAL;
+
+ return (*dev->intf->setup_bars)(dev);
+}
+
+int iocpt_probe(void *bus_dev, struct rte_device *rte_dev,
+ struct iocpt_dev_bars *bars, const struct iocpt_dev_intf *intf,
+ uint8_t driver_id, uint8_t socket_id);
+int iocpt_remove(struct rte_device *rte_dev);
+
+void iocpt_configure(struct iocpt_dev *dev);
+void iocpt_deinit(struct iocpt_dev *dev);
+
+static inline bool
+iocpt_is_embedded(void)
+{
+#if defined(RTE_LIBRTE_IONIC_PMD_EMBEDDED)
+ return true;
+#else
+ return false;
+#endif
+}
+
+#endif /* _IONIC_CRYPTO_H_ */
diff --git a/drivers/crypto/ionic/ionic_crypto_main.c b/drivers/crypto/ionic/ionic_crypto_main.c
new file mode 100644
index 0000000000..ecbb1cb161
--- /dev/null
+++ b/drivers/crypto/ionic/ionic_crypto_main.c
@@ -0,0 +1,148 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021-2024 Advanced Micro Devices, Inc.
+ */
+
+#include <inttypes.h>
+
+#include <rte_common.h>
+#include <rte_malloc.h>
+#include <rte_bitops.h>
+
+#include "ionic_crypto.h"
+
+static int
+iocpt_init(struct iocpt_dev *dev)
+{
+ dev->state |= IOCPT_DEV_F_INITED;
+
+ return 0;
+}
+
+void
+iocpt_configure(struct iocpt_dev *dev)
+{
+ RTE_SET_USED(dev);
+}
+
+void
+iocpt_deinit(struct iocpt_dev *dev)
+{
+ IOCPT_PRINT_CALL();
+
+ if (!(dev->state & IOCPT_DEV_F_INITED))
+ return;
+
+ dev->state &= ~IOCPT_DEV_F_INITED;
+}
+
+static int
+iocpt_devargs(struct rte_devargs *devargs, struct iocpt_dev *dev)
+{
+ RTE_SET_USED(devargs);
+ RTE_SET_USED(dev);
+
+ return 0;
+}
+
+int
+iocpt_probe(void *bus_dev, struct rte_device *rte_dev,
+ struct iocpt_dev_bars *bars, const struct iocpt_dev_intf *intf,
+ uint8_t driver_id, uint8_t socket_id)
+{
+ struct rte_cryptodev_pmd_init_params init_params = {
+ "iocpt",
+ sizeof(struct iocpt_dev),
+ socket_id,
+ RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
+ };
+ struct rte_cryptodev *cdev;
+ struct iocpt_dev *dev;
+ uint32_t i;
+ int err;
+
+ /* Multi-process not supported */
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+ err = -EPERM;
+ goto err;
+ }
+
+ cdev = rte_cryptodev_pmd_create(rte_dev->name, rte_dev, &init_params);
+ if (cdev == NULL) {
+ IOCPT_PRINT(ERR, "OOM");
+ err = -ENOMEM;
+ goto err;
+ }
+
+ dev = cdev->data->dev_private;
+ dev->crypto_dev = cdev;
+ dev->bus_dev = bus_dev;
+ dev->intf = intf;
+ dev->driver_id = driver_id;
+ dev->socket_id = socket_id;
+
+ for (i = 0; i < bars->num_bars; i++) {
+ struct ionic_dev_bar *bar = &bars->bar[i];
+
+ IOCPT_PRINT(DEBUG,
+ "bar[%u] = { .va = %p, .pa = %#jx, .len = %lu }",
+ i, bar->vaddr, bar->bus_addr, bar->len);
+ if (bar->vaddr == NULL) {
+ IOCPT_PRINT(ERR, "Null bar found, aborting");
+ err = -EFAULT;
+ goto err_destroy_crypto_dev;
+ }
+
+ dev->bars.bar[i].vaddr = bar->vaddr;
+ dev->bars.bar[i].bus_addr = bar->bus_addr;
+ dev->bars.bar[i].len = bar->len;
+ }
+ dev->bars.num_bars = bars->num_bars;
+
+ err = iocpt_devargs(rte_dev->devargs, dev);
+ if (err != 0) {
+ IOCPT_PRINT(ERR, "Cannot parse device arguments");
+ goto err_destroy_crypto_dev;
+ }
+
+ err = iocpt_setup_bars(dev);
+ if (err != 0) {
+ IOCPT_PRINT(ERR, "Cannot setup BARs: %d, aborting", err);
+ goto err_destroy_crypto_dev;
+ }
+
+ err = iocpt_init(dev);
+ if (err != 0) {
+ IOCPT_PRINT(ERR, "Cannot init device: %d, aborting", err);
+ goto err_destroy_crypto_dev;
+ }
+
+ return 0;
+
+err_destroy_crypto_dev:
+ rte_cryptodev_pmd_destroy(cdev);
+err:
+ return err;
+}
+
+int
+iocpt_remove(struct rte_device *rte_dev)
+{
+ struct rte_cryptodev *cdev;
+ struct iocpt_dev *dev;
+
+ cdev = rte_cryptodev_pmd_get_named_dev(rte_dev->name);
+ if (cdev == NULL) {
+ IOCPT_PRINT(DEBUG, "Cannot find device %s", rte_dev->name);
+ return -ENODEV;
+ }
+
+ dev = cdev->data->dev_private;
+
+ iocpt_deinit(dev);
+
+ rte_cryptodev_pmd_destroy(cdev);
+
+ return 0;
+}
+
+RTE_LOG_REGISTER_DEFAULT(iocpt_logtype, NOTICE);
diff --git a/drivers/crypto/ionic/ionic_crypto_vdev.c b/drivers/crypto/ionic/ionic_crypto_vdev.c
new file mode 100644
index 0000000000..a915aa06aa
--- /dev/null
+++ b/drivers/crypto/ionic/ionic_crypto_vdev.c
@@ -0,0 +1,91 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021-2024 Advanced Micro Devices, Inc.
+ */
+
+#include <stdint.h>
+#include <errno.h>
+
+#include <rte_errno.h>
+#include <rte_common.h>
+#include <rte_log.h>
+#include <rte_eal.h>
+#include <bus_vdev_driver.h>
+#include <rte_dev.h>
+#include <rte_string_fns.h>
+
+#include "ionic_crypto.h"
+
+#define IOCPT_VDEV_DEV_BAR 0
+#define IOCPT_VDEV_INTR_CTL_BAR 1
+#define IOCPT_VDEV_INTR_CFG_BAR 2
+#define IOCPT_VDEV_DB_BAR 3
+#define IOCPT_VDEV_BARS_MAX 4
+
+#define IOCPT_VDEV_DEV_INFO_REGS_OFFSET 0x0000
+#define IOCPT_VDEV_DEV_CMD_REGS_OFFSET 0x0800
+
+static int
+iocpt_vdev_setup_bars(struct iocpt_dev *dev)
+{
+ IOCPT_PRINT_CALL();
+
+ dev->name = rte_vdev_device_name(dev->bus_dev);
+
+ return 0;
+}
+
+static void
+iocpt_vdev_unmap_bars(struct iocpt_dev *dev)
+{
+ struct iocpt_dev_bars *bars = &dev->bars;
+ uint32_t i;
+
+ for (i = 0; i < IOCPT_VDEV_BARS_MAX; i++)
+ ionic_uio_rel_rsrc(dev->name, i, &bars->bar[i]);
+}
+
+static uint8_t iocpt_vdev_driver_id;
+static const struct iocpt_dev_intf iocpt_vdev_intf = {
+ .setup_bars = iocpt_vdev_setup_bars,
+ .unmap_bars = iocpt_vdev_unmap_bars,
+};
+
+static int
+iocpt_vdev_probe(struct rte_vdev_device *vdev)
+{
+ struct iocpt_dev_bars bars = {};
+ const char *name = rte_vdev_device_name(vdev);
+ unsigned int i;
+
+ IOCPT_PRINT(NOTICE, "Initializing device %s%s", name,
+ rte_eal_process_type() == RTE_PROC_SECONDARY ?
+ " [SECONDARY]" : "");
+
+ ionic_uio_scan_mcrypt_devices();
+
+ for (i = 0; i < IOCPT_VDEV_BARS_MAX; i++)
+ ionic_uio_get_rsrc(name, i, &bars.bar[i]);
+
+ bars.num_bars = IOCPT_VDEV_BARS_MAX;
+
+ return iocpt_probe((void *)vdev, &vdev->device,
+ &bars, &iocpt_vdev_intf,
+ iocpt_vdev_driver_id, rte_socket_id());
+}
+
+static int
+iocpt_vdev_remove(struct rte_vdev_device *vdev)
+{
+ return iocpt_remove(&vdev->device);
+}
+
+static struct rte_vdev_driver rte_vdev_iocpt_pmd = {
+ .probe = iocpt_vdev_probe,
+ .remove = iocpt_vdev_remove,
+};
+
+static struct cryptodev_driver rte_vdev_iocpt_drv;
+
+RTE_PMD_REGISTER_VDEV(crypto_ionic, rte_vdev_iocpt_pmd);
+RTE_PMD_REGISTER_CRYPTO_DRIVER(rte_vdev_iocpt_drv, rte_vdev_iocpt_pmd.driver,
+ iocpt_vdev_driver_id);
diff --git a/drivers/crypto/ionic/meson.build b/drivers/crypto/ionic/meson.build
new file mode 100644
index 0000000000..4114e13e53
--- /dev/null
+++ b/drivers/crypto/ionic/meson.build
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2021-2024 Advanced Micro Devices, Inc.
+
+deps += ['bus_vdev']
+deps += ['common_ionic']
+
+sources = files(
+ 'ionic_crypto_main.c',
+ 'ionic_crypto_vdev.c',
+)
+name = 'ionic_crypto'
+
+includes += include_directories('../../common/ionic')
diff --git a/drivers/crypto/meson.build b/drivers/crypto/meson.build
index ee5377deff..e799861bb6 100644
--- a/drivers/crypto/meson.build
+++ b/drivers/crypto/meson.build
@@ -10,6 +10,7 @@ drivers = [
'cnxk',
'dpaa_sec',
'dpaa2_sec',
+ 'ionic',
'ipsec_mb',
'mlx5',
'mvsam',
--
2.17.1
^ permalink raw reply [flat|nested] 19+ messages in thread* [PATCH v2 2/9] crypto/ionic: add the firmware interface definition file
2024-04-30 20:21 ` [PATCH v2 0/9] " Andrew Boyer
2024-04-30 20:21 ` [PATCH v2 1/9] " Andrew Boyer
@ 2024-04-30 20:21 ` Andrew Boyer
2024-04-30 20:21 ` [PATCH v2 3/9] crypto/ionic: add device commands Andrew Boyer
` (6 subsequent siblings)
8 siblings, 0 replies; 19+ messages in thread
From: Andrew Boyer @ 2024-04-30 20:21 UTC (permalink / raw)
To: dev; +Cc: Andrew Boyer
These definitions are shared between the PMD and firmware.
Teach the device to read the FW version and check for liveness.
Signed-off-by: Andrew Boyer <andrew.boyer@amd.com>
---
drivers/crypto/ionic/ionic_crypto.h | 53 ++
drivers/crypto/ionic/ionic_crypto_if.h | 1021 ++++++++++++++++++++++
drivers/crypto/ionic/ionic_crypto_main.c | 17 +-
drivers/crypto/ionic/ionic_crypto_vdev.c | 37 +
4 files changed, 1127 insertions(+), 1 deletion(-)
create mode 100644 drivers/crypto/ionic/ionic_crypto_if.h
diff --git a/drivers/crypto/ionic/ionic_crypto.h b/drivers/crypto/ionic/ionic_crypto.h
index 86750f0cbd..7f456b5154 100644
--- a/drivers/crypto/ionic/ionic_crypto.h
+++ b/drivers/crypto/ionic/ionic_crypto.h
@@ -16,6 +16,7 @@
#include <rte_log.h>
#include "ionic_common.h"
+#include "ionic_crypto_if.h"
#include "ionic_regs.h"
/* Devargs */
@@ -29,6 +30,51 @@ extern int iocpt_logtype;
#define IOCPT_PRINT_CALL() IOCPT_PRINT(DEBUG, " >>")
+static inline void iocpt_struct_size_checks(void)
+{
+ RTE_BUILD_BUG_ON(sizeof(struct ionic_doorbell) != 8);
+ RTE_BUILD_BUG_ON(sizeof(struct ionic_intr) != 32);
+ RTE_BUILD_BUG_ON(sizeof(struct ionic_intr_status) != 8);
+
+ RTE_BUILD_BUG_ON(sizeof(union iocpt_dev_regs) != 4096);
+ RTE_BUILD_BUG_ON(sizeof(union iocpt_dev_info_regs) != 2048);
+ RTE_BUILD_BUG_ON(sizeof(union iocpt_dev_cmd_regs) != 2048);
+
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_admin_cmd) != 64);
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_admin_comp) != 16);
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_nop_cmd) != 64);
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_nop_comp) != 16);
+
+ /* Device commands */
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_dev_identify_cmd) != 64);
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_dev_identify_comp) != 16);
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_dev_reset_cmd) != 64);
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_dev_reset_comp) != 16);
+
+ /* LIF commands */
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_lif_identify_cmd) != 64);
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_lif_identify_comp) != 16);
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_lif_init_cmd) != 64);
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_lif_init_comp) != 16);
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_lif_reset_cmd) != 64);
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_lif_getattr_cmd) != 64);
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_lif_getattr_comp) != 16);
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_lif_setattr_cmd) != 64);
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_lif_setattr_comp) != 16);
+
+ /* Queue commands */
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_q_identify_cmd) != 64);
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_q_identify_comp) != 16);
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_q_init_cmd) != 64);
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_q_init_comp) != 16);
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_q_control_cmd) != 64);
+
+ /* Crypto */
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_crypto_desc) != 32);
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_crypto_sg_desc) != 256);
+ RTE_BUILD_BUG_ON(sizeof(struct iocpt_crypto_comp) != 16);
+}
+
struct iocpt_dev_bars {
struct ionic_dev_bar bar[IONIC_BARS_MAX];
uint32_t num_bars;
@@ -41,12 +87,19 @@ struct iocpt_dev_bars {
/* Combined dev / LIF object */
struct iocpt_dev {
const char *name;
+ char fw_version[IOCPT_FWVERS_BUFLEN];
struct iocpt_dev_bars bars;
const struct iocpt_dev_intf *intf;
void *bus_dev;
struct rte_cryptodev *crypto_dev;
+ union iocpt_dev_info_regs __iomem *dev_info;
+ union iocpt_dev_cmd_regs __iomem *dev_cmd;
+
+ struct ionic_doorbell __iomem *db_pages;
+ struct ionic_intr __iomem *intr_ctrl;
+
uint32_t max_qps;
uint32_t max_sessions;
uint16_t state;
diff --git a/drivers/crypto/ionic/ionic_crypto_if.h b/drivers/crypto/ionic/ionic_crypto_if.h
new file mode 100644
index 0000000000..ea418f3d4b
--- /dev/null
+++ b/drivers/crypto/ionic/ionic_crypto_if.h
@@ -0,0 +1,1021 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021-2024 Advanced Micro Devices, Inc.
+ */
+
+#ifndef _IONIC_CRYPTO_IF_H_
+#define _IONIC_CRYPTO_IF_H_
+
+#define IOCPT_DEV_INFO_SIGNATURE 0x43585660 /* 'CRPT' */
+#define IOCPT_DEV_INFO_VERSION 1
+#define IOCPT_IFNAMSIZ 16
+
+/**
+ * enum iocpt_cmd_opcode - Device commands
+ */
+enum iocpt_cmd_opcode {
+ IOCPT_CMD_NOP = 0, /* D, A */
+
+ /* Device commands */
+ IOCPT_CMD_IDENTIFY = 1, /* D */
+ IOCPT_CMD_RESET = 3, /* D */
+
+ /* LIF commands */
+ IOCPT_CMD_LIF_IDENTIFY = 20, /* D */
+ IOCPT_CMD_LIF_INIT = 21, /* D */
+ IOCPT_CMD_LIF_RESET = 22, /* D */
+ IOCPT_CMD_LIF_GETATTR = 23, /* D, A */
+ IOCPT_CMD_LIF_SETATTR = 24, /* D, A */
+
+ /* Queue commands */
+ IOCPT_CMD_Q_IDENTIFY = 39, /* D, A */
+ IOCPT_CMD_Q_INIT = 40, /* D, A */
+ IOCPT_CMD_Q_CONTROL = 41, /* D, A */
+
+ /* Session commands */
+ IOCPT_CMD_SESS_CONTROL = 45, /* D, A */
+};
+
+/**
+ * enum iocpt_status_code - Device command return codes
+ */
+enum iocpt_status_code {
+ IOCPT_RC_SUCCESS = 0, /* Success */
+ IOCPT_RC_EVERSION = 1, /* Incorrect version for request */
+ IOCPT_RC_EOPCODE = 2, /* Invalid cmd opcode */
+ IOCPT_RC_EIO = 3, /* I/O error */
+ IOCPT_RC_EPERM = 4, /* Permission denied */
+ IOCPT_RC_EQID = 5, /* Bad qid */
+ IOCPT_RC_EQTYPE = 6, /* Bad qtype */
+ IOCPT_RC_ENOENT = 7, /* No such element */
+ IOCPT_RC_EINTR = 8, /* Operation interrupted */
+ IOCPT_RC_EAGAIN = 9, /* Try again */
+ IOCPT_RC_ENOMEM = 10, /* Out of memory */
+ IOCPT_RC_EFAULT = 11, /* Bad address */
+ IOCPT_RC_EBUSY = 12, /* Device or resource busy */
+ IOCPT_RC_EEXIST = 13, /* Object already exists */
+ IOCPT_RC_EINVAL = 14, /* Invalid argument */
+ IOCPT_RC_ENOSPC = 15, /* No space left or alloc failure */
+ IOCPT_RC_ERANGE = 16, /* Parameter out of range */
+ IOCPT_RC_BAD_ADDR = 17, /* Descriptor contains a bad ptr */
+ IOCPT_RC_DEV_CMD = 18, /* Device cmd attempted on AdminQ */
+ IOCPT_RC_ENOSUPP = 19, /* Operation not supported */
+ IOCPT_RC_ERROR = 29, /* Generic error */
+};
+
+enum iocpt_notifyq_opcode {
+ IOCPT_EVENT_RESET = 1,
+ IOCPT_EVENT_HEARTBEAT = 2,
+ IOCPT_EVENT_LOG = 3,
+};
+
+enum iocpt_lif_type {
+ IOCPT_LIF_TYPE_DEFAULT = 0,
+};
+
+/**
+ * struct iocpt_admin_cmd - General admin command format
+ * @opcode: Opcode for the command
+ * @lif_index: LIF index
+ * @cmd_data: Opcode-specific command bytes
+ */
+struct iocpt_admin_cmd {
+ u8 opcode;
+ u8 rsvd;
+ __le16 lif_index;
+ u8 cmd_data[60];
+};
+
+/**
+ * struct iocpt_admin_comp - General admin command completion format
+ * @status: Status of the command (enum iocpt_status_code)
+ * @comp_index: Index in the descriptor ring for which this is the completion
+ * @cmd_data: Command-specific bytes
+ * @color: Color bit (Always 0 for commands issued to the
+ * Device Cmd Registers)
+ */
+struct iocpt_admin_comp {
+ u8 status;
+ u8 rsvd;
+ __le16 comp_index;
+ u8 cmd_data[11];
+ u8 color;
+#define IOCPT_COMP_COLOR_MASK 0x80
+};
+
+static inline u8 iocpt_color_match(u8 color, u8 done_color)
+{
+ return (!!(color & IOCPT_COMP_COLOR_MASK)) == done_color;
+}
+
+/**
+ * struct iocpt_nop_cmd - NOP command
+ * @opcode: Opcode
+ */
+struct iocpt_nop_cmd {
+ u8 opcode;
+ u8 rsvd[63];
+};
+
+/**
+ * struct iocpt_nop_comp - NOP command completion
+ * @status: Status of the command (enum iocpt_status_code)
+ */
+struct iocpt_nop_comp {
+ u8 status;
+ u8 rsvd[15];
+};
+
+#define IOCPT_IDENTITY_VERSION_1 1
+
+/**
+ * struct iocpt_dev_identify_cmd - Driver/device identify command
+ * @opcode: Opcode
+ * @ver: Highest version of identify supported by driver
+ */
+struct iocpt_dev_identify_cmd {
+ u8 opcode;
+ u8 ver;
+ u8 rsvd[62];
+};
+
+/**
+ * struct iocpt_dev_identify_comp - Device identify command completion
+ * @status: Status of the command (enum iocpt_status_code)
+ * @ver: Version of identify returned by device
+ */
+struct iocpt_dev_identify_comp {
+ u8 status;
+ u8 ver;
+ u8 rsvd[14];
+};
+
+/**
+ * struct iocpt_dev_reset_cmd - Device reset command
+ * Will reset all LIFs on the device.
+ * @opcode: Opcode
+ */
+struct iocpt_dev_reset_cmd {
+ u8 opcode;
+ u8 rsvd[63];
+};
+
+/**
+ * struct iocpt_dev_reset_comp - Reset command completion
+ * @status: Status of the command (enum iocpt_status_code)
+ */
+struct iocpt_dev_reset_comp {
+ u8 status;
+ u8 rsvd[15];
+};
+
+/**
+ * struct iocpt_lif_identify_cmd - LIF identify command
+ * @opcode: Opcode
+ * @type: LIF type (enum iocpt_lif_type)
+ * @lif_index: LIF index
+ * @ver: Version of identify returned by device
+ */
+struct iocpt_lif_identify_cmd {
+ u8 opcode;
+ u8 type;
+ __le16 lif_index;
+ u8 ver;
+ u8 rsvd[59];
+};
+
+/**
+ * struct iocpt_lif_identify_comp - LIF identify command completion
+ * @status: Status of the command (enum iocpt_status_code)
+ * @ver: Version of identify returned by device
+ */
+struct iocpt_lif_identify_comp {
+ u8 status;
+ u8 ver;
+ u8 rsvd2[14];
+};
+
+/**
+ * struct iocpt_lif_init_cmd - LIF init command
+ * @opcode: Opcode
+ * @type: LIF type (enum iocpt_lif_type)
+ * @lif_index: LIF index
+ * @info_pa: Destination address for LIF info (struct iocpt_lif_info)
+ */
+struct iocpt_lif_init_cmd {
+ u8 opcode;
+ u8 type;
+ __le16 lif_index;
+ __le32 rsvd;
+ __le64 info_pa;
+ u8 rsvd2[48];
+};
+
+/**
+ * struct iocpt_lif_init_comp - LIF init command completion
+ * @status: Status of the command (enum iocpt_status_code)
+ * @hw_index: Hardware index of the initialized LIF
+ */
+struct iocpt_lif_init_comp {
+ u8 status;
+ u8 rsvd;
+ __le16 hw_index;
+ u8 rsvd2[12];
+};
+
+/**
+ * struct iocpt_lif_reset_cmd - LIF reset command
+ * Will reset only the specified LIF.
+ * @opcode: Opcode
+ * @lif_index: LIF index
+ */
+struct iocpt_lif_reset_cmd {
+ u8 opcode;
+ u8 rsvd;
+ __le16 lif_index;
+ __le32 rsvd2[15];
+};
+
+/**
+ * enum iocpt_lif_attr - List of LIF attributes
+ * @IOCPT_LIF_ATTR_STATE: LIF state attribute
+ * @IOCPT_LIF_ATTR_NAME: LIF name attribute
+ * @IOCPT_LIF_ATTR_FEATURES: LIF features attribute
+ * @IOCPT_LIF_ATTR_STATS_CTRL: LIF statistics control attribute
+ */
+enum iocpt_lif_attr {
+ IOCPT_LIF_ATTR_STATE = 0,
+ IOCPT_LIF_ATTR_NAME = 1,
+ IOCPT_LIF_ATTR_FEATURES = 4,
+ IOCPT_LIF_ATTR_STATS_CTRL = 6,
+};
+
+/**
+ * struct iocpt_lif_setattr_cmd - Set LIF attributes on the NIC
+ * @opcode: Opcode
+ * @attr: Attribute type (enum iocpt_lif_attr)
+ * @lif_index: LIF index
+ * @state: LIF state (enum iocpt_lif_state)
+ * @name: The name string, 0 terminated
+ * @features: Features (enum iocpt_hw_features)
+ * @stats_ctl: Stats control commands (enum iocpt_stats_ctl_cmd)
+ */
+struct iocpt_lif_setattr_cmd {
+ u8 opcode;
+ u8 attr;
+ __le16 lif_index;
+ union {
+ u8 state;
+ char name[IOCPT_IFNAMSIZ];
+ __le64 features;
+ u8 stats_ctl;
+ u8 rsvd[60];
+ } __rte_packed;
+};
+
+/**
+ * struct iocpt_lif_setattr_comp - LIF set attr command completion
+ * @status: Status of the command (enum iocpt_status_code)
+ * @comp_index: Index in the descriptor ring for which this is the completion
+ * @features: Features (enum iocpt_hw_features)
+ * @color: Color bit
+ */
+struct iocpt_lif_setattr_comp {
+ u8 status;
+ u8 rsvd;
+ __le16 comp_index;
+ union {
+ __le64 features;
+ u8 rsvd2[11];
+ } __rte_packed;
+ u8 color;
+};
+
+/**
+ * struct iocpt_lif_getattr_cmd - Get LIF attributes from the NIC
+ * @opcode: Opcode
+ * @attr: Attribute type (enum iocpt_lif_attr)
+ * @lif_index: LIF index
+ */
+struct iocpt_lif_getattr_cmd {
+ u8 opcode;
+ u8 attr;
+ __le16 lif_index;
+ u8 rsvd[60];
+};
+
+/**
+ * struct iocpt_lif_getattr_comp - LIF get attr command completion
+ * @status: Status of the command (enum iocpt_status_code)
+ * @comp_index: Index in the descriptor ring for which this is the completion
+ * @state: LIF state (enum iocpt_lif_state)
+ * @name: LIF name string, 0 terminated
+ * @features: Features (enum iocpt_hw_features)
+ * @color: Color bit
+ */
+struct iocpt_lif_getattr_comp {
+ u8 status;
+ u8 rsvd;
+ __le16 comp_index;
+ union {
+ u8 state;
+ __le64 features;
+ u8 rsvd2[11];
+ } __rte_packed;
+ u8 color;
+};
+
+/**
+ * enum iocpt_logical_qtype - Logical Queue Types
+ * @IOCPT_QTYPE_ADMINQ: Administrative Queue
+ * @IOCPT_QTYPE_NOTIFYQ: Notify Queue
+ * @IOCPT_QTYPE_CRYPTOQ: Cryptographic Queue
+ * @IOCPT_QTYPE_MAX: Max queue type supported
+ */
+enum iocpt_logical_qtype {
+ IOCPT_QTYPE_ADMINQ = 0,
+ IOCPT_QTYPE_NOTIFYQ = 1,
+ IOCPT_QTYPE_CRYPTOQ = 2,
+ IOCPT_QTYPE_MAX = 8,
+};
+
+/**
+ * struct iocpt_q_identify_cmd - queue identify command
+ * @opcode: Opcode
+ * @type: Logical queue type (enum iocpt_logical_qtype)
+ * @lif_index: LIF index
+ * @ver: Highest queue type version that the driver supports
+ */
+struct iocpt_q_identify_cmd {
+ u8 opcode;
+ u8 type;
+ __le16 lif_index;
+ u8 ver;
+ u8 rsvd2[59];
+};
+
+/**
+ * struct iocpt_q_identify_comp - queue identify command completion
+ * @status: Status of the command (enum iocpt_status_code)
+ * @comp_index: Index in the descriptor ring for which this is the completion
+ * @ver: Queue type version that can be used with FW
+ */
+struct iocpt_q_identify_comp {
+ u8 status;
+ u8 rsvd;
+ __le16 comp_index;
+ u8 ver;
+ u8 rsvd2[11];
+};
+
+/**
+ * struct iocpt_q_init_cmd - Queue init command
+ * @opcode: Opcode
+ * @type: Logical queue type
+ * @lif_index: LIF index
+ * @ver: Queue type version
+ * @index: (LIF, qtype) relative admin queue index
+ * @intr_index: Interrupt control register index, or Event queue index
+ * @pid: Process ID
+ * @flags:
+ * IRQ: Interrupt requested on completion
+ * ENA: Enable the queue. If ENA=0 the queue is initialized
+ * but remains disabled, to be later enabled with the
+ * Queue Enable command. If ENA=1, then queue is
+ * initialized and then enabled.
+ * SG: Enable Scatter-Gather on the queue.
+ * @cos: Class of service for this queue
+ * @ring_size: Queue ring size, encoded as a log2(size), in
+ * number of descriptors. The actual ring size is
+ * (1 << ring_size). For example, to select a ring size
+ * of 64 descriptors write ring_size = 6. The minimum
+ * ring_size value is 2 for a ring of 4 descriptors.
+ * The maximum ring_size value is 12 for a ring of 4k
+ * descriptors. Values of ring_size <2 and >12 are
+ * reserved.
+ * @ring_base: Queue ring base address
+ * @cq_ring_base: Completion queue ring base address
+ * @sg_ring_base: Scatter/Gather ring base address
+ */
+struct iocpt_q_init_cmd {
+ u8 opcode;
+ u8 type;
+ __le16 lif_index;
+ u8 ver;
+ u8 rsvd[3];
+ __le32 index;
+ __le16 pid;
+ __le16 intr_index;
+ __le16 flags;
+#define IOCPT_QINIT_F_IRQ 0x01 /* Request interrupt on completion */
+#define IOCPT_QINIT_F_ENA 0x02 /* Enable the queue */
+#define IOCPT_QINIT_F_SG 0x04 /* Enable scatter/gather on queue */
+ u8 cos;
+#define IOCPT_QSIZE_MIN_LG2 2
+#define IOCPT_QSIZE_MAX_LG2 12
+ u8 ring_size;
+ __le64 ring_base;
+ __le64 cq_ring_base;
+ __le64 sg_ring_base;
+ u8 rsvd2[20];
+} __rte_packed;
+
+/**
+ * struct iocpt_q_init_comp - Queue init command completion
+ * @status: Status of the command (enum iocpt_status_code)
+ * @comp_index: Index in the descriptor ring for which this is the completion
+ * @hw_index: Hardware Queue ID
+ * @hw_type: Hardware Queue type
+ * @color: Color
+ */
+struct iocpt_q_init_comp {
+ u8 status;
+ u8 rsvd;
+ __le16 comp_index;
+ __le32 hw_index;
+ u8 hw_type;
+ u8 rsvd2[6];
+ u8 color;
+};
+
+enum iocpt_desc_opcode {
+ IOCPT_DESC_OPCODE_GCM_AEAD_ENCRYPT = 0,
+ IOCPT_DESC_OPCODE_GCM_AEAD_DECRYPT = 1,
+ IOCPT_DESC_OPCODE_XTS_ENCRYPT = 2,
+ IOCPT_DESC_OPCODE_XTS_DECRYPT = 3,
+};
+
+#define IOCPT_DESC_F_AAD_VALID 0x1
+
+/**
+ * struct iocpt_desc - Crypto queue descriptor format
+ * @opcode:
+ * IOCPT_DESC_OPCODE_GCM_AEAD_ENCRYPT:
+ * Perform a GCM-AES encrypt operation
+ *
+ * IOCPT_DESC_OPCODE_GCM_AEAD_DECRYPT:
+ * Perform a GCM-AES decrypt operation
+ *
+ * IOCPT_DESC_OPCODE_XTS_ENCRYPT:
+ * Perform an XTS encrypt operation
+ *
+ * IOCPT_DESC_OPCODE_XTS_DECRYPT:
+ * Perform an XTS decrypt operation
+ * @flags:
+ * IOCPT_DESC_F_AAD_VALID:
+ * Source SGL contains an AAD addr and length
+ * @num_src_dst_sgs: Number of scatter-gather elements in SG
+ * descriptor (4 bits for source, 4 bits for destination)
+ * @session_tag: Session tag (key index)
+ * @intr_ctx_addr: Completion interrupt context address
+ * @intr_ctx_data: Completion interrupt context data
+ */
+struct iocpt_crypto_desc {
+ uint8_t opcode;
+ uint8_t flags;
+ uint8_t num_src_dst_sgs;
+#define IOCPT_DESC_NSGE_SRC_MASK 0xf
+#define IOCPT_DESC_NSGE_SRC_SHIFT 0
+#define IOCPT_DESC_NSGE_DST_MASK 0xf
+#define IOCPT_DESC_NSGE_DST_SHIFT 4
+ uint8_t rsvd[9];
+ __le32 session_tag;
+ __le64 intr_ctx_addr;
+ __le64 intr_ctx_data;
+} __rte_packed;
+
+static inline uint8_t iocpt_encode_nsge_src_dst(uint8_t src, uint8_t dst)
+{
+ uint8_t nsge_src_dst;
+
+ nsge_src_dst = (src & IOCPT_DESC_NSGE_SRC_MASK) <<
+ IOCPT_DESC_NSGE_SRC_SHIFT;
+ nsge_src_dst |= (dst & IOCPT_DESC_NSGE_DST_MASK) <<
+ IOCPT_DESC_NSGE_DST_SHIFT;
+
+ return nsge_src_dst;
+};
+
+static inline void iocpt_decode_nsge_src_dst(uint8_t nsge_src_dst,
+ uint8_t *src, uint8_t *dst)
+{
+ *src = (nsge_src_dst >> IOCPT_DESC_NSGE_SRC_SHIFT) &
+ IOCPT_DESC_NSGE_SRC_MASK;
+ *dst = (nsge_src_dst >> IOCPT_DESC_NSGE_DST_SHIFT) &
+ IOCPT_DESC_NSGE_DST_MASK;
+};
+
+/**
+ * struct iocpt_crypto_sg_elem - Crypto scatter-gather (SG) descriptor element
+ * @addr: DMA address of SG element data buffer
+ * @len: Length of SG element data buffer, in bytes
+ */
+struct iocpt_crypto_sg_elem {
+ __le64 addr;
+ __le16 len;
+ uint8_t rsvd[6];
+};
+
+/**
+ * struct iocpt_crypto_sg_desc - Crypto scatter-gather (SG) list
+ * @src_elems: Source SG elements; also destination in IP case
+ * AES_GCM:
+ * SGE0: Nonce
+ * SGE1: AAD (see IOCPT_DESC_F_AAD_VALID)
+ * SGE2 to SGE(N): Payload
+ * SGE(N+1): Auth tag
+ * @dst_elems: Destination SG elements for OOP case; unused in IP case
+ */
+struct iocpt_crypto_sg_desc {
+#define IOCPT_CRYPTO_MAX_SG_ELEMS 8
+#define IOCPT_CRYPTO_NONCE_ELEM 0
+#define IOCPT_CRYPTO_AAD_ELEM 1
+ struct iocpt_crypto_sg_elem src_elems[IOCPT_CRYPTO_MAX_SG_ELEMS];
+ struct iocpt_crypto_sg_elem dst_elems[IOCPT_CRYPTO_MAX_SG_ELEMS];
+};
+
+/**
+ * struct iocpt_crypto_comp - Crypto queue completion descriptor
+ * @status: Status of the command (enum iocpt_status_code)
+ * @comp_index: Index in the descriptor ring for which this is the completion
+ * @color: Color bit
+ */
+struct iocpt_crypto_comp {
+#define IOCPT_COMP_SUCCESS 0
+#define IOCPT_COMP_INVAL_OPCODE_ERROR 1
+#define IOCPT_COMP_UNSUPP_OPCODE_ERROR 2
+#define IOCPT_COMP_SYMM_SRC_SG_ERROR 3
+#define IOCPT_COMP_SYMM_DST_SG_ERROR 4
+#define IOCPT_COMP_SYMM_SRC_DST_LEN_MISMATCH 5
+#define IOCPT_COMP_SYMM_HW_QAVAIL_ERROR 6
+#define IOCPT_COMP_SYMM_AUTH_VERIFY_ERROR 7
+#define IOCPT_COMP_SYMM_OTHER_VERIFY_ERROR 8
+#define IOCPT_COMP_SYMM_PI_MODE_CHKSUM_ERROR 9
+#define IOCPT_COMP_SYMM_HARDWARE_ERROR 10
+#define IOCPT_COMP_SYMM_KEY_IDX_ERROR 11
+ u8 status;
+ u8 rsvd;
+ __le16 comp_index;
+ u8 rsvd2[11];
+ u8 color;
+};
+
+/**
+ * enum iocpt_hw_features - Feature flags supported by hardware
+ * @IOCPT_HW_SYM: Symmetric crypto operations
+ * @IOCPT_HW_ASYM: Asymmetric crypto operations
+ * @IOCPT_HW_CHAIN: Chained crypto operations
+ * @IOCPT_HW_IP: In-Place (destination same as source)
+ * @IOCPT_HW_OOP: Out-Of-Place (destination differs from source)
+ */
+enum iocpt_hw_features {
+ IOCPT_HW_SYM = BIT(0),
+ IOCPT_HW_ASYM = BIT(1),
+ IOCPT_HW_CHAIN = BIT(2),
+ IOCPT_HW_IP = BIT(3),
+ IOCPT_HW_OOP = BIT(4),
+};
+
+/**
+ * struct iocpt_q_control_cmd - Queue control command
+ * @opcode: Opcode
+ * @type: Queue type
+ * @lif_index: LIF index
+ * @index: Queue index
+ * @oper: Operation (enum iocpt_q_control_oper)
+ */
+struct iocpt_q_control_cmd {
+ u8 opcode;
+ u8 type;
+ __le16 lif_index;
+ __le32 index;
+ u8 oper;
+ u8 rsvd2[55];
+};
+
+enum iocpt_q_control_oper {
+ IOCPT_Q_DISABLE = 0,
+ IOCPT_Q_ENABLE = 1,
+};
+
+/* NB: It will take 64 transfers to update a 2048B key */
+#define IOCPT_SESS_KEY_LEN_MIN 16
+#define IOCPT_SESS_KEY_LEN_MAX_SYMM 32
+#define IOCPT_SESS_KEY_LEN_MAX_ASYM 2048
+#define IOCPT_SESS_KEY_SEG_LEN 32
+#define IOCPT_SESS_KEY_SEG_SHFT 5
+#define IOCPT_SESS_KEY_SEG_CNT \
+ (IOCPT_SESS_KEY_LEN_MAX_SYMM >> IOCPT_SESS_KEY_SEG_SHFT)
+
+enum iocpt_sess_type {
+ IOCPT_SESS_NONE = 0,
+ IOCPT_SESS_AEAD_AES_GCM = 1,
+};
+
+enum iocpt_sess_control_oper {
+ IOCPT_SESS_INIT = 0,
+ IOCPT_SESS_UPDATE_KEY = 2,
+ IOCPT_SESS_DISABLE = 3,
+};
+
+/**
+ * struct iocpt_sess_control_cmd - Session control command
+ * @opcode: Opcode
+ * @type: Session type (enum iocpt_sess_type)
+ * @lif_index: LIF index
+ * @oper: Operation (enum iocpt_sess_control_oper)
+ * @flags:
+ * END: Indicates that this is the final segment of the key.
+ * When this flag is set, a write will be triggered from the
+ * controller's memory into the dedicated key-storage area.
+ * @key_len: Crypto key length in bytes
+ * @index: Session index, as allocated by PMD
+ * @key_seg_len: Crypto key segment length in bytes
+ * @key_seg_idx: Crypto key segment index
+ * @key: Crypto key
+ */
+struct iocpt_sess_control_cmd {
+ u8 opcode;
+ u8 type;
+ __le16 lif_index;
+ u8 oper;
+ u8 flags;
+#define IOCPT_SCTL_F_END 0x01 /* Final segment of key */
+ __le16 key_len;
+ __le32 index;
+ u8 key_seg_len;
+ u8 key_seg_idx;
+ u8 rsvd[18];
+ u8 key[IOCPT_SESS_KEY_SEG_LEN];
+};
+
+/**
+ * struct iocpt_sess_control_comp - Session control command completion
+ * @status: Status of the command (enum iocpt_status_code)
+ * @comp_index: Index in the descriptor ring for which this is the completion
+ * @index: Session index
+ * @hw_type: Hardware Session type
+ * @color: Color
+ */
+struct iocpt_sess_control_comp {
+ u8 status;
+ u8 rsvd;
+ __le16 comp_index;
+ __le32 index;
+ u8 hw_type;
+ u8 rsvd2[6];
+ u8 color;
+};
+
+/**
+ * enum iocpt_stats_ctl_cmd - List of commands for stats control
+ * @IOCPT_STATS_CTL_RESET: Reset statistics
+ */
+enum iocpt_stats_ctl_cmd {
+ IOCPT_STATS_CTL_RESET = 0,
+};
+
+/**
+ * struct iocpt_dev_status - Device status register
+ * @eid: most recent NotifyQ event id
+ */
+struct iocpt_dev_status {
+ __le64 eid;
+ u8 rsvd2[56];
+};
+
+enum iocpt_dev_state {
+ IOCPT_DEV_DISABLE = 0,
+ IOCPT_DEV_ENABLE = 1,
+ IOCPT_DEV_HANG_RESET = 2,
+};
+
+/**
+ * enum iocpt_dev_attr - List of device attributes
+ * @IOCPT_DEV_ATTR_STATE: Device state attribute
+ * @IOCPT_DEV_ATTR_NAME: Device name attribute
+ * @IOCPT_DEV_ATTR_FEATURES: Device feature attributes
+ */
+enum iocpt_dev_attr {
+ IOCPT_DEV_ATTR_STATE = 0,
+ IOCPT_DEV_ATTR_NAME = 1,
+ IOCPT_DEV_ATTR_FEATURES = 2,
+};
+
+/**
+ * struct iocpt_notify_event - Generic event reporting structure
+ * @eid: event number
+ * @ecode: event code
+ * @data: unspecified data about the event
+ *
+ * This is the generic event report struct from which the other
+ * actual events will be formed.
+ */
+struct iocpt_notify_event {
+ __le64 eid;
+ __le16 ecode;
+ u8 data[54];
+};
+
+/**
+ * struct iocpt_reset_event - Reset event notification
+ * @eid: event number
+ * @ecode: event code = IOCPT_EVENT_RESET
+ * @reset_code: reset type
+ * @state: 0=pending, 1=complete, 2=error
+ *
+ * Sent when the NIC or some subsystem is going to be or
+ * has been reset.
+ */
+struct iocpt_reset_event {
+ __le64 eid;
+ __le16 ecode;
+ u8 reset_code;
+ u8 state;
+ u8 rsvd[52];
+};
+
+/**
+ * struct iocpt_heartbeat_event - Sent periodically by NIC to indicate health
+ * @eid: event number
+ * @ecode: event code = IOCPT_EVENT_HEARTBEAT
+ */
+struct iocpt_heartbeat_event {
+ __le64 eid;
+ __le16 ecode;
+ u8 rsvd[54];
+};
+
+/**
+ * struct iocpt_log_event - Sent to notify the driver of an internal error
+ * @eid: event number
+ * @ecode: event code = IOCPT_EVENT_LOG
+ * @data: log data
+ */
+struct iocpt_log_event {
+ __le64 eid;
+ __le16 ecode;
+ u8 data[54];
+};
+
+/**
+ * union iocpt_lif_config - LIF configuration
+ * @state: LIF state (enum iocpt_lif_state)
+ * @name: LIF name
+ * @features: LIF features active (enum iocpt_hw_features)
+ * @queue_count: Queue counts per queue-type
+ */
+union iocpt_lif_config {
+ struct {
+ u8 state;
+ u8 rsvd[3];
+ char name[IOCPT_IFNAMSIZ];
+ u8 rsvd2[12];
+ __le64 features;
+ __le32 queue_count[IOCPT_QTYPE_MAX];
+ } __rte_packed;
+ __le32 words[56];
+};
+
+/**
+ * struct iocpt_lif_status - LIF status register
+ * @eid: most recent NotifyQ event id
+ */
+struct iocpt_lif_status {
+ __le64 eid;
+ u8 rsvd[56];
+};
+
+/**
+ * struct iocpt_lif_info - LIF info structure
+ * @config: LIF configuration structure
+ * @status: LIF status structure
+ * @stats: LIF statistics structure
+ */
+struct iocpt_lif_info {
+ union iocpt_lif_config config;
+ struct iocpt_lif_status status;
+};
+
+union iocpt_dev_cmd {
+ u32 words[16];
+ struct iocpt_admin_cmd cmd;
+ struct iocpt_nop_cmd nop;
+
+ struct iocpt_dev_identify_cmd identify;
+ struct iocpt_dev_reset_cmd reset;
+
+ struct iocpt_lif_identify_cmd lif_identify;
+ struct iocpt_lif_init_cmd lif_init;
+ struct iocpt_lif_reset_cmd lif_reset;
+ struct iocpt_lif_getattr_cmd lif_getattr;
+ struct iocpt_lif_setattr_cmd lif_setattr;
+
+ struct iocpt_q_identify_cmd q_identify;
+ struct iocpt_q_init_cmd q_init;
+ struct iocpt_q_control_cmd q_control;
+
+ struct iocpt_sess_control_cmd sess_control;
+};
+
+union iocpt_dev_cmd_comp {
+ u32 words[4];
+ u8 status;
+ struct iocpt_admin_comp comp;
+ struct iocpt_nop_comp nop;
+
+ struct iocpt_dev_identify_comp identify;
+ struct iocpt_dev_reset_comp reset;
+
+ struct iocpt_lif_identify_comp lif_identify;
+ struct iocpt_lif_init_comp lif_init;
+ struct iocpt_lif_getattr_comp lif_getattr;
+ struct iocpt_lif_setattr_comp lif_setattr;
+
+ struct iocpt_q_identify_comp q_identify;
+ struct iocpt_q_init_comp q_init;
+
+ struct iocpt_sess_control_comp sess_control;
+};
+
+/**
+ * union iocpt_dev_info_regs - Device info register format (read-only)
+ * @signature: Signature value of 0x43585660 ('CRPT')
+ * @version: Current version of info
+ * @asic_type: Asic type
+ * @asic_rev: Asic revision
+ * @fw_status: Firmware status
+ * @fw_heartbeat: Firmware heartbeat counter
+ * @serial_num: Serial number
+ * @fw_version: Firmware version
+ */
+union iocpt_dev_info_regs {
+#define IOCPT_FWVERS_BUFLEN 32
+#define IOCPT_SERIAL_BUFLEN 32
+ struct {
+ u32 signature;
+ u8 version;
+ u8 asic_type;
+ u8 asic_rev;
+#define IOCPT_FW_STS_F_RUNNING 0x1
+ u8 fw_status;
+ u32 fw_heartbeat;
+ char fw_version[IOCPT_FWVERS_BUFLEN];
+ char serial_num[IOCPT_SERIAL_BUFLEN];
+ };
+ u32 words[512];
+};
+
+/**
+ * union iocpt_dev_cmd_regs - Device command register format (read-write)
+ * @doorbell: Device Cmd Doorbell, write-only
+ * Write a 1 to signal device to process cmd,
+ * poll done for completion.
+ * @done: Done indicator, bit 0 == 1 when command is complete
+ * @cmd: Opcode-specific command bytes
+ * @comp: Opcode-specific response bytes
+ * @data: Opcode-specific side-data
+ */
+union iocpt_dev_cmd_regs {
+ struct {
+ u32 doorbell;
+ u32 done;
+ union iocpt_dev_cmd cmd;
+ union iocpt_dev_cmd_comp comp;
+ u8 rsvd[48];
+ u32 data[478];
+ } __rte_packed;
+ u32 words[512];
+};
+
+/**
+ * union iocpt_dev_regs - Device register format for bar 0 page 0
+ * @info: Device info registers
+ * @devcmd: Device command registers
+ */
+union iocpt_dev_regs {
+ struct {
+ union iocpt_dev_info_regs info;
+ union iocpt_dev_cmd_regs devcmd;
+ } __rte_packed;
+ __le32 words[1024];
+};
+
+union iocpt_adminq_cmd {
+ struct iocpt_admin_cmd cmd;
+ struct iocpt_nop_cmd nop;
+ struct iocpt_q_identify_cmd q_identify;
+ struct iocpt_q_init_cmd q_init;
+ struct iocpt_q_control_cmd q_control;
+ struct iocpt_lif_setattr_cmd lif_setattr;
+ struct iocpt_lif_getattr_cmd lif_getattr;
+ struct iocpt_sess_control_cmd sess_control;
+};
+
+union iocpt_adminq_comp {
+ struct iocpt_admin_comp comp;
+ struct iocpt_nop_comp nop;
+ struct iocpt_q_identify_comp q_identify;
+ struct iocpt_q_init_comp q_init;
+ struct iocpt_lif_setattr_comp lif_setattr;
+ struct iocpt_lif_getattr_comp lif_getattr;
+ struct iocpt_sess_control_comp sess_control;
+};
+
+union iocpt_notify_comp {
+ struct iocpt_notify_event event;
+ struct iocpt_reset_event reset;
+ struct iocpt_heartbeat_event heartbeat;
+ struct iocpt_log_event log;
+};
+
+/**
+ * union iocpt_dev_identity - device identity information
+ * @version: Version of device identify
+ * @type: Identify type (0 for now)
+ * @state: Device state
+ * @nlifs: Number of LIFs provisioned
+ * @nintrs: Number of interrupts provisioned
+ * @ndbpgs_per_lif: Number of doorbell pages per LIF
+ * @intr_coal_mult: Interrupt coalescing multiplication factor
+ * Scale user-supplied interrupt coalescing
+ * value in usecs to device units using:
+ * device units = usecs * mult / div
+ * @intr_coal_div: Interrupt coalescing division factor
+ * Scale user-supplied interrupt coalescing
+ * value in usecs to device units using:
+ * device units = usecs * mult / div
+ */
+union iocpt_dev_identity {
+ struct {
+ u8 version;
+ u8 type;
+ u8 state;
+ u8 rsvd;
+ __le32 nlifs;
+ __le32 nintrs;
+ __le32 ndbpgs_per_lif;
+ __le32 intr_coal_mult;
+ __le32 intr_coal_div;
+ u8 rsvd2[8];
+ };
+ __le32 words[8];
+};
+
+/**
+ * union iocpt_lif_identity - LIF identity information (type-specific)
+ *
+ * @features: LIF features (see enum iocpt_hw_features)
+ * @version: Identify structure version
+ * @hw_index: LIF hardware index
+ * @max_nb_sessions: Maximum number of sessions supported
+ * @config: LIF config struct with features, q counts
+ */
+union iocpt_lif_identity {
+ struct {
+ __le64 features;
+
+ u8 version;
+ u8 hw_index;
+ u8 rsvd[2];
+ __le32 max_nb_sessions;
+ u8 rsvd2[120];
+ union iocpt_lif_config config;
+ } __rte_packed;
+ __le32 words[90];
+};
+
+/**
+ * union iocpt_q_identity - queue identity information
+ * @version: Queue type version that can be used with FW
+ * @supported: Bitfield of queue versions, first bit = ver 0
+ * @features: Queue features
+ * @desc_sz: Descriptor size
+ * @comp_sz: Completion descriptor size
+ * @sg_desc_sz: Scatter/Gather descriptor size
+ * @max_sg_elems: Maximum number of Scatter/Gather elements
+ * @sg_desc_stride: Number of Scatter/Gather elements per descriptor
+ */
+union iocpt_q_identity {
+ struct {
+ u8 version;
+ u8 supported;
+ u8 rsvd[6];
+#define IOCPT_QIDENT_F_CQ 0x01 /* queue has completion ring */
+#define IOCPT_QIDENT_F_SG 0x02 /* queue has scatter/gather ring */
+ __le64 features;
+ __le16 desc_sz;
+ __le16 comp_sz;
+ __le16 sg_desc_sz;
+ __le16 max_sg_elems;
+ __le16 sg_desc_stride;
+ };
+ __le32 words[20];
+};
+
+struct iocpt_identity {
+ union iocpt_dev_identity dev;
+ union iocpt_lif_identity lif;
+ union iocpt_q_identity q;
+};
+
+#endif /* _IONIC_CRYPTO_IF_H_ */
diff --git a/drivers/crypto/ionic/ionic_crypto_main.c b/drivers/crypto/ionic/ionic_crypto_main.c
index ecbb1cb161..64b7cae03c 100644
--- a/drivers/crypto/ionic/ionic_crypto_main.c
+++ b/drivers/crypto/ionic/ionic_crypto_main.c
@@ -57,9 +57,12 @@ iocpt_probe(void *bus_dev, struct rte_device *rte_dev,
};
struct rte_cryptodev *cdev;
struct iocpt_dev *dev;
- uint32_t i;
+ uint32_t i, sig;
int err;
+ /* Check structs (trigger error at compilation time) */
+ iocpt_struct_size_checks();
+
/* Multi-process not supported */
if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
err = -EPERM;
@@ -110,6 +113,18 @@ iocpt_probe(void *bus_dev, struct rte_device *rte_dev,
goto err_destroy_crypto_dev;
}
+ sig = ioread32(&dev->dev_info->signature);
+ if (sig != IOCPT_DEV_INFO_SIGNATURE) {
+ IOCPT_PRINT(ERR, "Incompatible firmware signature %#x", sig);
+ err = -EFAULT;
+ goto err_destroy_crypto_dev;
+ }
+
+ for (i = 0; i < IOCPT_FWVERS_BUFLEN; i++)
+ dev->fw_version[i] = ioread8(&dev->dev_info->fw_version[i]);
+ dev->fw_version[IOCPT_FWVERS_BUFLEN - 1] = '\0';
+ IOCPT_PRINT(DEBUG, "%s firmware: %s", dev->name, dev->fw_version);
+
err = iocpt_init(dev);
if (err != 0) {
IOCPT_PRINT(ERR, "Cannot init device: %d, aborting", err);
diff --git a/drivers/crypto/ionic/ionic_crypto_vdev.c b/drivers/crypto/ionic/ionic_crypto_vdev.c
index a915aa06aa..d15acf660a 100644
--- a/drivers/crypto/ionic/ionic_crypto_vdev.c
+++ b/drivers/crypto/ionic/ionic_crypto_vdev.c
@@ -24,11 +24,48 @@
#define IOCPT_VDEV_DEV_INFO_REGS_OFFSET 0x0000
#define IOCPT_VDEV_DEV_CMD_REGS_OFFSET 0x0800
+#define IOCPT_VDEV_FW_WAIT_US 1000 /* 1ms */
+#define IOCPT_VDEV_FW_WAIT_MAX 5000 /* 5s */
+
static int
iocpt_vdev_setup_bars(struct iocpt_dev *dev)
{
+ struct iocpt_dev_bars *bars = &dev->bars;
+ uint8_t *bar0_base;
+ uint32_t fw_waits = 0;
+ uint8_t fw;
+
IOCPT_PRINT_CALL();
+ /* BAR0: dev_cmd */
+ bar0_base = bars->bar[IOCPT_VDEV_DEV_BAR].vaddr;
+ dev->dev_info = (union iocpt_dev_info_regs *)
+ &bar0_base[IOCPT_VDEV_DEV_INFO_REGS_OFFSET];
+ dev->dev_cmd = (union iocpt_dev_cmd_regs *)
+ &bar0_base[IOCPT_VDEV_DEV_CMD_REGS_OFFSET];
+
+ /* BAR1: interrupts */
+ dev->intr_ctrl = (void *)bars->bar[IOCPT_VDEV_INTR_CTL_BAR].vaddr;
+
+ /* BAR3: doorbells */
+ dev->db_pages = (void *)bars->bar[IOCPT_VDEV_DB_BAR].vaddr;
+
+ /* Wait for the FW to indicate readiness */
+ while (1) {
+ fw = ioread8(&dev->dev_info->fw_status);
+ if ((fw & IOCPT_FW_STS_F_RUNNING) != 0)
+ break;
+
+ if (fw_waits > IOCPT_VDEV_FW_WAIT_MAX) {
+ IOCPT_PRINT(ERR, "Firmware readiness bit not set");
+ return -ETIMEDOUT;
+ }
+
+ fw_waits++;
+ rte_delay_us_block(IOCPT_VDEV_FW_WAIT_US);
+ }
+ IOCPT_PRINT(DEBUG, "Firmware ready (%u waits)", fw_waits);
+
dev->name = rte_vdev_device_name(dev->bus_dev);
return 0;
--
2.17.1
^ permalink raw reply [flat|nested] 19+ messages in thread* [PATCH v2 3/9] crypto/ionic: add device commands
2024-04-30 20:21 ` [PATCH v2 0/9] " Andrew Boyer
2024-04-30 20:21 ` [PATCH v2 1/9] " Andrew Boyer
2024-04-30 20:21 ` [PATCH v2 2/9] crypto/ionic: add the firmware interface definition file Andrew Boyer
@ 2024-04-30 20:21 ` Andrew Boyer
2024-04-30 20:21 ` [PATCH v2 4/9] crypto/ionic: add adminq command support Andrew Boyer
` (5 subsequent siblings)
8 siblings, 0 replies; 19+ messages in thread
From: Andrew Boyer @ 2024-04-30 20:21 UTC (permalink / raw)
To: dev; +Cc: Andrew Boyer
This defines the device (register-based) commands. They are used for
device identification, setup, and teardown.
Signed-off-by: Andrew Boyer <andrew.boyer@amd.com>
---
drivers/crypto/ionic/ionic_crypto.h | 24 ++
drivers/crypto/ionic/ionic_crypto_cmds.c | 348 +++++++++++++++++++++++
drivers/crypto/ionic/ionic_crypto_main.c | 89 +++++-
drivers/crypto/ionic/meson.build | 1 +
4 files changed, 461 insertions(+), 1 deletion(-)
create mode 100644 drivers/crypto/ionic/ionic_crypto_cmds.c
diff --git a/drivers/crypto/ionic/ionic_crypto.h b/drivers/crypto/ionic/ionic_crypto.h
index 7f456b5154..a7999a0f68 100644
--- a/drivers/crypto/ionic/ionic_crypto.h
+++ b/drivers/crypto/ionic/ionic_crypto.h
@@ -80,6 +80,17 @@ struct iocpt_dev_bars {
uint32_t num_bars;
};
+struct iocpt_qtype_info {
+ uint8_t version;
+ uint8_t supported;
+ uint64_t features;
+ uint16_t desc_sz;
+ uint16_t comp_sz;
+ uint16_t sg_desc_sz;
+ uint16_t max_sg_elems;
+ uint16_t sg_desc_stride;
+};
+
#define IOCPT_DEV_F_INITED BIT(0)
#define IOCPT_DEV_F_UP BIT(1)
#define IOCPT_DEV_F_FW_RESET BIT(2)
@@ -89,6 +100,7 @@ struct iocpt_dev {
const char *name;
char fw_version[IOCPT_FWVERS_BUFLEN];
struct iocpt_dev_bars bars;
+ struct iocpt_identity ident;
const struct iocpt_dev_intf *intf;
void *bus_dev;
@@ -108,6 +120,14 @@ struct iocpt_dev {
uint64_t features;
uint32_t hw_features;
+
+ uint32_t info_sz;
+ struct iocpt_lif_info *info;
+ rte_iova_t info_pa;
+ const struct rte_memzone *info_z;
+
+ struct iocpt_qtype_info qtype_info[IOCPT_QTYPE_MAX];
+ uint8_t qtype_ver[IOCPT_QTYPE_MAX];
};
struct iocpt_dev_intf {
@@ -132,6 +152,10 @@ int iocpt_remove(struct rte_device *rte_dev);
void iocpt_configure(struct iocpt_dev *dev);
void iocpt_deinit(struct iocpt_dev *dev);
+int iocpt_dev_identify(struct iocpt_dev *dev);
+int iocpt_dev_init(struct iocpt_dev *dev, rte_iova_t info_pa);
+void iocpt_dev_reset(struct iocpt_dev *dev);
+
static inline bool
iocpt_is_embedded(void)
{
diff --git a/drivers/crypto/ionic/ionic_crypto_cmds.c b/drivers/crypto/ionic/ionic_crypto_cmds.c
new file mode 100644
index 0000000000..105005539b
--- /dev/null
+++ b/drivers/crypto/ionic/ionic_crypto_cmds.c
@@ -0,0 +1,348 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021-2024 Advanced Micro Devices, Inc.
+ */
+
+#include <stdbool.h>
+
+#include <rte_malloc.h>
+#include <rte_memzone.h>
+
+#include "ionic_crypto.h"
+
+/* queuetype support level */
+static const uint8_t iocpt_qtype_vers[IOCPT_QTYPE_MAX] = {
+ [IOCPT_QTYPE_ADMINQ] = 0, /* 0 = Base version */
+ [IOCPT_QTYPE_NOTIFYQ] = 0, /* 0 = Base version */
+ [IOCPT_QTYPE_CRYPTOQ] = 0, /* 0 = Base version */
+};
+
+static const char *
+iocpt_opcode_to_str(enum iocpt_cmd_opcode opcode)
+{
+ switch (opcode) {
+ case IOCPT_CMD_NOP:
+ return "IOCPT_CMD_NOP";
+ case IOCPT_CMD_IDENTIFY:
+ return "IOCPT_CMD_IDENTIFY";
+ case IOCPT_CMD_RESET:
+ return "IOCPT_CMD_RESET";
+ case IOCPT_CMD_LIF_IDENTIFY:
+ return "IOCPT_CMD_LIF_IDENTIFY";
+ case IOCPT_CMD_LIF_INIT:
+ return "IOCPT_CMD_LIF_INIT";
+ case IOCPT_CMD_LIF_RESET:
+ return "IOCPT_CMD_LIF_RESET";
+ case IOCPT_CMD_LIF_GETATTR:
+ return "IOCPT_CMD_LIF_GETATTR";
+ case IOCPT_CMD_LIF_SETATTR:
+ return "IOCPT_CMD_LIF_SETATTR";
+ case IOCPT_CMD_Q_IDENTIFY:
+ return "IOCPT_CMD_Q_IDENTIFY";
+ case IOCPT_CMD_Q_INIT:
+ return "IOCPT_CMD_Q_INIT";
+ case IOCPT_CMD_Q_CONTROL:
+ return "IOCPT_CMD_Q_CONTROL";
+ case IOCPT_CMD_SESS_CONTROL:
+ return "IOCPT_CMD_SESS_CONTROL";
+ default:
+ return "DEVCMD_UNKNOWN";
+ }
+}
+
+/* Dev_cmd Interface */
+
+static void
+iocpt_dev_cmd_go(struct iocpt_dev *dev, union iocpt_dev_cmd *cmd)
+{
+ uint32_t cmd_size = RTE_DIM(cmd->words);
+ uint32_t i;
+
+ IOCPT_PRINT(DEBUG, "Sending %s (%d) via dev_cmd",
+ iocpt_opcode_to_str(cmd->cmd.opcode), cmd->cmd.opcode);
+
+ for (i = 0; i < cmd_size; i++)
+ iowrite32(cmd->words[i], &dev->dev_cmd->cmd.words[i]);
+
+ iowrite32(0, &dev->dev_cmd->done);
+ iowrite32(1, &dev->dev_cmd->doorbell);
+}
+
+static int
+iocpt_dev_cmd_wait(struct iocpt_dev *dev, unsigned long max_wait)
+{
+ unsigned long step_usec = IONIC_DEVCMD_CHECK_PERIOD_US;
+ unsigned long max_wait_usec = max_wait * 1000000L;
+ unsigned long elapsed_usec = 0;
+ int done;
+
+ /* Wait for dev cmd to complete.. but no more than max_wait sec */
+
+ do {
+ done = ioread32(&dev->dev_cmd->done) & IONIC_DEV_CMD_DONE;
+ if (done != 0) {
+ IOCPT_PRINT(DEBUG, "DEVCMD %d done took %lu usecs",
+ ioread8(&dev->dev_cmd->cmd.cmd.opcode),
+ elapsed_usec);
+ return 0;
+ }
+
+ rte_delay_us_block(step_usec);
+
+ elapsed_usec += step_usec;
+ } while (elapsed_usec < max_wait_usec);
+
+ IOCPT_PRINT(ERR, "DEVCMD %d timeout after %lu usecs",
+ ioread8(&dev->dev_cmd->cmd.cmd.opcode), elapsed_usec);
+
+ return -ETIMEDOUT;
+}
+
+static int
+iocpt_dev_cmd_wait_check(struct iocpt_dev *dev, unsigned long max_wait)
+{
+ uint8_t status;
+ int err;
+
+ err = iocpt_dev_cmd_wait(dev, max_wait);
+ if (err == 0) {
+ status = ioread8(&dev->dev_cmd->comp.comp.status);
+ if (status == IOCPT_RC_EAGAIN)
+ err = -EAGAIN;
+ else if (status != 0)
+ err = -EIO;
+ }
+
+ IOCPT_PRINT(DEBUG, "dev_cmd returned %d", err);
+ return err;
+}
+
+/* Dev_cmds */
+
+static void
+iocpt_dev_cmd_reset(struct iocpt_dev *dev)
+{
+ union iocpt_dev_cmd cmd = {
+ .reset.opcode = IOCPT_CMD_RESET,
+ };
+
+ iocpt_dev_cmd_go(dev, &cmd);
+}
+
+static void
+iocpt_dev_cmd_lif_identify(struct iocpt_dev *dev, uint8_t ver)
+{
+ union iocpt_dev_cmd cmd = {
+ .lif_identify.opcode = IOCPT_CMD_LIF_IDENTIFY,
+ .lif_identify.type = IOCPT_LIF_TYPE_DEFAULT,
+ .lif_identify.ver = ver,
+ };
+
+ iocpt_dev_cmd_go(dev, &cmd);
+}
+
+static void
+iocpt_dev_cmd_lif_init(struct iocpt_dev *dev, rte_iova_t info_pa)
+{
+ union iocpt_dev_cmd cmd = {
+ .lif_init.opcode = IOCPT_CMD_LIF_INIT,
+ .lif_init.type = IOCPT_LIF_TYPE_DEFAULT,
+ .lif_init.info_pa = info_pa,
+ };
+
+ iocpt_dev_cmd_go(dev, &cmd);
+}
+
+static void
+iocpt_dev_cmd_lif_reset(struct iocpt_dev *dev)
+{
+ union iocpt_dev_cmd cmd = {
+ .lif_reset.opcode = IOCPT_CMD_LIF_RESET,
+ };
+
+ iocpt_dev_cmd_go(dev, &cmd);
+}
+
+static void
+iocpt_dev_cmd_queue_identify(struct iocpt_dev *dev,
+ uint8_t qtype, uint8_t qver)
+{
+ union iocpt_dev_cmd cmd = {
+ .q_identify.opcode = IOCPT_CMD_Q_IDENTIFY,
+ .q_identify.type = qtype,
+ .q_identify.ver = qver,
+ };
+
+ iocpt_dev_cmd_go(dev, &cmd);
+}
+
+/* Dev_cmd consumers */
+
+static void
+iocpt_queue_identify(struct iocpt_dev *dev)
+{
+ union iocpt_q_identity *q_ident = &dev->ident.q;
+ uint32_t q_words = RTE_DIM(q_ident->words);
+ uint32_t cmd_words = RTE_DIM(dev->dev_cmd->data);
+ uint32_t i, nwords, qtype;
+ int err;
+
+ for (qtype = 0; qtype < RTE_DIM(iocpt_qtype_vers); qtype++) {
+ struct iocpt_qtype_info *qti = &dev->qtype_info[qtype];
+
+ /* Filter out the types this driver knows about */
+ switch (qtype) {
+ case IOCPT_QTYPE_ADMINQ:
+ case IOCPT_QTYPE_NOTIFYQ:
+ case IOCPT_QTYPE_CRYPTOQ:
+ break;
+ default:
+ continue;
+ }
+
+ memset(qti, 0, sizeof(*qti));
+
+ if (iocpt_is_embedded()) {
+ /* When embedded, FW will always match the driver */
+ qti->version = iocpt_qtype_vers[qtype];
+ continue;
+ }
+
+ /* On the host, query the FW for info */
+ iocpt_dev_cmd_queue_identify(dev,
+ qtype, iocpt_qtype_vers[qtype]);
+ err = iocpt_dev_cmd_wait_check(dev, IONIC_DEVCMD_TIMEOUT);
+ if (err == -EINVAL) {
+ IOCPT_PRINT(ERR, "qtype %d not supported", qtype);
+ continue;
+ } else if (err == -EIO) {
+ IOCPT_PRINT(ERR, "q_ident failed, older FW");
+ return;
+ } else if (err != 0) {
+ IOCPT_PRINT(ERR, "q_ident failed, qtype %d: %d",
+ qtype, err);
+ return;
+ }
+
+ nwords = RTE_MIN(q_words, cmd_words);
+ for (i = 0; i < nwords; i++)
+ q_ident->words[i] = ioread32(&dev->dev_cmd->data[i]);
+
+ qti->version = q_ident->version;
+ qti->supported = q_ident->supported;
+ qti->features = rte_le_to_cpu_64(q_ident->features);
+ qti->desc_sz = rte_le_to_cpu_16(q_ident->desc_sz);
+ qti->comp_sz = rte_le_to_cpu_16(q_ident->comp_sz);
+ qti->sg_desc_sz = rte_le_to_cpu_16(q_ident->sg_desc_sz);
+ qti->max_sg_elems = rte_le_to_cpu_16(q_ident->max_sg_elems);
+ qti->sg_desc_stride =
+ rte_le_to_cpu_16(q_ident->sg_desc_stride);
+
+ IOCPT_PRINT(DEBUG, " qtype[%d].version = %d",
+ qtype, qti->version);
+ IOCPT_PRINT(DEBUG, " qtype[%d].supported = %#x",
+ qtype, qti->supported);
+ IOCPT_PRINT(DEBUG, " qtype[%d].features = %#jx",
+ qtype, qti->features);
+ IOCPT_PRINT(DEBUG, " qtype[%d].desc_sz = %d",
+ qtype, qti->desc_sz);
+ IOCPT_PRINT(DEBUG, " qtype[%d].comp_sz = %d",
+ qtype, qti->comp_sz);
+ IOCPT_PRINT(DEBUG, " qtype[%d].sg_desc_sz = %d",
+ qtype, qti->sg_desc_sz);
+ IOCPT_PRINT(DEBUG, " qtype[%d].max_sg_elems = %d",
+ qtype, qti->max_sg_elems);
+ IOCPT_PRINT(DEBUG, " qtype[%d].sg_desc_stride = %d",
+ qtype, qti->sg_desc_stride);
+ }
+}
+
+int
+iocpt_dev_identify(struct iocpt_dev *dev)
+{
+ union iocpt_lif_identity *ident = &dev->ident.lif;
+ union iocpt_lif_config *cfg = &ident->config;
+ uint64_t features;
+ uint32_t cmd_size = RTE_DIM(dev->dev_cmd->data);
+ uint32_t dev_size = RTE_DIM(ident->words);
+ uint32_t i, nwords;
+ int err;
+
+ memset(ident, 0, sizeof(*ident));
+
+ iocpt_dev_cmd_lif_identify(dev, IOCPT_IDENTITY_VERSION_1);
+ err = iocpt_dev_cmd_wait_check(dev, IONIC_DEVCMD_TIMEOUT);
+ if (err != 0)
+ return err;
+
+ nwords = RTE_MIN(dev_size, cmd_size);
+ for (i = 0; i < nwords; i++)
+ ident->words[i] = ioread32(&dev->dev_cmd->data[i]);
+
+ dev->max_qps =
+ rte_le_to_cpu_32(cfg->queue_count[IOCPT_QTYPE_CRYPTOQ]);
+ dev->max_sessions =
+ rte_le_to_cpu_32(ident->max_nb_sessions);
+
+ features = rte_le_to_cpu_64(ident->features);
+ dev->features = RTE_CRYPTODEV_FF_HW_ACCELERATED;
+ if (features & IOCPT_HW_SYM)
+ dev->features |= RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO;
+ if (features & IOCPT_HW_ASYM)
+ dev->features |= RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO;
+ if (features & IOCPT_HW_CHAIN)
+ dev->features |= RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING;
+ if (features & IOCPT_HW_IP)
+ dev->features |= RTE_CRYPTODEV_FF_IN_PLACE_SGL;
+ if (features & IOCPT_HW_OOP) {
+ dev->features |= RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT;
+ dev->features |= RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT;
+ dev->features |= RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
+ dev->features |= RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT;
+ }
+
+ IOCPT_PRINT(INFO, "crypto.features %#jx",
+ rte_le_to_cpu_64(ident->features));
+ IOCPT_PRINT(INFO, "crypto.features_active %#jx",
+ rte_le_to_cpu_64(cfg->features));
+ IOCPT_PRINT(INFO, "crypto.queue_count[IOCPT_QTYPE_ADMINQ] %#x",
+ rte_le_to_cpu_32(cfg->queue_count[IOCPT_QTYPE_ADMINQ]));
+ IOCPT_PRINT(INFO, "crypto.queue_count[IOCPT_QTYPE_NOTIFYQ] %#x",
+ rte_le_to_cpu_32(cfg->queue_count[IOCPT_QTYPE_NOTIFYQ]));
+ IOCPT_PRINT(INFO, "crypto.queue_count[IOCPT_QTYPE_CRYPTOQ] %#x",
+ rte_le_to_cpu_32(cfg->queue_count[IOCPT_QTYPE_CRYPTOQ]));
+ IOCPT_PRINT(INFO, "crypto.max_sessions %u",
+ rte_le_to_cpu_32(ident->max_nb_sessions));
+
+ iocpt_queue_identify(dev);
+
+ return 0;
+}
+
+int
+iocpt_dev_init(struct iocpt_dev *dev, rte_iova_t info_pa)
+{
+ uint32_t retries = 5;
+ int err;
+
+retry_lif_init:
+ iocpt_dev_cmd_lif_init(dev, info_pa);
+
+ err = iocpt_dev_cmd_wait_check(dev, IONIC_DEVCMD_TIMEOUT);
+ if (err == -EAGAIN && retries > 0) {
+ retries--;
+ rte_delay_us_block(IONIC_DEVCMD_RETRY_WAIT_US);
+ goto retry_lif_init;
+ }
+
+ return err;
+}
+
+void
+iocpt_dev_reset(struct iocpt_dev *dev)
+{
+ iocpt_dev_cmd_lif_reset(dev);
+ (void)iocpt_dev_cmd_wait_check(dev, IONIC_DEVCMD_TIMEOUT);
+
+ iocpt_dev_cmd_reset(dev);
+ (void)iocpt_dev_cmd_wait_check(dev, IONIC_DEVCMD_TIMEOUT);
+}
diff --git a/drivers/crypto/ionic/ionic_crypto_main.c b/drivers/crypto/ionic/ionic_crypto_main.c
index 64b7cae03c..4f782f3fe4 100644
--- a/drivers/crypto/ionic/ionic_crypto_main.c
+++ b/drivers/crypto/ionic/ionic_crypto_main.c
@@ -10,9 +10,64 @@
#include "ionic_crypto.h"
+static const struct rte_memzone *
+iocpt_dma_zone_reserve(const char *type_name, uint16_t qid, size_t size,
+ unsigned int align, int socket_id)
+{
+ char zone_name[RTE_MEMZONE_NAMESIZE];
+ const struct rte_memzone *mz;
+ int err;
+
+ err = snprintf(zone_name, sizeof(zone_name),
+ "iocpt_%s_%u", type_name, qid);
+ if (err >= RTE_MEMZONE_NAMESIZE) {
+ IOCPT_PRINT(ERR, "Name %s too long", type_name);
+ return NULL;
+ }
+
+ mz = rte_memzone_lookup(zone_name);
+ if (mz != NULL)
+ return mz;
+
+ return rte_memzone_reserve_aligned(zone_name, size, socket_id,
+ RTE_MEMZONE_IOVA_CONTIG, align);
+}
+
+static int
+iocpt_alloc_objs(struct iocpt_dev *dev)
+{
+ int err;
+
+ IOCPT_PRINT(DEBUG, "Crypto: %s", dev->name);
+
+ dev->info_sz = RTE_ALIGN(sizeof(*dev->info), rte_mem_page_size());
+ dev->info_z = iocpt_dma_zone_reserve("info", 0, dev->info_sz,
+ IONIC_ALIGN, dev->socket_id);
+ if (dev->info_z == NULL) {
+ IOCPT_PRINT(ERR, "Cannot allocate dev info memory");
+ err = -ENOMEM;
+ goto err_out;
+ }
+
+ dev->info = dev->info_z->addr;
+ dev->info_pa = dev->info_z->iova;
+
+ return 0;
+
+err_out:
+ return err;
+}
+
static int
iocpt_init(struct iocpt_dev *dev)
{
+ int err;
+
+ /* Uses dev_cmds */
+ err = iocpt_dev_init(dev, dev->info_pa);
+ if (err != 0)
+ return err;
+
dev->state |= IOCPT_DEV_F_INITED;
return 0;
@@ -35,6 +90,19 @@ iocpt_deinit(struct iocpt_dev *dev)
dev->state &= ~IOCPT_DEV_F_INITED;
}
+static void
+iocpt_free_objs(struct iocpt_dev *dev)
+{
+ IOCPT_PRINT_CALL();
+
+ if (dev->info != NULL) {
+ rte_memzone_free(dev->info_z);
+ dev->info_z = NULL;
+ dev->info = NULL;
+ dev->info_pa = 0;
+ }
+}
+
static int
iocpt_devargs(struct rte_devargs *devargs, struct iocpt_dev *dev)
{
@@ -125,14 +193,29 @@ iocpt_probe(void *bus_dev, struct rte_device *rte_dev,
dev->fw_version[IOCPT_FWVERS_BUFLEN - 1] = '\0';
IOCPT_PRINT(DEBUG, "%s firmware: %s", dev->name, dev->fw_version);
+ err = iocpt_dev_identify(dev);
+ if (err != 0) {
+ IOCPT_PRINT(ERR, "Cannot identify device: %d, aborting",
+ err);
+ goto err_destroy_crypto_dev;
+ }
+
+ err = iocpt_alloc_objs(dev);
+ if (err != 0) {
+ IOCPT_PRINT(ERR, "Cannot alloc device objects: %d", err);
+ goto err_destroy_crypto_dev;
+ }
+
err = iocpt_init(dev);
if (err != 0) {
IOCPT_PRINT(ERR, "Cannot init device: %d, aborting", err);
- goto err_destroy_crypto_dev;
+ goto err_free_objs;
}
return 0;
+err_free_objs:
+ iocpt_free_objs(dev);
err_destroy_crypto_dev:
rte_cryptodev_pmd_destroy(cdev);
err:
@@ -155,6 +238,10 @@ iocpt_remove(struct rte_device *rte_dev)
iocpt_deinit(dev);
+ iocpt_dev_reset(dev);
+
+ iocpt_free_objs(dev);
+
rte_cryptodev_pmd_destroy(cdev);
return 0;
diff --git a/drivers/crypto/ionic/meson.build b/drivers/crypto/ionic/meson.build
index 4114e13e53..a6e0a1d415 100644
--- a/drivers/crypto/ionic/meson.build
+++ b/drivers/crypto/ionic/meson.build
@@ -5,6 +5,7 @@ deps += ['bus_vdev']
deps += ['common_ionic']
sources = files(
+ 'ionic_crypto_cmds.c',
'ionic_crypto_main.c',
'ionic_crypto_vdev.c',
)
--
2.17.1
^ permalink raw reply [flat|nested] 19+ messages in thread* [PATCH v2 4/9] crypto/ionic: add adminq command support
2024-04-30 20:21 ` [PATCH v2 0/9] " Andrew Boyer
` (2 preceding siblings ...)
2024-04-30 20:21 ` [PATCH v2 3/9] crypto/ionic: add device commands Andrew Boyer
@ 2024-04-30 20:21 ` Andrew Boyer
2024-04-30 20:21 ` [PATCH v2 5/9] crypto/ionic: add capabilities and basic ops Andrew Boyer
` (4 subsequent siblings)
8 siblings, 0 replies; 19+ messages in thread
From: Andrew Boyer @ 2024-04-30 20:21 UTC (permalink / raw)
To: dev; +Cc: Andrew Boyer
This defines the adminq used for control path commands. The adminq is
faster and more flexible than the device command interface.
Signed-off-by: Andrew Boyer <andrew.boyer@amd.com>
---
drivers/crypto/ionic/ionic_crypto.h | 115 +++++++++
drivers/crypto/ionic/ionic_crypto_cmds.c | 302 +++++++++++++++++++++++
drivers/crypto/ionic/ionic_crypto_main.c | 295 +++++++++++++++++++++-
3 files changed, 711 insertions(+), 1 deletion(-)
diff --git a/drivers/crypto/ionic/ionic_crypto.h b/drivers/crypto/ionic/ionic_crypto.h
index a7999a0f68..065e1bd826 100644
--- a/drivers/crypto/ionic/ionic_crypto.h
+++ b/drivers/crypto/ionic/ionic_crypto.h
@@ -22,6 +22,10 @@
/* Devargs */
/* NONE */
+#define IOCPT_MAX_RING_DESC 32768
+#define IOCPT_MIN_RING_DESC 16
+#define IOCPT_ADMINQ_LENGTH 16 /* must be a power of two */
+
extern int iocpt_logtype;
#define RTE_LOGTYPE_IOCPT iocpt_logtype
@@ -91,6 +95,63 @@ struct iocpt_qtype_info {
uint16_t sg_desc_stride;
};
+#define IOCPT_Q_F_INITED BIT(0)
+#define IOCPT_Q_F_DEFERRED BIT(1)
+#define IOCPT_Q_F_SG BIT(2)
+
+#define Q_NEXT_TO_POST(_q, _n) (((_q)->head_idx + (_n)) & ((_q)->size_mask))
+#define Q_NEXT_TO_SRVC(_q, _n) (((_q)->tail_idx + (_n)) & ((_q)->size_mask))
+
+#define IOCPT_INFO_SZ(_q) ((_q)->num_segs * sizeof(void *))
+#define IOCPT_INFO_IDX(_q, _i) ((_i) * (_q)->num_segs)
+#define IOCPT_INFO_PTR(_q, _i) (&(_q)->info[IOCPT_INFO_IDX((_q), _i)])
+
+struct iocpt_queue {
+ uint16_t num_descs;
+ uint16_t num_segs;
+ uint16_t head_idx;
+ uint16_t tail_idx;
+ uint16_t size_mask;
+ uint8_t type;
+ uint8_t hw_type;
+ void *base;
+ void *sg_base;
+ struct ionic_doorbell __iomem *db;
+ void **info;
+
+ uint32_t index;
+ uint32_t hw_index;
+ rte_iova_t base_pa;
+ rte_iova_t sg_base_pa;
+};
+
+struct iocpt_cq {
+ uint16_t tail_idx;
+ uint16_t num_descs;
+ uint16_t size_mask;
+ bool done_color;
+ void *base;
+ rte_iova_t base_pa;
+};
+
+#define IOCPT_COMMON_FIELDS \
+ struct iocpt_queue q; \
+ struct iocpt_cq cq; \
+ struct iocpt_dev *dev; \
+ const struct rte_memzone *base_z; \
+ void *base; \
+ rte_iova_t base_pa
+
+struct iocpt_common_q {
+ IOCPT_COMMON_FIELDS;
+};
+
+struct iocpt_admin_q {
+ IOCPT_COMMON_FIELDS;
+
+ uint16_t flags;
+};
+
#define IOCPT_DEV_F_INITED BIT(0)
#define IOCPT_DEV_F_UP BIT(1)
#define IOCPT_DEV_F_FW_RESET BIT(2)
@@ -118,6 +179,11 @@ struct iocpt_dev {
uint8_t driver_id;
uint8_t socket_id;
+ rte_spinlock_t adminq_lock;
+ rte_spinlock_t adminq_service_lock;
+
+ struct iocpt_admin_q *adminq;
+
uint64_t features;
uint32_t hw_features;
@@ -144,6 +210,17 @@ iocpt_setup_bars(struct iocpt_dev *dev)
return (*dev->intf->setup_bars)(dev);
}
+/** iocpt_admin_ctx - Admin command context.
+ * @pending_work: Flag that indicates a completion.
+ * @cmd: Admin command (64B) to be copied to the queue.
+ * @comp: Admin completion (16B) copied from the queue.
+ */
+struct iocpt_admin_ctx {
+ bool pending_work;
+ union iocpt_adminq_cmd cmd;
+ union iocpt_adminq_comp comp;
+};
+
int iocpt_probe(void *bus_dev, struct rte_device *rte_dev,
struct iocpt_dev_bars *bars, const struct iocpt_dev_intf *intf,
uint8_t driver_id, uint8_t socket_id);
@@ -154,8 +231,46 @@ void iocpt_deinit(struct iocpt_dev *dev);
int iocpt_dev_identify(struct iocpt_dev *dev);
int iocpt_dev_init(struct iocpt_dev *dev, rte_iova_t info_pa);
+int iocpt_dev_adminq_init(struct iocpt_dev *dev);
void iocpt_dev_reset(struct iocpt_dev *dev);
+int iocpt_adminq_post_wait(struct iocpt_dev *dev, struct iocpt_admin_ctx *ctx);
+
+struct ionic_doorbell __iomem *iocpt_db_map(struct iocpt_dev *dev,
+ struct iocpt_queue *q);
+
+typedef bool (*iocpt_cq_cb)(struct iocpt_cq *cq, uint16_t cq_desc_index,
+ void *cb_arg);
+uint32_t iocpt_cq_service(struct iocpt_cq *cq, uint32_t work_to_do,
+ iocpt_cq_cb cb, void *cb_arg);
+
+static inline uint16_t
+iocpt_q_space_avail(struct iocpt_queue *q)
+{
+ uint16_t avail = q->tail_idx;
+
+ if (q->head_idx >= avail)
+ avail += q->num_descs - q->head_idx - 1;
+ else
+ avail -= q->head_idx + 1;
+
+ return avail;
+}
+
+static inline void
+iocpt_q_flush(struct iocpt_queue *q)
+{
+ uint64_t val = IONIC_DBELL_QID(q->hw_index) | q->head_idx;
+
+#if defined(RTE_LIBRTE_IONIC_PMD_BARRIER_ERRATA)
+ /* On some devices the standard 'dmb' barrier is insufficient */
+ asm volatile("dsb st" : : : "memory");
+ rte_write64_relaxed(rte_cpu_to_le_64(val), q->db);
+#else
+ rte_write64(rte_cpu_to_le_64(val), q->db);
+#endif
+}
+
static inline bool
iocpt_is_embedded(void)
{
diff --git a/drivers/crypto/ionic/ionic_crypto_cmds.c b/drivers/crypto/ionic/ionic_crypto_cmds.c
index 105005539b..8165a6a7c8 100644
--- a/drivers/crypto/ionic/ionic_crypto_cmds.c
+++ b/drivers/crypto/ionic/ionic_crypto_cmds.c
@@ -16,6 +16,55 @@ static const uint8_t iocpt_qtype_vers[IOCPT_QTYPE_MAX] = {
[IOCPT_QTYPE_CRYPTOQ] = 0, /* 0 = Base version */
};
+static const char *
+iocpt_error_to_str(enum iocpt_status_code code)
+{
+ switch (code) {
+ case IOCPT_RC_SUCCESS:
+ return "IOCPT_RC_SUCCESS";
+ case IOCPT_RC_EVERSION:
+ return "IOCPT_RC_EVERSION";
+ case IOCPT_RC_EOPCODE:
+ return "IOCPT_RC_EOPCODE";
+ case IOCPT_RC_EIO:
+ return "IOCPT_RC_EIO";
+ case IOCPT_RC_EPERM:
+ return "IOCPT_RC_EPERM";
+ case IOCPT_RC_EQID:
+ return "IOCPT_RC_EQID";
+ case IOCPT_RC_EQTYPE:
+ return "IOCPT_RC_EQTYPE";
+ case IOCPT_RC_ENOENT:
+ return "IOCPT_RC_ENOENT";
+ case IOCPT_RC_EINTR:
+ return "IOCPT_RC_EINTR";
+ case IOCPT_RC_EAGAIN:
+ return "IOCPT_RC_EAGAIN";
+ case IOCPT_RC_ENOMEM:
+ return "IOCPT_RC_ENOMEM";
+ case IOCPT_RC_EFAULT:
+ return "IOCPT_RC_EFAULT";
+ case IOCPT_RC_EBUSY:
+ return "IOCPT_RC_EBUSY";
+ case IOCPT_RC_EEXIST:
+ return "IOCPT_RC_EEXIST";
+ case IOCPT_RC_EINVAL:
+ return "IOCPT_RC_EINVAL";
+ case IOCPT_RC_ENOSPC:
+ return "IOCPT_RC_ENOSPC";
+ case IOCPT_RC_ERANGE:
+ return "IOCPT_RC_ERANGE";
+ case IOCPT_RC_BAD_ADDR:
+ return "IOCPT_RC_BAD_ADDR";
+ case IOCPT_RC_DEV_CMD:
+ return "IOCPT_RC_DEV_CMD";
+ case IOCPT_RC_ERROR:
+ return "IOCPT_RC_ERROR";
+ default:
+ return "IOCPT_RC_UNKNOWN";
+ }
+}
+
static const char *
iocpt_opcode_to_str(enum iocpt_cmd_opcode opcode)
{
@@ -97,6 +146,17 @@ iocpt_dev_cmd_wait(struct iocpt_dev *dev, unsigned long max_wait)
return -ETIMEDOUT;
}
+static void
+iocpt_dev_cmd_comp(struct iocpt_dev *dev, void *mem)
+{
+ union iocpt_dev_cmd_comp *comp = mem;
+ uint32_t comp_size = RTE_DIM(comp->words);
+ uint32_t i;
+
+ for (i = 0; i < comp_size; i++)
+ comp->words[i] = ioread32(&dev->dev_cmd->comp.words[i]);
+}
+
static int
iocpt_dev_cmd_wait_check(struct iocpt_dev *dev, unsigned long max_wait)
{
@@ -175,6 +235,29 @@ iocpt_dev_cmd_queue_identify(struct iocpt_dev *dev,
iocpt_dev_cmd_go(dev, &cmd);
}
+static void
+iocpt_dev_cmd_adminq_init(struct iocpt_dev *dev)
+{
+ struct iocpt_queue *q = &dev->adminq->q;
+ struct iocpt_cq *cq = &dev->adminq->cq;
+
+ union iocpt_dev_cmd cmd = {
+ .q_init.opcode = IOCPT_CMD_Q_INIT,
+ .q_init.type = q->type,
+ .q_init.ver = dev->qtype_info[q->type].version,
+ .q_init.index = rte_cpu_to_le_32(q->index),
+ .q_init.flags = rte_cpu_to_le_16(IOCPT_QINIT_F_ENA),
+ .q_init.intr_index = rte_cpu_to_le_16(IONIC_INTR_NONE),
+ .q_init.ring_size = rte_log2_u32(q->num_descs),
+ .q_init.ring_base = rte_cpu_to_le_64(q->base_pa),
+ .q_init.cq_ring_base = rte_cpu_to_le_64(cq->base_pa),
+ };
+
+ IOCPT_PRINT(DEBUG, "adminq.q_init.ver %u", cmd.q_init.ver);
+
+ iocpt_dev_cmd_go(dev, &cmd);
+}
+
/* Dev_cmd consumers */
static void
@@ -346,3 +429,222 @@ iocpt_dev_reset(struct iocpt_dev *dev)
iocpt_dev_cmd_reset(dev);
(void)iocpt_dev_cmd_wait_check(dev, IONIC_DEVCMD_TIMEOUT);
}
+
+int
+iocpt_dev_adminq_init(struct iocpt_dev *dev)
+{
+ struct iocpt_queue *q = &dev->adminq->q;
+ struct iocpt_q_init_comp comp;
+ uint32_t retries = 5;
+ int err;
+
+retry_adminq_init:
+ iocpt_dev_cmd_adminq_init(dev);
+
+ err = iocpt_dev_cmd_wait_check(dev, IONIC_DEVCMD_TIMEOUT);
+ if (err == -EAGAIN && retries > 0) {
+ retries--;
+ rte_delay_us_block(IONIC_DEVCMD_RETRY_WAIT_US);
+ goto retry_adminq_init;
+ }
+ if (err != 0)
+ return err;
+
+ iocpt_dev_cmd_comp(dev, &comp);
+
+ q->hw_type = comp.hw_type;
+ q->hw_index = rte_le_to_cpu_32(comp.hw_index);
+ q->db = iocpt_db_map(dev, q);
+
+ IOCPT_PRINT(DEBUG, "adminq->hw_type %d", q->hw_type);
+ IOCPT_PRINT(DEBUG, "adminq->hw_index %d", q->hw_index);
+ IOCPT_PRINT(DEBUG, "adminq->db %p", q->db);
+
+ dev->adminq->flags |= IOCPT_Q_F_INITED;
+
+ return 0;
+}
+
+/* Admin_cmd interface */
+
+static bool
+iocpt_adminq_service(struct iocpt_cq *cq, uint16_t cq_desc_index,
+ void *cb_arg __rte_unused)
+{
+ struct iocpt_admin_comp *cq_desc_base = cq->base;
+ struct iocpt_admin_comp *cq_desc = &cq_desc_base[cq_desc_index];
+ struct iocpt_admin_q *adminq =
+ container_of(cq, struct iocpt_admin_q, cq);
+ struct iocpt_queue *q = &adminq->q;
+ struct iocpt_admin_ctx *ctx;
+ uint16_t curr_q_tail_idx;
+ uint16_t stop_index;
+ void **info;
+
+ if (!iocpt_color_match(cq_desc->color, cq->done_color))
+ return false;
+
+ stop_index = rte_le_to_cpu_16(cq_desc->comp_index);
+
+ do {
+ info = IOCPT_INFO_PTR(q, q->tail_idx);
+
+ ctx = info[0];
+ if (ctx != NULL) {
+ memcpy(&ctx->comp, cq_desc, sizeof(*cq_desc));
+ ctx->pending_work = false; /* done */
+ }
+
+ curr_q_tail_idx = q->tail_idx;
+ q->tail_idx = Q_NEXT_TO_SRVC(q, 1);
+ } while (curr_q_tail_idx != stop_index);
+
+ return true;
+}
+
+/** iocpt_adminq_post - Post an admin command.
+ * @dev: Handle to dev.
+ * @cmd_ctx: Api admin command context.
+ *
+ * Return: zero or negative error status.
+ */
+static int
+iocpt_adminq_post(struct iocpt_dev *dev, struct iocpt_admin_ctx *ctx)
+{
+ struct iocpt_queue *q = &dev->adminq->q;
+ struct iocpt_admin_cmd *q_desc_base = q->base;
+ struct iocpt_admin_cmd *q_desc;
+ void **info;
+ int err = 0;
+
+ rte_spinlock_lock(&dev->adminq_lock);
+
+ if (iocpt_q_space_avail(q) < 1) {
+ err = -ENOSPC;
+ goto err_out;
+ }
+
+ q_desc = &q_desc_base[q->head_idx];
+
+ memcpy(q_desc, &ctx->cmd, sizeof(ctx->cmd));
+
+ info = IOCPT_INFO_PTR(q, q->head_idx);
+ info[0] = ctx;
+
+ q->head_idx = Q_NEXT_TO_POST(q, 1);
+
+ /* Ring doorbell */
+ iocpt_q_flush(q);
+
+err_out:
+ rte_spinlock_unlock(&dev->adminq_lock);
+
+ return err;
+}
+
+static int
+iocpt_adminq_wait_for_completion(struct iocpt_dev *dev,
+ struct iocpt_admin_ctx *ctx, unsigned long max_wait)
+{
+ struct iocpt_queue *q = &dev->adminq->q;
+ unsigned long step_usec = IONIC_DEVCMD_CHECK_PERIOD_US;
+ unsigned long step_deadline;
+ unsigned long max_wait_usec = max_wait * 1000000L;
+ unsigned long elapsed_usec = 0;
+ int budget = 8;
+ uint16_t idx;
+ void **info;
+
+ step_deadline = IONIC_ADMINQ_WDOG_MS * 1000 / step_usec;
+
+ while (ctx->pending_work && elapsed_usec < max_wait_usec) {
+ /*
+ * Locking here as adminq is served inline and could be
+ * called from multiple places
+ */
+ rte_spinlock_lock(&dev->adminq_service_lock);
+
+ iocpt_cq_service(&dev->adminq->cq, budget,
+ iocpt_adminq_service, NULL);
+
+ /*
+ * Ring the doorbell again if work is pending after step_usec.
+ */
+ if (ctx->pending_work && !step_deadline) {
+ step_deadline = IONIC_ADMINQ_WDOG_MS *
+ 1000 / step_usec;
+
+ rte_spinlock_lock(&dev->adminq_lock);
+ idx = Q_NEXT_TO_POST(q, -1);
+ info = IOCPT_INFO_PTR(q, idx);
+ if (info[0] == ctx)
+ iocpt_q_flush(q);
+ rte_spinlock_unlock(&dev->adminq_lock);
+ }
+
+ rte_spinlock_unlock(&dev->adminq_service_lock);
+
+ rte_delay_us_block(step_usec);
+ elapsed_usec += step_usec;
+ step_deadline--;
+ }
+
+ return (!ctx->pending_work);
+}
+
+static int
+iocpt_adminq_check_err(struct iocpt_admin_ctx *ctx, bool timeout)
+{
+ const char *name;
+ const char *status;
+
+ name = iocpt_opcode_to_str(ctx->cmd.cmd.opcode);
+
+ if (ctx->comp.comp.status == IOCPT_RC_EAGAIN) {
+ IOCPT_PRINT(DEBUG, "%s (%d) returned EAGAIN (%d)",
+ name, ctx->cmd.cmd.opcode,
+ ctx->comp.comp.status);
+ return -EAGAIN;
+ }
+ if (ctx->comp.comp.status != 0 || timeout) {
+ status = iocpt_error_to_str(ctx->comp.comp.status);
+ IOCPT_PRINT(ERR, "%s (%d) failed: %s (%d)",
+ name,
+ ctx->cmd.cmd.opcode,
+ timeout ? "TIMEOUT" : status,
+ timeout ? -1 : ctx->comp.comp.status);
+ return -EIO;
+ }
+
+ if (ctx->cmd.cmd.opcode != IOCPT_CMD_SESS_CONTROL) {
+ IOCPT_PRINT(DEBUG, "%s (%d) succeeded",
+ name, ctx->cmd.cmd.opcode);
+ }
+
+ return 0;
+}
+
+int
+iocpt_adminq_post_wait(struct iocpt_dev *dev, struct iocpt_admin_ctx *ctx)
+{
+ bool done;
+ int err;
+
+ if (ctx->cmd.cmd.opcode != IOCPT_CMD_SESS_CONTROL) {
+ IOCPT_PRINT(DEBUG, "Sending %s (%d) via the admin queue",
+ iocpt_opcode_to_str(ctx->cmd.cmd.opcode),
+ ctx->cmd.cmd.opcode);
+ }
+
+ err = iocpt_adminq_post(dev, ctx);
+ if (err != 0) {
+ IOCPT_PRINT(ERR, "Failure posting %d to the admin queue (%d)",
+ ctx->cmd.cmd.opcode, err);
+ return err;
+ }
+
+ done = iocpt_adminq_wait_for_completion(dev, ctx,
+ IONIC_DEVCMD_TIMEOUT);
+
+ return iocpt_adminq_check_err(ctx, !done /* timed out */);
+}
diff --git a/drivers/crypto/ionic/ionic_crypto_main.c b/drivers/crypto/ionic/ionic_crypto_main.c
index 4f782f3fe4..8d4ee7f29f 100644
--- a/drivers/crypto/ionic/ionic_crypto_main.c
+++ b/drivers/crypto/ionic/ionic_crypto_main.c
@@ -10,6 +10,108 @@
#include "ionic_crypto.h"
+static int
+iocpt_cq_init(struct iocpt_cq *cq, uint16_t num_descs)
+{
+ if (!rte_is_power_of_2(num_descs) ||
+ num_descs < IOCPT_MIN_RING_DESC ||
+ num_descs > IOCPT_MAX_RING_DESC) {
+ IOCPT_PRINT(ERR, "%u descriptors (min: %u max: %u)",
+ num_descs, IOCPT_MIN_RING_DESC, IOCPT_MAX_RING_DESC);
+ return -EINVAL;
+ }
+
+ cq->num_descs = num_descs;
+ cq->size_mask = num_descs - 1;
+ cq->tail_idx = 0;
+ cq->done_color = 1;
+
+ return 0;
+}
+
+static void
+iocpt_cq_map(struct iocpt_cq *cq, void *base, rte_iova_t base_pa)
+{
+ cq->base = base;
+ cq->base_pa = base_pa;
+}
+
+uint32_t
+iocpt_cq_service(struct iocpt_cq *cq, uint32_t work_to_do,
+ iocpt_cq_cb cb, void *cb_arg)
+{
+ uint32_t work_done = 0;
+
+ if (work_to_do == 0)
+ return 0;
+
+ while (cb(cq, cq->tail_idx, cb_arg)) {
+ cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1);
+ if (cq->tail_idx == 0)
+ cq->done_color = !cq->done_color;
+
+ if (++work_done == work_to_do)
+ break;
+ }
+
+ return work_done;
+}
+
+static int
+iocpt_q_init(struct iocpt_queue *q, uint8_t type, uint32_t index,
+ uint16_t num_descs, uint16_t num_segs, uint32_t socket_id)
+{
+ uint32_t ring_size;
+
+ if (!rte_is_power_of_2(num_descs))
+ return -EINVAL;
+
+ ring_size = rte_log2_u32(num_descs);
+ if (ring_size < 2 || ring_size > 16)
+ return -EINVAL;
+
+ q->type = type;
+ q->index = index;
+ q->num_descs = num_descs;
+ q->num_segs = num_segs;
+ q->size_mask = num_descs - 1;
+ q->head_idx = 0;
+ q->tail_idx = 0;
+
+ q->info = rte_calloc_socket("iocpt",
+ num_descs * num_segs, sizeof(void *),
+ rte_mem_page_size(), socket_id);
+ if (q->info == NULL) {
+ IOCPT_PRINT(ERR, "Cannot allocate queue info");
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+static void
+iocpt_q_map(struct iocpt_queue *q, void *base, rte_iova_t base_pa)
+{
+ q->base = base;
+ q->base_pa = base_pa;
+}
+
+static void
+iocpt_q_sg_map(struct iocpt_queue *q, void *base, rte_iova_t base_pa)
+{
+ q->sg_base = base;
+ q->sg_base_pa = base_pa;
+}
+
+static void
+iocpt_q_free(struct iocpt_queue *q)
+{
+ if (q->info != NULL) {
+ rte_free(q->info);
+ q->info = NULL;
+ }
+}
+
static const struct rte_memzone *
iocpt_dma_zone_reserve(const char *type_name, uint16_t qid, size_t size,
unsigned int align, int socket_id)
@@ -33,6 +135,173 @@ iocpt_dma_zone_reserve(const char *type_name, uint16_t qid, size_t size,
RTE_MEMZONE_IOVA_CONTIG, align);
}
+static int
+iocpt_commonq_alloc(struct iocpt_dev *dev,
+ uint8_t type,
+ size_t struct_size,
+ uint32_t socket_id,
+ uint32_t index,
+ const char *type_name,
+ uint16_t flags,
+ uint16_t num_descs,
+ uint16_t num_segs,
+ uint16_t desc_size,
+ uint16_t cq_desc_size,
+ uint16_t sg_desc_size,
+ struct iocpt_common_q **comq)
+{
+ struct iocpt_common_q *new;
+ uint32_t q_size, cq_size, sg_size, total_size;
+ void *q_base, *cq_base, *sg_base;
+ rte_iova_t q_base_pa = 0;
+ rte_iova_t cq_base_pa = 0;
+ rte_iova_t sg_base_pa = 0;
+ size_t page_size = rte_mem_page_size();
+ int err;
+
+ *comq = NULL;
+
+ q_size = num_descs * desc_size;
+ cq_size = num_descs * cq_desc_size;
+ sg_size = num_descs * sg_desc_size;
+
+ /*
+ * Note: aligning q_size/cq_size is not enough due to cq_base address
+ * aligning as q_base could be not aligned to the page.
+ * Adding page_size.
+ */
+ total_size = RTE_ALIGN(q_size, page_size) +
+ RTE_ALIGN(cq_size, page_size) + page_size;
+ if (flags & IOCPT_Q_F_SG)
+ total_size += RTE_ALIGN(sg_size, page_size) + page_size;
+
+ new = rte_zmalloc_socket("iocpt", struct_size,
+ RTE_CACHE_LINE_SIZE, socket_id);
+ if (new == NULL) {
+ IOCPT_PRINT(ERR, "Cannot allocate queue structure");
+ return -ENOMEM;
+ }
+
+ new->dev = dev;
+
+ err = iocpt_q_init(&new->q, type, index, num_descs, num_segs,
+ socket_id);
+ if (err != 0) {
+ IOCPT_PRINT(ERR, "Queue initialization failed");
+ goto err_free_q;
+ }
+
+ err = iocpt_cq_init(&new->cq, num_descs);
+ if (err != 0) {
+ IOCPT_PRINT(ERR, "Completion queue initialization failed");
+ goto err_deinit_q;
+ }
+
+ new->base_z = iocpt_dma_zone_reserve(type_name, index, total_size,
+ IONIC_ALIGN, socket_id);
+ if (new->base_z == NULL) {
+ IOCPT_PRINT(ERR, "Cannot reserve queue DMA memory");
+ err = -ENOMEM;
+ goto err_deinit_cq;
+ }
+
+ new->base = new->base_z->addr;
+ new->base_pa = new->base_z->iova;
+
+ q_base = new->base;
+ q_base_pa = new->base_pa;
+ iocpt_q_map(&new->q, q_base, q_base_pa);
+
+ cq_base = (void *)RTE_ALIGN((uintptr_t)q_base + q_size, page_size);
+ cq_base_pa = RTE_ALIGN(q_base_pa + q_size, page_size);
+ iocpt_cq_map(&new->cq, cq_base, cq_base_pa);
+
+ if (flags & IOCPT_Q_F_SG) {
+ sg_base = (void *)RTE_ALIGN((uintptr_t)cq_base + cq_size,
+ page_size);
+ sg_base_pa = RTE_ALIGN(cq_base_pa + cq_size, page_size);
+ iocpt_q_sg_map(&new->q, sg_base, sg_base_pa);
+ }
+
+ IOCPT_PRINT(DEBUG, "q_base_pa %#jx cq_base_pa %#jx sg_base_pa %#jx",
+ q_base_pa, cq_base_pa, sg_base_pa);
+
+ *comq = new;
+
+ return 0;
+
+err_deinit_cq:
+err_deinit_q:
+ iocpt_q_free(&new->q);
+err_free_q:
+ rte_free(new);
+ return err;
+}
+
+struct ionic_doorbell *
+iocpt_db_map(struct iocpt_dev *dev, struct iocpt_queue *q)
+{
+ return dev->db_pages + q->hw_type;
+}
+
+static int
+iocpt_adminq_alloc(struct iocpt_dev *dev)
+{
+ struct iocpt_admin_q *aq;
+ uint16_t num_descs = IOCPT_ADMINQ_LENGTH;
+ uint16_t flags = 0;
+ int err;
+
+ err = iocpt_commonq_alloc(dev,
+ IOCPT_QTYPE_ADMINQ,
+ sizeof(struct iocpt_admin_q),
+ rte_socket_id(),
+ 0,
+ "admin",
+ flags,
+ num_descs,
+ 1,
+ sizeof(struct iocpt_admin_cmd),
+ sizeof(struct iocpt_admin_comp),
+ 0,
+ (struct iocpt_common_q **)&aq);
+ if (err != 0)
+ return err;
+
+ aq->flags = flags;
+
+ dev->adminq = aq;
+
+ return 0;
+}
+
+static int
+iocpt_adminq_init(struct iocpt_dev *dev)
+{
+ return iocpt_dev_adminq_init(dev);
+}
+
+static void
+iocpt_adminq_deinit(struct iocpt_dev *dev)
+{
+ dev->adminq->flags &= ~IOCPT_Q_F_INITED;
+}
+
+static void
+iocpt_adminq_free(struct iocpt_admin_q *aq)
+{
+ if (aq->base_z != NULL) {
+ rte_memzone_free(aq->base_z);
+ aq->base_z = NULL;
+ aq->base = NULL;
+ aq->base_pa = 0;
+ }
+
+ iocpt_q_free(&aq->q);
+
+ rte_free(aq);
+}
+
static int
iocpt_alloc_objs(struct iocpt_dev *dev)
{
@@ -40,13 +309,23 @@ iocpt_alloc_objs(struct iocpt_dev *dev)
IOCPT_PRINT(DEBUG, "Crypto: %s", dev->name);
+ rte_spinlock_init(&dev->adminq_lock);
+ rte_spinlock_init(&dev->adminq_service_lock);
+
+ err = iocpt_adminq_alloc(dev);
+ if (err != 0) {
+ IOCPT_PRINT(ERR, "Cannot allocate admin queue");
+ err = -ENOMEM;
+ goto err_out;
+ }
+
dev->info_sz = RTE_ALIGN(sizeof(*dev->info), rte_mem_page_size());
dev->info_z = iocpt_dma_zone_reserve("info", 0, dev->info_sz,
IONIC_ALIGN, dev->socket_id);
if (dev->info_z == NULL) {
IOCPT_PRINT(ERR, "Cannot allocate dev info memory");
err = -ENOMEM;
- goto err_out;
+ goto err_free_adminq;
}
dev->info = dev->info_z->addr;
@@ -54,6 +333,9 @@ iocpt_alloc_objs(struct iocpt_dev *dev)
return 0;
+err_free_adminq:
+ iocpt_adminq_free(dev->adminq);
+ dev->adminq = NULL;
err_out:
return err;
}
@@ -68,6 +350,10 @@ iocpt_init(struct iocpt_dev *dev)
if (err != 0)
return err;
+ err = iocpt_adminq_init(dev);
+ if (err != 0)
+ return err;
+
dev->state |= IOCPT_DEV_F_INITED;
return 0;
@@ -87,6 +373,8 @@ iocpt_deinit(struct iocpt_dev *dev)
if (!(dev->state & IOCPT_DEV_F_INITED))
return;
+ iocpt_adminq_deinit(dev);
+
dev->state &= ~IOCPT_DEV_F_INITED;
}
@@ -95,6 +383,11 @@ iocpt_free_objs(struct iocpt_dev *dev)
{
IOCPT_PRINT_CALL();
+ if (dev->adminq != NULL) {
+ iocpt_adminq_free(dev->adminq);
+ dev->adminq = NULL;
+ }
+
if (dev->info != NULL) {
rte_memzone_free(dev->info_z);
dev->info_z = NULL;
--
2.17.1
^ permalink raw reply [flat|nested] 19+ messages in thread* [PATCH v2 5/9] crypto/ionic: add capabilities and basic ops
2024-04-30 20:21 ` [PATCH v2 0/9] " Andrew Boyer
` (3 preceding siblings ...)
2024-04-30 20:21 ` [PATCH v2 4/9] crypto/ionic: add adminq command support Andrew Boyer
@ 2024-04-30 20:21 ` Andrew Boyer
2024-04-30 20:21 ` [PATCH v2 6/9] crypto/ionic: add session support Andrew Boyer
` (3 subsequent siblings)
8 siblings, 0 replies; 19+ messages in thread
From: Andrew Boyer @ 2024-04-30 20:21 UTC (permalink / raw)
To: dev; +Cc: Andrew Boyer
This exposes the supported capabilities to the stack.
Signed-off-by: Andrew Boyer <andrew.boyer@amd.com>
---
drivers/crypto/ionic/ionic_crypto.h | 3 ++
drivers/crypto/ionic/ionic_crypto_caps.c | 25 +++++++++
drivers/crypto/ionic/ionic_crypto_main.c | 8 +++
drivers/crypto/ionic/ionic_crypto_ops.c | 66 ++++++++++++++++++++++++
drivers/crypto/ionic/meson.build | 2 +
5 files changed, 104 insertions(+)
create mode 100644 drivers/crypto/ionic/ionic_crypto_caps.c
create mode 100644 drivers/crypto/ionic/ionic_crypto_ops.c
diff --git a/drivers/crypto/ionic/ionic_crypto.h b/drivers/crypto/ionic/ionic_crypto.h
index 065e1bd826..f487768c10 100644
--- a/drivers/crypto/ionic/ionic_crypto.h
+++ b/drivers/crypto/ionic/ionic_crypto.h
@@ -34,6 +34,8 @@ extern int iocpt_logtype;
#define IOCPT_PRINT_CALL() IOCPT_PRINT(DEBUG, " >>")
+const struct rte_cryptodev_capabilities *iocpt_get_caps(uint64_t flags);
+
static inline void iocpt_struct_size_checks(void)
{
RTE_BUILD_BUG_ON(sizeof(struct ionic_doorbell) != 8);
@@ -227,6 +229,7 @@ int iocpt_probe(void *bus_dev, struct rte_device *rte_dev,
int iocpt_remove(struct rte_device *rte_dev);
void iocpt_configure(struct iocpt_dev *dev);
+int iocpt_assign_ops(struct rte_cryptodev *cdev);
void iocpt_deinit(struct iocpt_dev *dev);
int iocpt_dev_identify(struct iocpt_dev *dev);
diff --git a/drivers/crypto/ionic/ionic_crypto_caps.c b/drivers/crypto/ionic/ionic_crypto_caps.c
new file mode 100644
index 0000000000..c22681fabc
--- /dev/null
+++ b/drivers/crypto/ionic/ionic_crypto_caps.c
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021-2024 Advanced Micro Devices, Inc.
+ */
+
+#include <rte_cryptodev.h>
+
+#include "ionic_crypto.h"
+
+static const struct rte_cryptodev_capabilities iocpt_sym_caps[] = {
+ RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
+};
+
+static const struct rte_cryptodev_capabilities iocpt_asym_caps[] = {
+ /* None */
+ RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
+};
+
+const struct rte_cryptodev_capabilities *
+iocpt_get_caps(uint64_t flags)
+{
+ if (flags & RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO)
+ return iocpt_asym_caps;
+ else
+ return iocpt_sym_caps;
+}
diff --git a/drivers/crypto/ionic/ionic_crypto_main.c b/drivers/crypto/ionic/ionic_crypto_main.c
index 8d4ee7f29f..3ff2d51950 100644
--- a/drivers/crypto/ionic/ionic_crypto_main.c
+++ b/drivers/crypto/ionic/ionic_crypto_main.c
@@ -505,8 +505,16 @@ iocpt_probe(void *bus_dev, struct rte_device *rte_dev,
goto err_free_objs;
}
+ err = iocpt_assign_ops(cdev);
+ if (err != 0) {
+ IOCPT_PRINT(ERR, "Failed to configure opts");
+ goto err_deinit_dev;
+ }
+
return 0;
+err_deinit_dev:
+ iocpt_deinit(dev);
err_free_objs:
iocpt_free_objs(dev);
err_destroy_crypto_dev:
diff --git a/drivers/crypto/ionic/ionic_crypto_ops.c b/drivers/crypto/ionic/ionic_crypto_ops.c
new file mode 100644
index 0000000000..74a6ce56ea
--- /dev/null
+++ b/drivers/crypto/ionic/ionic_crypto_ops.c
@@ -0,0 +1,66 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2021-2024 Advanced Micro Devices, Inc.
+ */
+
+#include <rte_cryptodev.h>
+#include <cryptodev_pmd.h>
+#include <rte_errno.h>
+#include <rte_malloc.h>
+#include <rte_mempool.h>
+
+#include "ionic_crypto.h"
+
+static int
+iocpt_op_config(struct rte_cryptodev *cdev,
+ struct rte_cryptodev_config *config __rte_unused)
+{
+ struct iocpt_dev *dev = cdev->data->dev_private;
+
+ iocpt_configure(dev);
+
+ return 0;
+}
+
+static int
+iocpt_op_close(struct rte_cryptodev *cdev)
+{
+ struct iocpt_dev *dev = cdev->data->dev_private;
+
+ iocpt_deinit(dev);
+
+ return 0;
+}
+
+static void
+iocpt_op_info_get(struct rte_cryptodev *cdev, struct rte_cryptodev_info *info)
+{
+ struct iocpt_dev *dev = cdev->data->dev_private;
+
+ if (info == NULL)
+ return;
+
+ info->max_nb_queue_pairs = dev->max_qps;
+ info->feature_flags = dev->features;
+ info->capabilities = iocpt_get_caps(info->feature_flags);
+ info->sym.max_nb_sessions = dev->max_sessions;
+ info->driver_id = dev->driver_id;
+ info->min_mbuf_headroom_req = 0;
+ info->min_mbuf_tailroom_req = 0;
+}
+
+static struct rte_cryptodev_ops iocpt_ops = {
+ .dev_configure = iocpt_op_config,
+ .dev_close = iocpt_op_close,
+ .dev_infos_get = iocpt_op_info_get,
+};
+
+int
+iocpt_assign_ops(struct rte_cryptodev *cdev)
+{
+ struct iocpt_dev *dev = cdev->data->dev_private;
+
+ cdev->dev_ops = &iocpt_ops;
+ cdev->feature_flags = dev->features;
+
+ return 0;
+}
diff --git a/drivers/crypto/ionic/meson.build b/drivers/crypto/ionic/meson.build
index a6e0a1d415..b63428fa9b 100644
--- a/drivers/crypto/ionic/meson.build
+++ b/drivers/crypto/ionic/meson.build
@@ -5,8 +5,10 @@ deps += ['bus_vdev']
deps += ['common_ionic']
sources = files(
+ 'ionic_crypto_caps.c',
'ionic_crypto_cmds.c',
'ionic_crypto_main.c',
+ 'ionic_crypto_ops.c',
'ionic_crypto_vdev.c',
)
name = 'ionic_crypto'
--
2.17.1
^ permalink raw reply [flat|nested] 19+ messages in thread* [PATCH v2 6/9] crypto/ionic: add session support
2024-04-30 20:21 ` [PATCH v2 0/9] " Andrew Boyer
` (4 preceding siblings ...)
2024-04-30 20:21 ` [PATCH v2 5/9] crypto/ionic: add capabilities and basic ops Andrew Boyer
@ 2024-04-30 20:21 ` Andrew Boyer
2024-04-30 20:21 ` [PATCH v2 7/9] crypto/ionic: add datapath Andrew Boyer
` (2 subsequent siblings)
8 siblings, 0 replies; 19+ messages in thread
From: Andrew Boyer @ 2024-04-30 20:21 UTC (permalink / raw)
To: dev; +Cc: Andrew Boyer
This defines the session object and related commands.
Signed-off-by: Andrew Boyer <andrew.boyer@amd.com>
---
drivers/crypto/ionic/ionic_crypto.h | 33 ++++++
drivers/crypto/ionic/ionic_crypto_main.c | 143 +++++++++++++++++++++++
drivers/crypto/ionic/ionic_crypto_ops.c | 123 +++++++++++++++++++
3 files changed, 299 insertions(+)
diff --git a/drivers/crypto/ionic/ionic_crypto.h b/drivers/crypto/ionic/ionic_crypto.h
index f487768c10..f50c4b4291 100644
--- a/drivers/crypto/ionic/ionic_crypto.h
+++ b/drivers/crypto/ionic/ionic_crypto.h
@@ -14,6 +14,7 @@
#include <rte_cryptodev.h>
#include <cryptodev_pmd.h>
#include <rte_log.h>
+#include <rte_bitmap.h>
#include "ionic_common.h"
#include "ionic_crypto_if.h"
@@ -154,6 +155,32 @@ struct iocpt_admin_q {
uint16_t flags;
};
+#define IOCPT_S_F_INITED BIT(0)
+
+struct iocpt_session_priv {
+ struct iocpt_dev *dev;
+
+ uint32_t index;
+
+ uint16_t iv_offset;
+ uint16_t iv_length;
+ uint16_t digest_length;
+ uint16_t aad_length;
+
+ uint8_t flags;
+ uint8_t op;
+ uint8_t type;
+
+ uint16_t key_len;
+ uint8_t key[IOCPT_SESS_KEY_LEN_MAX_SYMM];
+};
+
+static inline uint32_t
+iocpt_session_size(void)
+{
+ return sizeof(struct iocpt_session_priv);
+}
+
#define IOCPT_DEV_F_INITED BIT(0)
#define IOCPT_DEV_F_UP BIT(1)
#define IOCPT_DEV_F_FW_RESET BIT(2)
@@ -186,6 +213,8 @@ struct iocpt_dev {
struct iocpt_admin_q *adminq;
+ struct rte_bitmap *sess_bm; /* SET bit indicates index is free */
+
uint64_t features;
uint32_t hw_features;
@@ -239,6 +268,10 @@ void iocpt_dev_reset(struct iocpt_dev *dev);
int iocpt_adminq_post_wait(struct iocpt_dev *dev, struct iocpt_admin_ctx *ctx);
+int iocpt_session_init(struct iocpt_session_priv *priv);
+int iocpt_session_update(struct iocpt_session_priv *priv);
+void iocpt_session_deinit(struct iocpt_session_priv *priv);
+
struct ionic_doorbell __iomem *iocpt_db_map(struct iocpt_dev *dev,
struct iocpt_queue *q);
diff --git a/drivers/crypto/ionic/ionic_crypto_main.c b/drivers/crypto/ionic/ionic_crypto_main.c
index 3ff2d51950..cc893ad8e9 100644
--- a/drivers/crypto/ionic/ionic_crypto_main.c
+++ b/drivers/crypto/ionic/ionic_crypto_main.c
@@ -112,6 +112,116 @@ iocpt_q_free(struct iocpt_queue *q)
}
}
+static int
+iocpt_session_write(struct iocpt_session_priv *priv,
+ enum iocpt_sess_control_oper oper)
+{
+ struct iocpt_dev *dev = priv->dev;
+ struct iocpt_admin_ctx ctx = {
+ .pending_work = true,
+ .cmd.sess_control = {
+ .opcode = IOCPT_CMD_SESS_CONTROL,
+ .type = priv->type,
+ .oper = oper,
+ .index = rte_cpu_to_le_32(priv->index),
+ .key_len = rte_cpu_to_le_16(priv->key_len),
+ .key_seg_len = (uint8_t)RTE_MIN(priv->key_len,
+ IOCPT_SESS_KEY_SEG_LEN),
+ },
+ };
+ struct iocpt_sess_control_cmd *cmd = &ctx.cmd.sess_control;
+ uint16_t key_offset;
+ uint8_t key_segs, seg;
+ int err;
+
+ key_segs = ((priv->key_len - 1) >> IOCPT_SESS_KEY_SEG_SHFT) + 1;
+
+ for (seg = 0; seg < key_segs; seg++) {
+ ctx.pending_work = true;
+
+ key_offset = seg * cmd->key_seg_len;
+ memcpy(cmd->key, &priv->key[key_offset],
+ IOCPT_SESS_KEY_SEG_LEN);
+ cmd->key_seg_idx = seg;
+
+ /* Mark final segment */
+ if (seg + 1 == key_segs)
+ cmd->flags |= rte_cpu_to_le_16(IOCPT_SCTL_F_END);
+
+ err = iocpt_adminq_post_wait(dev, &ctx);
+ if (err != 0)
+ return err;
+ }
+
+ return 0;
+}
+
+int
+iocpt_session_init(struct iocpt_session_priv *priv)
+{
+ struct iocpt_dev *dev = priv->dev;
+ uint64_t bm_slab = 0;
+ uint32_t bm_pos = 0;
+ int err = 0;
+
+ rte_spinlock_lock(&dev->adminq_lock);
+
+ if (rte_bitmap_scan(dev->sess_bm, &bm_pos, &bm_slab) > 0) {
+ priv->index = bm_pos + rte_ctz64(bm_slab);
+ rte_bitmap_clear(dev->sess_bm, priv->index);
+ } else
+ err = -ENOSPC;
+
+ rte_spinlock_unlock(&dev->adminq_lock);
+
+ if (err != 0) {
+ IOCPT_PRINT(ERR, "session index space exhausted");
+ return err;
+ }
+
+ err = iocpt_session_write(priv, IOCPT_SESS_INIT);
+ if (err != 0) {
+ rte_spinlock_lock(&dev->adminq_lock);
+ rte_bitmap_set(dev->sess_bm, priv->index);
+ rte_spinlock_unlock(&dev->adminq_lock);
+ return err;
+ }
+
+ priv->flags |= IOCPT_S_F_INITED;
+
+ return 0;
+}
+
+int
+iocpt_session_update(struct iocpt_session_priv *priv)
+{
+ return iocpt_session_write(priv, IOCPT_SESS_UPDATE_KEY);
+}
+
+void
+iocpt_session_deinit(struct iocpt_session_priv *priv)
+{
+ struct iocpt_dev *dev = priv->dev;
+ struct iocpt_admin_ctx ctx = {
+ .pending_work = true,
+ .cmd.sess_control = {
+ .opcode = IOCPT_CMD_SESS_CONTROL,
+ .type = priv->type,
+ .oper = IOCPT_SESS_DISABLE,
+ .index = rte_cpu_to_le_32(priv->index),
+ .key_len = rte_cpu_to_le_16(priv->key_len),
+ },
+ };
+
+ (void)iocpt_adminq_post_wait(dev, &ctx);
+
+ rte_spinlock_lock(&dev->adminq_lock);
+ rte_bitmap_set(dev->sess_bm, priv->index);
+ rte_spinlock_unlock(&dev->adminq_lock);
+
+ priv->flags &= ~IOCPT_S_F_INITED;
+}
+
static const struct rte_memzone *
iocpt_dma_zone_reserve(const char *type_name, uint16_t qid, size_t size,
unsigned int align, int socket_id)
@@ -305,6 +415,8 @@ iocpt_adminq_free(struct iocpt_admin_q *aq)
static int
iocpt_alloc_objs(struct iocpt_dev *dev)
{
+ uint32_t bmsize, i;
+ uint8_t *bm;
int err;
IOCPT_PRINT(DEBUG, "Crypto: %s", dev->name);
@@ -331,8 +443,33 @@ iocpt_alloc_objs(struct iocpt_dev *dev)
dev->info = dev->info_z->addr;
dev->info_pa = dev->info_z->iova;
+ bmsize = rte_bitmap_get_memory_footprint(dev->max_sessions);
+ bm = rte_malloc_socket("iocpt", bmsize,
+ RTE_CACHE_LINE_SIZE, dev->socket_id);
+ if (bm == NULL) {
+ IOCPT_PRINT(ERR, "Cannot allocate %uB bitmap memory", bmsize);
+ err = -ENOMEM;
+ goto err_free_dmazone;
+ }
+
+ dev->sess_bm = rte_bitmap_init(dev->max_sessions, bm, bmsize);
+ if (dev->sess_bm == NULL) {
+ IOCPT_PRINT(ERR, "Cannot initialize bitmap");
+ err = -EFAULT;
+ goto err_free_bm;
+ }
+ for (i = 0; i < dev->max_sessions; i++)
+ rte_bitmap_set(dev->sess_bm, i);
+
return 0;
+err_free_bm:
+ rte_free(bm);
+err_free_dmazone:
+ rte_memzone_free(dev->info_z);
+ dev->info_z = NULL;
+ dev->info = NULL;
+ dev->info_pa = 0;
err_free_adminq:
iocpt_adminq_free(dev->adminq);
dev->adminq = NULL;
@@ -383,6 +520,12 @@ iocpt_free_objs(struct iocpt_dev *dev)
{
IOCPT_PRINT_CALL();
+ if (dev->sess_bm != NULL) {
+ rte_bitmap_free(dev->sess_bm);
+ rte_free(dev->sess_bm);
+ dev->sess_bm = NULL;
+ }
+
if (dev->adminq != NULL) {
iocpt_adminq_free(dev->adminq);
dev->adminq = NULL;
diff --git a/drivers/crypto/ionic/ionic_crypto_ops.c b/drivers/crypto/ionic/ionic_crypto_ops.c
index 74a6ce56ea..e6b3402b63 100644
--- a/drivers/crypto/ionic/ionic_crypto_ops.c
+++ b/drivers/crypto/ionic/ionic_crypto_ops.c
@@ -48,10 +48,133 @@ iocpt_op_info_get(struct rte_cryptodev *cdev, struct rte_cryptodev_info *info)
info->min_mbuf_tailroom_req = 0;
}
+static unsigned int
+iocpt_op_get_session_size(struct rte_cryptodev *cdev __rte_unused)
+{
+ return iocpt_session_size();
+}
+
+static inline int
+iocpt_is_algo_supported(struct rte_crypto_sym_xform *xform)
+{
+ if (xform->next != NULL) {
+ IOCPT_PRINT(ERR, "chaining not supported");
+ return -ENOTSUP;
+ }
+
+ if (xform->type != RTE_CRYPTO_SYM_XFORM_AEAD) {
+ IOCPT_PRINT(ERR, "xform->type %d not supported", xform->type);
+ return -ENOTSUP;
+ }
+
+ return 0;
+}
+
+static __rte_always_inline int
+iocpt_fill_sess_aead(struct rte_crypto_sym_xform *xform,
+ struct iocpt_session_priv *priv)
+{
+ struct rte_crypto_aead_xform *aead_form = &xform->aead;
+
+ if (aead_form->algo != RTE_CRYPTO_AEAD_AES_GCM) {
+ IOCPT_PRINT(ERR, "Unknown algo");
+ return -EINVAL;
+ }
+ if (aead_form->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) {
+ priv->op = IOCPT_DESC_OPCODE_GCM_AEAD_ENCRYPT;
+ } else if (aead_form->op == RTE_CRYPTO_AEAD_OP_DECRYPT) {
+ priv->op = IOCPT_DESC_OPCODE_GCM_AEAD_DECRYPT;
+ } else {
+ IOCPT_PRINT(ERR, "Unknown cipher operations");
+ return -1;
+ }
+
+ if (aead_form->key.length < IOCPT_SESS_KEY_LEN_MIN ||
+ aead_form->key.length > IOCPT_SESS_KEY_LEN_MAX_SYMM) {
+ IOCPT_PRINT(ERR, "Invalid cipher keylen %u",
+ aead_form->key.length);
+ return -1;
+ }
+ priv->key_len = aead_form->key.length;
+ memcpy(priv->key, aead_form->key.data, priv->key_len);
+
+ priv->type = IOCPT_SESS_AEAD_AES_GCM;
+ priv->iv_offset = aead_form->iv.offset;
+ priv->iv_length = aead_form->iv.length;
+ priv->digest_length = aead_form->digest_length;
+ priv->aad_length = aead_form->aad_length;
+
+ return 0;
+}
+
+static int
+iocpt_session_cfg(struct iocpt_dev *dev,
+ struct rte_crypto_sym_xform *xform,
+ struct rte_cryptodev_sym_session *sess)
+{
+ struct rte_crypto_sym_xform *chain;
+ struct iocpt_session_priv *priv = NULL;
+
+ if (iocpt_is_algo_supported(xform) < 0)
+ return -ENOTSUP;
+
+ if (unlikely(sess == NULL)) {
+ IOCPT_PRINT(ERR, "invalid session");
+ return -EINVAL;
+ }
+
+ priv = CRYPTODEV_GET_SYM_SESS_PRIV(sess);
+ priv->dev = dev;
+
+ chain = xform;
+ while (chain) {
+ switch (chain->type) {
+ case RTE_CRYPTO_SYM_XFORM_AEAD:
+ if (iocpt_fill_sess_aead(chain, priv))
+ return -EIO;
+ break;
+ default:
+ IOCPT_PRINT(ERR, "invalid crypto xform type %d",
+ chain->type);
+ return -ENOTSUP;
+ }
+ chain = chain->next;
+ }
+
+ return iocpt_session_init(priv);
+}
+
+static int
+iocpt_op_session_cfg(struct rte_cryptodev *cdev,
+ struct rte_crypto_sym_xform *xform,
+ struct rte_cryptodev_sym_session *sess)
+{
+ struct iocpt_dev *dev = cdev->data->dev_private;
+
+ return iocpt_session_cfg(dev, xform, sess);
+}
+
+static void
+iocpt_session_clear(struct rte_cryptodev_sym_session *sess)
+{
+ iocpt_session_deinit(CRYPTODEV_GET_SYM_SESS_PRIV(sess));
+}
+
+static void
+iocpt_op_session_clear(struct rte_cryptodev *cdev __rte_unused,
+ struct rte_cryptodev_sym_session *sess)
+{
+ iocpt_session_clear(sess);
+}
+
static struct rte_cryptodev_ops iocpt_ops = {
.dev_configure = iocpt_op_config,
.dev_close = iocpt_op_close,
.dev_infos_get = iocpt_op_info_get,
+
+ .sym_session_get_size = iocpt_op_get_session_size,
+ .sym_session_configure = iocpt_op_session_cfg,
+ .sym_session_clear = iocpt_op_session_clear,
};
int
--
2.17.1
^ permalink raw reply [flat|nested] 19+ messages in thread* [PATCH v2 7/9] crypto/ionic: add datapath
2024-04-30 20:21 ` [PATCH v2 0/9] " Andrew Boyer
` (5 preceding siblings ...)
2024-04-30 20:21 ` [PATCH v2 6/9] crypto/ionic: add session support Andrew Boyer
@ 2024-04-30 20:21 ` Andrew Boyer
2024-04-30 20:21 ` [PATCH v2 8/9] crypto/ionic: add a watchdog operation Andrew Boyer
2024-04-30 20:21 ` [PATCH v2 9/9] crypto/ionic: add stats support Andrew Boyer
8 siblings, 0 replies; 19+ messages in thread
From: Andrew Boyer @ 2024-04-30 20:21 UTC (permalink / raw)
To: dev; +Cc: Andrew Boyer
This defines the main crypto operation enqueue and dequeue handlers.
Signed-off-by: Andrew Boyer <andrew.boyer@amd.com>
---
doc/guides/cryptodevs/features/ionic.ini | 8 +
doc/guides/cryptodevs/ionic.rst | 11 +
drivers/crypto/ionic/ionic_crypto.h | 17 ++
drivers/crypto/ionic/ionic_crypto_caps.c | 30 +++
drivers/crypto/ionic/ionic_crypto_main.c | 222 ++++++++++++++++-
drivers/crypto/ionic/ionic_crypto_ops.c | 295 +++++++++++++++++++++++
6 files changed, 581 insertions(+), 2 deletions(-)
diff --git a/doc/guides/cryptodevs/features/ionic.ini b/doc/guides/cryptodevs/features/ionic.ini
index 62b7e9e8f2..d3e00bd795 100644
--- a/doc/guides/cryptodevs/features/ionic.ini
+++ b/doc/guides/cryptodevs/features/ionic.ini
@@ -4,6 +4,12 @@
; Refer to default.ini for the full list of available PMD features.
;
[Features]
+Symmetric crypto = Y
+HW Accelerated = Y
+In Place SGL = Y
+OOP SGL In LB Out = Y
+OOP SGL In SGL Out = Y
+OOP LB In LB Out = Y
;
; Supported crypto algorithms of 'ionic' crypto driver.
@@ -19,6 +25,8 @@
; Supported AEAD algorithms of 'ionic' crypto driver.
;
[AEAD]
+AES GCM (128) = Y
+AES GCM (256) = Y
;
; Supported Asymmetric algorithms of the 'ionic' crypto driver.
diff --git a/doc/guides/cryptodevs/ionic.rst b/doc/guides/cryptodevs/ionic.rst
index c9173deb2f..9d557f7cc2 100644
--- a/doc/guides/cryptodevs/ionic.rst
+++ b/doc/guides/cryptodevs/ionic.rst
@@ -26,3 +26,14 @@ Runtime Configuration
None
+Features
+--------
+
+The ionic crypto PMD has support for:
+
+Symmetric Crypto Algorithms
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+AEAD algorithms:
+
+* ``RTE_CRYPTO_AEAD_AES_GCM``
diff --git a/drivers/crypto/ionic/ionic_crypto.h b/drivers/crypto/ionic/ionic_crypto.h
index f50c4b4291..e05b458926 100644
--- a/drivers/crypto/ionic/ionic_crypto.h
+++ b/drivers/crypto/ionic/ionic_crypto.h
@@ -27,6 +27,8 @@
#define IOCPT_MIN_RING_DESC 16
#define IOCPT_ADMINQ_LENGTH 16 /* must be a power of two */
+#define IOCPT_CRYPTOQ_WAIT 10 /* 1s */
+
extern int iocpt_logtype;
#define RTE_LOGTYPE_IOCPT iocpt_logtype
@@ -155,6 +157,14 @@ struct iocpt_admin_q {
uint16_t flags;
};
+struct iocpt_crypto_q {
+ /* cacheline0, cacheline1 */
+ IOCPT_COMMON_FIELDS;
+
+ /* cacheline2 */
+ uint16_t flags;
+};
+
#define IOCPT_S_F_INITED BIT(0)
struct iocpt_session_priv {
@@ -212,6 +222,7 @@ struct iocpt_dev {
rte_spinlock_t adminq_service_lock;
struct iocpt_admin_q *adminq;
+ struct iocpt_crypto_q **cryptoqs;
struct rte_bitmap *sess_bm; /* SET bit indicates index is free */
@@ -259,6 +270,8 @@ int iocpt_remove(struct rte_device *rte_dev);
void iocpt_configure(struct iocpt_dev *dev);
int iocpt_assign_ops(struct rte_cryptodev *cdev);
+int iocpt_start(struct iocpt_dev *dev);
+void iocpt_stop(struct iocpt_dev *dev);
void iocpt_deinit(struct iocpt_dev *dev);
int iocpt_dev_identify(struct iocpt_dev *dev);
@@ -268,6 +281,10 @@ void iocpt_dev_reset(struct iocpt_dev *dev);
int iocpt_adminq_post_wait(struct iocpt_dev *dev, struct iocpt_admin_ctx *ctx);
+int iocpt_cryptoq_alloc(struct iocpt_dev *dev, uint32_t socket_id,
+ uint32_t index, uint16_t ndescs);
+void iocpt_cryptoq_free(struct iocpt_crypto_q *cptq);
+
int iocpt_session_init(struct iocpt_session_priv *priv);
int iocpt_session_update(struct iocpt_session_priv *priv);
void iocpt_session_deinit(struct iocpt_session_priv *priv);
diff --git a/drivers/crypto/ionic/ionic_crypto_caps.c b/drivers/crypto/ionic/ionic_crypto_caps.c
index c22681fabc..da5a69be3d 100644
--- a/drivers/crypto/ionic/ionic_crypto_caps.c
+++ b/drivers/crypto/ionic/ionic_crypto_caps.c
@@ -7,6 +7,36 @@
#include "ionic_crypto.h"
static const struct rte_cryptodev_capabilities iocpt_sym_caps[] = {
+ { /* AES GCM */
+ .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+ {.sym = {
+ .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
+ {.aead = {
+ .algo = RTE_CRYPTO_AEAD_AES_GCM,
+ .block_size = 16,
+ .key_size = {
+ .min = 16,
+ .max = 32,
+ .increment = 16
+ },
+ .digest_size = {
+ .min = 16,
+ .max = 16,
+ .increment = 0
+ },
+ .aad_size = {
+ .min = 0,
+ .max = 1024,
+ .increment = 1
+ },
+ .iv_size = {
+ .min = 12,
+ .max = 12,
+ .increment = 0
+ }
+ }, }
+ }, }
+ },
RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
};
diff --git a/drivers/crypto/ionic/ionic_crypto_main.c b/drivers/crypto/ionic/ionic_crypto_main.c
index cc893ad8e9..7693377831 100644
--- a/drivers/crypto/ionic/ionic_crypto_main.c
+++ b/drivers/crypto/ionic/ionic_crypto_main.c
@@ -29,6 +29,15 @@ iocpt_cq_init(struct iocpt_cq *cq, uint16_t num_descs)
return 0;
}
+static void
+iocpt_cq_reset(struct iocpt_cq *cq)
+{
+ cq->tail_idx = 0;
+ cq->done_color = 1;
+
+ memset(cq->base, 0, sizeof(struct iocpt_nop_comp) * cq->num_descs);
+}
+
static void
iocpt_cq_map(struct iocpt_cq *cq, void *base, rte_iova_t base_pa)
{
@@ -89,6 +98,13 @@ iocpt_q_init(struct iocpt_queue *q, uint8_t type, uint32_t index,
return 0;
}
+static void
+iocpt_q_reset(struct iocpt_queue *q)
+{
+ q->head_idx = 0;
+ q->tail_idx = 0;
+}
+
static void
iocpt_q_map(struct iocpt_queue *q, void *base, rte_iova_t base_pa)
{
@@ -348,12 +364,154 @@ iocpt_commonq_alloc(struct iocpt_dev *dev,
return err;
}
+int
+iocpt_cryptoq_alloc(struct iocpt_dev *dev, uint32_t socket_id, uint32_t index,
+ uint16_t num_descs)
+{
+ struct iocpt_crypto_q *cptq;
+ uint16_t flags = 0;
+ int err;
+
+ /* CryptoQ always supports scatter-gather */
+ flags |= IOCPT_Q_F_SG;
+
+ IOCPT_PRINT(DEBUG, "cptq %u num_descs %u num_segs %u",
+ index, num_descs, 1);
+
+ err = iocpt_commonq_alloc(dev,
+ IOCPT_QTYPE_CRYPTOQ,
+ sizeof(struct iocpt_crypto_q),
+ socket_id,
+ index,
+ "crypto",
+ flags,
+ num_descs,
+ 1,
+ sizeof(struct iocpt_crypto_desc),
+ sizeof(struct iocpt_crypto_comp),
+ sizeof(struct iocpt_crypto_sg_desc),
+ (struct iocpt_common_q **)&cptq);
+ if (err != 0)
+ return err;
+
+ cptq->flags = flags;
+
+ dev->cryptoqs[index] = cptq;
+
+ return 0;
+}
+
struct ionic_doorbell *
iocpt_db_map(struct iocpt_dev *dev, struct iocpt_queue *q)
{
return dev->db_pages + q->hw_type;
}
+static int
+iocpt_cryptoq_init(struct iocpt_crypto_q *cptq)
+{
+ struct iocpt_queue *q = &cptq->q;
+ struct iocpt_dev *dev = cptq->dev;
+ struct iocpt_cq *cq = &cptq->cq;
+ struct iocpt_admin_ctx ctx = {
+ .pending_work = true,
+ .cmd.q_init = {
+ .opcode = IOCPT_CMD_Q_INIT,
+ .type = IOCPT_QTYPE_CRYPTOQ,
+ .ver = dev->qtype_info[IOCPT_QTYPE_CRYPTOQ].version,
+ .index = rte_cpu_to_le_32(q->index),
+ .flags = rte_cpu_to_le_16(IOCPT_QINIT_F_ENA |
+ IOCPT_QINIT_F_SG),
+ .intr_index = rte_cpu_to_le_16(IONIC_INTR_NONE),
+ .ring_size = rte_log2_u32(q->num_descs),
+ .ring_base = rte_cpu_to_le_64(q->base_pa),
+ .cq_ring_base = rte_cpu_to_le_64(cq->base_pa),
+ .sg_ring_base = rte_cpu_to_le_64(q->sg_base_pa),
+ },
+ };
+ int err;
+
+ IOCPT_PRINT(DEBUG, "cptq_init.index %d", q->index);
+ IOCPT_PRINT(DEBUG, "cptq_init.ring_base %#jx", q->base_pa);
+ IOCPT_PRINT(DEBUG, "cptq_init.ring_size %d",
+ ctx.cmd.q_init.ring_size);
+ IOCPT_PRINT(DEBUG, "cptq_init.ver %u", ctx.cmd.q_init.ver);
+
+ iocpt_q_reset(q);
+ iocpt_cq_reset(cq);
+
+ err = iocpt_adminq_post_wait(dev, &ctx);
+ if (err != 0)
+ return err;
+
+ q->hw_type = ctx.comp.q_init.hw_type;
+ q->hw_index = rte_le_to_cpu_32(ctx.comp.q_init.hw_index);
+ q->db = iocpt_db_map(dev, q);
+
+ IOCPT_PRINT(DEBUG, "cptq->hw_type %d", q->hw_type);
+ IOCPT_PRINT(DEBUG, "cptq->hw_index %d", q->hw_index);
+ IOCPT_PRINT(DEBUG, "cptq->db %p", q->db);
+
+ cptq->flags |= IOCPT_Q_F_INITED;
+
+ return 0;
+}
+
+static void
+iocpt_cryptoq_deinit(struct iocpt_crypto_q *cptq)
+{
+ struct iocpt_dev *dev = cptq->dev;
+ struct iocpt_admin_ctx ctx = {
+ .pending_work = true,
+ .cmd.q_control = {
+ .opcode = IOCPT_CMD_Q_CONTROL,
+ .type = IOCPT_QTYPE_CRYPTOQ,
+ .index = rte_cpu_to_le_32(cptq->q.index),
+ .oper = IOCPT_Q_DISABLE,
+ },
+ };
+ unsigned long sleep_usec = 100UL * 1000;
+ uint32_t sleep_cnt, sleep_max = IOCPT_CRYPTOQ_WAIT;
+ int err;
+
+ for (sleep_cnt = 0; sleep_cnt < sleep_max; sleep_cnt++) {
+ ctx.pending_work = true;
+
+ err = iocpt_adminq_post_wait(dev, &ctx);
+ if (err != -EAGAIN)
+ break;
+
+ rte_delay_us_block(sleep_usec);
+ }
+
+ if (err != 0)
+ IOCPT_PRINT(ERR, "Deinit queue %u returned %d after %u ms",
+ cptq->q.index, err, sleep_cnt * 100);
+ else
+ IOCPT_PRINT(DEBUG, "Deinit queue %u returned %d after %u ms",
+ cptq->q.index, err, sleep_cnt * 100);
+
+ cptq->flags &= ~IOCPT_Q_F_INITED;
+}
+
+void
+iocpt_cryptoq_free(struct iocpt_crypto_q *cptq)
+{
+ if (cptq == NULL)
+ return;
+
+ if (cptq->base_z != NULL) {
+ rte_memzone_free(cptq->base_z);
+ cptq->base_z = NULL;
+ cptq->base = NULL;
+ cptq->base_pa = 0;
+ }
+
+ iocpt_q_free(&cptq->q);
+
+ rte_free(cptq);
+}
+
static int
iocpt_adminq_alloc(struct iocpt_dev *dev)
{
@@ -421,6 +579,14 @@ iocpt_alloc_objs(struct iocpt_dev *dev)
IOCPT_PRINT(DEBUG, "Crypto: %s", dev->name);
+ dev->cryptoqs = rte_calloc_socket("iocpt",
+ dev->max_qps, sizeof(*dev->cryptoqs),
+ RTE_CACHE_LINE_SIZE, dev->socket_id);
+ if (dev->cryptoqs == NULL) {
+ IOCPT_PRINT(ERR, "Cannot allocate tx queues array");
+ return -ENOMEM;
+ }
+
rte_spinlock_init(&dev->adminq_lock);
rte_spinlock_init(&dev->adminq_service_lock);
@@ -428,7 +594,7 @@ iocpt_alloc_objs(struct iocpt_dev *dev)
if (err != 0) {
IOCPT_PRINT(ERR, "Cannot allocate admin queue");
err = -ENOMEM;
- goto err_out;
+ goto err_free_cryptoqs;
}
dev->info_sz = RTE_ALIGN(sizeof(*dev->info), rte_mem_page_size());
@@ -473,7 +639,9 @@ iocpt_alloc_objs(struct iocpt_dev *dev)
err_free_adminq:
iocpt_adminq_free(dev->adminq);
dev->adminq = NULL;
-err_out:
+err_free_cryptoqs:
+ rte_free(dev->cryptoqs);
+ dev->cryptoqs = NULL;
return err;
}
@@ -502,6 +670,43 @@ iocpt_configure(struct iocpt_dev *dev)
RTE_SET_USED(dev);
}
+int
+iocpt_start(struct iocpt_dev *dev)
+{
+ uint32_t i;
+ int err;
+
+ IOCPT_PRINT(DEBUG, "Starting %u queues",
+ dev->crypto_dev->data->nb_queue_pairs);
+
+ for (i = 0; i < dev->crypto_dev->data->nb_queue_pairs; i++) {
+ err = iocpt_cryptoq_init(dev->cryptoqs[i]);
+ if (err != 0)
+ return err;
+ }
+
+ dev->state |= IOCPT_DEV_F_UP;
+
+ return 0;
+}
+
+void
+iocpt_stop(struct iocpt_dev *dev)
+{
+ uint32_t i;
+
+ IOCPT_PRINT_CALL();
+
+ dev->state &= ~IOCPT_DEV_F_UP;
+
+ for (i = 0; i < dev->crypto_dev->data->nb_queue_pairs; i++) {
+ struct iocpt_crypto_q *cptq = dev->cryptoqs[i];
+
+ if (cptq->flags & IOCPT_Q_F_INITED)
+ (void)iocpt_cryptoq_deinit(cptq);
+ }
+}
+
void
iocpt_deinit(struct iocpt_dev *dev)
{
@@ -518,8 +723,16 @@ iocpt_deinit(struct iocpt_dev *dev)
static void
iocpt_free_objs(struct iocpt_dev *dev)
{
+ void **queue_pairs = dev->crypto_dev->data->queue_pairs;
+ uint32_t i;
+
IOCPT_PRINT_CALL();
+ for (i = 0; i < dev->crypto_dev->data->nb_queue_pairs; i++) {
+ iocpt_cryptoq_free(queue_pairs[i]);
+ queue_pairs[i] = NULL;
+ }
+
if (dev->sess_bm != NULL) {
rte_bitmap_free(dev->sess_bm);
rte_free(dev->sess_bm);
@@ -531,6 +744,11 @@ iocpt_free_objs(struct iocpt_dev *dev)
dev->adminq = NULL;
}
+ if (dev->cryptoqs != NULL) {
+ rte_free(dev->cryptoqs);
+ dev->cryptoqs = NULL;
+ }
+
if (dev->info != NULL) {
rte_memzone_free(dev->info_z);
dev->info_z = NULL;
diff --git a/drivers/crypto/ionic/ionic_crypto_ops.c b/drivers/crypto/ionic/ionic_crypto_ops.c
index e6b3402b63..28b099dea2 100644
--- a/drivers/crypto/ionic/ionic_crypto_ops.c
+++ b/drivers/crypto/ionic/ionic_crypto_ops.c
@@ -21,6 +21,22 @@ iocpt_op_config(struct rte_cryptodev *cdev,
return 0;
}
+static int
+iocpt_op_start(struct rte_cryptodev *cdev)
+{
+ struct iocpt_dev *dev = cdev->data->dev_private;
+
+ return iocpt_start(dev);
+}
+
+static void
+iocpt_op_stop(struct rte_cryptodev *cdev)
+{
+ struct iocpt_dev *dev = cdev->data->dev_private;
+
+ return iocpt_stop(dev);
+}
+
static int
iocpt_op_close(struct rte_cryptodev *cdev)
{
@@ -48,6 +64,53 @@ iocpt_op_info_get(struct rte_cryptodev *cdev, struct rte_cryptodev_info *info)
info->min_mbuf_tailroom_req = 0;
}
+static int
+iocpt_op_queue_release(struct rte_cryptodev *cdev, uint16_t queue_id)
+{
+ struct iocpt_crypto_q *cptq = cdev->data->queue_pairs[queue_id];
+
+ IOCPT_PRINT(DEBUG, "queue_id %u", queue_id);
+
+ assert(!(cptq->flags & IOCPT_Q_F_INITED));
+
+ iocpt_cryptoq_free(cptq);
+
+ cdev->data->queue_pairs[queue_id] = NULL;
+
+ return 0;
+}
+
+static int
+iocpt_op_queue_setup(struct rte_cryptodev *cdev, uint16_t queue_id,
+ const struct rte_cryptodev_qp_conf *qp_conf,
+ int socket_id)
+{
+ struct iocpt_dev *dev = cdev->data->dev_private;
+ int err;
+
+ if (cdev->data->queue_pairs[queue_id] != NULL)
+ iocpt_op_queue_release(cdev, queue_id);
+
+ if (qp_conf->nb_descriptors < (1 << IOCPT_QSIZE_MIN_LG2) ||
+ qp_conf->nb_descriptors > (1 << IOCPT_QSIZE_MAX_LG2)) {
+ IOCPT_PRINT(ERR, "invalid nb_descriptors %u, use range %u..%u",
+ qp_conf->nb_descriptors,
+ 1 << IOCPT_QSIZE_MIN_LG2, 1 << IOCPT_QSIZE_MAX_LG2);
+ return -ERANGE;
+ }
+
+ IOCPT_PRINT(DEBUG, "queue_id %u", queue_id);
+
+ err = iocpt_cryptoq_alloc(dev, socket_id, queue_id,
+ qp_conf->nb_descriptors);
+ if (err != 0)
+ return err;
+
+ cdev->data->queue_pairs[queue_id] = dev->cryptoqs[queue_id];
+
+ return 0;
+}
+
static unsigned int
iocpt_op_get_session_size(struct rte_cryptodev *cdev __rte_unused)
{
@@ -167,11 +230,238 @@ iocpt_op_session_clear(struct rte_cryptodev *cdev __rte_unused,
iocpt_session_clear(sess);
}
+static inline void
+iocpt_fill_sge(struct iocpt_crypto_sg_elem *arr, uint8_t idx,
+ uint64_t addr, uint16_t len)
+{
+ arr[idx].addr = rte_cpu_to_le_64(addr);
+ arr[idx].len = rte_cpu_to_le_16(len);
+}
+
+static __rte_always_inline int
+iocpt_enq_one_aead(struct iocpt_crypto_q *cptq,
+ struct iocpt_session_priv *priv, struct rte_crypto_op *op)
+{
+ struct rte_crypto_sym_op *sym_op = op->sym;
+ struct iocpt_queue *q = &cptq->q;
+ struct iocpt_crypto_desc *desc, *desc_base = q->base;
+ struct iocpt_crypto_sg_desc *sg_desc, *sg_desc_base = q->sg_base;
+ struct iocpt_crypto_sg_elem *src, *dst;
+ rte_iova_t aad_addr, digest_addr, iv_addr, seg_addr;
+ uint32_t data_len, data_offset, seg_len;
+ uint8_t nsge_src = 0, nsge_dst = 0, flags = 0;
+ struct rte_mbuf *m;
+
+ desc = &desc_base[q->head_idx];
+ sg_desc = &sg_desc_base[q->head_idx];
+ src = sg_desc->src_elems;
+ dst = sg_desc->dst_elems;
+
+ /* Fill the first SGE with the IV / Nonce */
+ iv_addr = rte_crypto_op_ctophys_offset(op, priv->iv_offset);
+ iocpt_fill_sge(src, nsge_src++, iv_addr, priv->iv_length);
+
+ /* Fill the second SGE with the AAD, if applicable */
+ if (priv->aad_length > 0) {
+ aad_addr = sym_op->aead.aad.phys_addr;
+ iocpt_fill_sge(src, nsge_src++, aad_addr, priv->aad_length);
+ flags |= IOCPT_DESC_F_AAD_VALID;
+ }
+
+ m = sym_op->m_src;
+ data_len = sym_op->aead.data.length;
+
+ /* Fast-forward through mbuf chain to account for data offset */
+ data_offset = sym_op->aead.data.offset;
+ while (m != NULL && data_offset >= m->data_len) {
+ data_offset -= m->data_len;
+ m = m->next;
+ }
+
+ /* Fill the next SGEs with the payload segments */
+ while (m != NULL && data_len > 0) {
+ seg_addr = rte_mbuf_data_iova(m) + data_offset;
+ seg_len = RTE_MIN(m->data_len - data_offset, data_len);
+ data_offset = 0;
+ data_len -= seg_len;
+
+ /* Use -1 to save room for digest */
+ if (nsge_src >= IOCPT_CRYPTO_MAX_SG_ELEMS - 1)
+ return -ERANGE;
+
+ iocpt_fill_sge(src, nsge_src++, seg_addr, seg_len);
+
+ m = m->next;
+ }
+
+ /* AEAD AES-GCM: digest == authentication tag */
+ digest_addr = sym_op->aead.digest.phys_addr;
+ iocpt_fill_sge(src, nsge_src++, digest_addr, priv->digest_length);
+
+ /* Process Out-Of-Place destination SGL */
+ if (sym_op->m_dst != NULL) {
+ /* Put the AAD here, too */
+ if (priv->aad_length > 0)
+ iocpt_fill_sge(dst, nsge_dst++,
+ sym_op->aead.aad.phys_addr, priv->aad_length);
+
+ m = sym_op->m_dst;
+ data_len = sym_op->aead.data.length;
+
+ /* Fast-forward through chain to account for data offset */
+ data_offset = sym_op->aead.data.offset;
+ while (m != NULL && data_offset >= m->data_len) {
+ data_offset -= m->data_len;
+ m = m->next;
+ }
+
+ /* Fill in the SGEs with the payload segments */
+ while (m != NULL && data_len > 0) {
+ seg_addr = rte_mbuf_data_iova(m) + data_offset;
+ seg_len = RTE_MIN(m->data_len - data_offset, data_len);
+ data_offset = 0;
+ data_len -= seg_len;
+
+ if (nsge_dst >= IOCPT_CRYPTO_MAX_SG_ELEMS)
+ return -ERANGE;
+
+ iocpt_fill_sge(dst, nsge_dst++, seg_addr, seg_len);
+
+ m = m->next;
+ }
+ }
+
+ desc->opcode = priv->op;
+ desc->flags = flags;
+ desc->num_src_dst_sgs = iocpt_encode_nsge_src_dst(nsge_src, nsge_dst);
+ desc->session_tag = rte_cpu_to_le_32(priv->index);
+
+ op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
+ q->info[q->head_idx] = op;
+ q->head_idx = Q_NEXT_TO_POST(q, 1);
+
+ return 0;
+}
+
+static uint16_t
+iocpt_enqueue_sym(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)
+{
+ struct iocpt_crypto_q *cptq = qp;
+ struct rte_crypto_op *op;
+ struct iocpt_session_priv *priv;
+ uint16_t avail, count;
+ int err;
+
+ avail = iocpt_q_space_avail(&cptq->q);
+ if (unlikely(nb_ops > avail))
+ nb_ops = avail;
+
+ count = 0;
+ while (likely(count < nb_ops)) {
+ op = ops[count];
+
+ if (unlikely(op->sess_type != RTE_CRYPTO_OP_WITH_SESSION)) {
+ op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+ break;
+ }
+
+ priv = CRYPTODEV_GET_SYM_SESS_PRIV(op->sym->session);
+ if (unlikely(priv == NULL)) {
+ op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
+ break;
+ }
+
+ err = iocpt_enq_one_aead(cptq, priv, op);
+ if (unlikely(err != 0)) {
+ op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+ break;
+ }
+
+ count++;
+ }
+
+ if (likely(count > 0))
+ iocpt_q_flush(&cptq->q);
+
+ return count;
+}
+
+static uint16_t
+iocpt_dequeue_sym(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)
+{
+ struct iocpt_crypto_q *cptq = qp;
+ struct iocpt_queue *q = &cptq->q;
+ struct iocpt_cq *cq = &cptq->cq;
+ struct rte_crypto_op *op;
+ struct iocpt_crypto_comp *cq_desc_base = cq->base;
+ volatile struct iocpt_crypto_comp *cq_desc;
+ uint16_t count = 0;
+
+ cq_desc = &cq_desc_base[cq->tail_idx];
+
+ /* First walk the CQ to update any completed op's status
+ * NB: These can arrive out of order!
+ */
+ while ((cq_desc->color & 0x1) == cq->done_color) {
+ cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1);
+ if (unlikely(cq->tail_idx == 0))
+ cq->done_color = !cq->done_color;
+
+ op = q->info[rte_le_to_cpu_16(cq_desc->comp_index)];
+
+ /* Process returned CQ descriptor status */
+ if (unlikely(cq_desc->status)) {
+ switch (cq_desc->status) {
+ case IOCPT_COMP_SYMM_AUTH_VERIFY_ERROR:
+ op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
+ break;
+ case IOCPT_COMP_INVAL_OPCODE_ERROR:
+ case IOCPT_COMP_UNSUPP_OPCODE_ERROR:
+ case IOCPT_COMP_SYMM_SRC_SG_ERROR:
+ case IOCPT_COMP_SYMM_DST_SG_ERROR:
+ case IOCPT_COMP_SYMM_SRC_DST_LEN_MISMATCH:
+ case IOCPT_COMP_SYMM_KEY_IDX_ERROR:
+ op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+ break;
+ default:
+ op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+ break;
+ }
+ } else
+ op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+
+ cq_desc = &cq_desc_base[cq->tail_idx];
+ }
+
+ /* Next walk the SQ to pop off completed ops in-order */
+ while (count < nb_ops) {
+ op = q->info[q->tail_idx];
+
+ /* No more completions */
+ if (op == NULL ||
+ op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
+ break;
+
+ ops[count] = op;
+ q->info[q->tail_idx] = NULL;
+
+ q->tail_idx = Q_NEXT_TO_SRVC(q, 1);
+ count++;
+ }
+
+ return count;
+}
+
static struct rte_cryptodev_ops iocpt_ops = {
.dev_configure = iocpt_op_config,
+ .dev_start = iocpt_op_start,
+ .dev_stop = iocpt_op_stop,
.dev_close = iocpt_op_close,
.dev_infos_get = iocpt_op_info_get,
+ .queue_pair_setup = iocpt_op_queue_setup,
+ .queue_pair_release = iocpt_op_queue_release,
+
.sym_session_get_size = iocpt_op_get_session_size,
.sym_session_configure = iocpt_op_session_cfg,
.sym_session_clear = iocpt_op_session_clear,
@@ -185,5 +475,10 @@ iocpt_assign_ops(struct rte_cryptodev *cdev)
cdev->dev_ops = &iocpt_ops;
cdev->feature_flags = dev->features;
+ if (dev->features & RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO) {
+ cdev->enqueue_burst = iocpt_enqueue_sym;
+ cdev->dequeue_burst = iocpt_dequeue_sym;
+ }
+
return 0;
}
--
2.17.1
^ permalink raw reply [flat|nested] 19+ messages in thread* [PATCH v2 8/9] crypto/ionic: add a watchdog operation
2024-04-30 20:21 ` [PATCH v2 0/9] " Andrew Boyer
` (6 preceding siblings ...)
2024-04-30 20:21 ` [PATCH v2 7/9] crypto/ionic: add datapath Andrew Boyer
@ 2024-04-30 20:21 ` Andrew Boyer
2024-04-30 20:21 ` [PATCH v2 9/9] crypto/ionic: add stats support Andrew Boyer
8 siblings, 0 replies; 19+ messages in thread
From: Andrew Boyer @ 2024-04-30 20:21 UTC (permalink / raw)
To: dev; +Cc: Andrew Boyer
If no progress has been made within the timeout, post a dummy operation
using session 0. This will restart the queue in the rare case that a
doorbell is lost inside the device.
Signed-off-by: Andrew Boyer <andrew.boyer@amd.com>
---
drivers/crypto/ionic/ionic_crypto.h | 16 ++++
drivers/crypto/ionic/ionic_crypto_main.c | 31 ++++++++
drivers/crypto/ionic/ionic_crypto_ops.c | 97 +++++++++++++++++++++++-
3 files changed, 142 insertions(+), 2 deletions(-)
diff --git a/drivers/crypto/ionic/ionic_crypto.h b/drivers/crypto/ionic/ionic_crypto.h
index e05b458926..69c17887fb 100644
--- a/drivers/crypto/ionic/ionic_crypto.h
+++ b/drivers/crypto/ionic/ionic_crypto.h
@@ -89,6 +89,14 @@ struct iocpt_dev_bars {
uint32_t num_bars;
};
+/* Queue watchdog */
+#define IOCPT_Q_WDOG_SESS_IDX 0
+#define IOCPT_Q_WDOG_KEY_LEN 16
+#define IOCPT_Q_WDOG_IV_LEN 12
+#define IOCPT_Q_WDOG_PLD_LEN 4
+#define IOCPT_Q_WDOG_TAG_LEN 16
+#define IOCPT_Q_WDOG_OP_TYPE RTE_CRYPTO_OP_TYPE_UNDEFINED
+
struct iocpt_qtype_info {
uint8_t version;
uint8_t supported;
@@ -162,7 +170,15 @@ struct iocpt_crypto_q {
IOCPT_COMMON_FIELDS;
/* cacheline2 */
+ uint64_t last_wdog_cycles;
uint16_t flags;
+
+ /* cacheline3 */
+ uint64_t enqueued_wdogs;
+ uint64_t dequeued_wdogs;
+ uint8_t wdog_iv[IOCPT_Q_WDOG_IV_LEN];
+ uint8_t wdog_pld[IOCPT_Q_WDOG_PLD_LEN];
+ uint8_t wdog_tag[IOCPT_Q_WDOG_TAG_LEN];
};
#define IOCPT_S_F_INITED BIT(0)
diff --git a/drivers/crypto/ionic/ionic_crypto_main.c b/drivers/crypto/ionic/ionic_crypto_main.c
index 7693377831..113347c57a 100644
--- a/drivers/crypto/ionic/ionic_crypto_main.c
+++ b/drivers/crypto/ionic/ionic_crypto_main.c
@@ -172,6 +172,22 @@ iocpt_session_write(struct iocpt_session_priv *priv,
return 0;
}
+static int
+iocpt_session_wdog(struct iocpt_dev *dev)
+{
+ struct iocpt_session_priv priv = {
+ .dev = dev,
+ .index = IOCPT_Q_WDOG_SESS_IDX,
+ .type = IOCPT_SESS_AEAD_AES_GCM,
+ .key_len = IOCPT_Q_WDOG_KEY_LEN,
+ };
+
+ /* Reserve session 0 for queue watchdog */
+ rte_bitmap_clear(dev->sess_bm, IOCPT_Q_WDOG_SESS_IDX);
+
+ return iocpt_session_write(&priv, IOCPT_SESS_INIT);
+}
+
int
iocpt_session_init(struct iocpt_session_priv *priv)
{
@@ -491,6 +507,9 @@ iocpt_cryptoq_deinit(struct iocpt_crypto_q *cptq)
IOCPT_PRINT(DEBUG, "Deinit queue %u returned %d after %u ms",
cptq->q.index, err, sleep_cnt * 100);
+ IOCPT_PRINT(DEBUG, "Queue %u watchdog: enq %"PRIu64" deq %"PRIu64,
+ cptq->q.index, cptq->enqueued_wdogs, cptq->dequeued_wdogs);
+
cptq->flags &= ~IOCPT_Q_F_INITED;
}
@@ -659,9 +678,21 @@ iocpt_init(struct iocpt_dev *dev)
if (err != 0)
return err;
+ /* Write the queue watchdog key */
+ err = iocpt_session_wdog(dev);
+ if (err != 0) {
+ IOCPT_PRINT(ERR, "Cannot setup watchdog session");
+ goto err_out_adminq_deinit;
+ }
+
dev->state |= IOCPT_DEV_F_INITED;
return 0;
+
+err_out_adminq_deinit:
+ iocpt_adminq_deinit(dev);
+
+ return err;
}
void
diff --git a/drivers/crypto/ionic/ionic_crypto_ops.c b/drivers/crypto/ionic/ionic_crypto_ops.c
index 28b099dea2..0330fd76ad 100644
--- a/drivers/crypto/ionic/ionic_crypto_ops.c
+++ b/drivers/crypto/ionic/ionic_crypto_ops.c
@@ -58,7 +58,8 @@ iocpt_op_info_get(struct rte_cryptodev *cdev, struct rte_cryptodev_info *info)
info->max_nb_queue_pairs = dev->max_qps;
info->feature_flags = dev->features;
info->capabilities = iocpt_get_caps(info->feature_flags);
- info->sym.max_nb_sessions = dev->max_sessions;
+ /* Reserve one session for watchdog */
+ info->sym.max_nb_sessions = dev->max_sessions - 1;
info->driver_id = dev->driver_id;
info->min_mbuf_headroom_req = 0;
info->min_mbuf_tailroom_req = 0;
@@ -380,12 +381,72 @@ iocpt_enqueue_sym(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)
count++;
}
- if (likely(count > 0))
+ if (likely(count > 0)) {
iocpt_q_flush(&cptq->q);
+ /* Restart timer if ops are being enqueued */
+ cptq->last_wdog_cycles = rte_get_timer_cycles();
+ }
+
return count;
}
+static void
+iocpt_enqueue_wdog(struct iocpt_crypto_q *cptq)
+{
+ struct iocpt_queue *q = &cptq->q;
+ struct iocpt_crypto_desc *desc, *desc_base = q->base;
+ struct iocpt_crypto_sg_desc *sg_desc, *sg_desc_base = q->sg_base;
+ struct iocpt_crypto_sg_elem *src;
+ struct rte_crypto_op *wdog_op;
+ rte_iova_t iv_addr, pld_addr, tag_addr;
+ uint8_t nsge_src = 0;
+ uint16_t avail;
+
+ avail = iocpt_q_space_avail(&cptq->q);
+ if (avail < 1)
+ goto out_flush;
+
+ wdog_op = rte_zmalloc_socket("iocpt", sizeof(*wdog_op),
+ RTE_CACHE_LINE_SIZE, rte_socket_id());
+ if (wdog_op == NULL)
+ goto out_flush;
+
+ wdog_op->type = IOCPT_Q_WDOG_OP_TYPE;
+ wdog_op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
+
+ desc = &desc_base[q->head_idx];
+ sg_desc = &sg_desc_base[q->head_idx];
+ src = sg_desc->src_elems;
+
+ /* Fill the first SGE with the IV / Nonce */
+ iv_addr = rte_mem_virt2iova(cptq->wdog_iv);
+ iocpt_fill_sge(src, nsge_src++, iv_addr, IOCPT_Q_WDOG_IV_LEN);
+
+ /* Fill the second SGE with the payload segment */
+ pld_addr = rte_mem_virt2iova(cptq->wdog_pld);
+ iocpt_fill_sge(src, nsge_src++, pld_addr, IOCPT_Q_WDOG_PLD_LEN);
+
+ /* AEAD AES-GCM: digest == authentication tag */
+ tag_addr = rte_mem_virt2iova(cptq->wdog_tag);
+ iocpt_fill_sge(src, nsge_src++, tag_addr, IOCPT_Q_WDOG_TAG_LEN);
+
+ desc->opcode = IOCPT_DESC_OPCODE_GCM_AEAD_ENCRYPT;
+ desc->flags = 0;
+ desc->num_src_dst_sgs = iocpt_encode_nsge_src_dst(nsge_src, 0);
+ desc->session_tag = rte_cpu_to_le_32(IOCPT_Q_WDOG_SESS_IDX);
+
+ q->info[q->head_idx] = wdog_op;
+ q->head_idx = Q_NEXT_TO_POST(q, 1);
+
+ IOCPT_PRINT(DEBUG, "Queue %u wdog enq %p",
+ q->index, wdog_op);
+ cptq->enqueued_wdogs++;
+
+out_flush:
+ iocpt_q_flush(q);
+}
+
static uint16_t
iocpt_dequeue_sym(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)
{
@@ -395,6 +456,7 @@ iocpt_dequeue_sym(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)
struct rte_crypto_op *op;
struct iocpt_crypto_comp *cq_desc_base = cq->base;
volatile struct iocpt_crypto_comp *cq_desc;
+ uint64_t then, now, hz, delta;
uint16_t count = 0;
cq_desc = &cq_desc_base[cq->tail_idx];
@@ -442,6 +504,17 @@ iocpt_dequeue_sym(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)
op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
break;
+ /* Handle watchdog operations */
+ if (unlikely(op->type == IOCPT_Q_WDOG_OP_TYPE)) {
+ IOCPT_PRINT(DEBUG, "Queue %u wdog deq %p st %d",
+ q->index, op, op->status);
+ q->info[q->tail_idx] = NULL;
+ q->tail_idx = Q_NEXT_TO_SRVC(q, 1);
+ cptq->dequeued_wdogs++;
+ rte_free(op);
+ continue;
+ }
+
ops[count] = op;
q->info[q->tail_idx] = NULL;
@@ -449,6 +522,26 @@ iocpt_dequeue_sym(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)
count++;
}
+ if (!count) {
+ /*
+ * Ring the doorbell again if no work was dequeued and work
+ * is still pending after the deadline.
+ */
+ if (q->head_idx != q->tail_idx) {
+ then = cptq->last_wdog_cycles;
+ now = rte_get_timer_cycles();
+ hz = rte_get_timer_hz();
+ delta = (now - then) * 1000;
+
+ if (delta >= hz * IONIC_Q_WDOG_MS) {
+ iocpt_enqueue_wdog(cptq);
+ cptq->last_wdog_cycles = now;
+ }
+ }
+ } else
+ /* Restart timer if the queue is making progress */
+ cptq->last_wdog_cycles = rte_get_timer_cycles();
+
return count;
}
--
2.17.1
^ permalink raw reply [flat|nested] 19+ messages in thread* [PATCH v2 9/9] crypto/ionic: add stats support
2024-04-30 20:21 ` [PATCH v2 0/9] " Andrew Boyer
` (7 preceding siblings ...)
2024-04-30 20:21 ` [PATCH v2 8/9] crypto/ionic: add a watchdog operation Andrew Boyer
@ 2024-04-30 20:21 ` Andrew Boyer
8 siblings, 0 replies; 19+ messages in thread
From: Andrew Boyer @ 2024-04-30 20:21 UTC (permalink / raw)
To: dev; +Cc: Andrew Boyer
This defines the stats handlers and exposes them to the stack.
Signed-off-by: Andrew Boyer <andrew.boyer@amd.com>
---
drivers/crypto/ionic/ionic_crypto.h | 8 ++++
drivers/crypto/ionic/ionic_crypto_main.c | 48 ++++++++++++++++++++++++
drivers/crypto/ionic/ionic_crypto_ops.c | 33 +++++++++++++++-
3 files changed, 87 insertions(+), 2 deletions(-)
diff --git a/drivers/crypto/ionic/ionic_crypto.h b/drivers/crypto/ionic/ionic_crypto.h
index 69c17887fb..db87ea0490 100644
--- a/drivers/crypto/ionic/ionic_crypto.h
+++ b/drivers/crypto/ionic/ionic_crypto.h
@@ -174,6 +174,8 @@ struct iocpt_crypto_q {
uint16_t flags;
/* cacheline3 */
+ struct rte_cryptodev_stats stats;
+
uint64_t enqueued_wdogs;
uint64_t dequeued_wdogs;
uint8_t wdog_iv[IOCPT_Q_WDOG_IV_LEN];
@@ -252,6 +254,8 @@ struct iocpt_dev {
struct iocpt_qtype_info qtype_info[IOCPT_QTYPE_MAX];
uint8_t qtype_ver[IOCPT_QTYPE_MAX];
+
+ struct rte_cryptodev_stats stats_base;
};
struct iocpt_dev_intf {
@@ -313,6 +317,10 @@ typedef bool (*iocpt_cq_cb)(struct iocpt_cq *cq, uint16_t cq_desc_index,
uint32_t iocpt_cq_service(struct iocpt_cq *cq, uint32_t work_to_do,
iocpt_cq_cb cb, void *cb_arg);
+void iocpt_get_stats(const struct iocpt_dev *dev,
+ struct rte_cryptodev_stats *stats);
+void iocpt_reset_stats(struct iocpt_dev *dev);
+
static inline uint16_t
iocpt_q_space_avail(struct iocpt_queue *q)
{
diff --git a/drivers/crypto/ionic/ionic_crypto_main.c b/drivers/crypto/ionic/ionic_crypto_main.c
index 113347c57a..0685ddd457 100644
--- a/drivers/crypto/ionic/ionic_crypto_main.c
+++ b/drivers/crypto/ionic/ionic_crypto_main.c
@@ -128,6 +128,52 @@ iocpt_q_free(struct iocpt_queue *q)
}
}
+static void
+iocpt_get_abs_stats(const struct iocpt_dev *dev,
+ struct rte_cryptodev_stats *stats)
+{
+ uint32_t i;
+
+ memset(stats, 0, sizeof(*stats));
+
+ /* Sum up the per-queue stats counters */
+ for (i = 0; i < dev->crypto_dev->data->nb_queue_pairs; i++) {
+ struct rte_cryptodev_stats *q_stats = &dev->cryptoqs[i]->stats;
+
+ stats->enqueued_count += q_stats->enqueued_count;
+ stats->dequeued_count += q_stats->dequeued_count;
+ stats->enqueue_err_count += q_stats->enqueue_err_count;
+ stats->dequeue_err_count += q_stats->dequeue_err_count;
+ }
+}
+
+void
+iocpt_get_stats(const struct iocpt_dev *dev, struct rte_cryptodev_stats *stats)
+{
+ /* Retrieve the new absolute stats values */
+ iocpt_get_abs_stats(dev, stats);
+
+ /* Subtract the base stats values to get relative values */
+ stats->enqueued_count -= dev->stats_base.enqueued_count;
+ stats->dequeued_count -= dev->stats_base.dequeued_count;
+ stats->enqueue_err_count -= dev->stats_base.enqueue_err_count;
+ stats->dequeue_err_count -= dev->stats_base.dequeue_err_count;
+}
+
+void
+iocpt_reset_stats(struct iocpt_dev *dev)
+{
+ uint32_t i;
+
+ /* Erase the per-queue stats counters */
+ for (i = 0; i < dev->crypto_dev->data->nb_queue_pairs; i++)
+ memset(&dev->cryptoqs[i]->stats, 0,
+ sizeof(dev->cryptoqs[i]->stats));
+
+ /* Update the base stats values */
+ iocpt_get_abs_stats(dev, &dev->stats_base);
+}
+
static int
iocpt_session_write(struct iocpt_session_priv *priv,
enum iocpt_sess_control_oper oper)
@@ -669,6 +715,8 @@ iocpt_init(struct iocpt_dev *dev)
{
int err;
+ memset(&dev->stats_base, 0, sizeof(dev->stats_base));
+
/* Uses dev_cmds */
err = iocpt_dev_init(dev, dev->info_pa);
if (err != 0)
diff --git a/drivers/crypto/ionic/ionic_crypto_ops.c b/drivers/crypto/ionic/ionic_crypto_ops.c
index 0330fd76ad..839bbf69d1 100644
--- a/drivers/crypto/ionic/ionic_crypto_ops.c
+++ b/drivers/crypto/ionic/ionic_crypto_ops.c
@@ -65,6 +65,23 @@ iocpt_op_info_get(struct rte_cryptodev *cdev, struct rte_cryptodev_info *info)
info->min_mbuf_tailroom_req = 0;
}
+static void
+iocpt_op_stats_get(struct rte_cryptodev *cdev,
+ struct rte_cryptodev_stats *stats)
+{
+ struct iocpt_dev *dev = cdev->data->dev_private;
+
+ iocpt_get_stats(dev, stats);
+}
+
+static void
+iocpt_op_stats_reset(struct rte_cryptodev *cdev)
+{
+ struct iocpt_dev *dev = cdev->data->dev_private;
+
+ iocpt_reset_stats(dev);
+}
+
static int
iocpt_op_queue_release(struct rte_cryptodev *cdev, uint16_t queue_id)
{
@@ -350,6 +367,7 @@ iocpt_enqueue_sym(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)
struct iocpt_crypto_q *cptq = qp;
struct rte_crypto_op *op;
struct iocpt_session_priv *priv;
+ struct rte_cryptodev_stats *stats = &cptq->stats;
uint16_t avail, count;
int err;
@@ -375,6 +393,7 @@ iocpt_enqueue_sym(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)
err = iocpt_enq_one_aead(cptq, priv, op);
if (unlikely(err != 0)) {
op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+ stats->enqueue_err_count++;
break;
}
@@ -386,6 +405,8 @@ iocpt_enqueue_sym(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)
/* Restart timer if ops are being enqueued */
cptq->last_wdog_cycles = rte_get_timer_cycles();
+
+ stats->enqueued_count += count;
}
return count;
@@ -439,8 +460,8 @@ iocpt_enqueue_wdog(struct iocpt_crypto_q *cptq)
q->info[q->head_idx] = wdog_op;
q->head_idx = Q_NEXT_TO_POST(q, 1);
- IOCPT_PRINT(DEBUG, "Queue %u wdog enq %p",
- q->index, wdog_op);
+ IOCPT_PRINT(DEBUG, "Queue %u wdog enq %p ops %"PRIu64,
+ q->index, wdog_op, cptq->stats.enqueued_count);
cptq->enqueued_wdogs++;
out_flush:
@@ -456,6 +477,7 @@ iocpt_dequeue_sym(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)
struct rte_crypto_op *op;
struct iocpt_crypto_comp *cq_desc_base = cq->base;
volatile struct iocpt_crypto_comp *cq_desc;
+ struct rte_cryptodev_stats *stats = &cptq->stats;
uint64_t then, now, hz, delta;
uint16_t count = 0;
@@ -515,6 +537,9 @@ iocpt_dequeue_sym(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)
continue;
}
+ if (unlikely(op->status != RTE_CRYPTO_OP_STATUS_SUCCESS))
+ stats->dequeue_err_count++;
+
ops[count] = op;
q->info[q->tail_idx] = NULL;
@@ -542,6 +567,8 @@ iocpt_dequeue_sym(void *qp, struct rte_crypto_op **ops, uint16_t nb_ops)
/* Restart timer if the queue is making progress */
cptq->last_wdog_cycles = rte_get_timer_cycles();
+ stats->dequeued_count += count;
+
return count;
}
@@ -552,6 +579,8 @@ static struct rte_cryptodev_ops iocpt_ops = {
.dev_close = iocpt_op_close,
.dev_infos_get = iocpt_op_info_get,
+ .stats_get = iocpt_op_stats_get,
+ .stats_reset = iocpt_op_stats_reset,
.queue_pair_setup = iocpt_op_queue_setup,
.queue_pair_release = iocpt_op_queue_release,
--
2.17.1
^ permalink raw reply [flat|nested] 19+ messages in thread