[dpdk-dev] [PATCH v6 03/10] crypto/virtio: support basic PMD ops

[Jay Zhou <jianjay.zhou@huawei.com>]

Signed-off-by: Jay Zhou <jianjay.zhou@huawei.com>
---
 drivers/crypto/virtio/virtio_cryptodev.c | 434 ++++++++++++++++++++++++++++++-
 drivers/crypto/virtio/virtio_cryptodev.h |  32 ++-
 drivers/crypto/virtio/virtio_rxtx.c      |  68 +++++
 3 files changed, 525 insertions(+), 9 deletions(-)

diff --git a/drivers/crypto/virtio/virtio_cryptodev.c b/drivers/crypto/virtio/virtio_cryptodev.c
index 4550834..0be99fd 100644
--- a/drivers/crypto/virtio/virtio_cryptodev.c
+++ b/drivers/crypto/virtio/virtio_cryptodev.c
@@ -1,6 +1,7 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2018 HUAWEI TECHNOLOGIES CO., LTD.
  */
+#include <rte_errno.h>
 #include <rte_pci.h>
 #include <rte_bus_pci.h>
 #include <rte_cryptodev.h>
@@ -15,6 +16,22 @@
 int virtio_crypto_logtype_tx;
 int virtio_crypto_logtype_driver;
 
+static int virtio_crypto_dev_configure(struct rte_cryptodev *dev,
+		struct rte_cryptodev_config *config);
+static int virtio_crypto_dev_start(struct rte_cryptodev *dev);
+static void virtio_crypto_dev_stop(struct rte_cryptodev *dev);
+static int virtio_crypto_dev_close(struct rte_cryptodev *dev);
+static void virtio_crypto_dev_info_get(struct rte_cryptodev *dev,
+		struct rte_cryptodev_info *dev_info);
+static int virtio_crypto_qp_setup(struct rte_cryptodev *dev,
+		uint16_t queue_pair_id,
+		const struct rte_cryptodev_qp_conf *qp_conf,
+		int socket_id,
+		struct rte_mempool *session_pool);
+static int virtio_crypto_qp_release(struct rte_cryptodev *dev,
+		uint16_t queue_pair_id);
+static void virtio_crypto_dev_free_mbufs(struct rte_cryptodev *dev);
+
 /*
  * The set of PCI devices this driver supports
  */
@@ -26,22 +43,249 @@
 
 uint8_t cryptodev_virtio_driver_id;
 
+void virtio_crypto_queue_release(struct virtqueue *vq)
+{
+	struct virtio_crypto_hw *hw;
+
+	PMD_INIT_FUNC_TRACE();
+
+	if (vq) {
+		hw = vq->hw;
+		/* Select and deactivate the queue */
+		VTPCI_OPS(hw)->del_queue(hw, vq);
+
+		rte_memzone_free(vq->mz);
+		rte_mempool_free(vq->mpool);
+		rte_free(vq);
+	}
+}
+
+#define MPOOL_MAX_NAME_SZ 32
+
+int virtio_crypto_queue_setup(struct rte_cryptodev *dev,
+		int queue_type,
+		uint16_t vtpci_queue_idx,
+		uint16_t nb_desc,
+		int socket_id,
+		struct virtqueue **pvq)
+{
+	char vq_name[VIRTQUEUE_MAX_NAME_SZ];
+	char mpool_name[MPOOL_MAX_NAME_SZ];
+	const struct rte_memzone *mz;
+	unsigned int vq_size, size;
+	struct virtio_crypto_hw *hw = dev->data->dev_private;
+	struct virtqueue *vq = NULL;
+	uint32_t i = 0;
+	uint32_t j;
+
+	PMD_INIT_FUNC_TRACE();
+
+	VIRTIO_CRYPTO_INIT_LOG_DBG("setting up queue: %u", vtpci_queue_idx);
+
+	/*
+	 * Read the virtqueue size from the Queue Size field
+	 * Always power of 2 and if 0 virtqueue does not exist
+	 */
+	vq_size = VTPCI_OPS(hw)->get_queue_num(hw, vtpci_queue_idx);
+	if (vq_size == 0) {
+		VIRTIO_CRYPTO_INIT_LOG_ERR("virtqueue does not exist");
+		return -EINVAL;
+	}
+	VIRTIO_CRYPTO_INIT_LOG_DBG("vq_size: %u", vq_size);
+
+	if (!rte_is_power_of_2(vq_size)) {
+		VIRTIO_CRYPTO_INIT_LOG_ERR("virtqueue size is not powerof 2");
+		return -EINVAL;
+	}
+
+	if (queue_type == VTCRYPTO_DATAQ) {
+		snprintf(vq_name, sizeof(vq_name), "dev%d_dataqueue%d",
+				dev->data->dev_id, vtpci_queue_idx);
+		snprintf(mpool_name, sizeof(mpool_name),
+				"dev%d_dataqueue%d_mpool",
+				dev->data->dev_id, vtpci_queue_idx);
+	} else if (queue_type == VTCRYPTO_CTRLQ) {
+		snprintf(vq_name, sizeof(vq_name), "dev%d_controlqueue",
+				dev->data->dev_id);
+		snprintf(mpool_name, sizeof(mpool_name),
+				"dev%d_controlqueue_mpool",
+				dev->data->dev_id);
+	}
+	size = RTE_ALIGN_CEIL(sizeof(*vq) +
+				vq_size * sizeof(struct vq_desc_extra),
+				RTE_CACHE_LINE_SIZE);
+	vq = rte_zmalloc_socket(vq_name, size, RTE_CACHE_LINE_SIZE,
+				socket_id);
+	if (vq == NULL) {
+		VIRTIO_CRYPTO_INIT_LOG_ERR("Can not allocate virtqueue");
+		return -ENOMEM;
+	}
+
+	if (queue_type == VTCRYPTO_DATAQ) {
+		/* pre-allocate a mempool and use it in the data plane to
+		 * improve performance
+		 */
+		vq->mpool = rte_mempool_lookup(mpool_name);
+		if (vq->mpool == NULL)
+			vq->mpool = rte_mempool_create(mpool_name,
+					vq_size,
+					sizeof(struct virtio_crypto_op_cookie),
+					RTE_CACHE_LINE_SIZE, 0,
+					NULL, NULL, NULL, NULL, socket_id,
+					0);
+		if (!vq->mpool) {
+			VIRTIO_CRYPTO_DRV_LOG_ERR("Virtio Crypto PMD "
+					"Cannot create mempool");
+			goto mpool_create_err;
+		}
+		for (i = 0; i < vq_size; i++) {
+			vq->vq_descx[i].cookie =
+				rte_zmalloc("crypto PMD op cookie pointer",
+					sizeof(struct virtio_crypto_op_cookie),
+					RTE_CACHE_LINE_SIZE);
+			if (vq->vq_descx[i].cookie == NULL) {
+				VIRTIO_CRYPTO_DRV_LOG_ERR("Failed to "
+						"alloc mem for cookie");
+				goto cookie_alloc_err;
+			}
+		}
+	}
+
+	vq->hw = hw;
+	vq->dev_id = dev->data->dev_id;
+	vq->vq_queue_index = vtpci_queue_idx;
+	vq->vq_nentries = vq_size;
+
+	/*
+	 * Using part of the vring entries is permitted, but the maximum
+	 * is vq_size
+	 */
+	if (nb_desc == 0 || nb_desc > vq_size)
+		nb_desc = vq_size;
+	vq->vq_free_cnt = nb_desc;
+
+	/*
+	 * Reserve a memzone for vring elements
+	 */
+	size = vring_size(vq_size, VIRTIO_PCI_VRING_ALIGN);
+	vq->vq_ring_size = RTE_ALIGN_CEIL(size, VIRTIO_PCI_VRING_ALIGN);
+	VIRTIO_CRYPTO_INIT_LOG_DBG("%s vring_size: %d, rounded_vring_size: %d",
+			(queue_type == VTCRYPTO_DATAQ) ? "dataq" : "ctrlq",
+			size, vq->vq_ring_size);
+
+	mz = rte_memzone_reserve_aligned(vq_name, vq->vq_ring_size,
+			socket_id, 0, VIRTIO_PCI_VRING_ALIGN);
+	if (mz == NULL) {
+		if (rte_errno == EEXIST)
+			mz = rte_memzone_lookup(vq_name);
+		if (mz == NULL) {
+			VIRTIO_CRYPTO_INIT_LOG_ERR("not enough memory");
+			goto mz_reserve_err;
+		}
+	}
+
+	/*
+	 * Virtio PCI device VIRTIO_PCI_QUEUE_PF register is 32bit,
+	 * and only accepts 32 bit page frame number.
+	 * Check if the allocated physical memory exceeds 16TB.
+	 */
+	if ((mz->phys_addr + vq->vq_ring_size - 1)
+				>> (VIRTIO_PCI_QUEUE_ADDR_SHIFT + 32)) {
+		VIRTIO_CRYPTO_INIT_LOG_ERR("vring address shouldn't be "
+					"above 16TB!");
+		goto vring_addr_err;
+	}
+
+	memset(mz->addr, 0, sizeof(mz->len));
+	vq->mz = mz;
+	vq->vq_ring_mem = mz->phys_addr;
+	vq->vq_ring_virt_mem = mz->addr;
+	VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_ring_mem(physical): 0x%"PRIx64,
+					(uint64_t)mz->phys_addr);
+	VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_ring_virt_mem: 0x%"PRIx64,
+					(uint64_t)(uintptr_t)mz->addr);
+
+	*pvq = vq;
+
+	return 0;
+
+vring_addr_err:
+	rte_memzone_free(mz);
+mz_reserve_err:
+cookie_alloc_err:
+	rte_mempool_free(vq->mpool);
+	if (i != 0) {
+		for (j = 0; j < i; j++)
+			rte_free(vq->vq_descx[j].cookie);
+	}
+mpool_create_err:
+	rte_free(vq);
+	return -ENOMEM;
+}
+
+static int
+virtio_crypto_ctrlq_setup(struct rte_cryptodev *dev, uint16_t queue_idx)
+{
+	int ret;
+	struct virtqueue *vq;
+	struct virtio_crypto_hw *hw = dev->data->dev_private;
+
+	/* if virtio device has started, do not touch the virtqueues */
+	if (dev->data->dev_started)
+		return 0;
+
+	PMD_INIT_FUNC_TRACE();
+
+	ret = virtio_crypto_queue_setup(dev, VTCRYPTO_CTRLQ, queue_idx,
+			0, SOCKET_ID_ANY, &vq);
+	if (ret < 0) {
+		VIRTIO_CRYPTO_INIT_LOG_ERR("control vq initialization failed");
+		return ret;
+	}
+
+	hw->cvq = vq;
+
+	return 0;
+}
+
+static void
+virtio_crypto_free_queues(struct rte_cryptodev *dev)
+{
+	unsigned int i;
+	struct virtio_crypto_hw *hw = dev->data->dev_private;
+
+	PMD_INIT_FUNC_TRACE();
+
+	/* control queue release */
+	virtio_crypto_queue_release(hw->cvq);
+
+	/* data queue release */
+	for (i = 0; i < hw->max_dataqueues; i++)
+		virtio_crypto_queue_release(dev->data->queue_pairs[i]);
+}
+
+static int
+virtio_crypto_dev_close(struct rte_cryptodev *dev __rte_unused)
+{
+	return 0;
+}
+
 /*
  * dev_ops for virtio, bare necessities for basic operation
  */
 static struct rte_cryptodev_ops virtio_crypto_dev_ops = {
 	/* Device related operations */
-	.dev_configure			 = NULL,
-	.dev_start			 = NULL,
-	.dev_stop			 = NULL,
-	.dev_close			 = NULL,
-	.dev_infos_get			 = NULL,
+	.dev_configure			 = virtio_crypto_dev_configure,
+	.dev_start			 = virtio_crypto_dev_start,
+	.dev_stop			 = virtio_crypto_dev_stop,
+	.dev_close			 = virtio_crypto_dev_close,
+	.dev_infos_get			 = virtio_crypto_dev_info_get,
 
 	.stats_get			 = NULL,
 	.stats_reset			 = NULL,
 
-	.queue_pair_setup                = NULL,
-	.queue_pair_release              = NULL,
+	.queue_pair_setup                = virtio_crypto_qp_setup,
+	.queue_pair_release              = virtio_crypto_qp_release,
 	.queue_pair_start                = NULL,
 	.queue_pair_stop                 = NULL,
 	.queue_pair_count                = NULL,
@@ -55,6 +299,51 @@
 };
 
 static int
+virtio_crypto_qp_setup(struct rte_cryptodev *dev, uint16_t queue_pair_id,
+		const struct rte_cryptodev_qp_conf *qp_conf,
+		int socket_id,
+		struct rte_mempool *session_pool __rte_unused)
+{
+	int ret;
+	struct virtqueue *vq;
+
+	PMD_INIT_FUNC_TRACE();
+
+	/* if virtio dev is started, do not touch the virtqueues */
+	if (dev->data->dev_started)
+		return 0;
+
+	ret = virtio_crypto_queue_setup(dev, VTCRYPTO_DATAQ, queue_pair_id,
+			qp_conf->nb_descriptors, socket_id, &vq);
+	if (ret < 0) {
+		VIRTIO_CRYPTO_INIT_LOG_ERR(
+			"virtio crypto data queue initialization failed\n");
+		return ret;
+	}
+
+	dev->data->queue_pairs[queue_pair_id] = vq;
+
+	return 0;
+}
+
+static int
+virtio_crypto_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id)
+{
+	struct virtqueue *vq
+		= (struct virtqueue *)dev->data->queue_pairs[queue_pair_id];
+
+	PMD_INIT_FUNC_TRACE();
+
+	if (vq == NULL) {
+		VIRTIO_CRYPTO_DRV_LOG_DBG("vq already freed");
+		return 0;
+	}
+
+	virtio_crypto_queue_release(vq);
+	return 0;
+}
+
+static int
 virtio_negotiate_features(struct virtio_crypto_hw *hw, uint64_t req_features)
 {
 	uint64_t host_features;
@@ -191,6 +480,135 @@
 	return 0;
 }
 
+static int
+virtio_crypto_dev_uninit(struct rte_cryptodev *cryptodev)
+{
+	struct virtio_crypto_hw *hw = cryptodev->data->dev_private;
+
+	PMD_INIT_FUNC_TRACE();
+
+	if (rte_eal_process_type() == RTE_PROC_SECONDARY)
+		return -EPERM;
+
+	if (cryptodev->data->dev_started) {
+		virtio_crypto_dev_stop(cryptodev);
+		virtio_crypto_dev_close(cryptodev);
+	}
+
+	cryptodev->dev_ops = NULL;
+	cryptodev->enqueue_burst = NULL;
+	cryptodev->dequeue_burst = NULL;
+
+	/* release control queue */
+	virtio_crypto_queue_release(hw->cvq);
+
+	rte_free(cryptodev->data);
+	cryptodev->data = NULL;
+
+	VIRTIO_CRYPTO_DRV_LOG_INFO("dev_uninit completed");
+
+	return 0;
+}
+
+static int
+virtio_crypto_dev_configure(struct rte_cryptodev *cryptodev,
+	struct rte_cryptodev_config *config __rte_unused)
+{
+	struct virtio_crypto_hw *hw = cryptodev->data->dev_private;
+
+	PMD_INIT_FUNC_TRACE();
+
+	if (virtio_crypto_init_device(cryptodev,
+			VIRTIO_CRYPTO_PMD_GUEST_FEATURES) < 0)
+		return -1;
+
+	/* setup control queue
+	 * [0, 1, ... ,(config->max_dataqueues - 1)] are data queues
+	 * config->max_dataqueues is the control queue
+	 */
+	if (virtio_crypto_ctrlq_setup(cryptodev, hw->max_dataqueues) < 0) {
+		VIRTIO_CRYPTO_INIT_LOG_ERR("control queue setup error");
+		return -1;
+	}
+	virtio_crypto_ctrlq_start(cryptodev);
+
+	return 0;
+}
+
+static void
+virtio_crypto_dev_stop(struct rte_cryptodev *dev)
+{
+	struct virtio_crypto_hw *hw = dev->data->dev_private;
+
+	PMD_INIT_FUNC_TRACE();
+	VIRTIO_CRYPTO_DRV_LOG_DBG("virtio_dev_stop");
+
+	vtpci_cryptodev_reset(hw);
+
+	virtio_crypto_dev_free_mbufs(dev);
+	virtio_crypto_free_queues(dev);
+
+	dev->data->dev_started = 0;
+}
+
+static int
+virtio_crypto_dev_start(struct rte_cryptodev *dev)
+{
+	struct virtio_crypto_hw *hw = dev->data->dev_private;
+
+	if (dev->data->dev_started)
+		return 0;
+
+	/* Do final configuration before queue engine starts */
+	virtio_crypto_dataq_start(dev);
+	vtpci_cryptodev_reinit_complete(hw);
+
+	dev->data->dev_started = 1;
+
+	return 0;
+}
+
+static void virtio_crypto_dev_free_mbufs(struct rte_cryptodev *dev)
+{
+	uint32_t i;
+	struct virtio_crypto_hw *hw = dev->data->dev_private;
+
+	for (i = 0; i < hw->max_dataqueues; i++) {
+		VIRTIO_CRYPTO_INIT_LOG_DBG("Before freeing dataq[%d] used "
+			"and unused buf", i);
+		VIRTQUEUE_DUMP((struct virtqueue *)
+			dev->data->queue_pairs[i]);
+
+		VIRTIO_CRYPTO_INIT_LOG_DBG("queue_pairs[%d]=%p",
+				i, dev->data->queue_pairs[i]);
+
+		virtqueue_detatch_unused(dev->data->queue_pairs[i]);
+
+		VIRTIO_CRYPTO_INIT_LOG_DBG("After freeing dataq[%d] used and "
+					"unused buf", i);
+		VIRTQUEUE_DUMP(
+			(struct virtqueue *)dev->data->queue_pairs[i]);
+	}
+}
+
+static void
+virtio_crypto_dev_info_get(struct rte_cryptodev *dev,
+		struct rte_cryptodev_info *info)
+{
+	struct virtio_crypto_hw *hw = dev->data->dev_private;
+
+	PMD_INIT_FUNC_TRACE();
+
+	if (info != NULL) {
+		info->driver_id = cryptodev_virtio_driver_id;
+		info->pci_dev = RTE_DEV_TO_PCI(dev->device);
+		info->feature_flags = dev->feature_flags;
+		info->max_nb_queue_pairs = hw->max_dataqueues;
+		info->sym.max_nb_sessions =
+			RTE_VIRTIO_CRYPTO_PMD_MAX_NB_SESSIONS;
+	}
+}
+
 static int crypto_virtio_pci_probe(
 	struct rte_pci_driver *pci_drv __rte_unused,
 	struct rte_pci_device *pci_dev)
@@ -229,7 +647,7 @@ static int crypto_virtio_pci_remove(
 	if (cryptodev == NULL)
 		return -ENODEV;
 
-	return 0;
+	return virtio_crypto_dev_uninit(cryptodev);
 }
 
 static struct rte_pci_driver rte_virtio_crypto_driver = {
diff --git a/drivers/crypto/virtio/virtio_cryptodev.h b/drivers/crypto/virtio/virtio_cryptodev.h
index 392db4a..159f58e 100644
--- a/drivers/crypto/virtio/virtio_cryptodev.h
+++ b/drivers/crypto/virtio/virtio_cryptodev.h
@@ -5,13 +5,43 @@
 #ifndef _VIRTIO_CRYPTODEV_H_
 #define _VIRTIO_CRYPTODEV_H_
 
-#include <rte_cryptodev.h>
+#include <linux/virtio_crypto.h>
+
+#include "virtio_pci.h"
+#include "virtio_ring.h"
 
 /* Features desired/implemented by this driver. */
 #define VIRTIO_CRYPTO_PMD_GUEST_FEATURES (1ULL << VIRTIO_F_VERSION_1)
 
 #define CRYPTODEV_NAME_VIRTIO_PMD crypto_virtio
 
+#define NUM_ENTRY_VIRTIO_CRYPTO_OP 7
+
+struct virtio_crypto_op_cookie {
+	struct virtio_crypto_op_data_req data_req;
+	struct virtio_crypto_inhdr inhdr;
+	struct vring_desc desc[NUM_ENTRY_VIRTIO_CRYPTO_OP];
+};
+
+/*
+ * Control queue function prototype
+ */
+void virtio_crypto_ctrlq_start(struct rte_cryptodev *dev);
+
+/*
+ * Data queue function prototype
+ */
+void virtio_crypto_dataq_start(struct rte_cryptodev *dev);
+
+int virtio_crypto_queue_setup(struct rte_cryptodev *dev,
+		int queue_type,
+		uint16_t vtpci_queue_idx,
+		uint16_t nb_desc,
+		int socket_id,
+		struct virtqueue **pvq);
+
+void virtio_crypto_queue_release(struct virtqueue *vq);
+
 uint16_t virtio_crypto_pkt_tx_burst(void *tx_queue,
 		struct rte_crypto_op **tx_pkts,
 		uint16_t nb_pkts);
diff --git a/drivers/crypto/virtio/virtio_rxtx.c b/drivers/crypto/virtio/virtio_rxtx.c
index 51f6e09..3b038a7 100644
--- a/drivers/crypto/virtio/virtio_rxtx.c
+++ b/drivers/crypto/virtio/virtio_rxtx.c
@@ -1,8 +1,76 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2018 HUAWEI TECHNOLOGIES CO., LTD.
  */
+#include "virtqueue.h"
 #include "virtio_cryptodev.h"
 
+static int
+virtio_crypto_vring_start(struct virtqueue *vq)
+{
+	struct virtio_crypto_hw *hw = vq->hw;
+	int i, size = vq->vq_nentries;
+	struct vring *vr = &vq->vq_ring;
+	uint8_t *ring_mem = vq->vq_ring_virt_mem;
+
+	PMD_INIT_FUNC_TRACE();
+
+	vring_init(vr, size, ring_mem, VIRTIO_PCI_VRING_ALIGN);
+	vq->vq_desc_tail_idx = (uint16_t)(vq->vq_nentries - 1);
+	vq->vq_free_cnt = vq->vq_nentries;
+
+	/* Chain all the descriptors in the ring with an END */
+	for (i = 0; i < size - 1; i++)
+		vr->desc[i].next = (uint16_t)(i + 1);
+	vr->desc[i].next = VQ_RING_DESC_CHAIN_END;
+
+	/*
+	 * Disable device(host) interrupting guest
+	 */
+	virtqueue_disable_intr(vq);
+
+	/*
+	 * Set guest physical address of the virtqueue
+	 * in VIRTIO_PCI_QUEUE_PFN config register of device
+	 * to share with the backend
+	 */
+	if (VTPCI_OPS(hw)->setup_queue(hw, vq) < 0) {
+		VIRTIO_CRYPTO_INIT_LOG_ERR("setup_queue failed");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+void
+virtio_crypto_ctrlq_start(struct rte_cryptodev *dev)
+{
+	struct virtio_crypto_hw *hw = dev->data->dev_private;
+
+	if (hw->cvq) {
+		virtio_crypto_vring_start(hw->cvq);
+		VIRTQUEUE_DUMP((struct virtqueue *)hw->cvq);
+	}
+}
+
+void
+virtio_crypto_dataq_start(struct rte_cryptodev *dev)
+{
+	/*
+	 * Start data vrings
+	 * -	Setup vring structure for data queues
+	 */
+	uint16_t i;
+	struct virtio_crypto_hw *hw = dev->data->dev_private;
+
+	PMD_INIT_FUNC_TRACE();
+
+	/* Start data vring. */
+	for (i = 0; i < hw->max_dataqueues; i++) {
+		virtio_crypto_vring_start(dev->data->queue_pairs[i]);
+		VIRTQUEUE_DUMP((struct virtqueue *)dev->data->queue_pairs[i]);
+	}
+}
+
 uint16_t
 virtio_crypto_pkt_rx_burst(
 	void *tx_queue __rte_unused,
-- 
1.8.3.1