[dpdk-dev] [PATCH v6 02/14] compressdev: add queue pair management

[Pablo de Lara <pablo.de.lara.guarch@intel.com>]

From: Fiona Trahe <fiona.trahe@intel.com>

Add functions to manage device queue pairs.

Signed-off-by: Fiona Trahe <fiona.trahe@intel.com>
Signed-off-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
Signed-off-by: Shally Verma <shally.verma@caviumnetworks.com>
Signed-off-by: Ashish Gupta <ashish.gupta@caviumnetworks.com>
---
 lib/librte_compressdev/rte_compressdev.c           | 178 ++++++++++++++++++++-
 lib/librte_compressdev/rte_compressdev.h           |  45 ++++++
 lib/librte_compressdev/rte_compressdev_internal.h  |   5 +
 lib/librte_compressdev/rte_compressdev_pmd.h       |  47 ++++++
 lib/librte_compressdev/rte_compressdev_version.map |   2 +
 5 files changed, 276 insertions(+), 1 deletion(-)

diff --git a/lib/librte_compressdev/rte_compressdev.c b/lib/librte_compressdev/rte_compressdev.c
index 751517c3d..6667528b5 100644
--- a/lib/librte_compressdev/rte_compressdev.c
+++ b/lib/librte_compressdev/rte_compressdev.c
@@ -227,10 +227,136 @@ rte_compressdev_pmd_release_device(struct rte_compressdev *compressdev)
 	return 0;
 }
 
+uint16_t __rte_experimental
+rte_compressdev_queue_pair_count(uint8_t dev_id)
+{
+	struct rte_compressdev *dev;
+
+	dev = &rte_comp_devices[dev_id];
+	return dev->data->nb_queue_pairs;
+}
+
+static int
+rte_compressdev_queue_pairs_config(struct rte_compressdev *dev,
+		uint16_t nb_qpairs, int socket_id)
+{
+	struct rte_compressdev_info dev_info;
+	void **qp;
+	unsigned int i;
+
+	if ((dev == NULL) || (nb_qpairs < 1)) {
+		COMPRESSDEV_LOG(ERR, "invalid param: dev %p, nb_queues %u",
+							dev, nb_qpairs);
+		return -EINVAL;
+	}
+
+	COMPRESSDEV_LOG(DEBUG, "Setup %d queues pairs on device %u",
+			nb_qpairs, dev->data->dev_id);
+
+	memset(&dev_info, 0, sizeof(struct rte_compressdev_info));
+
+	RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
+	(*dev->dev_ops->dev_infos_get)(dev, &dev_info);
+
+	if ((dev_info.max_nb_queue_pairs != 0) &&
+			(nb_qpairs > dev_info.max_nb_queue_pairs)) {
+		COMPRESSDEV_LOG(ERR, "Invalid num queue_pairs (%u) for dev %u",
+				nb_qpairs, dev->data->dev_id);
+		return -EINVAL;
+	}
+
+	if (dev->data->queue_pairs == NULL) { /* first time configuration */
+		dev->data->queue_pairs = rte_zmalloc_socket(
+				"compressdev->queue_pairs",
+				sizeof(dev->data->queue_pairs[0]) * nb_qpairs,
+				RTE_CACHE_LINE_SIZE, socket_id);
+
+		if (dev->data->queue_pairs == NULL) {
+			dev->data->nb_queue_pairs = 0;
+			COMPRESSDEV_LOG(ERR,
+			"failed to get memory for qp meta data, nb_queues %u",
+							nb_qpairs);
+			return -(ENOMEM);
+		}
+	} else { /* re-configure */
+		int ret;
+		uint16_t old_nb_queues = dev->data->nb_queue_pairs;
+
+		qp = dev->data->queue_pairs;
+
+		RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_pair_release,
+				-ENOTSUP);
+
+		for (i = nb_qpairs; i < old_nb_queues; i++) {
+			ret = (*dev->dev_ops->queue_pair_release)(dev, i);
+			if (ret < 0)
+				return ret;
+		}
+
+		qp = rte_realloc(qp, sizeof(qp[0]) * nb_qpairs,
+				RTE_CACHE_LINE_SIZE);
+		if (qp == NULL) {
+			COMPRESSDEV_LOG(ERR,
+			"failed to realloc qp meta data, nb_queues %u",
+						nb_qpairs);
+			return -(ENOMEM);
+		}
+
+		if (nb_qpairs > old_nb_queues) {
+			uint16_t new_qs = nb_qpairs - old_nb_queues;
+
+			memset(qp + old_nb_queues, 0,
+				sizeof(qp[0]) * new_qs);
+		}
+
+		dev->data->queue_pairs = qp;
+
+	}
+	dev->data->nb_queue_pairs = nb_qpairs;
+	return 0;
+}
+
+static int
+rte_compressdev_queue_pairs_release(struct rte_compressdev *dev)
+{
+	uint16_t num_qps, i;
+	int ret;
+
+	if (dev == NULL) {
+		COMPRESSDEV_LOG(ERR, "invalid param: dev %p", dev);
+		return -EINVAL;
+	}
+
+	num_qps = dev->data->nb_queue_pairs;
+
+	if (num_qps == 0)
+		return 0;
+
+	COMPRESSDEV_LOG(DEBUG, "Free %d queues pairs on device %u",
+			dev->data->nb_queue_pairs, dev->data->dev_id);
+
+	RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_pair_release,
+				-ENOTSUP);
+
+	for (i = 0; i < num_qps; i++) {
+		ret = (*dev->dev_ops->queue_pair_release)(dev, i);
+		if (ret < 0)
+			return ret;
+	}
+
+	if (dev->data->queue_pairs != NULL)
+		rte_free(dev->data->queue_pairs);
+	dev->data->queue_pairs = NULL;
+	dev->data->nb_queue_pairs = 0;
+
+	return 0;
+}
+
 int __rte_experimental
 rte_compressdev_configure(uint8_t dev_id, struct rte_compressdev_config *config)
 {
 	struct rte_compressdev *dev;
+	int diag;
 
 	if (!rte_compressdev_is_valid_dev(dev_id)) {
 		COMPRESSDEV_LOG(ERR, "Invalid dev_id=%" PRIu8, dev_id);
@@ -247,10 +373,19 @@ rte_compressdev_configure(uint8_t dev_id, struct rte_compressdev_config *config)
 
 	RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_configure, -ENOTSUP);
 
+	/* Setup new number of queue pairs and reconfigure device. */
+	diag = rte_compressdev_queue_pairs_config(dev, config->nb_queue_pairs,
+			config->socket_id);
+	if (diag != 0) {
+		COMPRESSDEV_LOG(ERR,
+			"dev%d rte_comp_dev_queue_pairs_config = %d",
+				dev_id, diag);
+		return diag;
+	}
+
 	return (*dev->dev_ops->dev_configure)(dev, config);
 }
 
-
 int __rte_experimental
 rte_compressdev_start(uint8_t dev_id)
 {
@@ -327,6 +462,12 @@ rte_compressdev_close(uint8_t dev_id)
 		return -EBUSY;
 	}
 
+	/* Free queue pairs memory */
+	retval = rte_compressdev_queue_pairs_release(dev);
+
+	if (retval < 0)
+		return retval;
+
 	RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_close, -ENOTSUP);
 	retval = (*dev->dev_ops->dev_close)(dev);
 
@@ -336,6 +477,41 @@ rte_compressdev_close(uint8_t dev_id)
 	return 0;
 }
 
+int __rte_experimental
+rte_compressdev_queue_pair_setup(uint8_t dev_id, uint16_t queue_pair_id,
+		uint32_t max_inflight_ops, int socket_id)
+{
+	struct rte_compressdev *dev;
+
+	if (!rte_compressdev_is_valid_dev(dev_id)) {
+		COMPRESSDEV_LOG(ERR, "Invalid dev_id=%" PRIu8, dev_id);
+		return -EINVAL;
+	}
+
+	dev = &rte_comp_devices[dev_id];
+	if (queue_pair_id >= dev->data->nb_queue_pairs) {
+		COMPRESSDEV_LOG(ERR, "Invalid queue_pair_id=%d", queue_pair_id);
+		return -EINVAL;
+	}
+
+	if (dev->data->dev_started) {
+		COMPRESSDEV_LOG(ERR,
+		    "device %d must be stopped to allow configuration", dev_id);
+		return -EBUSY;
+	}
+
+	if (max_inflight_ops == 0) {
+		COMPRESSDEV_LOG(ERR,
+			"Invalid maximum number of inflight operations");
+		return -EINVAL;
+	}
+
+	RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_pair_setup, -ENOTSUP);
+
+	return (*dev->dev_ops->queue_pair_setup)(dev, queue_pair_id,
+			max_inflight_ops, socket_id);
+}
+
 void __rte_experimental
 rte_compressdev_info_get(uint8_t dev_id, struct rte_compressdev_info *dev_info)
 {
diff --git a/lib/librte_compressdev/rte_compressdev.h b/lib/librte_compressdev/rte_compressdev.h
index 5be5973bf..81710bbea 100644
--- a/lib/librte_compressdev/rte_compressdev.h
+++ b/lib/librte_compressdev/rte_compressdev.h
@@ -22,6 +22,10 @@ extern "C" {
 /**  comp device information */
 struct rte_compressdev_info {
 	const char *driver_name;		/**< Driver name. */
+	uint16_t max_nb_queue_pairs;
+	/**< Maximum number of queues pairs supported by device.
+	 * (If 0, there is no limit in maximum number of queue pairs)
+	 */
 };
 
 /**
@@ -80,6 +84,9 @@ rte_compressdev_socket_id(uint8_t dev_id);
 /** Compress device configuration structure */
 struct rte_compressdev_config {
 	int socket_id;
+	/**< Socket on which to allocate resources */
+	uint16_t nb_queue_pairs;
+	/**< Total number of queue pairs to configure on a device */
 };
 
 /**
@@ -145,6 +152,44 @@ rte_compressdev_stop(uint8_t dev_id);
 int __rte_experimental
 rte_compressdev_close(uint8_t dev_id);
 
+/**
+ * Allocate and set up a receive queue pair for a device.
+ * This should only be called when the device is stopped.
+ *
+ *
+ * @param dev_id
+ *   Compress device identifier
+ * @param queue_pair_id
+ *   The index of the queue pairs to set up. The
+ *   value must be in the range [0, nb_queue_pair - 1]
+ *   previously supplied to rte_compressdev_configure()
+ * @param max_inflight_ops
+ *   Max number of ops which the qp will have to
+ *   accommodate simultaneously
+ * @param socket_id
+ *   The *socket_id* argument is the socket identifier
+ *   in case of NUMA. The value can be *SOCKET_ID_ANY*
+ *   if there is no NUMA constraint for the DMA memory
+ *   allocated for the receive queue pair
+ * @return
+ *   - 0: Success, queue pair correctly set up.
+ *   - <0: Queue pair configuration failed
+ */
+int __rte_experimental
+rte_compressdev_queue_pair_setup(uint8_t dev_id, uint16_t queue_pair_id,
+		uint32_t max_inflight_ops, int socket_id);
+
+/**
+ * Get the number of queue pairs on a specific comp device
+ *
+ * @param dev_id
+ *   Compress device identifier
+ * @return
+ *   - The number of configured queue pairs.
+ */
+uint16_t __rte_experimental
+rte_compressdev_queue_pair_count(uint8_t dev_id);
+
 /**
  * Retrieve the contextual information of a device.
  *
diff --git a/lib/librte_compressdev/rte_compressdev_internal.h b/lib/librte_compressdev/rte_compressdev_internal.h
index 0a2ddcb2b..57af163c1 100644
--- a/lib/librte_compressdev/rte_compressdev_internal.h
+++ b/lib/librte_compressdev/rte_compressdev_internal.h
@@ -58,6 +58,11 @@ struct rte_compressdev_data {
 	uint8_t dev_started : 1;
 	/**< Device state: STARTED(1)/STOPPED(0) */
 
+	void **queue_pairs;
+	/**< Array of pointers to queue pairs. */
+	uint16_t nb_queue_pairs;
+	/**< Number of device queue pairs */
+
 	void *dev_private;
 	/**< PMD-specific private data */
 } __rte_cache_aligned;
diff --git a/lib/librte_compressdev/rte_compressdev_pmd.h b/lib/librte_compressdev/rte_compressdev_pmd.h
index 43307ee8e..14ce76b44 100644
--- a/lib/librte_compressdev/rte_compressdev_pmd.h
+++ b/lib/librte_compressdev/rte_compressdev_pmd.h
@@ -125,6 +125,48 @@ typedef int (*compressdev_close_t)(struct rte_compressdev *dev);
 typedef void (*compressdev_info_get_t)(struct rte_compressdev *dev,
 				struct rte_compressdev_info *dev_info);
 
+/**
+ * Setup a queue pair for a device.
+ *
+ * @param dev
+ *   Compress device
+ * @param qp_id
+ *   Queue pair identifier
+ * @param max_inflight_ops
+ *   Max inflight ops which qp must accommodate
+ * @param socket_id
+ *   Socket identifier
+ * @return
+ *   Returns 0 on success.
+ */
+typedef int (*compressdev_queue_pair_setup_t)(struct rte_compressdev *dev,
+		uint16_t qp_id,	uint32_t max_inflight_ops, int socket_id);
+
+/**
+ * Release memory resources allocated by given queue pair.
+ *
+ * @param dev
+ *   Compress device
+ * @param qp_id
+ *   Queue pair identifier
+ * @return
+ * - 0 on success.
+ * - EAGAIN if can't close as device is busy
+ */
+typedef int (*compressdev_queue_pair_release_t)(struct rte_compressdev *dev,
+		uint16_t qp_id);
+
+/**
+ * Get number of available queue pairs of a device.
+ *
+ * @param dev
+ *   Compress device
+ * @return
+ *   Returns number of queue pairs on success.
+ */
+typedef uint32_t (*compressdev_queue_pair_count_t)(struct rte_compressdev *dev);
+
+
 /** comp device operations function pointer table */
 struct rte_compressdev_ops {
 	compressdev_configure_t dev_configure;	/**< Configure device. */
@@ -133,6 +175,11 @@ struct rte_compressdev_ops {
 	compressdev_close_t dev_close;		/**< Close device. */
 
 	compressdev_info_get_t dev_infos_get;	/**< Get device info. */
+
+	compressdev_queue_pair_setup_t queue_pair_setup;
+	/**< Set up a device queue pair. */
+	compressdev_queue_pair_release_t queue_pair_release;
+	/**< Release a queue pair. */
 };
 
 /**
diff --git a/lib/librte_compressdev/rte_compressdev_version.map b/lib/librte_compressdev/rte_compressdev_version.map
index a996abcd2..182a371d8 100644
--- a/lib/librte_compressdev/rte_compressdev_version.map
+++ b/lib/librte_compressdev/rte_compressdev_version.map
@@ -16,6 +16,8 @@ EXPERIMENTAL {
 	rte_compressdev_pmd_get_named_dev;
 	rte_compressdev_pmd_parse_input_args;
 	rte_compressdev_pmd_release_device;
+	rte_compressdev_queue_pair_count;
+	rte_compressdev_queue_pair_setup;
 	rte_compressdev_socket_id;
 	rte_compressdev_start;
 	rte_compressdev_stop;
-- 
2.14.3