[dpdk-dev v3 1/1] lib/cryptodev: multi-process IPC request handler

[Kai Ji <kai.ji@intel.com>]

This patch add a function to support queue-pair configuration
request to allow the primary or secondary process to setup/free the
queue-pair via IPC handler. Add in queue pair in-used by process id
array in rte_cryptodev_data for pid tracking.

Signed-off-by: Kai Ji <kai.ji@intel.com>
---
v3:
- addin missing free function for qp_in_use_by_pid

v2:
- code rework
---
 lib/cryptodev/cryptodev_pmd.h |  3 +-
 lib/cryptodev/rte_cryptodev.c | 92 +++++++++++++++++++++++++++++++++++
 lib/cryptodev/rte_cryptodev.h | 37 ++++++++++++++
 lib/cryptodev/version.map     |  2 +
 4 files changed, 133 insertions(+), 1 deletion(-)

diff --git a/lib/cryptodev/cryptodev_pmd.h b/lib/cryptodev/cryptodev_pmd.h
index 09ba952455..f404604963 100644
--- a/lib/cryptodev/cryptodev_pmd.h
+++ b/lib/cryptodev/cryptodev_pmd.h
@@ -78,7 +78,8 @@ struct rte_cryptodev_data {
 	void **queue_pairs;
 	/** Number of device queue pairs. */
 	uint16_t nb_queue_pairs;
-
+	/** Array of process id used for queue pairs **/
+	uint16_t *qp_in_use_by_pid;
 	/** PMD-specific private data */
 	void *dev_private;
 } __rte_cache_aligned;
diff --git a/lib/cryptodev/rte_cryptodev.c b/lib/cryptodev/rte_cryptodev.c
index 9e76a1c72d..dab6a37bff 100644
--- a/lib/cryptodev/rte_cryptodev.c
+++ b/lib/cryptodev/rte_cryptodev.c
@@ -49,6 +49,9 @@ struct rte_crypto_fp_ops rte_crypto_fp_ops[RTE_CRYPTO_MAX_DEVS];
 /* spinlock for crypto device callbacks */
 static rte_spinlock_t rte_cryptodev_cb_lock = RTE_SPINLOCK_INITIALIZER;
 
+/* crypto queue pair config */
+#define CRYPTODEV_MP_REQ "cryptodev_mp_request"
+
 /**
  * The user application callback description.
  *
@@ -1050,6 +1053,9 @@ rte_cryptodev_pmd_release_device(struct rte_cryptodev *cryptodev)
 			return ret;
 	}
 
+	if (cryptodev->data->qp_in_use_by_pid)
+		rte_free(cryptodev->data->qp_in_use_by_pid);
+
 	ret = rte_cryptodev_data_free(dev_id, &cryptodev_globals.data[dev_id]);
 	if (ret < 0)
 		return ret;
@@ -1138,6 +1144,21 @@ rte_cryptodev_queue_pairs_config(struct rte_cryptodev *dev, uint16_t nb_qpairs,
 
 	}
 	dev->data->nb_queue_pairs = nb_qpairs;
+
+	if (dev->data->qp_in_use_by_pid == NULL) {
+		dev->data->qp_in_use_by_pid = rte_zmalloc_socket(
+				"cryptodev->qp_in_use_by_pid",
+				sizeof(dev->data->qp_in_use_by_pid[0]) *
+				dev_info.max_nb_queue_pairs,
+				RTE_CACHE_LINE_SIZE, socket_id);
+		if (dev->data->qp_in_use_by_pid == NULL) {
+			CDEV_LOG_ERR("failed to get memory for qp meta data, "
+							"nb_queues %u",
+							nb_qpairs);
+			return -(ENOMEM);
+		}
+	}
+
 	return 0;
 }
 
@@ -1400,6 +1421,77 @@ rte_cryptodev_queue_pair_setup(uint8_t dev_id, uint16_t queue_pair_id,
 			socket_id);
 }
 
+static int
+rte_cryptodev_ipc_request(const struct rte_mp_msg *mp_msg, const void *peer)
+{
+	struct rte_mp_msg mp_res;
+	struct rte_cryptodev_mp_param *res =
+		(struct rte_cryptodev_mp_param *)mp_res.param;
+	const struct rte_cryptodev_mp_param *param =
+		(const struct rte_cryptodev_mp_param *)mp_msg->param;
+
+	int ret;
+	struct rte_cryptodev *dev;
+	uint16_t *qps_in_used_by_pid;
+	int dev_id = param->dev_id;
+	int qp_id = param->qp_id;
+	struct rte_cryptodev_qp_conf *queue_conf = param->queue_conf;
+
+	res->result = -EINVAL;
+	if (!rte_cryptodev_is_valid_dev(dev_id)) {
+		CDEV_LOG_ERR("Invalid dev_id=%" PRIu8, dev_id);
+		goto out;
+	}
+
+	if (!rte_cryptodev_get_qp_status(dev_id, qp_id))
+		goto out;
+
+	dev = &rte_crypto_devices[dev_id];
+	qps_in_used_by_pid = dev->data->qp_in_use_by_pid;
+
+	switch (param->type) {
+	case RTE_CRYPTODEV_MP_REQ_QP_SET:
+		ret = rte_cryptodev_queue_pair_setup(dev_id, qp_id,
+				queue_conf, param->socket_id);
+		if (!ret)
+			qps_in_used_by_pid[qp_id] = param->process_id;
+		res->result = ret;
+		break;
+	case RTE_CRYPTODEV_MP_REQ_QP_FREE:
+		if ((rte_eal_process_type() == RTE_PROC_SECONDARY) &&
+			(qps_in_used_by_pid[qp_id] != param->process_id)) {
+			CDEV_LOG_ERR("Unable to release qp_id=%" PRIu8, qp_id);
+			goto out;
+		}
+
+		ret = (*dev->dev_ops->queue_pair_release)(dev, qp_id);
+		if (!ret)
+			qps_in_used_by_pid[qp_id] = 0;
+
+		res->result = ret;
+		break;
+	default:
+		CDEV_LOG_ERR("invalid mp request type\n");
+	}
+
+out:
+	ret = rte_mp_reply(&mp_res, peer);
+	return ret;
+}
+
+int rte_cryptodev_mp_request_register(void)
+{
+	RTE_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
+	return rte_mp_action_register(CRYPTODEV_MP_REQ,
+				rte_cryptodev_ipc_request);
+}
+
+void rte_cryptodev_mp_request_unregister(void)
+{
+	RTE_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
+	rte_mp_action_unregister(CRYPTODEV_MP_REQ);
+}
+
 struct rte_cryptodev_cb *
 rte_cryptodev_add_enq_callback(uint8_t dev_id,
 			       uint16_t qp_id,
diff --git a/lib/cryptodev/rte_cryptodev.h b/lib/cryptodev/rte_cryptodev.h
index 56f459c6a0..d8cadebd0c 100644
--- a/lib/cryptodev/rte_cryptodev.h
+++ b/lib/cryptodev/rte_cryptodev.h
@@ -539,6 +539,24 @@ enum rte_cryptodev_event_type {
 	RTE_CRYPTODEV_EVENT_MAX		/**< max value of this enum */
 };
 
+/* Request types for IPC. */
+enum rte_cryptodev_mp_req_type {
+	RTE_CRYPTODEV_MP_REQ_NONE,
+	RTE_CRYPTODEV_MP_REQ_QP_SET,
+	RTE_CRYPTODEV_MP_REQ_QP_FREE
+};
+
+/* Parameters for IPC. */
+struct rte_cryptodev_mp_param {
+	enum rte_cryptodev_mp_req_type type;
+	int dev_id;
+	int qp_id;
+	int socket_id;
+	uint16_t process_id;
+	struct rte_cryptodev_qp_conf *queue_conf;
+	int result;
+};
+
 /** Crypto device queue pair configuration structure. */
 struct rte_cryptodev_qp_conf {
 	uint32_t nb_descriptors; /**< Number of descriptors per queue pair */
@@ -769,6 +787,25 @@ extern int
 rte_cryptodev_queue_pair_setup(uint8_t dev_id, uint16_t queue_pair_id,
 		const struct rte_cryptodev_qp_conf *qp_conf, int socket_id);
 
+/**
+ * Register multi process request IPC handler
+ *
+ * @return
+ *   - 0: Success registered
+ *	 - 1: Failed registration failed
+ *	 - -EINVAL: device was not configured
+ */
+__rte_experimental
+int
+rte_cryptodev_mp_request_register(void);
+
+/**
+ * Unregister multi process unrequest IPC handler
+ */
+__rte_experimental
+void
+rte_cryptodev_mp_request_unregister(void);
+
 /**
  * Get the status of queue pairs setup on a specific crypto device
  *
diff --git a/lib/cryptodev/version.map b/lib/cryptodev/version.map
index 6d9b3e01a6..4130c39e7c 100644
--- a/lib/cryptodev/version.map
+++ b/lib/cryptodev/version.map
@@ -157,6 +157,8 @@ EXPERIMENTAL {
 	__rte_cryptodev_trace_sym_session_get_user_data;
 	__rte_cryptodev_trace_sym_session_set_user_data;
 	__rte_cryptodev_trace_count;
+	rte_cryptodev_mp_request_register;
+	rte_cryptodev_mp_request_unregister;
 };
 
 INTERNAL {
-- 
2.17.1