From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id DF028A0543; Tue, 25 Oct 2022 01:39:16 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 7B50040DF7; Tue, 25 Oct 2022 01:39:16 +0200 (CEST) Received: from mga11.intel.com (mga11.intel.com [192.55.52.93]) by mails.dpdk.org (Postfix) with ESMTP id B53D040A8B for ; Tue, 25 Oct 2022 01:39:14 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=intel.com; i=@intel.com; q=dns/txt; s=Intel; t=1666654754; x=1698190754; h=from:to:cc:subject:date:message-id:in-reply-to: references; bh=JQE6sqGNO90hNb9+Gks1fVngwfBXnNrxVCUFt66nCSI=; b=TlWYTRtektC8T8zYSa/1sicagUBjyX5WrLj882hfAQxexN8crnzMHpgm vgNkCbafkRK19mUbThAm+rNb+csc5y9GNvPm7CeWCTlIW2nvjako39tuu VTZu1HYLY57A9JJI8pMMS8Tjk0CRPqjFi5wZYyMxJ6ntgT36Vwfds7GaT 90nnSLTD9CplM6AfL1RdD8L+Xb97nQIIP4RIg4i8qGH8vTBPir3PiAJ0Q TKpDuL/H6uawULidlRpcQGth0kF1QY88hHtBBBzxQaitTt6CP+/SYNC4e Mtsl6QR/aoLKQgZDuXig0Ci9BH+BV5ECUldw45x0R9zW5DVg3IaRlM+Jx Q==; X-IronPort-AV: E=McAfee;i="6500,9779,10510"; a="305157932" X-IronPort-AV: E=Sophos;i="5.95,210,1661842800"; d="scan'208";a="305157932" Received: from orsmga007.jf.intel.com ([10.7.209.58]) by fmsmga102.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 24 Oct 2022 16:38:58 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=McAfee;i="6500,9779,10510"; a="626233326" X-IronPort-AV: E=Sophos;i="5.95,210,1661842800"; d="scan'208";a="626233326" Received: from silpixa00400465.ir.intel.com ([10.55.128.22]) by orsmga007.jf.intel.com with ESMTP; 24 Oct 2022 16:38:56 -0700 From: Kai Ji To: dev@dpdk.org Cc: gakhil@marvell.com, Kai Ji , Pablo de Lara , Anatoly Burakov Subject: [dpdk-dev v2] crypto/ipsec_mb: multi-process IPC request handler Date: Tue, 25 Oct 2022 07:38:53 +0800 Message-Id: <20221024233854.87514-1-kai.ji@intel.com> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20221020235110.93834-1-kai.ji@intel.com> References: <20221020235110.93834-1-kai.ji@intel.com> X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org As the secondary process needs queue-pair to be configured by the primary process before use. This patch adds an IPC register function to allow secondary process to setup cryptodev queue-pair via IPC messages during the runtime. A new "qp_in_used_pid" param stores the PID to provide the ownership of the queue-pair so that only the PID matched queue-pair can be free'd in the request. Signed-off-by: Kai Ji --- v2: - add in share memzone for data exchange between multi-process --- drivers/crypto/ipsec_mb/ipsec_mb_ops.c | 135 ++++++++++++++++++++- drivers/crypto/ipsec_mb/ipsec_mb_private.c | 40 +++++- drivers/crypto/ipsec_mb/ipsec_mb_private.h | 44 +++++++ 3 files changed, 215 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/ipsec_mb/ipsec_mb_ops.c b/drivers/crypto/ipsec_mb/ipsec_mb_ops.c index cedcaa2742..a0e696819b 100644 --- a/drivers/crypto/ipsec_mb/ipsec_mb_ops.c +++ b/drivers/crypto/ipsec_mb/ipsec_mb_ops.c @@ -3,6 +3,7 @@ */ #include +#include #include #include @@ -93,6 +94,45 @@ ipsec_mb_info_get(struct rte_cryptodev *dev, } } +static int +ipsec_mb_secondary_qp_op(int dev_id, int qp_id, + struct rte_cryptodev_qp_conf *qp_conf, int socket_id, uint16_t pid, + enum ipsec_mb_mp_req_type op_type) +{ + int ret; + struct rte_mp_msg qp_req_msg; + struct rte_mp_msg qp_resp_msg; + struct rte_mp_reply qp_resp; + struct ipsec_mb_mp_param *req_param; + struct ipsec_mb_mp_param *resp_param; + + struct timespec ts = {.tv_sec = 5, .tv_nsec = 0}; + + memset(&qp_req_msg, 0, sizeof(IPSEC_MB_MP_REQ)); + memcpy(qp_req_msg.name, IPSEC_MB_MP_REQ, sizeof(IPSEC_MB_MP_REQ)); + req_param = (struct ipsec_mb_mp_param *)&qp_req_msg.param; + + qp_req_msg.len_param = sizeof(struct ipsec_mb_mp_param); + req_param->type = op_type; + req_param->dev_id = dev_id; + req_param->qp_id = qp_id; + req_param->socket_id = socket_id; + req_param->process_id = pid; + req_param->queue_conf = (void *)qp_conf; + + qp_req_msg.num_fds = 0; + qp_resp.msgs = &qp_resp_msg; + ret = rte_mp_request_sync(&qp_req_msg, &qp_resp, &ts); + if (ret) { + RTE_LOG(ERR, USER1, "Create MR request to primary process failed."); + return -1; + } + + resp_param = (struct ipsec_mb_mp_param *)&qp_resp_msg.param; + + return resp_param->result; +} + /** Release queue pair */ int ipsec_mb_qp_release(struct rte_cryptodev *dev, uint16_t qp_id) @@ -100,7 +140,10 @@ ipsec_mb_qp_release(struct rte_cryptodev *dev, uint16_t qp_id) struct ipsec_mb_qp *qp = dev->data->queue_pairs[qp_id]; struct rte_ring *r = NULL; - if (qp != NULL && rte_eal_process_type() == RTE_PROC_PRIMARY) { + if (qp != NULL) + return 0; + + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { r = rte_ring_lookup(qp->name); rte_ring_free(r); @@ -115,6 +158,9 @@ ipsec_mb_qp_release(struct rte_cryptodev *dev, uint16_t qp_id) #endif rte_free(qp); dev->data->queue_pairs[qp_id] = NULL; + } else { /* secondary process */ + return ipsec_mb_secondary_qp_op(dev->data->dev_id, qp_id, + NULL, 0, getpid(), RTE_IPSEC_MB_MP_REQ_QP_FREE); } return 0; } @@ -222,9 +268,23 @@ ipsec_mb_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, #endif qp = dev->data->queue_pairs[qp_id]; if (qp == NULL) { - IPSEC_MB_LOG(ERR, "Primary process hasn't configured device qp."); - return -EINVAL; + struct rte_cryptodev_qp_conf *shared_qp_conf; + IPSEC_MB_LOG(DEBUG, "Secondary process setting up device qp."); + if (!mp_shared_data) { + IPSEC_MB_LOG(ERR, "no multi-process share mz allocated."); + return -EINVAL; + } + shared_qp_conf = &(mp_shared_data->qp_conf); + + memcpy(shared_qp_conf, qp_conf, + sizeof(struct rte_cryptodev_qp_conf)); + return ipsec_mb_secondary_qp_op(dev->data->dev_id, qp_id, + shared_qp_conf, socket_id, getpid(), + RTE_IPSEC_MB_MP_REQ_QP_SET); } + + IPSEC_MB_LOG(ERR, "Queue pair already setup'ed."); + return -EINVAL; } else { /* Free memory prior to re-allocation if needed. */ if (dev->data->queue_pairs[qp_id] != NULL) @@ -296,6 +356,75 @@ ipsec_mb_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id, return ret; } +int +ipsec_mb_ipc_request(const struct rte_mp_msg *mp_msg, const void *peer) +{ + struct rte_mp_msg mp_res; + struct ipsec_mb_mp_param *resp_param = + (struct ipsec_mb_mp_param *)mp_res.param; + const struct ipsec_mb_mp_param *req_param = + (const struct ipsec_mb_mp_param *)mp_msg->param; + + int ret; + struct rte_cryptodev *dev; + struct ipsec_mb_qp *qp; + int dev_id = req_param->dev_id; + int qp_id = req_param->qp_id; + struct rte_cryptodev_qp_conf *queue_conf = + (struct rte_cryptodev_qp_conf *)req_param->queue_conf; + + resp_param->result = -EINVAL; + if (!rte_cryptodev_is_valid_dev(dev_id)) { + CDEV_LOG_ERR("Invalid dev_id=%d", dev_id); + goto out; + } + + dev = rte_cryptodev_pmd_get_dev(dev_id); + switch (req_param->type) { + case RTE_IPSEC_MB_MP_REQ_QP_SET: + qp = dev->data->queue_pairs[qp_id]; + if (qp) { + CDEV_LOG_DEBUG("qp %d on dev %d is initialised", qp_id, dev_id); + goto out; + } + + ret = ipsec_mb_qp_setup(dev, qp_id, queue_conf, req_param->socket_id); + if (!ret) { + qp = dev->data->queue_pairs[qp_id]; + if (!qp) { + CDEV_LOG_DEBUG("qp %d on dev %d is not initialised", + qp_id, dev_id); + goto out; + } + qp->qp_in_used_by_pid = req_param->process_id; + } + resp_param->result = ret; + break; + case RTE_IPSEC_MB_MP_REQ_QP_FREE: + qp = dev->data->queue_pairs[qp_id]; + if (!qp) { + CDEV_LOG_DEBUG("qp %d on dev %d is not initialised", + qp_id, dev_id); + goto out; + } + + if (qp->qp_in_used_by_pid != req_param->process_id) { + CDEV_LOG_ERR("Unable to release qp_id=%d", qp_id); + goto out; + } + + qp->qp_in_used_by_pid = 0; + resp_param->result = ipsec_mb_qp_release(dev, qp_id); + break; + default: + CDEV_LOG_ERR("invalid mp request type\n"); + } + +out: + ret = rte_mp_reply(&mp_res, peer); + return ret; +} + /** Return the size of the specific pmd session structure */ unsigned ipsec_mb_sym_session_get_size(struct rte_cryptodev *dev) diff --git a/drivers/crypto/ipsec_mb/ipsec_mb_private.c b/drivers/crypto/ipsec_mb/ipsec_mb_private.c index e56579596f..a02010881c 100644 --- a/drivers/crypto/ipsec_mb/ipsec_mb_private.c +++ b/drivers/crypto/ipsec_mb/ipsec_mb_private.c @@ -42,6 +42,38 @@ ipsec_mb_enqueue_burst(void *__qp, struct rte_crypto_op **ops, return nb_enqueued; } +static int +ipsec_mb_mp_request_register(void) +{ + RTE_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY); + /* Setup memzone for shared data */ + ipsec_mb_mp_mz = rte_memzone_reserve( + IPSEC_MB_MP_SHARED_MZ_NAME, + sizeof(struct ipsec_mb_mp_shared_data), 0, 0); + if (ipsec_mb_mp_mz == NULL) { + RTE_LOG(ERR, USER1, + "%s: cannot create multiprocess-shared memzone for ipsec_mb", + __func__); + return -1; + } + + mp_shared_data = ipsec_mb_mp_mz->addr; + IPSEC_MB_LOG(INFO, "Starting register MP IPC request\n"); + return rte_mp_action_register(IPSEC_MB_MP_REQ, + ipsec_mb_ipc_request); +} + +static void +ipsec_mb_mp_request_unregister(void) +{ + RTE_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY); + mp_shared_data = NULL; + if (ipsec_mb_mp_mz) + rte_memzone_free(ipsec_mb_mp_mz); + + rte_mp_action_unregister(IPSEC_MB_MP_REQ); +} + int ipsec_mb_create(struct rte_vdev_device *vdev, enum ipsec_mb_pmd_types pmd_type) @@ -152,7 +184,10 @@ ipsec_mb_create(struct rte_vdev_device *vdev, IPSEC_MB_LOG(INFO, "IPSec Multi-buffer library version used: %s\n", imb_get_version_str()); - return 0; + if (rte_eal_process_type() == RTE_PROC_PRIMARY) + retval = ipsec_mb_mp_request_register(); + + return retval; } int @@ -187,5 +222,8 @@ ipsec_mb_remove(struct rte_vdev_device *vdev) for (qp_id = 0; qp_id < cryptodev->data->nb_queue_pairs; qp_id++) ipsec_mb_qp_release(cryptodev, qp_id); + if (rte_eal_process_type() == RTE_PROC_PRIMARY) + ipsec_mb_mp_request_unregister(); + return rte_cryptodev_pmd_destroy(cryptodev); } diff --git a/drivers/crypto/ipsec_mb/ipsec_mb_private.h b/drivers/crypto/ipsec_mb/ipsec_mb_private.h index b56eaf061e..f27321b9c4 100644 --- a/drivers/crypto/ipsec_mb/ipsec_mb_private.h +++ b/drivers/crypto/ipsec_mb/ipsec_mb_private.h @@ -10,6 +10,7 @@ #else #include #endif +#include #include #include @@ -25,6 +26,9 @@ /* Maximum length for memzone name */ #define IPSEC_MB_MAX_MZ_NAME 32 +/* ipsec mb multi-process queue pair config */ +#define IPSEC_MB_MP_REQ "ipsec_mb_mp_request" + enum ipsec_mb_vector_mode { IPSEC_MB_NOT_SUPPORTED = 0, IPSEC_MB_SSE, @@ -63,6 +67,8 @@ extern int ipsec_mb_logtype_driver; rte_log(RTE_LOG_##level, ipsec_mb_logtype_driver, \ "%s() line %u: " fmt "\n", __func__, __LINE__, ##__VA_ARGS__) +#define IPSEC_MB_MP_SHARED_MZ_NAME "IPSEC_MB_MP_SHARED_MZ" + /** All supported device types */ enum ipsec_mb_pmd_types { IPSEC_MB_PMD_TYPE_AESNI_MB = 0, @@ -149,11 +155,49 @@ struct ipsec_mb_qp { IMB_MGR *mb_mgr; /* Multi buffer manager */ const struct rte_memzone *mb_mgr_mz; + /** Array of process id used for queue pairs **/ + uint16_t qp_in_used_by_pid; /* Shared memzone for storing mb_mgr */ __extension__ uint8_t additional_data[]; /**< Storing PMD specific additional data */ }; +/** Request types for IPC. */ +enum ipsec_mb_mp_req_type { + RTE_IPSEC_MB_MP_REQ_NONE, /**< unknown event type */ + RTE_IPSEC_MB_MP_REQ_QP_SET, /**< Queue pair setup request */ + RTE_IPSEC_MB_MP_REQ_QP_FREE /**< Queue pair free request */ +}; + +/* multi-process shared data */ +struct ipsec_mb_mp_shared_data { + struct rte_cryptodev_qp_conf qp_conf; +}; + +/** Parameters for IPC. */ +struct ipsec_mb_mp_param { + enum ipsec_mb_mp_req_type type; /**< IPC request type */ + int dev_id; + /**< The identifier of the device */ + int qp_id; + /**< The index of the queue pair to be configured */ + int socket_id; + /**< Socket to allocate resources on */ + uint16_t process_id; + /**< The pid who send out the requested */ + void *queue_conf; + /**< A pointer of Crypto device queue pair configuration structure */ + int result; + /**< The request result for response message */ +}; + +/* memzone for multi-process shared data */ +const struct rte_memzone *ipsec_mb_mp_mz; +struct ipsec_mb_mp_shared_data *mp_shared_data; + +int +ipsec_mb_ipc_request(const struct rte_mp_msg *mp_msg, const void *peer); + static __rte_always_inline void * ipsec_mb_get_qp_private_data(struct ipsec_mb_qp *qp) { -- 2.17.1