From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by dpdk.org (Postfix) with ESMTP id 426843572 for ; Mon, 25 Mar 2019 20:15:56 +0100 (CET) Received: from Internal Mail-Server by MTLPINE1 (envelope-from yskoh@mellanox.com) with ESMTPS (AES256-SHA encrypted); 25 Mar 2019 21:15:53 +0200 Received: from scfae-sc-2.mti.labs.mlnx (scfae-sc-2.mti.labs.mlnx [10.101.0.96]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x2PJFmis012764; Mon, 25 Mar 2019 21:15:52 +0200 From: Yongseok Koh To: shahafs@mellanox.com Cc: dev@dpdk.org Date: Mon, 25 Mar 2019 12:15:43 -0700 Message-Id: <20190325191545.20707-3-yskoh@mellanox.com> X-Mailer: git-send-email 2.11.0 In-Reply-To: <20190325191545.20707-1-yskoh@mellanox.com> References: <20190307073314.18324-1-yskoh@mellanox.com> <20190325191545.20707-1-yskoh@mellanox.com> Subject: [dpdk-dev] [PATCH v2 2/4] net/mlx5: replace IPC socket with EAL API X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 25 Mar 2019 19:15:56 -0000 Socket API is used for IPC in order for secondary process to acquire Verb command file descriptor. The FD is used to remap UAR address. The new multi-process APIs (rte_mp) in EAL are newly introduced. mlx5_socket.c is replaced with mlx5_mp.c, which uses the new APIs. As it is PMD global infrastructure, only one IPC channel is established. All the IPC message types may have port_id in the message if there is need to reference a specific device. Signed-off-by: Yongseok Koh --- drivers/net/mlx5/Makefile | 2 +- drivers/net/mlx5/meson.build | 2 +- drivers/net/mlx5/mlx5.c | 5 +- drivers/net/mlx5/mlx5.h | 29 ++-- drivers/net/mlx5/mlx5_ethdev.c | 29 ---- drivers/net/mlx5/mlx5_mp.c | 139 +++++++++++++++++++ drivers/net/mlx5/mlx5_socket.c | 306 ----------------------------------------- 7 files changed, 164 insertions(+), 348 deletions(-) create mode 100644 drivers/net/mlx5/mlx5_mp.c delete mode 100644 drivers/net/mlx5/mlx5_socket.c diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile index 9a7da18196..34dc957ac0 100644 --- a/drivers/net/mlx5/Makefile +++ b/drivers/net/mlx5/Makefile @@ -34,7 +34,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_flow.c SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_flow_dv.c SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_flow_tcf.c SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_flow_verbs.c -SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_socket.c +SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_mp.c SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_nl.c SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_devx_cmds.c diff --git a/drivers/net/mlx5/meson.build b/drivers/net/mlx5/meson.build index 0cf2f0873e..d99667670c 100644 --- a/drivers/net/mlx5/meson.build +++ b/drivers/net/mlx5/meson.build @@ -41,7 +41,7 @@ if build 'mlx5_rxmode.c', 'mlx5_rxq.c', 'mlx5_rxtx.c', - 'mlx5_socket.c', + 'mlx5_mp.c', 'mlx5_stats.c', 'mlx5_trigger.c', 'mlx5_txq.c', diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index dd29eba955..316f34cd05 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -160,6 +160,7 @@ mlx5_prepare_shared_data(void) rte_mem_event_callback_register("MLX5_MEM_EVENT_CB", mlx5_mr_mem_event_cb, NULL); + mlx5_mp_init(); } } rte_spinlock_unlock(&mlx5_shared_data_lock); @@ -291,8 +292,6 @@ mlx5_dev_close(struct rte_eth_dev *dev) rte_free(priv->rss_conf.rss_key); if (priv->reta_idx != NULL) rte_free(priv->reta_idx); - if (priv->primary_socket) - mlx5_socket_uninit(dev); if (priv->config.vf) mlx5_nl_mac_addr_flush(dev); if (priv->nl_socket_route >= 0) @@ -929,7 +928,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev, goto error; } /* Receive command fd from primary process */ - err = mlx5_socket_connect(eth_dev); + err = mlx5_mp_req_verbs_cmd_fd(eth_dev); if (err < 0) { err = rte_errno; goto error; diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index 88ffb19247..7030c6f7d7 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -56,6 +56,24 @@ enum { PCI_DEVICE_ID_MELLANOX_CONNECTX6VF = 0x101c, }; +/* Request types for IPC. */ +enum mlx5_mp_req_type { + MLX5_MP_REQ_VERBS_CMD_FD = 1, +}; + +/* Pameters for IPC. */ +struct mlx5_mp_param { + enum mlx5_mp_req_type type; + int port_id; + int result; +}; + +/** Request timeout for IPC. */ +#define MLX5_MP_REQ_TIMEOUT_SEC 5 + +/** Key string for IPC. */ +#define MLX5_MP_NAME "net_mlx5_mp" + /** Switch information returned by mlx5_nl_switch_info(). */ struct mlx5_switch_info { uint32_t master:1; /**< Master device. */ @@ -242,9 +260,7 @@ struct mlx5_priv { uint32_t link_speed_capa; /* Link speed capabilities. */ struct mlx5_xstats_ctrl xstats_ctrl; /* Extended stats control. */ struct mlx5_stats_ctrl stats_ctrl; /* Stats control. */ - int primary_socket; /* Unix socket for primary process. */ void *uar_base; /* Reserved address space for UAR mapping */ - struct rte_intr_handle intr_handle_socket; /* Interrupt handler. */ struct mlx5_dev_config config; /* Device configuration. */ struct mlx5_verbs_alloc_ctx verbs_alloc_ctx; /* Context for Verbs allocator. */ @@ -402,12 +418,9 @@ int mlx5_ctrl_flow(struct rte_eth_dev *dev, int mlx5_flow_create_drop_queue(struct rte_eth_dev *dev); void mlx5_flow_delete_drop_queue(struct rte_eth_dev *dev); -/* mlx5_socket.c */ - -int mlx5_socket_init(struct rte_eth_dev *priv); -void mlx5_socket_uninit(struct rte_eth_dev *priv); -void mlx5_socket_handle(struct rte_eth_dev *priv); -int mlx5_socket_connect(struct rte_eth_dev *priv); +/* mlx5_mp.c */ +int mlx5_mp_req_verbs_cmd_fd(struct rte_eth_dev *dev); +void mlx5_mp_init(void); /* mlx5_nl.c */ diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c index 84d761c8e9..82a77e0d2b 100644 --- a/drivers/net/mlx5/mlx5_ethdev.c +++ b/drivers/net/mlx5/mlx5_ethdev.c @@ -1091,20 +1091,6 @@ mlx5_dev_interrupt_handler(void *cb_arg) } /** - * Handle interrupts from the socket. - * - * @param cb_arg - * Callback argument. - */ -static void -mlx5_dev_handler_socket(void *cb_arg) -{ - struct rte_eth_dev *dev = cb_arg; - - mlx5_socket_handle(dev); -} - -/** * Uninstall interrupt handler. * * @param dev @@ -1119,13 +1105,8 @@ mlx5_dev_interrupt_handler_uninstall(struct rte_eth_dev *dev) dev->data->dev_conf.intr_conf.rmv) rte_intr_callback_unregister(&priv->intr_handle, mlx5_dev_interrupt_handler, dev); - if (priv->primary_socket) - rte_intr_callback_unregister(&priv->intr_handle_socket, - mlx5_dev_handler_socket, dev); priv->intr_handle.fd = 0; priv->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN; - priv->intr_handle_socket.fd = 0; - priv->intr_handle_socket.type = RTE_INTR_HANDLE_UNKNOWN; } /** @@ -1159,16 +1140,6 @@ mlx5_dev_interrupt_handler_install(struct rte_eth_dev *dev) rte_intr_callback_register(&priv->intr_handle, mlx5_dev_interrupt_handler, dev); } - ret = mlx5_socket_init(dev); - if (ret) - DRV_LOG(ERR, "port %u cannot initialise socket: %s", - dev->data->port_id, strerror(rte_errno)); - else if (priv->primary_socket) { - priv->intr_handle_socket.fd = priv->primary_socket; - priv->intr_handle_socket.type = RTE_INTR_HANDLE_EXT; - rte_intr_callback_register(&priv->intr_handle_socket, - mlx5_dev_handler_socket, dev); - } } /** diff --git a/drivers/net/mlx5/mlx5_mp.c b/drivers/net/mlx5/mlx5_mp.c new file mode 100644 index 0000000000..b8dd4b5fa7 --- /dev/null +++ b/drivers/net/mlx5/mlx5_mp.c @@ -0,0 +1,139 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2019 6WIND S.A. + * Copyright 2019 Mellanox Technologies, Ltd + */ + +#include +#include +#include + +#include +#include +#include + +#include "mlx5.h" +#include "mlx5_utils.h" + +/** + * Initialize IPC message. + * + * @param[in] dev + * Pointer to Ethernet structure. + * @param[out] msg + * Pointer to message to fill in. + * @param[in] type + * Message type. + */ +static inline void +mp_init_msg(struct rte_eth_dev *dev, struct rte_mp_msg *msg, + enum mlx5_mp_req_type type) +{ + struct mlx5_mp_param *param = (struct mlx5_mp_param *)msg->param; + + memset(msg, 0, sizeof(*msg)); + strlcpy(msg->name, MLX5_MP_NAME, sizeof(msg->name)); + msg->len_param = sizeof(*param); + param->type = type; + param->port_id = dev->data->port_id; +} + +/** + * IPC message handler of primary process. + * + * @param[in] dev + * Pointer to Ethernet structure. + * @param[in] peer + * Pointer to the peer socket path. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +mp_primary_handle(const struct rte_mp_msg *mp_msg, const void *peer) +{ + struct rte_mp_msg mp_res; + struct mlx5_mp_param *res = (struct mlx5_mp_param *)mp_res.param; + const struct mlx5_mp_param *param = + (const struct mlx5_mp_param *)mp_msg->param; + struct rte_eth_dev *dev; + struct mlx5_priv *priv; + int ret; + + assert(rte_eal_process_type() == RTE_PROC_PRIMARY); + if (!rte_eth_dev_is_valid_port(param->port_id)) { + rte_errno = ENODEV; + DRV_LOG(ERR, "port %u invalid port ID", param->port_id); + return -rte_errno; + } + dev = &rte_eth_devices[param->port_id]; + priv = dev->data->dev_private; + switch (param->type) { + case MLX5_MP_REQ_VERBS_CMD_FD: + mp_init_msg(dev, &mp_res, param->type); + mp_res.num_fds = 1; + mp_res.fds[0] = priv->ctx->cmd_fd; + res->result = 0; + ret = rte_mp_reply(&mp_res, peer); + break; + default: + rte_errno = EINVAL; + DRV_LOG(ERR, "port %u invalid mp request type", + dev->data->port_id); + return -rte_errno; + } + return ret; +} + +/** + * Request Verbs command file descriptor for mmap to the primary process. + * + * @param[in] dev + * Pointer to Ethernet structure. + * + * @return + * fd on success, a negative errno value otherwise and rte_errno is set. + */ +int +mlx5_mp_req_verbs_cmd_fd(struct rte_eth_dev *dev) +{ + struct rte_mp_msg mp_req; + struct rte_mp_msg *mp_res; + struct rte_mp_reply mp_rep; + struct mlx5_mp_param *res; + struct timespec ts = {.tv_sec = MLX5_MP_REQ_TIMEOUT_SEC, .tv_nsec = 0}; + int ret; + + assert(rte_eal_process_type() == RTE_PROC_SECONDARY); + mp_init_msg(dev, &mp_req, MLX5_MP_REQ_VERBS_CMD_FD); + ret = rte_mp_request_sync(&mp_req, &mp_rep, &ts); + if (ret) { + DRV_LOG(ERR, "port %u request to primary process failed", + dev->data->port_id); + return -rte_errno; + } + assert(mp_rep.nb_received == 1); + mp_res = &mp_rep.msgs[0]; + res = (struct mlx5_mp_param *)mp_res->param; + if (res->result) { + rte_errno = -res->result; + DRV_LOG(ERR, + "port %u failed to get command FD from primary process", + dev->data->port_id); + ret = -rte_errno; + goto exit; + } + assert(mp_res->num_fds == 1); + ret = mp_res->fds[0]; + DRV_LOG(DEBUG, "port %u command FD from primary is %d", + dev->data->port_id, ret); +exit: + free(mp_rep.msgs); + return ret; +} + +void +mlx5_mp_init(void) +{ + if (rte_eal_process_type() == RTE_PROC_PRIMARY) + rte_mp_action_register(MLX5_MP_NAME, mp_primary_handle); +} diff --git a/drivers/net/mlx5/mlx5_socket.c b/drivers/net/mlx5/mlx5_socket.c deleted file mode 100644 index 41cac3c6aa..0000000000 --- a/drivers/net/mlx5/mlx5_socket.c +++ /dev/null @@ -1,306 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright 2016 6WIND S.A. - * Copyright 2016 Mellanox Technologies, Ltd - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "mlx5.h" -#include "mlx5_utils.h" - -/** - * Initialise the socket to communicate with the secondary process - * - * @param[in] dev - * Pointer to Ethernet device. - * - * @return - * 0 on success, a negative errno value otherwise and rte_errno is set. - */ -int -mlx5_socket_init(struct rte_eth_dev *dev) -{ - struct mlx5_priv *priv = dev->data->dev_private; - struct sockaddr_un sun = { - .sun_family = AF_UNIX, - }; - int ret; - int flags; - - /* - * Close the last socket that was used to communicate - * with the secondary process - */ - if (priv->primary_socket) - mlx5_socket_uninit(dev); - /* - * Initialise the socket to communicate with the secondary - * process. - */ - ret = socket(AF_UNIX, SOCK_STREAM, 0); - if (ret < 0) { - rte_errno = errno; - DRV_LOG(WARNING, "port %u secondary process not supported: %s", - dev->data->port_id, strerror(errno)); - goto error; - } - priv->primary_socket = ret; - flags = fcntl(priv->primary_socket, F_GETFL, 0); - if (flags == -1) { - rte_errno = errno; - goto error; - } - ret = fcntl(priv->primary_socket, F_SETFL, flags | O_NONBLOCK); - if (ret < 0) { - rte_errno = errno; - goto error; - } - snprintf(sun.sun_path, sizeof(sun.sun_path), "/var/tmp/%s_%d", - MLX5_DRIVER_NAME, priv->primary_socket); - remove(sun.sun_path); - ret = bind(priv->primary_socket, (const struct sockaddr *)&sun, - sizeof(sun)); - if (ret < 0) { - rte_errno = errno; - DRV_LOG(WARNING, - "port %u cannot bind socket, secondary process not" - " supported: %s", - dev->data->port_id, strerror(errno)); - goto close; - } - ret = listen(priv->primary_socket, 0); - if (ret < 0) { - rte_errno = errno; - DRV_LOG(WARNING, "port %u secondary process not supported: %s", - dev->data->port_id, strerror(errno)); - goto close; - } - return 0; -close: - remove(sun.sun_path); -error: - claim_zero(close(priv->primary_socket)); - priv->primary_socket = 0; - return -rte_errno; -} - -/** - * Un-Initialise the socket to communicate with the secondary process - * - * @param[in] dev - */ -void -mlx5_socket_uninit(struct rte_eth_dev *dev) -{ - struct mlx5_priv *priv = dev->data->dev_private; - - MKSTR(path, "/var/tmp/%s_%d", MLX5_DRIVER_NAME, priv->primary_socket); - claim_zero(close(priv->primary_socket)); - priv->primary_socket = 0; - claim_zero(remove(path)); -} - -/** - * Handle socket interrupts. - * - * @param dev - * Pointer to Ethernet device. - */ -void -mlx5_socket_handle(struct rte_eth_dev *dev) -{ - struct mlx5_priv *priv = dev->data->dev_private; - int conn_sock; - int ret = 0; - struct cmsghdr *cmsg = NULL; - struct ucred *cred = NULL; - char buf[CMSG_SPACE(sizeof(struct ucred))] = { 0 }; - char vbuf[1024] = { 0 }; - struct iovec io = { - .iov_base = vbuf, - .iov_len = sizeof(*vbuf), - }; - struct msghdr msg = { - .msg_iov = &io, - .msg_iovlen = 1, - .msg_control = buf, - .msg_controllen = sizeof(buf), - }; - int *fd; - - /* Accept the connection from the client. */ - conn_sock = accept(priv->primary_socket, NULL, NULL); - if (conn_sock < 0) { - DRV_LOG(WARNING, "port %u connection failed: %s", - dev->data->port_id, strerror(errno)); - return; - } - ret = setsockopt(conn_sock, SOL_SOCKET, SO_PASSCRED, &(int){1}, - sizeof(int)); - if (ret < 0) { - ret = errno; - DRV_LOG(WARNING, "port %u cannot change socket options: %s", - dev->data->port_id, strerror(rte_errno)); - goto error; - } - ret = recvmsg(conn_sock, &msg, MSG_WAITALL); - if (ret < 0) { - ret = errno; - DRV_LOG(WARNING, "port %u received an empty message: %s", - dev->data->port_id, strerror(rte_errno)); - goto error; - } - /* Expect to receive credentials only. */ - cmsg = CMSG_FIRSTHDR(&msg); - if (cmsg == NULL) { - DRV_LOG(WARNING, "port %u no message", dev->data->port_id); - goto error; - } - if ((cmsg->cmsg_type == SCM_CREDENTIALS) && - (cmsg->cmsg_len >= sizeof(*cred))) { - cred = (struct ucred *)CMSG_DATA(cmsg); - assert(cred != NULL); - } - cmsg = CMSG_NXTHDR(&msg, cmsg); - if (cmsg != NULL) { - DRV_LOG(WARNING, "port %u message wrongly formatted", - dev->data->port_id); - goto error; - } - /* Make sure all the ancillary data was received and valid. */ - if ((cred == NULL) || (cred->uid != getuid()) || - (cred->gid != getgid())) { - DRV_LOG(WARNING, "port %u wrong credentials", - dev->data->port_id); - goto error; - } - /* Set-up the ancillary data. */ - cmsg = CMSG_FIRSTHDR(&msg); - assert(cmsg != NULL); - cmsg->cmsg_level = SOL_SOCKET; - cmsg->cmsg_type = SCM_RIGHTS; - cmsg->cmsg_len = CMSG_LEN(sizeof(priv->ctx->cmd_fd)); - fd = (int *)CMSG_DATA(cmsg); - *fd = priv->ctx->cmd_fd; - ret = sendmsg(conn_sock, &msg, 0); - if (ret < 0) - DRV_LOG(WARNING, "port %u cannot send response", - dev->data->port_id); -error: - close(conn_sock); -} - -/** - * Connect to the primary process. - * - * @param[in] dev - * Pointer to Ethernet structure. - * - * @return - * fd on success, negative errno value otherwise and rte_errno is set. - */ -int -mlx5_socket_connect(struct rte_eth_dev *dev) -{ - struct mlx5_priv *priv = dev->data->dev_private; - struct sockaddr_un sun = { - .sun_family = AF_UNIX, - }; - int socket_fd = -1; - int *fd = NULL; - int ret; - struct ucred *cred; - char buf[CMSG_SPACE(sizeof(*cred))] = { 0 }; - char vbuf[1024] = { 0 }; - struct iovec io = { - .iov_base = vbuf, - .iov_len = sizeof(*vbuf), - }; - struct msghdr msg = { - .msg_control = buf, - .msg_controllen = sizeof(buf), - .msg_iov = &io, - .msg_iovlen = 1, - }; - struct cmsghdr *cmsg; - - ret = socket(AF_UNIX, SOCK_STREAM, 0); - if (ret < 0) { - rte_errno = errno; - DRV_LOG(WARNING, "port %u cannot connect to primary", - dev->data->port_id); - goto error; - } - socket_fd = ret; - snprintf(sun.sun_path, sizeof(sun.sun_path), "/var/tmp/%s_%d", - MLX5_DRIVER_NAME, priv->primary_socket); - ret = connect(socket_fd, (const struct sockaddr *)&sun, sizeof(sun)); - if (ret < 0) { - rte_errno = errno; - DRV_LOG(WARNING, "port %u cannot connect to primary", - dev->data->port_id); - goto error; - } - cmsg = CMSG_FIRSTHDR(&msg); - if (cmsg == NULL) { - rte_errno = EINVAL; - DRV_LOG(DEBUG, "port %u cannot get first message", - dev->data->port_id); - goto error; - } - cmsg->cmsg_level = SOL_SOCKET; - cmsg->cmsg_type = SCM_CREDENTIALS; - cmsg->cmsg_len = CMSG_LEN(sizeof(*cred)); - cred = (struct ucred *)CMSG_DATA(cmsg); - if (cred == NULL) { - rte_errno = EINVAL; - DRV_LOG(DEBUG, "port %u no credentials received", - dev->data->port_id); - goto error; - } - cred->pid = getpid(); - cred->uid = getuid(); - cred->gid = getgid(); - ret = sendmsg(socket_fd, &msg, MSG_DONTWAIT); - if (ret < 0) { - rte_errno = errno; - DRV_LOG(WARNING, - "port %u cannot send credentials to primary: %s", - dev->data->port_id, strerror(errno)); - goto error; - } - ret = recvmsg(socket_fd, &msg, MSG_WAITALL); - if (ret <= 0) { - rte_errno = errno; - DRV_LOG(WARNING, "port %u no message from primary: %s", - dev->data->port_id, strerror(errno)); - goto error; - } - cmsg = CMSG_FIRSTHDR(&msg); - if (cmsg == NULL) { - rte_errno = EINVAL; - DRV_LOG(WARNING, "port %u no file descriptor received", - dev->data->port_id); - goto error; - } - fd = (int *)CMSG_DATA(cmsg); - if (*fd < 0) { - DRV_LOG(WARNING, "port %u no file descriptor received: %s", - dev->data->port_id, strerror(errno)); - rte_errno = *fd; - goto error; - } - ret = *fd; - close(socket_fd); - return ret; -error: - if (socket_fd != -1) - close(socket_fd); - return -rte_errno; -} -- 2.11.0 From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from dpdk.org (dpdk.org [92.243.14.124]) by dpdk.space (Postfix) with ESMTP id 51D42A05D3 for ; Mon, 25 Mar 2019 20:16:18 +0100 (CET) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 894034C88; Mon, 25 Mar 2019 20:16:03 +0100 (CET) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by dpdk.org (Postfix) with ESMTP id 426843572 for ; Mon, 25 Mar 2019 20:15:56 +0100 (CET) Received: from Internal Mail-Server by MTLPINE1 (envelope-from yskoh@mellanox.com) with ESMTPS (AES256-SHA encrypted); 25 Mar 2019 21:15:53 +0200 Received: from scfae-sc-2.mti.labs.mlnx (scfae-sc-2.mti.labs.mlnx [10.101.0.96]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x2PJFmis012764; Mon, 25 Mar 2019 21:15:52 +0200 From: Yongseok Koh To: shahafs@mellanox.com Cc: dev@dpdk.org Date: Mon, 25 Mar 2019 12:15:43 -0700 Message-Id: <20190325191545.20707-3-yskoh@mellanox.com> X-Mailer: git-send-email 2.11.0 In-Reply-To: <20190325191545.20707-1-yskoh@mellanox.com> References: <20190307073314.18324-1-yskoh@mellanox.com> <20190325191545.20707-1-yskoh@mellanox.com> Subject: [dpdk-dev] [PATCH v2 2/4] net/mlx5: replace IPC socket with EAL API X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Content-Type: text/plain; charset="UTF-8" Message-ID: <20190325191543.lc6NiycPQy2miNQuaEsxwhyV81aFOCQ6IUlauJukGco@z> Socket API is used for IPC in order for secondary process to acquire Verb command file descriptor. The FD is used to remap UAR address. The new multi-process APIs (rte_mp) in EAL are newly introduced. mlx5_socket.c is replaced with mlx5_mp.c, which uses the new APIs. As it is PMD global infrastructure, only one IPC channel is established. All the IPC message types may have port_id in the message if there is need to reference a specific device. Signed-off-by: Yongseok Koh --- drivers/net/mlx5/Makefile | 2 +- drivers/net/mlx5/meson.build | 2 +- drivers/net/mlx5/mlx5.c | 5 +- drivers/net/mlx5/mlx5.h | 29 ++-- drivers/net/mlx5/mlx5_ethdev.c | 29 ---- drivers/net/mlx5/mlx5_mp.c | 139 +++++++++++++++++++ drivers/net/mlx5/mlx5_socket.c | 306 ----------------------------------------- 7 files changed, 164 insertions(+), 348 deletions(-) create mode 100644 drivers/net/mlx5/mlx5_mp.c delete mode 100644 drivers/net/mlx5/mlx5_socket.c diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile index 9a7da18196..34dc957ac0 100644 --- a/drivers/net/mlx5/Makefile +++ b/drivers/net/mlx5/Makefile @@ -34,7 +34,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_flow.c SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_flow_dv.c SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_flow_tcf.c SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_flow_verbs.c -SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_socket.c +SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_mp.c SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_nl.c SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_devx_cmds.c diff --git a/drivers/net/mlx5/meson.build b/drivers/net/mlx5/meson.build index 0cf2f0873e..d99667670c 100644 --- a/drivers/net/mlx5/meson.build +++ b/drivers/net/mlx5/meson.build @@ -41,7 +41,7 @@ if build 'mlx5_rxmode.c', 'mlx5_rxq.c', 'mlx5_rxtx.c', - 'mlx5_socket.c', + 'mlx5_mp.c', 'mlx5_stats.c', 'mlx5_trigger.c', 'mlx5_txq.c', diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index dd29eba955..316f34cd05 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -160,6 +160,7 @@ mlx5_prepare_shared_data(void) rte_mem_event_callback_register("MLX5_MEM_EVENT_CB", mlx5_mr_mem_event_cb, NULL); + mlx5_mp_init(); } } rte_spinlock_unlock(&mlx5_shared_data_lock); @@ -291,8 +292,6 @@ mlx5_dev_close(struct rte_eth_dev *dev) rte_free(priv->rss_conf.rss_key); if (priv->reta_idx != NULL) rte_free(priv->reta_idx); - if (priv->primary_socket) - mlx5_socket_uninit(dev); if (priv->config.vf) mlx5_nl_mac_addr_flush(dev); if (priv->nl_socket_route >= 0) @@ -929,7 +928,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev, goto error; } /* Receive command fd from primary process */ - err = mlx5_socket_connect(eth_dev); + err = mlx5_mp_req_verbs_cmd_fd(eth_dev); if (err < 0) { err = rte_errno; goto error; diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index 88ffb19247..7030c6f7d7 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -56,6 +56,24 @@ enum { PCI_DEVICE_ID_MELLANOX_CONNECTX6VF = 0x101c, }; +/* Request types for IPC. */ +enum mlx5_mp_req_type { + MLX5_MP_REQ_VERBS_CMD_FD = 1, +}; + +/* Pameters for IPC. */ +struct mlx5_mp_param { + enum mlx5_mp_req_type type; + int port_id; + int result; +}; + +/** Request timeout for IPC. */ +#define MLX5_MP_REQ_TIMEOUT_SEC 5 + +/** Key string for IPC. */ +#define MLX5_MP_NAME "net_mlx5_mp" + /** Switch information returned by mlx5_nl_switch_info(). */ struct mlx5_switch_info { uint32_t master:1; /**< Master device. */ @@ -242,9 +260,7 @@ struct mlx5_priv { uint32_t link_speed_capa; /* Link speed capabilities. */ struct mlx5_xstats_ctrl xstats_ctrl; /* Extended stats control. */ struct mlx5_stats_ctrl stats_ctrl; /* Stats control. */ - int primary_socket; /* Unix socket for primary process. */ void *uar_base; /* Reserved address space for UAR mapping */ - struct rte_intr_handle intr_handle_socket; /* Interrupt handler. */ struct mlx5_dev_config config; /* Device configuration. */ struct mlx5_verbs_alloc_ctx verbs_alloc_ctx; /* Context for Verbs allocator. */ @@ -402,12 +418,9 @@ int mlx5_ctrl_flow(struct rte_eth_dev *dev, int mlx5_flow_create_drop_queue(struct rte_eth_dev *dev); void mlx5_flow_delete_drop_queue(struct rte_eth_dev *dev); -/* mlx5_socket.c */ - -int mlx5_socket_init(struct rte_eth_dev *priv); -void mlx5_socket_uninit(struct rte_eth_dev *priv); -void mlx5_socket_handle(struct rte_eth_dev *priv); -int mlx5_socket_connect(struct rte_eth_dev *priv); +/* mlx5_mp.c */ +int mlx5_mp_req_verbs_cmd_fd(struct rte_eth_dev *dev); +void mlx5_mp_init(void); /* mlx5_nl.c */ diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c index 84d761c8e9..82a77e0d2b 100644 --- a/drivers/net/mlx5/mlx5_ethdev.c +++ b/drivers/net/mlx5/mlx5_ethdev.c @@ -1091,20 +1091,6 @@ mlx5_dev_interrupt_handler(void *cb_arg) } /** - * Handle interrupts from the socket. - * - * @param cb_arg - * Callback argument. - */ -static void -mlx5_dev_handler_socket(void *cb_arg) -{ - struct rte_eth_dev *dev = cb_arg; - - mlx5_socket_handle(dev); -} - -/** * Uninstall interrupt handler. * * @param dev @@ -1119,13 +1105,8 @@ mlx5_dev_interrupt_handler_uninstall(struct rte_eth_dev *dev) dev->data->dev_conf.intr_conf.rmv) rte_intr_callback_unregister(&priv->intr_handle, mlx5_dev_interrupt_handler, dev); - if (priv->primary_socket) - rte_intr_callback_unregister(&priv->intr_handle_socket, - mlx5_dev_handler_socket, dev); priv->intr_handle.fd = 0; priv->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN; - priv->intr_handle_socket.fd = 0; - priv->intr_handle_socket.type = RTE_INTR_HANDLE_UNKNOWN; } /** @@ -1159,16 +1140,6 @@ mlx5_dev_interrupt_handler_install(struct rte_eth_dev *dev) rte_intr_callback_register(&priv->intr_handle, mlx5_dev_interrupt_handler, dev); } - ret = mlx5_socket_init(dev); - if (ret) - DRV_LOG(ERR, "port %u cannot initialise socket: %s", - dev->data->port_id, strerror(rte_errno)); - else if (priv->primary_socket) { - priv->intr_handle_socket.fd = priv->primary_socket; - priv->intr_handle_socket.type = RTE_INTR_HANDLE_EXT; - rte_intr_callback_register(&priv->intr_handle_socket, - mlx5_dev_handler_socket, dev); - } } /** diff --git a/drivers/net/mlx5/mlx5_mp.c b/drivers/net/mlx5/mlx5_mp.c new file mode 100644 index 0000000000..b8dd4b5fa7 --- /dev/null +++ b/drivers/net/mlx5/mlx5_mp.c @@ -0,0 +1,139 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2019 6WIND S.A. + * Copyright 2019 Mellanox Technologies, Ltd + */ + +#include +#include +#include + +#include +#include +#include + +#include "mlx5.h" +#include "mlx5_utils.h" + +/** + * Initialize IPC message. + * + * @param[in] dev + * Pointer to Ethernet structure. + * @param[out] msg + * Pointer to message to fill in. + * @param[in] type + * Message type. + */ +static inline void +mp_init_msg(struct rte_eth_dev *dev, struct rte_mp_msg *msg, + enum mlx5_mp_req_type type) +{ + struct mlx5_mp_param *param = (struct mlx5_mp_param *)msg->param; + + memset(msg, 0, sizeof(*msg)); + strlcpy(msg->name, MLX5_MP_NAME, sizeof(msg->name)); + msg->len_param = sizeof(*param); + param->type = type; + param->port_id = dev->data->port_id; +} + +/** + * IPC message handler of primary process. + * + * @param[in] dev + * Pointer to Ethernet structure. + * @param[in] peer + * Pointer to the peer socket path. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +mp_primary_handle(const struct rte_mp_msg *mp_msg, const void *peer) +{ + struct rte_mp_msg mp_res; + struct mlx5_mp_param *res = (struct mlx5_mp_param *)mp_res.param; + const struct mlx5_mp_param *param = + (const struct mlx5_mp_param *)mp_msg->param; + struct rte_eth_dev *dev; + struct mlx5_priv *priv; + int ret; + + assert(rte_eal_process_type() == RTE_PROC_PRIMARY); + if (!rte_eth_dev_is_valid_port(param->port_id)) { + rte_errno = ENODEV; + DRV_LOG(ERR, "port %u invalid port ID", param->port_id); + return -rte_errno; + } + dev = &rte_eth_devices[param->port_id]; + priv = dev->data->dev_private; + switch (param->type) { + case MLX5_MP_REQ_VERBS_CMD_FD: + mp_init_msg(dev, &mp_res, param->type); + mp_res.num_fds = 1; + mp_res.fds[0] = priv->ctx->cmd_fd; + res->result = 0; + ret = rte_mp_reply(&mp_res, peer); + break; + default: + rte_errno = EINVAL; + DRV_LOG(ERR, "port %u invalid mp request type", + dev->data->port_id); + return -rte_errno; + } + return ret; +} + +/** + * Request Verbs command file descriptor for mmap to the primary process. + * + * @param[in] dev + * Pointer to Ethernet structure. + * + * @return + * fd on success, a negative errno value otherwise and rte_errno is set. + */ +int +mlx5_mp_req_verbs_cmd_fd(struct rte_eth_dev *dev) +{ + struct rte_mp_msg mp_req; + struct rte_mp_msg *mp_res; + struct rte_mp_reply mp_rep; + struct mlx5_mp_param *res; + struct timespec ts = {.tv_sec = MLX5_MP_REQ_TIMEOUT_SEC, .tv_nsec = 0}; + int ret; + + assert(rte_eal_process_type() == RTE_PROC_SECONDARY); + mp_init_msg(dev, &mp_req, MLX5_MP_REQ_VERBS_CMD_FD); + ret = rte_mp_request_sync(&mp_req, &mp_rep, &ts); + if (ret) { + DRV_LOG(ERR, "port %u request to primary process failed", + dev->data->port_id); + return -rte_errno; + } + assert(mp_rep.nb_received == 1); + mp_res = &mp_rep.msgs[0]; + res = (struct mlx5_mp_param *)mp_res->param; + if (res->result) { + rte_errno = -res->result; + DRV_LOG(ERR, + "port %u failed to get command FD from primary process", + dev->data->port_id); + ret = -rte_errno; + goto exit; + } + assert(mp_res->num_fds == 1); + ret = mp_res->fds[0]; + DRV_LOG(DEBUG, "port %u command FD from primary is %d", + dev->data->port_id, ret); +exit: + free(mp_rep.msgs); + return ret; +} + +void +mlx5_mp_init(void) +{ + if (rte_eal_process_type() == RTE_PROC_PRIMARY) + rte_mp_action_register(MLX5_MP_NAME, mp_primary_handle); +} diff --git a/drivers/net/mlx5/mlx5_socket.c b/drivers/net/mlx5/mlx5_socket.c deleted file mode 100644 index 41cac3c6aa..0000000000 --- a/drivers/net/mlx5/mlx5_socket.c +++ /dev/null @@ -1,306 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * Copyright 2016 6WIND S.A. - * Copyright 2016 Mellanox Technologies, Ltd - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "mlx5.h" -#include "mlx5_utils.h" - -/** - * Initialise the socket to communicate with the secondary process - * - * @param[in] dev - * Pointer to Ethernet device. - * - * @return - * 0 on success, a negative errno value otherwise and rte_errno is set. - */ -int -mlx5_socket_init(struct rte_eth_dev *dev) -{ - struct mlx5_priv *priv = dev->data->dev_private; - struct sockaddr_un sun = { - .sun_family = AF_UNIX, - }; - int ret; - int flags; - - /* - * Close the last socket that was used to communicate - * with the secondary process - */ - if (priv->primary_socket) - mlx5_socket_uninit(dev); - /* - * Initialise the socket to communicate with the secondary - * process. - */ - ret = socket(AF_UNIX, SOCK_STREAM, 0); - if (ret < 0) { - rte_errno = errno; - DRV_LOG(WARNING, "port %u secondary process not supported: %s", - dev->data->port_id, strerror(errno)); - goto error; - } - priv->primary_socket = ret; - flags = fcntl(priv->primary_socket, F_GETFL, 0); - if (flags == -1) { - rte_errno = errno; - goto error; - } - ret = fcntl(priv->primary_socket, F_SETFL, flags | O_NONBLOCK); - if (ret < 0) { - rte_errno = errno; - goto error; - } - snprintf(sun.sun_path, sizeof(sun.sun_path), "/var/tmp/%s_%d", - MLX5_DRIVER_NAME, priv->primary_socket); - remove(sun.sun_path); - ret = bind(priv->primary_socket, (const struct sockaddr *)&sun, - sizeof(sun)); - if (ret < 0) { - rte_errno = errno; - DRV_LOG(WARNING, - "port %u cannot bind socket, secondary process not" - " supported: %s", - dev->data->port_id, strerror(errno)); - goto close; - } - ret = listen(priv->primary_socket, 0); - if (ret < 0) { - rte_errno = errno; - DRV_LOG(WARNING, "port %u secondary process not supported: %s", - dev->data->port_id, strerror(errno)); - goto close; - } - return 0; -close: - remove(sun.sun_path); -error: - claim_zero(close(priv->primary_socket)); - priv->primary_socket = 0; - return -rte_errno; -} - -/** - * Un-Initialise the socket to communicate with the secondary process - * - * @param[in] dev - */ -void -mlx5_socket_uninit(struct rte_eth_dev *dev) -{ - struct mlx5_priv *priv = dev->data->dev_private; - - MKSTR(path, "/var/tmp/%s_%d", MLX5_DRIVER_NAME, priv->primary_socket); - claim_zero(close(priv->primary_socket)); - priv->primary_socket = 0; - claim_zero(remove(path)); -} - -/** - * Handle socket interrupts. - * - * @param dev - * Pointer to Ethernet device. - */ -void -mlx5_socket_handle(struct rte_eth_dev *dev) -{ - struct mlx5_priv *priv = dev->data->dev_private; - int conn_sock; - int ret = 0; - struct cmsghdr *cmsg = NULL; - struct ucred *cred = NULL; - char buf[CMSG_SPACE(sizeof(struct ucred))] = { 0 }; - char vbuf[1024] = { 0 }; - struct iovec io = { - .iov_base = vbuf, - .iov_len = sizeof(*vbuf), - }; - struct msghdr msg = { - .msg_iov = &io, - .msg_iovlen = 1, - .msg_control = buf, - .msg_controllen = sizeof(buf), - }; - int *fd; - - /* Accept the connection from the client. */ - conn_sock = accept(priv->primary_socket, NULL, NULL); - if (conn_sock < 0) { - DRV_LOG(WARNING, "port %u connection failed: %s", - dev->data->port_id, strerror(errno)); - return; - } - ret = setsockopt(conn_sock, SOL_SOCKET, SO_PASSCRED, &(int){1}, - sizeof(int)); - if (ret < 0) { - ret = errno; - DRV_LOG(WARNING, "port %u cannot change socket options: %s", - dev->data->port_id, strerror(rte_errno)); - goto error; - } - ret = recvmsg(conn_sock, &msg, MSG_WAITALL); - if (ret < 0) { - ret = errno; - DRV_LOG(WARNING, "port %u received an empty message: %s", - dev->data->port_id, strerror(rte_errno)); - goto error; - } - /* Expect to receive credentials only. */ - cmsg = CMSG_FIRSTHDR(&msg); - if (cmsg == NULL) { - DRV_LOG(WARNING, "port %u no message", dev->data->port_id); - goto error; - } - if ((cmsg->cmsg_type == SCM_CREDENTIALS) && - (cmsg->cmsg_len >= sizeof(*cred))) { - cred = (struct ucred *)CMSG_DATA(cmsg); - assert(cred != NULL); - } - cmsg = CMSG_NXTHDR(&msg, cmsg); - if (cmsg != NULL) { - DRV_LOG(WARNING, "port %u message wrongly formatted", - dev->data->port_id); - goto error; - } - /* Make sure all the ancillary data was received and valid. */ - if ((cred == NULL) || (cred->uid != getuid()) || - (cred->gid != getgid())) { - DRV_LOG(WARNING, "port %u wrong credentials", - dev->data->port_id); - goto error; - } - /* Set-up the ancillary data. */ - cmsg = CMSG_FIRSTHDR(&msg); - assert(cmsg != NULL); - cmsg->cmsg_level = SOL_SOCKET; - cmsg->cmsg_type = SCM_RIGHTS; - cmsg->cmsg_len = CMSG_LEN(sizeof(priv->ctx->cmd_fd)); - fd = (int *)CMSG_DATA(cmsg); - *fd = priv->ctx->cmd_fd; - ret = sendmsg(conn_sock, &msg, 0); - if (ret < 0) - DRV_LOG(WARNING, "port %u cannot send response", - dev->data->port_id); -error: - close(conn_sock); -} - -/** - * Connect to the primary process. - * - * @param[in] dev - * Pointer to Ethernet structure. - * - * @return - * fd on success, negative errno value otherwise and rte_errno is set. - */ -int -mlx5_socket_connect(struct rte_eth_dev *dev) -{ - struct mlx5_priv *priv = dev->data->dev_private; - struct sockaddr_un sun = { - .sun_family = AF_UNIX, - }; - int socket_fd = -1; - int *fd = NULL; - int ret; - struct ucred *cred; - char buf[CMSG_SPACE(sizeof(*cred))] = { 0 }; - char vbuf[1024] = { 0 }; - struct iovec io = { - .iov_base = vbuf, - .iov_len = sizeof(*vbuf), - }; - struct msghdr msg = { - .msg_control = buf, - .msg_controllen = sizeof(buf), - .msg_iov = &io, - .msg_iovlen = 1, - }; - struct cmsghdr *cmsg; - - ret = socket(AF_UNIX, SOCK_STREAM, 0); - if (ret < 0) { - rte_errno = errno; - DRV_LOG(WARNING, "port %u cannot connect to primary", - dev->data->port_id); - goto error; - } - socket_fd = ret; - snprintf(sun.sun_path, sizeof(sun.sun_path), "/var/tmp/%s_%d", - MLX5_DRIVER_NAME, priv->primary_socket); - ret = connect(socket_fd, (const struct sockaddr *)&sun, sizeof(sun)); - if (ret < 0) { - rte_errno = errno; - DRV_LOG(WARNING, "port %u cannot connect to primary", - dev->data->port_id); - goto error; - } - cmsg = CMSG_FIRSTHDR(&msg); - if (cmsg == NULL) { - rte_errno = EINVAL; - DRV_LOG(DEBUG, "port %u cannot get first message", - dev->data->port_id); - goto error; - } - cmsg->cmsg_level = SOL_SOCKET; - cmsg->cmsg_type = SCM_CREDENTIALS; - cmsg->cmsg_len = CMSG_LEN(sizeof(*cred)); - cred = (struct ucred *)CMSG_DATA(cmsg); - if (cred == NULL) { - rte_errno = EINVAL; - DRV_LOG(DEBUG, "port %u no credentials received", - dev->data->port_id); - goto error; - } - cred->pid = getpid(); - cred->uid = getuid(); - cred->gid = getgid(); - ret = sendmsg(socket_fd, &msg, MSG_DONTWAIT); - if (ret < 0) { - rte_errno = errno; - DRV_LOG(WARNING, - "port %u cannot send credentials to primary: %s", - dev->data->port_id, strerror(errno)); - goto error; - } - ret = recvmsg(socket_fd, &msg, MSG_WAITALL); - if (ret <= 0) { - rte_errno = errno; - DRV_LOG(WARNING, "port %u no message from primary: %s", - dev->data->port_id, strerror(errno)); - goto error; - } - cmsg = CMSG_FIRSTHDR(&msg); - if (cmsg == NULL) { - rte_errno = EINVAL; - DRV_LOG(WARNING, "port %u no file descriptor received", - dev->data->port_id); - goto error; - } - fd = (int *)CMSG_DATA(cmsg); - if (*fd < 0) { - DRV_LOG(WARNING, "port %u no file descriptor received: %s", - dev->data->port_id, strerror(errno)); - rte_errno = *fd; - goto error; - } - ret = *fd; - close(socket_fd); - return ret; -error: - if (socket_fd != -1) - close(socket_fd); - return -rte_errno; -} -- 2.11.0