From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from dpdk.org (dpdk.org [92.243.14.124]) by dpdk.space (Postfix) with ESMTP id 262ECA05D3 for ; Mon, 25 Mar 2019 20:16:31 +0100 (CET) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 9188C37B0; Mon, 25 Mar 2019 20:16:11 +0100 (CET) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by dpdk.org (Postfix) with ESMTP id 375914C88 for ; Mon, 25 Mar 2019 20:16:01 +0100 (CET) Received: from Internal Mail-Server by MTLPINE1 (envelope-from yskoh@mellanox.com) with ESMTPS (AES256-SHA encrypted); 25 Mar 2019 21:15:56 +0200 Received: from scfae-sc-2.mti.labs.mlnx (scfae-sc-2.mti.labs.mlnx [10.101.0.96]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id x2PJFmiu012764; Mon, 25 Mar 2019 21:15:55 +0200 From: Yongseok Koh To: shahafs@mellanox.com Cc: dev@dpdk.org Date: Mon, 25 Mar 2019 12:15:45 -0700 Message-Id: <20190325191545.20707-5-yskoh@mellanox.com> X-Mailer: git-send-email 2.11.0 In-Reply-To: <20190325191545.20707-1-yskoh@mellanox.com> References: <20190307073314.18324-1-yskoh@mellanox.com> <20190325191545.20707-1-yskoh@mellanox.com> Subject: [dpdk-dev] [PATCH v2 4/4] net/mlx5: sync stop/start of datapath with secondary process X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Content-Type: text/plain; charset="UTF-8" Message-ID: <20190325191545.5Sf8CUkwoYBCcE0EFTWVIdmBovA2ZeywhONdmUb30c4@z> Rx/Tx burst function pointers are stored in the rte_eth_dev structure, which is local to a process. Even though primary process replaces the function pointers, secondary will not run the new ones. With rte_mp APIs, primary can easily broadcast a request to stop/start the datapath of secondary processes. Signed-off-by: Yongseok Koh --- drivers/net/mlx5/mlx5.c | 5 ++ drivers/net/mlx5/mlx5.h | 6 ++ drivers/net/mlx5/mlx5_mp.c | 156 ++++++++++++++++++++++++++++++++++++++++ drivers/net/mlx5/mlx5_rxtx.c | 2 + drivers/net/mlx5/mlx5_trigger.c | 5 ++ 5 files changed, 174 insertions(+) diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index 54a1896ea4..840cd3d307 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -305,6 +305,9 @@ mlx5_dev_close(struct rte_eth_dev *dev) /* Prevent crashes when queues are still in use. */ dev->rx_pkt_burst = removed_rx_burst; dev->tx_pkt_burst = removed_tx_burst; + rte_wmb(); + /* Disable datapath on secondary process. */ + mlx5_mp_req_stop_rxtx(dev); if (priv->rxqs != NULL) { /* XXX race condition if mlx5_rx_burst() is still running. */ usleep(1000); @@ -803,6 +806,7 @@ mlx5_init_once(void) case RTE_PROC_SECONDARY: if (ld->init_done) break; + mlx5_mp_init_secondary(); ret = mlx5_uar_init_secondary(); if (ret) goto error; @@ -823,6 +827,7 @@ mlx5_init_once(void) break; case RTE_PROC_SECONDARY: mlx5_uar_uninit_secondary(); + mlx5_mp_uninit_secondary(); break; default: break; diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index cb454e866a..d8a5162bdb 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -59,6 +59,8 @@ enum { /* Request types for IPC. */ enum mlx5_mp_req_type { MLX5_MP_REQ_VERBS_CMD_FD = 1, + MLX5_MP_REQ_START_RXTX, + MLX5_MP_REQ_STOP_RXTX, }; /* Pameters for IPC. */ @@ -431,9 +433,13 @@ int mlx5_flow_create_drop_queue(struct rte_eth_dev *dev); void mlx5_flow_delete_drop_queue(struct rte_eth_dev *dev); /* mlx5_mp.c */ +void mlx5_mp_req_start_rxtx(struct rte_eth_dev *dev); +void mlx5_mp_req_stop_rxtx(struct rte_eth_dev *dev); int mlx5_mp_req_verbs_cmd_fd(struct rte_eth_dev *dev); void mlx5_mp_init_primary(void); void mlx5_mp_uninit_primary(void); +void mlx5_mp_init_secondary(void); +void mlx5_mp_uninit_secondary(void); /* mlx5_nl.c */ diff --git a/drivers/net/mlx5/mlx5_mp.c b/drivers/net/mlx5/mlx5_mp.c index d0a38c3d52..657ab6872e 100644 --- a/drivers/net/mlx5/mlx5_mp.c +++ b/drivers/net/mlx5/mlx5_mp.c @@ -12,6 +12,7 @@ #include #include "mlx5.h" +#include "mlx5_rxtx.h" #include "mlx5_utils.h" /** @@ -85,6 +86,141 @@ mp_primary_handle(const struct rte_mp_msg *mp_msg, const void *peer) } /** + * IPC message handler of a secondary process. + * + * @param[in] dev + * Pointer to Ethernet structure. + * @param[in] peer + * Pointer to the peer socket path. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +mp_secondary_handle(const struct rte_mp_msg *mp_msg, const void *peer) +{ + struct rte_mp_msg mp_res; + struct mlx5_mp_param *res = (struct mlx5_mp_param *)mp_res.param; + const struct mlx5_mp_param *param = + (const struct mlx5_mp_param *)mp_msg->param; + struct rte_eth_dev *dev; + int ret; + + assert(rte_eal_process_type() == RTE_PROC_SECONDARY); + if (!rte_eth_dev_is_valid_port(param->port_id)) { + rte_errno = ENODEV; + DRV_LOG(ERR, "port %u invalid port ID", param->port_id); + return -rte_errno; + } + dev = &rte_eth_devices[param->port_id]; + switch (param->type) { + case MLX5_MP_REQ_START_RXTX: + DRV_LOG(INFO, "port %u starting datapath", dev->data->port_id); + rte_mb(); + dev->rx_pkt_burst = mlx5_select_rx_function(dev); + dev->tx_pkt_burst = mlx5_select_tx_function(dev); + mp_init_msg(dev, &mp_res, param->type); + res->result = 0; + ret = rte_mp_reply(&mp_res, peer); + break; + case MLX5_MP_REQ_STOP_RXTX: + DRV_LOG(INFO, "port %u stopping datapath", dev->data->port_id); + dev->rx_pkt_burst = removed_rx_burst; + dev->tx_pkt_burst = removed_tx_burst; + rte_mb(); + mp_init_msg(dev, &mp_res, param->type); + res->result = 0; + ret = rte_mp_reply(&mp_res, peer); + break; + default: + rte_errno = EINVAL; + DRV_LOG(ERR, "port %u invalid mp request type", + dev->data->port_id); + return -rte_errno; + } + return ret; +} + +/** + * Broadcast request of stopping/starting data-path to secondary processes. + * + * @param[in] dev + * Pointer to Ethernet structure. + * @param[in] type + * Request type. + */ +static void +mp_req_on_rxtx(struct rte_eth_dev *dev, enum mlx5_mp_req_type type) +{ + struct rte_mp_msg mp_req; + struct rte_mp_msg *mp_res; + struct rte_mp_reply mp_rep; + struct mlx5_mp_param *res; + struct timespec ts = {.tv_sec = MLX5_MP_REQ_TIMEOUT_SEC, .tv_nsec = 0}; + int ret; + int i; + + assert(rte_eal_process_type() == RTE_PROC_PRIMARY); + if (!mlx5_shared_data->secondary_cnt) + return; + if (type != MLX5_MP_REQ_START_RXTX && type != MLX5_MP_REQ_STOP_RXTX) { + DRV_LOG(ERR, "port %u unknown request (req_type %d)", + dev->data->port_id, type); + return; + } + mp_init_msg(dev, &mp_req, type); + ret = rte_mp_request_sync(&mp_req, &mp_rep, &ts); + if (ret) { + DRV_LOG(ERR, "port %u failed to request stop/start Rx/Tx (%d)", + dev->data->port_id, type); + goto exit; + } + if (mp_rep.nb_sent != mp_rep.nb_received) { + DRV_LOG(ERR, + "port %u not all secondaries responded (req_type %d)", + dev->data->port_id, type); + goto exit; + } + for (i = 0; i < mp_rep.nb_received; i++) { + mp_res = &mp_rep.msgs[i]; + res = (struct mlx5_mp_param *)mp_res->param; + if (res->result) { + DRV_LOG(ERR, "port %u request failed on secondary #%d", + dev->data->port_id, i); + goto exit; + } + } +exit: + free(mp_rep.msgs); +} + +/** + * Broadcast request of starting data-path to secondary processes. The request + * is synchronous. + * + * @param[in] dev + * Pointer to Ethernet structure. + */ +void +mlx5_mp_req_start_rxtx(struct rte_eth_dev *dev) +{ + mp_req_on_rxtx(dev, MLX5_MP_REQ_START_RXTX); +} + +/** + * Broadcast request of stopping data-path to secondary processes. The request + * is synchronous. + * + * @param[in] dev + * Pointer to Ethernet structure. + */ +void +mlx5_mp_req_stop_rxtx(struct rte_eth_dev *dev) +{ + mp_req_on_rxtx(dev, MLX5_MP_REQ_STOP_RXTX); +} + +/** * Request Verbs command file descriptor for mmap to the primary process. * * @param[in] dev @@ -150,3 +286,23 @@ mlx5_mp_uninit_primary(void) assert(rte_eal_process_type() == RTE_PROC_PRIMARY); rte_mp_action_unregister(MLX5_MP_NAME); } + +/** + * Initialize by secondary process. + */ +void +mlx5_mp_init_secondary(void) +{ + assert(rte_eal_process_type() == RTE_PROC_SECONDARY); + rte_mp_action_register(MLX5_MP_NAME, mp_secondary_handle); +} + +/** + * Un-initialize by secondary process. + */ +void +mlx5_mp_uninit_secondary(void) +{ + assert(rte_eal_process_type() == RTE_PROC_SECONDARY); + rte_mp_action_unregister(MLX5_MP_NAME); +} diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c index 38ce0e29a2..3da3f62fa9 100644 --- a/drivers/net/mlx5/mlx5_rxtx.c +++ b/drivers/net/mlx5/mlx5_rxtx.c @@ -2373,6 +2373,7 @@ removed_tx_burst(void *dpdk_txq __rte_unused, struct rte_mbuf **pkts __rte_unused, uint16_t pkts_n __rte_unused) { + rte_mb(); return 0; } @@ -2397,6 +2398,7 @@ removed_rx_burst(void *dpdk_txq __rte_unused, struct rte_mbuf **pkts __rte_unused, uint16_t pkts_n __rte_unused) { + rte_mb(); return 0; } diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c index 2137bdc461..7c9ff921ab 100644 --- a/drivers/net/mlx5/mlx5_trigger.c +++ b/drivers/net/mlx5/mlx5_trigger.c @@ -194,8 +194,11 @@ mlx5_dev_start(struct rte_eth_dev *dev) dev->data->port_id); goto error; } + rte_wmb(); dev->tx_pkt_burst = mlx5_select_tx_function(dev); dev->rx_pkt_burst = mlx5_select_rx_function(dev); + /* Enable datapath on secondary process. */ + mlx5_mp_req_start_rxtx(dev); mlx5_dev_interrupt_handler_install(dev); return 0; error: @@ -228,6 +231,8 @@ mlx5_dev_stop(struct rte_eth_dev *dev) dev->rx_pkt_burst = removed_rx_burst; dev->tx_pkt_burst = removed_tx_burst; rte_wmb(); + /* Disable datapath on secondary process. */ + mlx5_mp_req_stop_rxtx(dev); usleep(1000 * priv->rxqs_n); DRV_LOG(DEBUG, "port %u stopping device", dev->data->port_id); mlx5_flow_stop(dev, &priv->flows); -- 2.11.0