From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mail-wr0-f178.google.com (mail-wr0-f178.google.com [209.85.128.178]) by dpdk.org (Postfix) with ESMTP id 0DA00F92F for ; Fri, 3 Mar 2017 16:40:36 +0100 (CET) Received: by mail-wr0-f178.google.com with SMTP id u48so76286062wrc.0 for ; Fri, 03 Mar 2017 07:40:36 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=6wind-com.20150623.gappssmtp.com; s=20150623; h=from:to:cc:subject:date:message-id:in-reply-to:references; bh=4Jvii6L/nRqLedXAnoxEieRkmjvdHTERwoHJdcIGF3c=; b=qTXevXqfM8SfaYAAg6eEoJK5QeS9DSg9R6ljF9nwNEmTDPCkaecEWCw4IQGo77mhE1 D1F9hOqE3KwNBg3qbDxKQRs+iBRoHWFzBEv/EVLxKb0IJIg5NOaZPk6pzj6NN81ael5j A2aA2cw6GT0XRCs+d2D2JzshrcyxLbadF0ctFRcxjBJT0T/Ik1Xj4v3JZUHf2kTvko+C HG9c+mC01WTkKPejtSAXbHs3lPvUvQJKxYUVhzKzzqZBlBUGXE/0auW4fpB5JrlHXVXD xOtxDa8lu8ANZHwFvAFvHZVDuOr6IHORY8lovhsYmcPsa15uj06dNhDOqhPy808VpqIr q7/Q== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references; bh=4Jvii6L/nRqLedXAnoxEieRkmjvdHTERwoHJdcIGF3c=; b=c5o/llnyG1cJKSCCEJzhKLUMee2I63oGS5lICtd7w0uDE+Owf0zkfqbd5TM5G2Ywzg qcDebmUbchmn6uCHQ8Y1c3QXCze8n1m0DA7qSiP5fJxy0Mei74vF4DGlhKmiVqB6X7KN 4ddm6C3J0LqxTQu8Y4LxPPl7g56rnrSkJmgx3AEw/8+ejplCNF1adrMIeny3HibUxo4Y ptLt8zfP76EjkxYSXX1qfjAtIiA0f3hhQGvse+ljNNIdS9nsTQh1qYquWn9Btol4LwM1 hKIUaHoONUUIGGmTyKRTyn1ERCajaU4CkuX390BTxf13TdlNCd24kG11bk0HgejDlbD5 udmw== X-Gm-Message-State: AMke39lVCQLOTcx/Ukh2pxI6F9lgMn12miYPD6MCgSfhKTqyGEqLtZcJcHaV73Pdt5lEbimY X-Received: by 10.223.179.15 with SMTP id j15mr3042353wrd.159.1488555636270; Fri, 03 Mar 2017 07:40:36 -0800 (PST) Received: from bidouze.dev.6wind.com (host.78.145.23.62.rev.coltfrance.com. [62.23.145.78]) by smtp.gmail.com with ESMTPSA id i15sm3486899wmf.21.2017.03.03.07.40.35 (version=TLS1_2 cipher=ECDHE-RSA-AES128-SHA bits=128/128); Fri, 03 Mar 2017 07:40:35 -0800 (PST) From: Gaetan Rivet To: dev@dpdk.org Cc: Elad Persiko Date: Fri, 3 Mar 2017 16:40:20 +0100 Message-Id: X-Mailer: git-send-email 2.1.4 In-Reply-To: References: Subject: [dpdk-dev] [PATCH 2/5] net/mlx4: device removal event support X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 03 Mar 2017 15:40:37 -0000 Extend the LSC event handling to support the device removal as well. The Verbs library will send several related events, that can conflict with the LSC event itself. The event handling has thus been made capable of receiving and signaling several event types at once. Signed-off-by: Gaetan Rivet Signed-off-by: Elad Persiko --- doc/guides/nics/features/mlx4.ini | 1 + drivers/net/mlx4/mlx4.c | 258 ++++++++++++++++++++++++++++++++------ drivers/net/mlx4/mlx4.h | 1 + 3 files changed, 224 insertions(+), 36 deletions(-) diff --git a/doc/guides/nics/features/mlx4.ini b/doc/guides/nics/features/mlx4.ini index c9828f7..285f0ec 100644 --- a/doc/guides/nics/features/mlx4.ini +++ b/doc/guides/nics/features/mlx4.ini @@ -6,6 +6,7 @@ [Features] Link status = Y Link status event = Y +Removal event = Y Queue start/stop = Y MTU update = Y Jumbo frame = Y diff --git a/drivers/net/mlx4/mlx4.c b/drivers/net/mlx4/mlx4.c index c4b9fbd..db37711 100644 --- a/drivers/net/mlx4/mlx4.c +++ b/drivers/net/mlx4/mlx4.c @@ -3931,9 +3931,15 @@ mlx4_rx_queue_release(void *dpdk_rxq) priv_unlock(priv); } -static void +static int priv_dev_interrupt_handler_install(struct priv *, struct rte_eth_dev *); +static int +priv_dev_removal_interrupt_handler_install(struct priv *, struct rte_eth_dev *); + +static int +priv_dev_link_interrupt_handler_install(struct priv *, struct rte_eth_dev *); + /** * DPDK callback to start the device. * @@ -3986,7 +3992,18 @@ mlx4_dev_start(struct rte_eth_dev *dev) (void *)dev, strerror(ret)); goto err; } while ((--r) && ((rxq = (*priv->rxqs)[++i]), i)); - priv_dev_interrupt_handler_install(priv, dev); + ret = priv_dev_link_interrupt_handler_install(priv, dev); + if (ret) { + ERROR("%p: LSC handler install failed", + (void *)dev); + goto err; + } + ret = priv_dev_removal_interrupt_handler_install(priv, dev); + if (ret) { + ERROR("%p: RMV handler install failed", + (void *)dev); + goto err; + } ret = mlx4_priv_flow_start(priv); if (ret) { ERROR("%p: flow start failed: %s", @@ -4105,9 +4122,16 @@ removed_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) return 0; } -static void +static int priv_dev_interrupt_handler_uninstall(struct priv *, struct rte_eth_dev *); +static int +priv_dev_removal_interrupt_handler_uninstall(struct priv *, + struct rte_eth_dev *); + +static int +priv_dev_link_interrupt_handler_uninstall(struct priv *, struct rte_eth_dev *); + /** * DPDK callback to close the device. * @@ -4170,7 +4194,8 @@ mlx4_dev_close(struct rte_eth_dev *dev) claim_zero(ibv_close_device(priv->ctx)); } else assert(priv->ctx == NULL); - priv_dev_interrupt_handler_uninstall(priv, dev); + priv_dev_removal_interrupt_handler_uninstall(priv, dev); + priv_dev_link_interrupt_handler_uninstall(priv, dev); priv_unlock(priv); memset(priv, 0, sizeof(*priv)); } @@ -4730,6 +4755,10 @@ mlx4_link_update(struct rte_eth_dev *dev, int wait_to_complete) return -1; } +static int +mlx4_ibv_device_to_pci_addr(const struct ibv_device *device, + struct rte_pci_addr *pci_addr); + /** * DPDK callback to change the MTU. * @@ -5261,32 +5290,41 @@ static void mlx4_dev_interrupt_handler(struct rte_intr_handle *, void *); /** - * Link status handler. + * Link/device status handler. * * @param priv * Pointer to private structure. * @param dev * Pointer to the rte_eth_dev structure. + * @param events + * Pointer to event flags holder. * * @return - * Nonzero if the callback process can be called immediately. + * Number of events */ static int -priv_dev_link_status_handler(struct priv *priv, struct rte_eth_dev *dev) +priv_dev_status_handler(struct priv *priv, struct rte_eth_dev *dev, + uint32_t *events) { struct ibv_async_event event; int port_change = 0; int ret = 0; + *events = 0; /* Read all message and acknowledge them. */ for (;;) { if (ibv_get_async_event(priv->ctx, &event)) break; - - if (event.event_type == IBV_EVENT_PORT_ACTIVE || - event.event_type == IBV_EVENT_PORT_ERR) + if ((event.event_type == IBV_EVENT_PORT_ACTIVE || + event.event_type == IBV_EVENT_PORT_ERR) && + (priv->intr_conf.lsc == 1)) { port_change = 1; - else + ret++; + } else if (event.event_type == IBV_EVENT_DEVICE_FATAL && + priv->intr_conf.rmv == 1) { + *events |= (1 << RTE_ETH_EVENT_INTR_RMV); + ret++; + } else DEBUG("event type %d on port %d not handled", event.event_type, event.element.port_num); ibv_ack_async_event(&event); @@ -5304,8 +5342,9 @@ priv_dev_link_status_handler(struct priv *priv, struct rte_eth_dev *dev) rte_eal_alarm_set(MLX4_ALARM_TIMEOUT_US, mlx4_dev_link_status_handler, dev); - } else - ret = 1; + } else { + *events |= (1 << RTE_ETH_EVENT_INTR_LSC); + } } return ret; } @@ -5321,13 +5360,14 @@ mlx4_dev_link_status_handler(void *arg) { struct rte_eth_dev *dev = arg; struct priv *priv = dev->data->dev_private; + uint32_t events; int ret; priv_lock(priv); assert(priv->pending_alarm == 1); - ret = priv_dev_link_status_handler(priv, dev); + ret = priv_dev_status_handler(priv, dev, &events); priv_unlock(priv); - if (ret) + if (ret > 0 && events & (1 << RTE_ETH_EVENT_INTR_LSC)) _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL); } @@ -5345,13 +5385,27 @@ mlx4_dev_interrupt_handler(struct rte_intr_handle *intr_handle, void *cb_arg) struct rte_eth_dev *dev = cb_arg; struct priv *priv = dev->data->dev_private; int ret; + uint32_t ev; + int i; (void)intr_handle; priv_lock(priv); - ret = priv_dev_link_status_handler(priv, dev); + ret = priv_dev_status_handler(priv, dev, &ev); priv_unlock(priv); - if (ret) - _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL); + if (ret > 0) { + for (i = RTE_ETH_EVENT_UNKNOWN; + i < RTE_ETH_EVENT_MAX; + i++) { + if (ev & (1 << i)) { + ev &= ~(1 << i); + _rte_eth_dev_callback_process(dev, i, NULL); + ret--; + } + } + if (ret) + WARN("%d event%s not processed", ret, + (ret > 1 ? "s were" : " was")); + } } /** @@ -5361,20 +5415,30 @@ mlx4_dev_interrupt_handler(struct rte_intr_handle *intr_handle, void *cb_arg) * Pointer to private structure. * @param dev * Pointer to the rte_eth_dev structure. + * @return + * 0 on success, negative errno value on failure. */ -static void +static int priv_dev_interrupt_handler_uninstall(struct priv *priv, struct rte_eth_dev *dev) { - if (!dev->data->dev_conf.intr_conf.lsc) - return; - rte_intr_callback_unregister(&priv->intr_handle, - mlx4_dev_interrupt_handler, - dev); - if (priv->pending_alarm) - rte_eal_alarm_cancel(mlx4_dev_link_status_handler, dev); - priv->pending_alarm = 0; + int ret; + + if (priv->intr_conf.lsc || + priv->intr_conf.rmv) + return 0; + ret = rte_intr_callback_unregister(&priv->intr_handle, + mlx4_dev_interrupt_handler, + dev); + if (ret < 0) { + ERROR("rte_intr_callback_unregister failed with %d" + "%s%s%s", ret, + (errno ? " (errno: " : ""), + (errno ? strerror(errno) : ""), + (errno ? ")" : "")); + } priv->intr_handle.fd = 0; priv->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN; + return ret; } /** @@ -5384,27 +5448,148 @@ priv_dev_interrupt_handler_uninstall(struct priv *priv, struct rte_eth_dev *dev) * Pointer to private structure. * @param dev * Pointer to the rte_eth_dev structure. + * @return + * 0 on success, negative errno value on failure. */ -static void -priv_dev_interrupt_handler_install(struct priv *priv, struct rte_eth_dev *dev) +static int +priv_dev_interrupt_handler_install(struct priv *priv, + struct rte_eth_dev *dev) { - int rc, flags; + int flags; + int rc; - if (!dev->data->dev_conf.intr_conf.lsc) - return; + /* Check whether the interrupt handler has already been installed + * for either type of interrupt + */ + if (priv->intr_conf.lsc && + priv->intr_conf.rmv && + priv->intr_handle.fd) + return 0; assert(priv->ctx->async_fd > 0); flags = fcntl(priv->ctx->async_fd, F_GETFL); rc = fcntl(priv->ctx->async_fd, F_SETFL, flags | O_NONBLOCK); if (rc < 0) { INFO("failed to change file descriptor async event queue"); dev->data->dev_conf.intr_conf.lsc = 0; + dev->data->dev_conf.intr_conf.rmv = 0; + return -errno; } else { priv->intr_handle.fd = priv->ctx->async_fd; priv->intr_handle.type = RTE_INTR_HANDLE_EXT; - rte_intr_callback_register(&priv->intr_handle, - mlx4_dev_interrupt_handler, - dev); + rc = rte_intr_callback_register(&priv->intr_handle, + mlx4_dev_interrupt_handler, + dev); + if (rc) { + ERROR("rte_intr_callback_register failed " + " (errno: %s)", strerror(errno)); + return rc; + } } + return 0; +} + +/** + * Uninstall interrupt handler. + * + * @param priv + * Pointer to private structure. + * @param dev + * Pointer to the rte_eth_dev structure. + * @return + * 0 on success, negative value on error. + */ +static int +priv_dev_removal_interrupt_handler_uninstall(struct priv *priv, + struct rte_eth_dev *dev) +{ + if (dev->data->dev_conf.intr_conf.rmv) { + priv->intr_conf.rmv = 0; + return priv_dev_interrupt_handler_uninstall(priv, dev); + } + return 0; +} + +/** + * Uninstall interrupt handler. + * + * @param priv + * Pointer to private structure. + * @param dev + * Pointer to the rte_eth_dev structure. + * @return + * 0 on success, negative value on error, + */ +static int +priv_dev_link_interrupt_handler_uninstall(struct priv *priv, + struct rte_eth_dev *dev) +{ + int ret = 0; + + if (dev->data->dev_conf.intr_conf.lsc) { + priv->intr_conf.lsc = 0; + ret = priv_dev_interrupt_handler_uninstall(priv, dev); + if (ret) + return ret; + } + if (priv->pending_alarm) + if (rte_eal_alarm_cancel(mlx4_dev_link_status_handler, + dev)) { + ERROR("rte_eal_alarm_cancel failed " + " (errno: %s)", strerror(rte_errno)); + return -rte_errno; + } + priv->pending_alarm = 0; + return 0; +} + +/** + * Install link interrupt handler. + * + * @param priv + * Pointer to private structure. + * @param dev + * Pointer to the rte_eth_dev structure. + * @return + * 0 on success, negative value on error. + */ +static int +priv_dev_link_interrupt_handler_install(struct priv *priv, + struct rte_eth_dev *dev) +{ + int ret; + + if (dev->data->dev_conf.intr_conf.lsc) { + ret = priv_dev_interrupt_handler_install(priv, dev); + if (ret) + return ret; + priv->intr_conf.lsc = 1; + } + return 0; +} + +/** + * Install removal interrupt handler. + * + * @param priv + * Pointer to private structure. + * @param dev + * Pointer to the rte_eth_dev structure. + * @return + * 0 on success, negative value on error. + */ +static int +priv_dev_removal_interrupt_handler_install(struct priv *priv, + struct rte_eth_dev *dev) +{ + int ret; + + if (dev->data->dev_conf.intr_conf.rmv) { + ret = priv_dev_interrupt_handler_install(priv, dev); + if (ret) + return ret; + priv->intr_conf.rmv = 1; + } + return 0; } /** @@ -5889,7 +6074,8 @@ static struct eth_driver mlx4_driver = { }, .id_table = mlx4_pci_id_map, .probe = mlx4_pci_probe, - .drv_flags = RTE_PCI_DRV_INTR_LSC, + .drv_flags = RTE_PCI_DRV_INTR_LSC | + RTE_PCI_DRV_INTR_RMV, }, .dev_private_size = sizeof(struct priv) }; diff --git a/drivers/net/mlx4/mlx4.h b/drivers/net/mlx4/mlx4.h index e9659eb..063df5f 100644 --- a/drivers/net/mlx4/mlx4.h +++ b/drivers/net/mlx4/mlx4.h @@ -356,6 +356,7 @@ struct priv { struct rxq *(*rxqs)[]; /* RX queues. */ struct txq *(*txqs)[]; /* TX queues. */ struct rte_intr_handle intr_handle; /* Interrupt handler. */ + struct rte_intr_conf intr_conf; /* Active interrupt configuration. */ LIST_HEAD(mlx4_flows, rte_flow) flows; rte_spinlock_t lock; /* Lock for control functions. */ }; -- 2.1.4