From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 6B09BA054F; Sat, 24 Sep 2022 04:46:23 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 8848242BEE; Sat, 24 Sep 2022 04:46:05 +0200 (CEST) Received: from linux.microsoft.com (linux.microsoft.com [13.77.154.182]) by mails.dpdk.org (Postfix) with ESMTP id 3015742BD0 for ; Sat, 24 Sep 2022 04:45:55 +0200 (CEST) Received: by linux.microsoft.com (Postfix, from userid 1004) id DE83E20C32C4; Fri, 23 Sep 2022 19:45:54 -0700 (PDT) DKIM-Filter: OpenDKIM Filter v2.11.0 linux.microsoft.com DE83E20C32C4 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxonhyperv.com; s=default; t=1663987554; bh=WsoB6oW/UclKGxwRUHTtYZ315AJbks3/8ADoMtF/c9U=; h=From:To:Cc:Subject:Date:In-Reply-To:References:Reply-To:From; b=n4spRS2RjB5BdNnrme9hQ5Z6WdzJGSJ2N7i/8B8SQlfk9kdunW0xJX/5HB1IkZVgE 4NcFaIViWn7UJoeWMgq7yoRZXyvvzHhCvW5+En1KTx0bqxD7AR1GcgmH0zyQ/QNGfG w2R9Bq5IRE+EQWb4296phEBfITLE043rDK0iofm8= From: longli@linuxonhyperv.com To: Ferruh Yigit Cc: dev@dpdk.org, Ajay Sharma , Stephen Hemminger , Long Li Subject: [Patch v9 05/18] net/mana: support device removal interrupts Date: Fri, 23 Sep 2022 19:45:33 -0700 Message-Id: <1663987546-15982-6-git-send-email-longli@linuxonhyperv.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1663987546-15982-1-git-send-email-longli@linuxonhyperv.com> References: <1662674189-29524-1-git-send-email-longli@linuxonhyperv.com> <1663987546-15982-1-git-send-email-longli@linuxonhyperv.com> X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: longli@microsoft.com Errors-To: dev-bounces@dpdk.org From: Long Li MANA supports PCI hot plug events. Add this interrupt to DPDK core so its parent PMD can detect device removal during Azure servicing or live migration. Signed-off-by: Long Li --- v8: fix coding style of function definitions. v9: remove unused data fields. doc/guides/nics/features/mana.ini | 1 + drivers/net/mana/mana.c | 103 ++++++++++++++++++++++++++++++ drivers/net/mana/mana.h | 1 + 3 files changed, 105 insertions(+) diff --git a/doc/guides/nics/features/mana.ini b/doc/guides/nics/features/mana.ini index 62554b0a0a..8043e11f99 100644 --- a/doc/guides/nics/features/mana.ini +++ b/doc/guides/nics/features/mana.ini @@ -7,5 +7,6 @@ Link status = P Linux = Y Multiprocess aware = Y +Removal event = Y Usage doc = Y x86-64 = Y diff --git a/drivers/net/mana/mana.c b/drivers/net/mana/mana.c index 90e713318a..2ee8c2dbe9 100644 --- a/drivers/net/mana/mana.c +++ b/drivers/net/mana/mana.c @@ -103,12 +103,18 @@ mana_dev_configure(struct rte_eth_dev *dev) return 0; } +static int mana_intr_uninstall(struct mana_priv *priv); + static int mana_dev_close(struct rte_eth_dev *dev) { struct mana_priv *priv = dev->data->dev_private; int ret; + ret = mana_intr_uninstall(priv); + if (ret) + return ret; + ret = ibv_close_device(priv->ib_ctx); if (ret) { ret = errno; @@ -340,6 +346,96 @@ mana_ibv_device_to_pci_addr(const struct ibv_device *device, return 0; } +/* + * Interrupt handler from IB layer to notify this device is being removed. + */ +static void +mana_intr_handler(void *arg) +{ + struct mana_priv *priv = arg; + struct ibv_context *ctx = priv->ib_ctx; + struct ibv_async_event event; + + /* Read and ack all messages from IB device */ + while (true) { + if (ibv_get_async_event(ctx, &event)) + break; + + if (event.event_type == IBV_EVENT_DEVICE_FATAL) { + struct rte_eth_dev *dev; + + dev = &rte_eth_devices[priv->port_id]; + if (dev->data->dev_conf.intr_conf.rmv) + rte_eth_dev_callback_process(dev, + RTE_ETH_EVENT_INTR_RMV, NULL); + } + + ibv_ack_async_event(&event); + } +} + +static int +mana_intr_uninstall(struct mana_priv *priv) +{ + int ret; + + ret = rte_intr_callback_unregister(priv->intr_handle, + mana_intr_handler, priv); + if (ret <= 0) { + DRV_LOG(ERR, "Failed to unregister intr callback ret %d", ret); + return ret; + } + + rte_intr_instance_free(priv->intr_handle); + + return 0; +} + +static int +mana_intr_install(struct mana_priv *priv) +{ + int ret, flags; + struct ibv_context *ctx = priv->ib_ctx; + + priv->intr_handle = rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_SHARED); + if (!priv->intr_handle) { + DRV_LOG(ERR, "Failed to allocate intr_handle"); + rte_errno = ENOMEM; + return -ENOMEM; + } + + rte_intr_fd_set(priv->intr_handle, -1); + + flags = fcntl(ctx->async_fd, F_GETFL); + ret = fcntl(ctx->async_fd, F_SETFL, flags | O_NONBLOCK); + if (ret) { + DRV_LOG(ERR, "Failed to change async_fd to NONBLOCK"); + goto free_intr; + } + + rte_intr_fd_set(priv->intr_handle, ctx->async_fd); + rte_intr_type_set(priv->intr_handle, RTE_INTR_HANDLE_EXT); + + ret = rte_intr_callback_register(priv->intr_handle, + mana_intr_handler, priv); + if (ret) { + DRV_LOG(ERR, "Failed to register intr callback"); + rte_intr_fd_set(priv->intr_handle, -1); + goto restore_fd; + } + + return 0; + +restore_fd: + fcntl(ctx->async_fd, F_SETFL, flags); + +free_intr: + rte_intr_instance_free(priv->intr_handle); + priv->intr_handle = NULL; + + return ret; +} + static int mana_proc_priv_init(struct rte_eth_dev *dev) { @@ -667,6 +763,13 @@ mana_pci_probe_mac(struct rte_pci_device *pci_dev, name, priv->max_rx_queues, priv->max_rx_desc, priv->max_send_sge); + /* Create async interrupt handler */ + ret = mana_intr_install(priv); + if (ret) { + DRV_LOG(ERR, "Failed to install intr handler"); + goto failed; + } + rte_spinlock_lock(&mana_shared_data->lock); mana_shared_data->primary_cnt++; rte_spinlock_unlock(&mana_shared_data->lock); diff --git a/drivers/net/mana/mana.h b/drivers/net/mana/mana.h index 0b4a828b0a..d9eb19559d 100644 --- a/drivers/net/mana/mana.h +++ b/drivers/net/mana/mana.h @@ -40,6 +40,7 @@ struct mana_priv { struct ibv_pd *ib_pd; struct ibv_pd *ib_parent_pd; void *db_page; + struct rte_intr_handle *intr_handle; int max_rx_queues; int max_tx_queues; int max_rx_desc; -- 2.17.1