From mboxrd@z Thu Jan  1 00:00:00 1970
Return-Path: <dev-bounces@dpdk.org>
Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124])
	by inbox.dpdk.org (Postfix) with ESMTP id AAF56A0544;
	Wed,  6 Jul 2022 02:29:28 +0200 (CEST)
Received: from [217.70.189.124] (localhost [127.0.0.1])
	by mails.dpdk.org (Postfix) with ESMTP id 72A9042B76;
	Wed,  6 Jul 2022 02:29:03 +0200 (CEST)
Received: from linux.microsoft.com (linux.microsoft.com [13.77.154.182])
 by mails.dpdk.org (Postfix) with ESMTP id 9F7D6410E5
 for <dev@dpdk.org>; Wed,  6 Jul 2022 02:28:59 +0200 (CEST)
Received: by linux.microsoft.com (Postfix, from userid 1004)
 id D988320DDCB3; Tue,  5 Jul 2022 17:28:58 -0700 (PDT)
DKIM-Filter: OpenDKIM Filter v2.11.0 linux.microsoft.com D988320DDCB3
DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxonhyperv.com;
 s=default; t=1657067338;
 bh=FgHsgnrEx2y98Q1Pf9nDGUBI+ivqmleMTcKeOFHtItU=;
 h=From:To:Cc:Subject:Date:In-Reply-To:References:Reply-To:From;
 b=TxUG6vHBCg6734MR5zQ8nnsxym3zncAZfXCV5sk+0PUButMd/2R4SRPEzJhNbhD2F
 ZJDeJUpgK/E2R46i0gqV0nSGOgQxbUB/R0KPGzUD3ejTJpzwEjw5sjvRZi7pU+SjO9
 jC/Cb8rc83FQOadlZMy4L4SlXe9DAIe3pEvS5OWY=
From: longli@linuxonhyperv.com
To: Ferruh Yigit <ferruh.yigit@xilinx.com>
Cc: dev@dpdk.org, Ajay Sharma <sharmaajay@microsoft.com>,
 Stephen Hemminger <sthemmin@microsoft.com>, Long Li <longli@microsoft.com>
Subject: [Patch v2 05/17] net/mana: add function for device removal interrupts
Date: Tue,  5 Jul 2022 17:28:36 -0700
Message-Id: <1657067328-18374-6-git-send-email-longli@linuxonhyperv.com>
X-Mailer: git-send-email 1.8.3.1
In-Reply-To: <1657067328-18374-1-git-send-email-longli@linuxonhyperv.com>
References: <1657067328-18374-1-git-send-email-longli@linuxonhyperv.com>
X-BeenThere: dev@dpdk.org
X-Mailman-Version: 2.1.29
Precedence: list
List-Id: DPDK patches and discussions <dev.dpdk.org>
List-Unsubscribe: <https://mails.dpdk.org/options/dev>,
 <mailto:dev-request@dpdk.org?subject=unsubscribe>
List-Archive: <http://mails.dpdk.org/archives/dev/>
List-Post: <mailto:dev@dpdk.org>
List-Help: <mailto:dev-request@dpdk.org?subject=help>
List-Subscribe: <https://mails.dpdk.org/listinfo/dev>,
 <mailto:dev-request@dpdk.org?subject=subscribe>
Reply-To: longli@microsoft.com
Errors-To: dev-bounces@dpdk.org

From: Long Li <longli@microsoft.com>

MANA supports PCI hot plug events. Add this interrupt to DPDK core so its
parent PMD can detect device removal during Azure servicing or live
migration.

Signed-off-by: Long Li <longli@microsoft.com>
---
 doc/guides/nics/features/mana.ini |  1 +
 drivers/net/mana/mana.c           | 97 +++++++++++++++++++++++++++++++
 drivers/net/mana/mana.h           |  1 +
 3 files changed, 99 insertions(+)

diff --git a/doc/guides/nics/features/mana.ini b/doc/guides/nics/features/mana.ini
index 62554b0a0a..8043e11f99 100644
--- a/doc/guides/nics/features/mana.ini
+++ b/doc/guides/nics/features/mana.ini
@@ -7,5 +7,6 @@
 Link status          = P
 Linux                = Y
 Multiprocess aware   = Y
+Removal event        = Y
 Usage doc            = Y
 x86-64               = Y
diff --git a/drivers/net/mana/mana.c b/drivers/net/mana/mana.c
index 8c6491f045..e15ecb8ea6 100644
--- a/drivers/net/mana/mana.c
+++ b/drivers/net/mana/mana.c
@@ -95,12 +95,18 @@ static int mana_dev_configure(struct rte_eth_dev *dev)
 	return 0;
 }
 
+static int mana_intr_uninstall(struct mana_priv *priv);
+
 static int
 mana_dev_close(struct rte_eth_dev *dev)
 {
 	struct mana_priv *priv = dev->data->dev_private;
 	int ret;
 
+	ret = mana_intr_uninstall(priv);
+	if (ret)
+		return ret;
+
 	ret = ibv_close_device(priv->ib_ctx);
 	if (ret) {
 		ret = errno;
@@ -327,6 +333,90 @@ static int mana_ibv_device_to_pci_addr(const struct ibv_device *device,
 	return 0;
 }
 
+static void mana_intr_handler(void *arg)
+{
+	struct mana_priv *priv = arg;
+	struct ibv_context *ctx = priv->ib_ctx;
+	struct ibv_async_event event;
+
+	/* Read and ack all messages from IB device */
+	while (true) {
+		if (ibv_get_async_event(ctx, &event))
+			break;
+
+		if (event.event_type == IBV_EVENT_DEVICE_FATAL) {
+			struct rte_eth_dev *dev;
+
+			dev = &rte_eth_devices[priv->port_id];
+			if (dev->data->dev_conf.intr_conf.rmv)
+				rte_eth_dev_callback_process(dev,
+					RTE_ETH_EVENT_INTR_RMV, NULL);
+		}
+
+		ibv_ack_async_event(&event);
+	}
+}
+
+static int mana_intr_uninstall(struct mana_priv *priv)
+{
+	int ret;
+
+	ret = rte_intr_callback_unregister(priv->intr_handle,
+					   mana_intr_handler, priv);
+	if (ret <= 0) {
+		DRV_LOG(ERR, "Failed to unregister intr callback ret %d", ret);
+		return ret;
+	}
+
+	rte_intr_instance_free(priv->intr_handle);
+
+	return 0;
+}
+
+static int mana_intr_install(struct mana_priv *priv)
+{
+	int ret, flags;
+	struct ibv_context *ctx = priv->ib_ctx;
+
+	priv->intr_handle = rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_SHARED);
+	if (!priv->intr_handle) {
+		DRV_LOG(ERR, "Failed to allocate intr_handle");
+		rte_errno = ENOMEM;
+		return -ENOMEM;
+	}
+
+	rte_intr_fd_set(priv->intr_handle, -1);
+
+	flags = fcntl(ctx->async_fd, F_GETFL);
+	ret = fcntl(ctx->async_fd, F_SETFL, flags | O_NONBLOCK);
+	if (ret) {
+		DRV_LOG(ERR, "Failed to change async_fd to NONBLOCK");
+		goto free_intr;
+	}
+
+	rte_intr_fd_set(priv->intr_handle, ctx->async_fd);
+	rte_intr_type_set(priv->intr_handle, RTE_INTR_HANDLE_EXT);
+
+	ret = rte_intr_callback_register(priv->intr_handle,
+					 mana_intr_handler, priv);
+	if (ret) {
+		DRV_LOG(ERR, "Failed to register intr callback");
+		rte_intr_fd_set(priv->intr_handle, -1);
+		goto restore_fd;
+	}
+
+	return 0;
+
+restore_fd:
+	fcntl(ctx->async_fd, F_SETFL, flags);
+
+free_intr:
+	rte_intr_instance_free(priv->intr_handle);
+	priv->intr_handle = NULL;
+
+	return ret;
+}
+
 static int mana_proc_priv_init(struct rte_eth_dev *dev)
 {
 	struct mana_process_priv *priv;
@@ -640,6 +730,13 @@ static int mana_pci_probe_mac(struct rte_pci_driver *pci_drv __rte_unused,
 				name, priv->max_rx_queues, priv->max_rx_desc,
 				priv->max_send_sge);
 
+			/* Create async interrupt handler */
+			ret = mana_intr_install(priv);
+			if (ret) {
+				DRV_LOG(ERR, "Failed to install intr handler");
+				goto failed;
+			}
+
 			rte_spinlock_lock(&mana_shared_data->lock);
 			mana_shared_data->primary_cnt++;
 			rte_spinlock_unlock(&mana_shared_data->lock);
diff --git a/drivers/net/mana/mana.h b/drivers/net/mana/mana.h
index c433940022..f97eed2e81 100644
--- a/drivers/net/mana/mana.h
+++ b/drivers/net/mana/mana.h
@@ -72,6 +72,7 @@ struct mana_priv {
 	uint8_t ind_table_key[40];
 	struct ibv_qp *rwq_qp;
 	void *db_page;
+	struct rte_intr_handle *intr_handle;
 	int max_rx_queues;
 	int max_tx_queues;
 	int max_rx_desc;
-- 
2.17.1