patch 'net/mana: fix multi-process tracking' has been queued to stable release 23.11.4

[Xueming Li <xuemingl@nvidia.com>]

Hi,

FYI, your patch has been queued to stable release 23.11.4

Note it hasn't been pushed to http://dpdk.org/browse/dpdk-stable yet.
It will be pushed if I get no objections before 04/10/25. So please
shout if anyone has objections.

Also note that after the patch there's a diff of the upstream commit vs the
patch applied to the branch. This will indicate if there was any rebasing
needed to apply to the stable branch. If there were code changes for rebasing
(ie: not only metadata diffs), please double check that the rebase was
correctly done.

Queued patches are on a temporary branch at:
https://git.dpdk.org/dpdk-stable/log/?h=23.11-staging

This queued commit can be viewed at:
https://git.dpdk.org/dpdk-stable/commit/?h=23.11-staging&id=60357a01f1fd647417e59ed35c16e25023ea620e

Thanks.

Xueming Li <xuemingl@nvidia.com>

---
From 60357a01f1fd647417e59ed35c16e25023ea620e Mon Sep 17 00:00:00 2001
From: Long Li <longli@microsoft.com>
Date: Thu, 20 Feb 2025 15:32:02 -0800
Subject: [PATCH] net/mana: fix multi-process tracking
Cc: Xueming Li <xuemingl@nvidia.com>

[ upstream commit 57aa3ec91ecf13ab2f11e4dc0dc74c50a2afa0cc ]

The driver uses mana_shared_data for tracking usage count for primary
process. This is not correct as the mana_shared_data is allocated
by the primary and is meant to track usage of secondary process by the
primary process. And it creates a race condition when the device is
removed because the counter is no longer available if this shared
memory is freed.

Move the usage count tracking to mana_local_data and fix the race
condition in mana_pci_remove().

Fixes: 517ed6e2d590 ("net/mana: add basic driver with build environment")

Signed-off-by: Long Li <longli@microsoft.com>
---
 drivers/net/mana/mana.c | 103 +++++++++++++++++++++++-----------------
 drivers/net/mana/mana.h |   6 +--
 drivers/net/mana/mp.c   |   2 +-
 3 files changed, 61 insertions(+), 50 deletions(-)

diff --git a/drivers/net/mana/mana.c b/drivers/net/mana/mana.c
index 65ca139be5..c0ff132fe1 100644
--- a/drivers/net/mana/mana.c
+++ b/drivers/net/mana/mana.c
@@ -23,9 +23,14 @@
 #include "mana.h"
 
 /* Shared memory between primary/secondary processes, per driver */
-/* Data to track primary/secondary usage */
 struct mana_shared_data *mana_shared_data;
-static struct mana_shared_data mana_local_data;
+
+/* Local data to track device instance usage for primary/secondary processes */
+static struct mana_local_data {
+	int init_done;
+	unsigned int primary_cnt;
+	unsigned int secondary_cnt;
+} mana_local_data;
 
 /* The memory region for the above data */
 static const struct rte_memzone *mana_shared_mz;
@@ -1163,8 +1168,12 @@ mana_init_shared_data(void)
 	rte_spinlock_lock(&mana_shared_data_lock);
 
 	/* Skip if shared data is already initialized */
-	if (mana_shared_data)
+	if (mana_shared_data) {
+		DRV_LOG(INFO, "shared data is already initialized");
 		goto exit;
+	}
+
+	memset(&mana_local_data, 0, sizeof(mana_local_data));
 
 	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 		mana_shared_mz = rte_memzone_reserve(MZ_MANA_SHARED_DATA,
@@ -1177,8 +1186,8 @@ mana_init_shared_data(void)
 		}
 
 		mana_shared_data = mana_shared_mz->addr;
-		memset(mana_shared_data, 0, sizeof(*mana_shared_data));
-		rte_spinlock_init(&mana_shared_data->lock);
+		rte_atomic_store_explicit(&mana_shared_data->secondary_cnt, 0,
+					  rte_memory_order_relaxed);
 	} else {
 		secondary_mz = rte_memzone_lookup(MZ_MANA_SHARED_DATA);
 		if (!secondary_mz) {
@@ -1188,7 +1197,6 @@ mana_init_shared_data(void)
 		}
 
 		mana_shared_data = secondary_mz->addr;
-		memset(&mana_local_data, 0, sizeof(mana_local_data));
 	}
 
 exit:
@@ -1209,11 +1217,11 @@ mana_init_once(void)
 	if (ret)
 		return ret;
 
-	rte_spinlock_lock(&mana_shared_data->lock);
+	rte_spinlock_lock(&mana_shared_data_lock);
 
 	switch (rte_eal_process_type()) {
 	case RTE_PROC_PRIMARY:
-		if (mana_shared_data->init_done)
+		if (mana_local_data.init_done)
 			break;
 
 		ret = mana_mp_init_primary();
@@ -1221,7 +1229,7 @@ mana_init_once(void)
 			break;
 		DRV_LOG(ERR, "MP INIT PRIMARY");
 
-		mana_shared_data->init_done = 1;
+		mana_local_data.init_done = 1;
 		break;
 
 	case RTE_PROC_SECONDARY:
@@ -1244,7 +1252,7 @@ mana_init_once(void)
 		break;
 	}
 
-	rte_spinlock_unlock(&mana_shared_data->lock);
+	rte_spinlock_unlock(&mana_shared_data_lock);
 
 	return ret;
 }
@@ -1315,11 +1323,6 @@ mana_probe_port(struct ibv_device *ibdev, struct ibv_device_attr_ex *dev_attr,
 		eth_dev->tx_pkt_burst = mana_tx_burst_removed;
 		eth_dev->rx_pkt_burst = mana_rx_burst_removed;
 
-		rte_spinlock_lock(&mana_shared_data->lock);
-		mana_shared_data->secondary_cnt++;
-		mana_local_data.secondary_cnt++;
-		rte_spinlock_unlock(&mana_shared_data->lock);
-
 		rte_eth_copy_pci_info(eth_dev, pci_dev);
 		rte_eth_dev_probing_finish(eth_dev);
 
@@ -1402,10 +1405,6 @@ mana_probe_port(struct ibv_device *ibdev, struct ibv_device_attr_ex *dev_attr,
 		goto failed;
 	}
 
-	rte_spinlock_lock(&mana_shared_data->lock);
-	mana_shared_data->primary_cnt++;
-	rte_spinlock_unlock(&mana_shared_data->lock);
-
 	eth_dev->device = &pci_dev->device;
 
 	DRV_LOG(INFO, "device %s at port %u", name, eth_dev->data->port_id);
@@ -1548,13 +1547,38 @@ mana_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 		count = mana_pci_probe_mac(pci_dev, NULL);
 	}
 
+	/* If no device is found, clean up resources if this is the last one */
 	if (!count) {
-		rte_memzone_free(mana_shared_mz);
-		mana_shared_mz = NULL;
-		ret = -ENODEV;
+		rte_spinlock_lock(&mana_shared_data_lock);
+		if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+			if (!mana_local_data.primary_cnt) {
+				mana_mp_uninit_primary();
+				rte_memzone_free(mana_shared_mz);
+				mana_shared_mz = NULL;
+				mana_shared_data = NULL;
+			}
+		} else {
+			if (!mana_local_data.secondary_cnt) {
+				mana_mp_uninit_secondary();
+				mana_shared_data = NULL;
+			}
+		}
+		rte_spinlock_unlock(&mana_shared_data_lock);
+		return -ENODEV;
 	}
 
-	return ret;
+	/* At least one eth_dev is probed, increase counter for shared data */
+	rte_spinlock_lock(&mana_shared_data_lock);
+	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+		mana_local_data.primary_cnt++;
+	} else {
+		rte_atomic_fetch_add_explicit(&mana_shared_data->secondary_cnt, 1,
+					      rte_memory_order_relaxed);
+		mana_local_data.secondary_cnt++;
+	}
+	rte_spinlock_unlock(&mana_shared_data_lock);
+
+	return 0;
 }
 
 static int
@@ -1569,45 +1593,36 @@ mana_dev_uninit(struct rte_eth_dev *dev)
 static int
 mana_pci_remove(struct rte_pci_device *pci_dev)
 {
+	rte_spinlock_lock(&mana_shared_data_lock);
 	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
-		rte_spinlock_lock(&mana_shared_data_lock);
-
-		rte_spinlock_lock(&mana_shared_data->lock);
+		RTE_VERIFY(mana_local_data.primary_cnt > 0);
+		mana_local_data.primary_cnt--;
 
-		RTE_VERIFY(mana_shared_data->primary_cnt > 0);
-		mana_shared_data->primary_cnt--;
-		if (!mana_shared_data->primary_cnt) {
+		if (!mana_local_data.primary_cnt) {
 			DRV_LOG(DEBUG, "mp uninit primary");
 			mana_mp_uninit_primary();
-		}
-
-		rte_spinlock_unlock(&mana_shared_data->lock);
 
-		/* Also free the shared memory if this is the last */
-		if (!mana_shared_data->primary_cnt) {
+			/* Also free the shared memory if this is the last */
 			DRV_LOG(DEBUG, "free shared memezone data");
 			rte_memzone_free(mana_shared_mz);
 			mana_shared_mz = NULL;
+			mana_shared_data = NULL;
 		}
-
-		rte_spinlock_unlock(&mana_shared_data_lock);
 	} else {
-		rte_spinlock_lock(&mana_shared_data_lock);
-
-		rte_spinlock_lock(&mana_shared_data->lock);
-		RTE_VERIFY(mana_shared_data->secondary_cnt > 0);
-		mana_shared_data->secondary_cnt--;
-		rte_spinlock_unlock(&mana_shared_data->lock);
+		RTE_VERIFY(rte_atomic_load_explicit(&mana_shared_data->secondary_cnt,
+						    rte_memory_order_relaxed) > 0);
+		rte_atomic_fetch_sub_explicit(&mana_shared_data->secondary_cnt, 1,
+					      rte_memory_order_relaxed);
 
 		RTE_VERIFY(mana_local_data.secondary_cnt > 0);
 		mana_local_data.secondary_cnt--;
 		if (!mana_local_data.secondary_cnt) {
 			DRV_LOG(DEBUG, "mp uninit secondary");
 			mana_mp_uninit_secondary();
+			mana_shared_data = NULL;
 		}
-
-		rte_spinlock_unlock(&mana_shared_data_lock);
 	}
+	rte_spinlock_unlock(&mana_shared_data_lock);
 
 	return rte_eth_dev_pci_generic_remove(pci_dev, mana_dev_uninit);
 }
diff --git a/drivers/net/mana/mana.h b/drivers/net/mana/mana.h
index 822b8a1f15..7931a443a4 100644
--- a/drivers/net/mana/mana.h
+++ b/drivers/net/mana/mana.h
@@ -8,12 +8,8 @@
 #define	PCI_VENDOR_ID_MICROSOFT		0x1414
 #define PCI_DEVICE_ID_MICROSOFT_MANA	0x00ba
 
-/* Shared data between primary/secondary processes */
 struct mana_shared_data {
-	rte_spinlock_t lock;
-	int init_done;
-	unsigned int primary_cnt;
-	unsigned int secondary_cnt;
+	RTE_ATOMIC(uint32_t) secondary_cnt;
 };
 
 #define MANA_MAX_MTU	9000
diff --git a/drivers/net/mana/mp.c b/drivers/net/mana/mp.c
index 738487f65a..3d0aa0d493 100644
--- a/drivers/net/mana/mp.c
+++ b/drivers/net/mana/mp.c
@@ -305,7 +305,7 @@ mana_mp_req_on_rxtx(struct rte_eth_dev *dev, enum mana_mp_req_type type)
 		return;
 	}
 
-	if (!mana_shared_data->secondary_cnt)
+	if (rte_atomic_load_explicit(&mana_shared_data->secondary_cnt, rte_memory_order_relaxed) == 0)
 		return;
 
 	mp_init_msg(&mp_req, type, dev->data->port_id);
-- 
2.34.1

---
  Diff of the applied patch vs upstream commit (please double-check if non-empty:
---
--- -	2025-04-08 15:39:06.259162174 +0800
+++ 0007-net-mana-fix-multi-process-tracking.patch	2025-04-08 15:39:05.876436597 +0800
@@ -1 +1 @@
-From 57aa3ec91ecf13ab2f11e4dc0dc74c50a2afa0cc Mon Sep 17 00:00:00 2001
+From 60357a01f1fd647417e59ed35c16e25023ea620e Mon Sep 17 00:00:00 2001
@@ -4,0 +5,3 @@
+Cc: Xueming Li <xuemingl@nvidia.com>
+
+[ upstream commit 57aa3ec91ecf13ab2f11e4dc0dc74c50a2afa0cc ]
@@ -17 +19,0 @@
-Cc: stable@dpdk.org
@@ -27 +29 @@
-index c37c4e3444..2934da29f7 100644
+index 65ca139be5..c0ff132fe1 100644
@@ -47 +49 @@
-@@ -1167,8 +1172,12 @@ mana_init_shared_data(void)
+@@ -1163,8 +1168,12 @@ mana_init_shared_data(void)
@@ -61 +63 @@
-@@ -1181,8 +1190,8 @@ mana_init_shared_data(void)
+@@ -1177,8 +1186,8 @@ mana_init_shared_data(void)
@@ -72 +74 @@
-@@ -1192,7 +1201,6 @@ mana_init_shared_data(void)
+@@ -1188,7 +1197,6 @@ mana_init_shared_data(void)
@@ -80 +82 @@
-@@ -1213,11 +1221,11 @@ mana_init_once(void)
+@@ -1209,11 +1217,11 @@ mana_init_once(void)
@@ -94 +96 @@
-@@ -1225,7 +1233,7 @@ mana_init_once(void)
+@@ -1221,7 +1229,7 @@ mana_init_once(void)
@@ -103 +105 @@
-@@ -1248,7 +1256,7 @@ mana_init_once(void)
+@@ -1244,7 +1252,7 @@ mana_init_once(void)
@@ -112,3 +114,3 @@
-@@ -1319,11 +1327,6 @@ mana_probe_port(struct ibv_device *ibdev, struct ibv_device_attr_ex *dev_attr,
- 		eth_dev->tx_pkt_burst = mana_tx_burst;
- 		eth_dev->rx_pkt_burst = mana_rx_burst;
+@@ -1315,11 +1323,6 @@ mana_probe_port(struct ibv_device *ibdev, struct ibv_device_attr_ex *dev_attr,
+ 		eth_dev->tx_pkt_burst = mana_tx_burst_removed;
+ 		eth_dev->rx_pkt_burst = mana_rx_burst_removed;
@@ -124 +126 @@
-@@ -1406,10 +1409,6 @@ mana_probe_port(struct ibv_device *ibdev, struct ibv_device_attr_ex *dev_attr,
+@@ -1402,10 +1405,6 @@ mana_probe_port(struct ibv_device *ibdev, struct ibv_device_attr_ex *dev_attr,
@@ -135 +137 @@
-@@ -1552,13 +1551,38 @@ mana_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
+@@ -1548,13 +1547,38 @@ mana_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
@@ -178 +180 @@
-@@ -1573,45 +1597,36 @@ mana_dev_uninit(struct rte_eth_dev *dev)
+@@ -1569,45 +1593,36 @@ mana_dev_uninit(struct rte_eth_dev *dev)
@@ -237 +239 @@
-index 41a0ca6dfe..855d98911b 100644
+index 822b8a1f15..7931a443a4 100644
@@ -255 +257 @@
-index 34b45ed832..5467d385ce 100644
+index 738487f65a..3d0aa0d493 100644
@@ -258 +260 @@
-@@ -306,7 +306,7 @@ mana_mp_req_on_rxtx(struct rte_eth_dev *dev, enum mana_mp_req_type type)
+@@ -305,7 +305,7 @@ mana_mp_req_on_rxtx(struct rte_eth_dev *dev, enum mana_mp_req_type type)