DPDK patches and discussions
 help / color / mirror / Atom feed
From: Nithin Dabilpuram <ndabilpuram@marvell.com>
To: Nithin Kumar Dabilpuram <ndabilpuram@marvell.com>,
	Kiran Kumar K <kirankumark@marvell.com>,
	Sunil Kumar Kori <skori@marvell.com>,
	Satha Rao <skoteshwar@marvell.com>
Cc: <jerinj@marvell.com>, <dev@dpdk.org>, Harman Kalra <hkalra@marvell.com>
Subject: [PATCH v3 17/32] common/cnxk: sync between mbox up and down messages
Date: Thu, 25 May 2023 15:28:49 +0530	[thread overview]
Message-ID: <20230525095904.3967080-17-ndabilpuram@marvell.com> (raw)
In-Reply-To: <20230525095904.3967080-1-ndabilpuram@marvell.com>

From: Harman Kalra <hkalra@marvell.com>

An issue is observed where if PF is with DPDK and VF as kernel
netdev does not responds to link events. It was due to recent
design change in kernel where sender checks whether previous
interrupt is received before triggering current interrupt by
waiting for mailbox data register to become zero.

Signed-off-by: Harman Kalra <hkalra@marvell.com>
---
 drivers/common/cnxk/roc_dev.c       | 20 ++++++++-
 drivers/common/cnxk/roc_mbox.c      | 64 +++++++++++++++++++++--------
 drivers/common/cnxk/roc_mbox.h      | 15 +++++++
 drivers/common/cnxk/roc_mbox_priv.h |  6 ++-
 4 files changed, 84 insertions(+), 21 deletions(-)

diff --git a/drivers/common/cnxk/roc_dev.c b/drivers/common/cnxk/roc_dev.c
index 5e4e564ebe..e5a5cd7c10 100644
--- a/drivers/common/cnxk/roc_dev.c
+++ b/drivers/common/cnxk/roc_dev.c
@@ -195,7 +195,8 @@ af_pf_wait_msg(struct dev *dev, uint16_t vf, int num_msg)
 				vf_msg->rc = msg->rc;
 				vf_msg->pcifunc = msg->pcifunc;
 				/* Send to VF */
-				mbox_msg_send(&dev->mbox_vfpf_up, vf);
+				mbox_msg_send_up(&dev->mbox_vfpf_up, vf);
+				mbox_wait_for_zero(&dev->mbox_vfpf_up, vf);
 			}
 		}
 
@@ -498,6 +499,7 @@ pf_vf_mbox_send_up_msg(struct dev *dev, void *rec_msg)
 
 		/* Send to VF */
 		mbox_msg_send(vf_mbox, vf);
+		mbox_wait_for_zero(&dev->mbox_vfpf_up, vf);
 	}
 }
 
@@ -631,6 +633,7 @@ static void
 roc_pf_vf_mbox_irq(void *param)
 {
 	struct dev *dev = param;
+	uint64_t mbox_data;
 	uint64_t intr;
 
 	intr = plt_read64(dev->bar2 + RVU_VF_INT);
@@ -640,6 +643,13 @@ roc_pf_vf_mbox_irq(void *param)
 	plt_write64(intr, dev->bar2 + RVU_VF_INT);
 	plt_base_dbg("Irq 0x%" PRIx64 "(pf:%d,vf:%d)", intr, dev->pf, dev->vf);
 
+	/* Reading for UP/DOWN message, next message sending will be delayed
+	 * by 1ms until this region is zeroed mbox_wait_for_zero()
+	 */
+	mbox_data = plt_read64(dev->bar2 + RVU_VF_VFPF_MBOX0);
+	if (mbox_data)
+		plt_write64(!mbox_data, dev->bar2 + RVU_VF_VFPF_MBOX0);
+
 	/* First process all configuration messages */
 	process_msgs(dev, dev->mbox);
 
@@ -651,6 +661,7 @@ static void
 roc_af_pf_mbox_irq(void *param)
 {
 	struct dev *dev = param;
+	uint64_t mbox_data;
 	uint64_t intr;
 
 	intr = plt_read64(dev->bar2 + RVU_PF_INT);
@@ -660,6 +671,13 @@ roc_af_pf_mbox_irq(void *param)
 	plt_write64(intr, dev->bar2 + RVU_PF_INT);
 	plt_base_dbg("Irq 0x%" PRIx64 "(pf:%d,vf:%d)", intr, dev->pf, dev->vf);
 
+	/* Reading for UP/DOWN message, next message sending will be delayed
+	 * by 1ms until this region is zeroed mbox_wait_for_zero()
+	 */
+	mbox_data = plt_read64(dev->bar2 + RVU_PF_PFAF_MBOX0);
+	if (mbox_data)
+		plt_write64(!mbox_data, dev->bar2 + RVU_PF_PFAF_MBOX0);
+
 	/* First process all configuration messages */
 	process_msgs(dev, dev->mbox);
 
diff --git a/drivers/common/cnxk/roc_mbox.c b/drivers/common/cnxk/roc_mbox.c
index 7dcd188ca7..5338a960d9 100644
--- a/drivers/common/cnxk/roc_mbox.c
+++ b/drivers/common/cnxk/roc_mbox.c
@@ -10,18 +10,6 @@
 #include "roc_api.h"
 #include "roc_priv.h"
 
-#define RVU_AF_AFPF_MBOX0 (0x02000)
-#define RVU_AF_AFPF_MBOX1 (0x02008)
-
-#define RVU_PF_PFAF_MBOX0 (0xC00)
-#define RVU_PF_PFAF_MBOX1 (0xC08)
-
-#define RVU_PF_VFX_PFVF_MBOX0 (0x0000)
-#define RVU_PF_VFX_PFVF_MBOX1 (0x0008)
-
-#define RVU_VF_VFPF_MBOX0 (0x0000)
-#define RVU_VF_VFPF_MBOX1 (0x0008)
-
 /* RCLK, SCLK in MHz */
 uint16_t dev_rclk_freq;
 uint16_t dev_sclk_freq;
@@ -194,10 +182,31 @@ mbox_alloc_msg_rsp(struct mbox *mbox, int devid, int size, int size_rsp)
 
 /**
  * @internal
- * Send a mailbox message
+ * Synchronization between UP and DOWN messages
  */
-void
-mbox_msg_send(struct mbox *mbox, int devid)
+bool
+mbox_wait_for_zero(struct mbox *mbox, int devid)
+{
+	uint64_t data;
+
+	data = plt_read64((volatile void *)(mbox->reg_base +
+				(mbox->trigger | (devid << mbox->tr_shift))));
+
+	/* If data is non-zero wait for ~1ms and return to caller
+	 * whether data has changed to zero or not after the wait.
+	 */
+	if (data)
+		usleep(1000);
+	else
+		return true;
+
+	data = plt_read64((volatile void *)(mbox->reg_base +
+				(mbox->trigger | (devid << mbox->tr_shift))));
+	return data == 0;
+}
+
+static void
+mbox_msg_send_data(struct mbox *mbox, int devid, uint8_t data)
 {
 	struct mbox_dev *mdev = &mbox->dev[devid];
 	struct mbox_hdr *tx_hdr =
@@ -223,9 +232,28 @@ mbox_msg_send(struct mbox *mbox, int devid)
 	/* The interrupt should be fired after num_msgs is written
 	 * to the shared memory
 	 */
-	plt_write64(1, (volatile void *)(mbox->reg_base +
-					 (mbox->trigger |
-					  (devid << mbox->tr_shift))));
+	plt_write64(data, (volatile void *)(mbox->reg_base +
+				(mbox->trigger | (devid << mbox->tr_shift))));
+}
+
+/**
+ * @internal
+ * Send a mailbox message
+ */
+void
+mbox_msg_send(struct mbox *mbox, int devid)
+{
+	mbox_msg_send_data(mbox, devid, MBOX_DOWN_MSG);
+}
+
+/**
+ * @internal
+ * Send an UP mailbox message
+ */
+void
+mbox_msg_send_up(struct mbox *mbox, int devid)
+{
+	mbox_msg_send_data(mbox, devid, MBOX_UP_MSG);
 }
 
 /**
diff --git a/drivers/common/cnxk/roc_mbox.h b/drivers/common/cnxk/roc_mbox.h
index 3d5746b9b8..93c5451c0f 100644
--- a/drivers/common/cnxk/roc_mbox.h
+++ b/drivers/common/cnxk/roc_mbox.h
@@ -35,6 +35,21 @@ struct mbox_msghdr {
 	int __io rc; /* Msg processed response code */
 };
 
+#define RVU_AF_AFPF_MBOX0 (0x02000)
+#define RVU_AF_AFPF_MBOX1 (0x02008)
+
+#define RVU_PF_PFAF_MBOX0 (0xC00)
+#define RVU_PF_PFAF_MBOX1 (0xC08)
+
+#define RVU_PF_VFX_PFVF_MBOX0 (0x0000)
+#define RVU_PF_VFX_PFVF_MBOX1 (0x0008)
+
+#define RVU_VF_VFPF_MBOX0 (0x0000)
+#define RVU_VF_VFPF_MBOX1 (0x0008)
+
+#define MBOX_DOWN_MSG 1
+#define MBOX_UP_MSG   2
+
 /* Mailbox message types */
 #define MBOX_MSG_MASK	 0xFFFF
 #define MBOX_MSG_INVALID 0xFFFE
diff --git a/drivers/common/cnxk/roc_mbox_priv.h b/drivers/common/cnxk/roc_mbox_priv.h
index 4fafca6f72..354c8fa52a 100644
--- a/drivers/common/cnxk/roc_mbox_priv.h
+++ b/drivers/common/cnxk/roc_mbox_priv.h
@@ -71,10 +71,12 @@ struct mbox {
 const char *mbox_id2name(uint16_t id);
 int mbox_id2size(uint16_t id);
 void mbox_reset(struct mbox *mbox, int devid);
-int mbox_init(struct mbox *mbox, uintptr_t hwbase, uintptr_t reg_base,
-	      int direction, int ndevsi, uint64_t intr_offset);
+int mbox_init(struct mbox *mbox, uintptr_t hwbase, uintptr_t reg_base, int direction, int ndevsi,
+	      uint64_t intr_offset);
 void mbox_fini(struct mbox *mbox);
 void mbox_msg_send(struct mbox *mbox, int devid);
+void mbox_msg_send_up(struct mbox *mbox, int devid);
+bool mbox_wait_for_zero(struct mbox *mbox, int devid);
 int mbox_wait_for_rsp(struct mbox *mbox, int devid);
 int mbox_wait_for_rsp_tmo(struct mbox *mbox, int devid, uint32_t tmo);
 int mbox_get_rsp(struct mbox *mbox, int devid, void **msg);
-- 
2.25.1


  parent reply	other threads:[~2023-05-25 10:10 UTC|newest]

Thread overview: 89+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2023-04-11  9:11 [PATCH 01/21] common/cnxk: allocate dynamic BPIDs Nithin Dabilpuram
2023-04-11  9:11 ` [PATCH 02/21] common/cnxk: add pool BPID to RQ while using common pool Nithin Dabilpuram
2023-05-18  5:52   ` Jerin Jacob
2023-04-11  9:11 ` [PATCH 03/21] common/cnxk: skip flow ctrl set on non-existent meta aura Nithin Dabilpuram
2023-04-11  9:11 ` [PATCH 04/21] common/cnxk: reduce sqes per sqb by one Nithin Dabilpuram
2023-04-11  9:11 ` [PATCH 05/21] common/cnxk: dump SW SSO work count as xstat Nithin Dabilpuram
2023-04-11  9:11 ` [PATCH 06/21] common/cnxk: add percent drop threshold to pool Nithin Dabilpuram
2023-04-11  9:11 ` [PATCH 07/21] common/cnxk: make aura flow control config more predictable Nithin Dabilpuram
2023-04-11  9:11 ` [PATCH 08/21] common/cnxk: update age drop statistics Nithin Dabilpuram
2023-04-11  9:11 ` [PATCH 09/21] common/cnxk: fetch eng caps for inl outb inst format Nithin Dabilpuram
2023-04-11  9:11 ` [PATCH 10/21] common/cnxk: add receive error mask Nithin Dabilpuram
2023-04-11  9:11 ` [PATCH 11/21] common/cnxk: fix null pointer dereference Nithin Dabilpuram
2023-04-11  9:11 ` [PATCH 12/21] common/cnxk: fix parameter in NIX dump Nithin Dabilpuram
2023-04-11  9:11 ` [PATCH 13/21] common/cnxk: set relchan in TL4 config for each SDP queue Nithin Dabilpuram
2023-04-11  9:11 ` [PATCH 14/21] common/cnxk: avoid STALL with dual rate on CNF95N Nithin Dabilpuram
2023-04-11  9:11 ` [PATCH 15/21] common/cnxk: update errata info Nithin Dabilpuram
2023-04-11  9:11 ` [PATCH 16/21] common/cnxk: sync between mbox up and down messages Nithin Dabilpuram
2023-04-11  9:11 ` [PATCH 17/21] common/cnxk: add more comments to mbox code Nithin Dabilpuram
2023-04-11  9:11 ` [PATCH 18/21] common/cnxk: add CN105xxN B0 model Nithin Dabilpuram
2023-04-11  9:11 ` [PATCH 19/21] common/cnxk: access valid pass value Nithin Dabilpuram
2023-04-11  9:11 ` [PATCH 20/21] net/cnxk: add receive error mask Nithin Dabilpuram
2023-04-11  9:11 ` [PATCH 21/21] common/cnxk: support of 1:n pool:aura per NIX LF Nithin Dabilpuram
2023-05-18  5:50   ` Jerin Jacob
2023-05-24 10:03 ` [PATCH v2 01/32] common/cnxk: allocate dynamic BPIDs Nithin Dabilpuram
2023-05-24 10:03   ` [PATCH v2 02/32] common/cnxk: add pool BPID to RQ while using common pool Nithin Dabilpuram
2023-05-24 10:03   ` [PATCH v2 03/32] common/cnxk: fix CPT backpressure disable on LBK Nithin Dabilpuram
2023-05-24 10:03   ` [PATCH v2 04/32] common/cnxk: skip flow ctrl set on non-existent meta aura Nithin Dabilpuram
2023-05-24 10:03   ` [PATCH v2 05/32] common/cnxk: reduce sqes per sqb by one Nithin Dabilpuram
2023-05-24 10:03   ` [PATCH v2 06/32] common/cnxk: dump SW SSO work count as xstat Nithin Dabilpuram
2023-05-24 10:03   ` [PATCH v2 07/32] common/cnxk: add percent drop threshold to pool Nithin Dabilpuram
2023-05-24 10:03   ` [PATCH v2 08/32] common/cnxk: make aura flow control config more predictable Nithin Dabilpuram
2023-05-24 10:03   ` [PATCH v2 09/32] common/cnxk: update age drop statistics Nithin Dabilpuram
2023-05-24 10:03   ` [PATCH v2 10/32] common/cnxk: fetch eng caps for inl outb inst format Nithin Dabilpuram
2023-05-24 10:03   ` [PATCH v2 11/32] common/cnxk: add receive error mask Nithin Dabilpuram
2023-05-24 10:03   ` [PATCH v2 12/32] common/cnxk: fix null pointer dereference Nithin Dabilpuram
2023-05-24 10:03   ` [PATCH v2 13/32] common/cnxk: fix parameter in NIX dump Nithin Dabilpuram
2023-05-24 10:03   ` [PATCH v2 14/32] common/cnxk: set relchan in TL4 config for each SDP queue Nithin Dabilpuram
2023-05-24 10:03   ` [PATCH v2 15/32] common/cnxk: avoid STALL with dual rate on CNF95N Nithin Dabilpuram
2023-05-24 10:03   ` [PATCH v2 16/32] common/cnxk: update errata info Nithin Dabilpuram
2023-05-24 10:03   ` [PATCH v2 17/32] common/cnxk: sync between mbox up and down messages Nithin Dabilpuram
2023-05-24 10:03   ` [PATCH v2 18/32] common/cnxk: add more comments to mbox code Nithin Dabilpuram
2023-05-24 10:03   ` [PATCH v2 19/32] common/cnxk: add CN105xxN B0 model Nithin Dabilpuram
2023-05-24 10:03   ` [PATCH v2 20/32] common/cnxk: access valid pass value Nithin Dabilpuram
2023-05-24 10:03   ` [PATCH v2 21/32] net/cnxk: add receive error mask Nithin Dabilpuram
2023-05-24 10:03   ` [PATCH v2 22/32] common/cnxk: support of 1-N pool-aura per NIX LF Nithin Dabilpuram
2023-05-24 10:03   ` [PATCH v2 23/32] net/cnxk: support for inbound without inline dev mode Nithin Dabilpuram
2023-05-24 10:03   ` [PATCH v2 24/32] common/cnxk: fix inline device VF identification Nithin Dabilpuram
2023-05-24 10:04   ` [PATCH v2 25/32] common/cnxk: avoid inline dev CPT lf detach multiple times Nithin Dabilpuram
2023-05-24 10:04   ` [PATCH v2 26/32] common/cnxk: skip CGX promisc mode with NPC exact match Nithin Dabilpuram
2023-05-24 10:04   ` [PATCH v2 27/32] common/cnxk: configure PFC on SPB aura Nithin Dabilpuram
2023-05-24 10:04   ` [PATCH v2 28/32] common/nix: check for null derefernce Nithin Dabilpuram
2023-05-24 10:04   ` [PATCH v2 29/32] common/cnxk: fix receive queue with multiple mask Nithin Dabilpuram
2023-05-24 10:04   ` [PATCH v2 30/32] net/cnxk: handle extbuf completion on ethdev stop Nithin Dabilpuram
2023-05-24 10:04   ` [PATCH v2 31/32] net/cnxk: add aes-ccm to inline IPsec capabilities Nithin Dabilpuram
2023-05-24 10:04   ` [PATCH v2 32/32] common/cnxk: add check for null auth and anti-replay Nithin Dabilpuram
2023-05-25  9:28     ` Jerin Jacob
2023-05-25  9:58 ` [PATCH v3 01/32] common/cnxk: allocate dynamic BPIDs Nithin Dabilpuram
2023-05-25  9:58   ` [PATCH v3 02/32] common/cnxk: add pool BPID to RQ while using common pool Nithin Dabilpuram
2023-05-25  9:58   ` [PATCH v3 03/32] common/cnxk: fix CPT backpressure disable on LBK Nithin Dabilpuram
2023-05-25  9:58   ` [PATCH v3 04/32] common/cnxk: skip flow ctrl set on non-existent meta aura Nithin Dabilpuram
2023-05-25  9:58   ` [PATCH v3 05/32] common/cnxk: reduce sqes per sqb by one Nithin Dabilpuram
2023-05-25  9:58   ` [PATCH v3 06/32] common/cnxk: dump SW SSO work count as xstat Nithin Dabilpuram
2023-05-25  9:58   ` [PATCH v3 07/32] common/cnxk: add percent drop threshold to pool Nithin Dabilpuram
2023-05-25  9:58   ` [PATCH v3 08/32] common/cnxk: make aura flow control config more predictable Nithin Dabilpuram
2023-05-25  9:58   ` [PATCH v3 09/32] common/cnxk: update age drop statistics Nithin Dabilpuram
2023-05-25  9:58   ` [PATCH v3 10/32] common/cnxk: fetch eng caps for inl outb inst format Nithin Dabilpuram
2023-05-25  9:58   ` [PATCH v3 11/32] common/cnxk: add receive error mask Nithin Dabilpuram
2023-05-25  9:58   ` [PATCH v3 12/32] common/cnxk: fix null pointer dereference Nithin Dabilpuram
2023-05-25  9:58   ` [PATCH v3 13/32] common/cnxk: fix parameter in NIX dump Nithin Dabilpuram
2023-05-25  9:58   ` [PATCH v3 14/32] common/cnxk: set relchan in TL4 config for each SDP queue Nithin Dabilpuram
2023-05-25  9:58   ` [PATCH v3 15/32] common/cnxk: avoid STALL with dual rate on CNF95N Nithin Dabilpuram
2023-05-25  9:58   ` [PATCH v3 16/32] common/cnxk: update errata info Nithin Dabilpuram
2023-05-25  9:58   ` Nithin Dabilpuram [this message]
2023-05-25  9:58   ` [PATCH v3 18/32] common/cnxk: add more comments to mbox code Nithin Dabilpuram
2023-05-25  9:58   ` [PATCH v3 19/32] common/cnxk: add CN105xxN B0 model Nithin Dabilpuram
2023-05-25  9:58   ` [PATCH v3 20/32] common/cnxk: access valid pass value Nithin Dabilpuram
2023-05-25  9:58   ` [PATCH v3 21/32] net/cnxk: add receive error mask Nithin Dabilpuram
2023-05-25  9:58   ` [PATCH v3 22/32] common/cnxk: support of 1-N pool-aura per NIX LF Nithin Dabilpuram
2023-05-25  9:58   ` [PATCH v3 23/32] net/cnxk: support for inbound without inline dev mode Nithin Dabilpuram
2023-05-25  9:58   ` [PATCH v3 24/32] common/cnxk: fix inline device VF identification Nithin Dabilpuram
2023-05-25  9:58   ` [PATCH v3 25/32] common/cnxk: avoid inline dev CPT lf detach multiple times Nithin Dabilpuram
2023-05-25  9:58   ` [PATCH v3 26/32] common/cnxk: skip CGX promisc mode with NPC exact match Nithin Dabilpuram
2023-05-25  9:58   ` [PATCH v3 27/32] common/cnxk: configure PFC on SPB aura Nithin Dabilpuram
2023-05-25  9:59   ` [PATCH v3 28/32] common/nix: check for null dereference Nithin Dabilpuram
2023-05-25  9:59   ` [PATCH v3 29/32] common/cnxk: fix receive queue with multiple mask Nithin Dabilpuram
2023-05-25  9:59   ` [PATCH v3 30/32] net/cnxk: handle extbuf completion on ethdev stop Nithin Dabilpuram
2023-05-25  9:59   ` [PATCH v3 31/32] net/cnxk: add aes-ccm to inline IPsec capabilities Nithin Dabilpuram
2023-05-25  9:59   ` [PATCH v3 32/32] common/cnxk: add check for null auth and anti-replay Nithin Dabilpuram
2023-05-26  8:55     ` Jerin Jacob

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20230525095904.3967080-17-ndabilpuram@marvell.com \
    --to=ndabilpuram@marvell.com \
    --cc=dev@dpdk.org \
    --cc=hkalra@marvell.com \
    --cc=jerinj@marvell.com \
    --cc=kirankumark@marvell.com \
    --cc=skori@marvell.com \
    --cc=skoteshwar@marvell.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).