From: Aman Kumar <aman.kumar@vvdntech.in>
To: dev@dpdk.org
Cc: maxime.coquelin@redhat.com, david.marchand@redhat.com,
aman.kumar@vvdntech.in
Subject: [RFC PATCH 22/29] net/qdma: mbox API adaptation in Rx/Tx init
Date: Wed, 6 Jul 2022 13:22:12 +0530 [thread overview]
Message-ID: <20220706075219.517046-23-aman.kumar@vvdntech.in> (raw)
In-Reply-To: <20220706075219.517046-1-aman.kumar@vvdntech.in>
add mbox initialization and handling to enable virtual
functions queue setup.
Signed-off-by: Aman Kumar <aman.kumar@vvdntech.in>
---
drivers/net/qdma/qdma.h | 1 +
drivers/net/qdma/qdma_devops.c | 196 ++++++++++++++++++++++++++++++---
drivers/net/qdma/qdma_ethdev.c | 47 ++++++++
3 files changed, 226 insertions(+), 18 deletions(-)
diff --git a/drivers/net/qdma/qdma.h b/drivers/net/qdma/qdma.h
index 8fb64c21b0..20a1b72dd1 100644
--- a/drivers/net/qdma/qdma.h
+++ b/drivers/net/qdma/qdma.h
@@ -303,6 +303,7 @@ struct qdma_pci_dev {
void qdma_dev_ops_init(struct rte_eth_dev *dev);
void qdma_txq_pidx_update(void *arg);
int qdma_pf_csr_read(struct rte_eth_dev *dev);
+int qdma_vf_csr_read(struct rte_eth_dev *dev);
uint8_t qmda_get_desc_sz_idx(enum rte_pmd_qdma_bypass_desc_len);
diff --git a/drivers/net/qdma/qdma_devops.c b/drivers/net/qdma/qdma_devops.c
index 7f525773d0..e6803dd86f 100644
--- a/drivers/net/qdma/qdma_devops.c
+++ b/drivers/net/qdma/qdma_devops.c
@@ -22,6 +22,8 @@
#include "qdma.h"
#include "qdma_access_common.h"
+#include "qdma_mbox_protocol.h"
+#include "qdma_mbox.h"
#include "qdma_reg_dump.h"
#include "qdma_platform.h"
#include "qdma_devops.h"
@@ -64,6 +66,39 @@ static void qdma_sort_c2h_cntr_th_values(struct qdma_pci_dev *qdma_dev)
}
#endif /* QDMA_LATENCY_OPTIMIZED */
+int qdma_vf_csr_read(struct rte_eth_dev *dev)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ struct qdma_mbox_msg *m = qdma_mbox_msg_alloc();
+ int rv, i;
+ struct qdma_csr_info csr_info;
+
+ if (!m)
+ return -ENOMEM;
+
+ qdma_mbox_compose_csr_read(qdma_dev->func_id, m->raw_data);
+ rv = qdma_mbox_msg_send(dev, m, MBOX_OP_RSP_TIMEOUT);
+ if (rv < 0)
+ goto free_msg;
+
+ rv = qdma_mbox_vf_csr_get(m->raw_data, &csr_info);
+ if (rv < 0)
+ goto free_msg;
+ for (i = 0; i < QDMA_NUM_RING_SIZES; i++) {
+ qdma_dev->g_ring_sz[i] = (uint32_t)csr_info.ringsz[i];
+ qdma_dev->g_c2h_buf_sz[i] = (uint32_t)csr_info.bufsz[i];
+ qdma_dev->g_c2h_timer_cnt[i] = (uint32_t)csr_info.timer_cnt[i];
+ qdma_dev->g_c2h_cnt_th[i] = (uint32_t)csr_info.cnt_thres[i];
+ #ifdef QDMA_LATENCY_OPTIMIZED
+ qdma_sort_c2h_cntr_th_values(qdma_dev);
+ #endif /* QDMA_LATENCY_OPTIMIZED */
+ }
+
+free_msg:
+ qdma_mbox_msg_free(m);
+ return rv;
+}
+
int qdma_pf_csr_read(struct rte_eth_dev *dev)
{
int ret = 0;
@@ -131,6 +166,44 @@ static int qdma_pf_fmap_prog(struct rte_eth_dev *dev)
return ret;
}
+int qdma_dev_notify_qadd(struct rte_eth_dev *dev, uint32_t qidx_hw,
+ enum qdma_dev_q_type q_type)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ struct qdma_mbox_msg *m;
+ int rv = 0;
+
+ m = qdma_mbox_msg_alloc();
+ if (!m)
+ return -ENOMEM;
+
+ qdma_mbox_compose_vf_notify_qadd(qdma_dev->func_id, qidx_hw,
+ q_type, m->raw_data);
+ rv = qdma_mbox_msg_send(dev, m, MBOX_OP_RSP_TIMEOUT);
+
+ qdma_mbox_msg_free(m);
+ return rv;
+}
+
+int qdma_dev_notify_qdel(struct rte_eth_dev *dev, uint32_t qidx_hw,
+ enum qdma_dev_q_type q_type)
+{
+ struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ struct qdma_mbox_msg *m;
+ int rv = 0;
+
+ m = qdma_mbox_msg_alloc();
+ if (!m)
+ return -ENOMEM;
+
+ qdma_mbox_compose_vf_notify_qdel(qdma_dev->func_id, qidx_hw,
+ q_type, m->raw_data);
+ rv = qdma_mbox_msg_send(dev, m, MBOX_OP_RSP_TIMEOUT);
+
+ qdma_mbox_msg_free(m);
+ return rv;
+}
+
uint8_t qmda_get_desc_sz_idx(enum rte_pmd_qdma_bypass_desc_len size)
{
uint8_t ret;
@@ -243,9 +316,33 @@ int qdma_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t rx_queue_id,
return -EINVAL;
}
}
+ } else {
+ err = qdma_dev_notify_qadd(dev, rx_queue_id +
+ qdma_dev->queue_base,
+ QDMA_DEV_Q_TYPE_C2H);
+ if (err < 0)
+ return -EINVAL;
+
+ if (qdma_dev->q_info[rx_queue_id].queue_mode ==
+ RTE_PMD_QDMA_STREAMING_MODE) {
+ err = qdma_dev_notify_qadd(dev, rx_queue_id +
+ qdma_dev->queue_base,
+ QDMA_DEV_Q_TYPE_CMPT);
+ if (err < 0) {
+ qdma_dev_notify_qdel(dev, rx_queue_id +
+ qdma_dev->queue_base,
+ QDMA_DEV_Q_TYPE_C2H);
+ return -EINVAL;
+ }
+ }
}
+
if (!qdma_dev->init_q_range) {
- if (!qdma_dev->is_vf) {
+ if (qdma_dev->is_vf) {
+ err = qdma_vf_csr_read(dev);
+ if (err < 0)
+ goto rx_setup_err;
+ } else {
err = qdma_pf_csr_read(dev);
if (err < 0)
goto rx_setup_err;
@@ -534,18 +631,27 @@ int qdma_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t rx_queue_id,
QDMA_DEV_Q_TYPE_C2H);
if (qdma_dev->q_info[rx_queue_id].queue_mode ==
- RTE_PMD_QDMA_STREAMING_MODE)
+ RTE_PMD_QDMA_STREAMING_MODE) {
qdma_dev_decrement_active_queue
(qdma_dev->dma_device_index,
qdma_dev->func_id,
QDMA_DEV_Q_TYPE_CMPT);
- }
- if (rxq) {
- if (rxq->rx_mz)
- rte_memzone_free(rxq->rx_mz);
- if (rxq->sw_ring)
- rte_free(rxq->sw_ring);
- rte_free(rxq);
+ } else {
+ qdma_dev_notify_qdel(dev, rx_queue_id +
+ qdma_dev->queue_base, QDMA_DEV_Q_TYPE_C2H);
+
+ if (qdma_dev->q_info[rx_queue_id].queue_mode ==
+ RTE_PMD_QDMA_STREAMING_MODE)
+ qdma_dev_notify_qdel(dev, rx_queue_id +
+ qdma_dev->queue_base, QDMA_DEV_Q_TYPE_CMPT);
+ }
+ if (rxq) {
+ if (rxq->rx_mz)
+ rte_memzone_free(rxq->rx_mz);
+ if (rxq->sw_ring)
+ rte_free(rxq->sw_ring);
+ rte_free(rxq);
+ }
}
return err;
}
@@ -591,9 +697,21 @@ int qdma_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t tx_queue_id,
QDMA_DEV_Q_TYPE_H2C);
if (err != QDMA_SUCCESS)
return -EINVAL;
+ } else {
+ err = qdma_dev_notify_qadd(dev, tx_queue_id +
+ qdma_dev->queue_base,
+ QDMA_DEV_Q_TYPE_H2C);
+ if (err < 0)
+ return -EINVAL;
}
if (!qdma_dev->init_q_range) {
- if (!qdma_dev->is_vf) {
+ if (qdma_dev->is_vf) {
+ err = qdma_vf_csr_read(dev);
+ if (err < 0) {
+ PMD_DRV_LOG(ERR, "CSR read failed\n");
+ goto tx_setup_err;
+ }
+ } else {
err = qdma_pf_csr_read(dev);
if (err < 0) {
PMD_DRV_LOG(ERR, "CSR read failed\n");
@@ -751,16 +869,28 @@ int qdma_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t tx_queue_id,
tx_setup_err:
PMD_DRV_LOG(ERR, " Tx queue setup failed");
- if (!qdma_dev->is_vf)
+ if (!qdma_dev->is_vf) {
qdma_dev_decrement_active_queue(qdma_dev->dma_device_index,
qdma_dev->func_id,
QDMA_DEV_Q_TYPE_H2C);
+ } else {
+ qdma_dev_notify_qdel(dev, tx_queue_id +
+ qdma_dev->queue_base, QDMA_DEV_Q_TYPE_H2C);
+ }
if (txq) {
- if (txq->tx_mz)
- rte_memzone_free(txq->tx_mz);
- if (txq->sw_ring)
- rte_free(txq->sw_ring);
- rte_free(txq);
+ if (qdma_dev->is_vf) {
+ err = qdma_vf_csr_read(dev);
+ if (err < 0) {
+ PMD_DRV_LOG(ERR, "CSR read failed\n");
+ goto tx_setup_err;
+ }
+ } else {
+ if (txq->tx_mz)
+ rte_memzone_free(txq->tx_mz);
+ if (txq->sw_ring)
+ rte_free(txq->sw_ring);
+ rte_free(txq);
+ }
}
return err;
}
@@ -802,11 +932,16 @@ void qdma_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t q_id)
PMD_DRV_LOG(INFO, "Remove H2C queue: %d", txq->queue_id);
qdma_dev = txq->dev->data->dev_private;
- if (!qdma_dev->is_vf)
+ if (!qdma_dev->is_vf) {
qdma_dev_decrement_active_queue
(qdma_dev->dma_device_index,
qdma_dev->func_id,
- QDMA_DEV_Q_TYPE_H2C);
+ QDMA_DEV_Q_TYPE_H2C);
+ } else {
+ qdma_dev_notify_qdel(txq->dev, txq->queue_id +
+ qdma_dev->queue_base,
+ QDMA_DEV_Q_TYPE_H2C);
+ }
if (txq->sw_ring)
rte_free(txq->sw_ring);
if (txq->tx_mz)
@@ -837,6 +972,15 @@ void qdma_dev_rx_queue_release(struct rte_eth_dev *dev, uint16_t q_id)
(qdma_dev->dma_device_index,
qdma_dev->func_id,
QDMA_DEV_Q_TYPE_CMPT);
+ } else {
+ qdma_dev_notify_qdel(rxq->dev, rxq->queue_id +
+ qdma_dev->queue_base,
+ QDMA_DEV_Q_TYPE_C2H);
+
+ if (rxq->st_mode)
+ qdma_dev_notify_qdel(rxq->dev, rxq->queue_id +
+ qdma_dev->queue_base,
+ QDMA_DEV_Q_TYPE_CMPT);
}
if (rxq->sw_ring)
@@ -1111,6 +1255,7 @@ int qdma_dev_reset(struct rte_eth_dev *dev)
{
struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct qdma_mbox_msg *m = NULL;
uint32_t vf_device_count = 0;
uint32_t i = 0;
int ret = 0;
@@ -1141,6 +1286,21 @@ int qdma_dev_reset(struct rte_eth_dev *dev)
for (i = 0; i < pci_dev->max_vfs; i++) {
if (qdma_dev->vfinfo[i].func_id == QDMA_FUNC_ID_INVALID)
continue;
+
+ m = qdma_mbox_msg_alloc();
+ if (!m)
+ return -ENOMEM;
+ qdma_mbox_compose_pf_reset_done_message(m->raw_data, qdma_dev->func_id,
+ qdma_dev->vfinfo[i].func_id);
+ ret = qdma_mbox_msg_send(dev, m, 0);
+ if (ret < 0)
+ PMD_DRV_LOG(ERR, "Sending reset failed from PF:%d to VF:%d\n",
+ qdma_dev->func_id, qdma_dev->vfinfo[i].func_id);
+
+ /* Mark VFs with invalid function id mapping,
+ * and this gets updated when VF comes online again
+ */
+ qdma_dev->vfinfo[i].func_id = QDMA_FUNC_ID_INVALID;
}
/* Start waiting for a maximum of 60 secs to get all its VFs
diff --git a/drivers/net/qdma/qdma_ethdev.c b/drivers/net/qdma/qdma_ethdev.c
index cc1e8eee71..466a9e9284 100644
--- a/drivers/net/qdma/qdma_ethdev.c
+++ b/drivers/net/qdma/qdma_ethdev.c
@@ -25,6 +25,7 @@
#include "qdma_version.h"
#include "qdma_access_common.h"
#include "qdma_access_export.h"
+#include "qdma_mbox.h"
#include "qdma_devops.h"
/* Poll for QDMA errors every 1 second */
@@ -546,6 +547,8 @@ int qdma_eth_dev_init(struct rte_eth_dev *dev)
}
pcie_perf_enable(pci_dev);
+ if (dma_priv->dev_cap.mailbox_en && pci_dev->max_vfs)
+ qdma_mbox_init(dev);
if (!dma_priv->reset_in_progress) {
num_vfs = pci_dev->max_vfs;
@@ -581,13 +584,57 @@ int qdma_eth_dev_init(struct rte_eth_dev *dev)
int qdma_eth_dev_uninit(struct rte_eth_dev *dev)
{
struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+ struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+ struct qdma_mbox_msg *m = NULL;
+ int i, rv;
/* only uninitialize in the primary process */
if (rte_eal_process_type() != RTE_PROC_PRIMARY)
return -EPERM;
+ if (qdma_dev->vf_online_count) {
+ for (i = 0; i < pci_dev->max_vfs; i++) {
+ if (qdma_dev->vfinfo[i].func_id == QDMA_FUNC_ID_INVALID)
+ continue;
+ m = qdma_mbox_msg_alloc();
+ if (!m)
+ return -ENOMEM;
+ if (!qdma_dev->reset_in_progress)
+ qdma_mbox_compose_pf_offline(m->raw_data,
+ qdma_dev->func_id,
+ qdma_dev->vfinfo[i].func_id);
+ else
+ qdma_mbox_compose_vf_reset_message(m->raw_data,
+ qdma_dev->func_id,
+ qdma_dev->vfinfo[i].func_id);
+ rv = qdma_mbox_msg_send(dev, m, 0);
+ if (rv < 0)
+ PMD_DRV_LOG(ERR, "Send bye failed from PF:%d to VF:%d\n",
+ qdma_dev->func_id,
+ qdma_dev->vfinfo[i].func_id);
+ }
+ PMD_DRV_LOG(INFO, "%s: Wait till all VFs shutdown for PF-%d(DEVFN)\n",
+ __func__, qdma_dev->func_id);
+ i = 0;
+ while (i < SHUTDOWN_TIMEOUT) {
+ if (!qdma_dev->vf_online_count) {
+ PMD_DRV_LOG(INFO, "%s: VFs shutdown completed for PF-%d(DEVFN)\n",
+ __func__, qdma_dev->func_id);
+ break;
+ }
+ rte_delay_ms(1);
+ i++;
+ }
+
+ if (i >= SHUTDOWN_TIMEOUT) {
+ PMD_DRV_LOG(ERR, "%s: Failed VFs shutdown for PF-%d(DEVFN)\n",
+ __func__, qdma_dev->func_id);
+ }
+ }
if (qdma_dev->dev_configured)
qdma_dev_close(dev);
+ if (qdma_dev->dev_cap.mailbox_en && pci_dev->max_vfs)
+ qdma_mbox_uninit(dev);
/* cancel pending polls */
if (qdma_dev->is_master)
--
2.36.1
next prev parent reply other threads:[~2022-07-06 7:58 UTC|newest]
Thread overview: 43+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-07-06 7:51 [RFC PATCH 00/29] cover letter for net/qdma PMD Aman Kumar
2022-07-06 7:51 ` [RFC PATCH 01/29] net/qdma: add net PMD template Aman Kumar
2022-07-06 7:51 ` [RFC PATCH 02/29] maintainers: add maintainer for net/qdma PMD Aman Kumar
2022-07-06 7:51 ` [RFC PATCH 03/29] net/meson.build: add support to compile net qdma Aman Kumar
2022-07-06 7:51 ` [RFC PATCH 04/29] net/qdma: add logging support Aman Kumar
2022-07-06 15:27 ` Stephen Hemminger
2022-07-07 2:32 ` Aman Kumar
2022-07-06 7:51 ` [RFC PATCH 05/29] net/qdma: add device init and uninit functions Aman Kumar
2022-07-06 15:35 ` Stephen Hemminger
2022-07-07 2:41 ` Aman Kumar
2022-07-06 7:51 ` [RFC PATCH 06/29] net/qdma: add qdma access library Aman Kumar
2022-07-06 7:51 ` [RFC PATCH 07/29] net/qdma: add supported qdma version Aman Kumar
2022-07-06 7:51 ` [RFC PATCH 08/29] net/qdma: qdma hardware initialization Aman Kumar
2022-07-06 7:51 ` [RFC PATCH 09/29] net/qdma: define device modes and data structure Aman Kumar
2022-07-06 7:52 ` [RFC PATCH 10/29] net/qdma: add net PMD ops template Aman Kumar
2022-07-06 7:52 ` [RFC PATCH 11/29] net/qdma: add configure close and reset ethdev ops Aman Kumar
2022-07-06 7:52 ` [RFC PATCH 12/29] net/qdma: add routine for Rx queue initialization Aman Kumar
2022-07-06 7:52 ` [RFC PATCH 13/29] net/qdma: add callback support for Rx queue count Aman Kumar
2022-07-06 7:52 ` [RFC PATCH 14/29] net/qdma: add routine for Tx queue initialization Aman Kumar
2022-07-06 7:52 ` [RFC PATCH 15/29] net/qdma: add queue cleanup PMD ops Aman Kumar
2022-07-06 7:52 ` [RFC PATCH 16/29] net/qdma: add start and stop apis Aman Kumar
2022-07-06 7:52 ` [RFC PATCH 17/29] net/qdma: add Tx burst API Aman Kumar
2022-07-06 7:52 ` [RFC PATCH 18/29] net/qdma: add Tx queue reclaim routine Aman Kumar
2022-07-06 7:52 ` [RFC PATCH 19/29] net/qdma: add callback function for Tx desc status Aman Kumar
2022-07-06 7:52 ` [RFC PATCH 20/29] net/qdma: add Rx burst API Aman Kumar
2022-07-06 7:52 ` [RFC PATCH 21/29] net/qdma: add mailbox communication library Aman Kumar
2022-07-06 7:52 ` Aman Kumar [this message]
2022-07-06 7:52 ` [RFC PATCH 23/29] net/qdma: add support for VF interfaces Aman Kumar
2022-07-06 7:52 ` [RFC PATCH 24/29] net/qdma: add Rx/Tx queue setup routine for VF devices Aman Kumar
2022-07-06 7:52 ` [RFC PATCH 25/29] net/qdma: add basic PMD ops for VF Aman Kumar
2022-07-06 7:52 ` [RFC PATCH 26/29] net/qdma: add datapath burst API " Aman Kumar
2022-07-06 7:52 ` [RFC PATCH 27/29] net/qdma: add device specific APIs for export Aman Kumar
2022-07-06 7:52 ` [RFC PATCH 28/29] net/qdma: add additional debug APIs Aman Kumar
2022-07-06 7:52 ` [RFC PATCH 29/29] net/qdma: add stats PMD ops for PF and VF Aman Kumar
2022-07-07 6:57 ` [RFC PATCH 00/29] cover letter for net/qdma PMD Thomas Monjalon
2022-07-07 13:55 ` Aman Kumar
2022-07-07 14:15 ` Thomas Monjalon
2022-07-07 14:19 ` Hemant Agrawal
2022-07-18 18:15 ` aman.kumar
2022-07-19 12:12 ` Thomas Monjalon
2022-07-19 17:22 ` aman.kumar
2023-07-02 23:36 ` Stephen Hemminger
2023-07-03 9:15 ` Ferruh Yigit
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20220706075219.517046-23-aman.kumar@vvdntech.in \
--to=aman.kumar@vvdntech.in \
--cc=david.marchand@redhat.com \
--cc=dev@dpdk.org \
--cc=maxime.coquelin@redhat.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).