* [PATCH v1 0/4] net/ntnic: implement start, stop and deferred start for Rx/Tx queues
@ 2025-06-20 11:27 Oleksandr Kolomeiets
2025-06-20 11:27 ` [PATCH v1 1/4] net/ntnic: implement start/stop " Oleksandr Kolomeiets
` (3 more replies)
0 siblings, 4 replies; 5+ messages in thread
From: Oleksandr Kolomeiets @ 2025-06-20 11:27 UTC (permalink / raw)
To: dev; +Cc: mko-plv, sil-plv, ckm, stephen
This patchset includes:
* feature start/stop queues on HW layer.
* feature deferred start for queues.
* Improvement for memory mappings when IOMMU is unoptimized.
* Improvement for logging
Oleksandr Kolomeiets (4):
net/ntnic: implement start/stop for Rx/Tx queues
net/ntnic: implement deferred start for Rx/Tx queues
net/ntnic: unmap DMA during queue release
net/ntnic: add warning when sending on a stopped queue
drivers/net/ntnic/dbsconfig/ntnic_dbsconfig.c | 58 +++++---
drivers/net/ntnic/include/ntnic_dbs.h | 2 +
drivers/net/ntnic/include/ntos_drv.h | 4 +-
drivers/net/ntnic/nthw/dbs/nthw_dbs.c | 20 +++
drivers/net/ntnic/ntnic_ethdev.c | 132 ++++++++++++++++--
drivers/net/ntnic/ntnic_mod_reg.h | 14 +-
drivers/net/ntnic/ntnic_vfio.c | 3 -
7 files changed, 193 insertions(+), 40 deletions(-)
--
2.47.1
^ permalink raw reply [flat|nested] 5+ messages in thread
* [PATCH v1 1/4] net/ntnic: implement start/stop for Rx/Tx queues
2025-06-20 11:27 [PATCH v1 0/4] net/ntnic: implement start, stop and deferred start for Rx/Tx queues Oleksandr Kolomeiets
@ 2025-06-20 11:27 ` Oleksandr Kolomeiets
2025-06-20 11:27 ` [PATCH v1 2/4] net/ntnic: implement deferred start " Oleksandr Kolomeiets
` (2 subsequent siblings)
3 siblings, 0 replies; 5+ messages in thread
From: Oleksandr Kolomeiets @ 2025-06-20 11:27 UTC (permalink / raw)
To: dev; +Cc: mko-plv, sil-plv, ckm, stephen
The following functions exported by the driver were stubs
which merely changed the status flags:
* rx_queue_start
* rx_queue_stop
* tx_queue_start
* tx_queue_stop
Proper implementation was added to control queues's state.
Signed-off-by: Oleksandr Kolomeiets <okl-plv@napatech.com>
---
drivers/net/ntnic/dbsconfig/ntnic_dbsconfig.c | 12 +++
drivers/net/ntnic/include/ntnic_dbs.h | 2 +
drivers/net/ntnic/nthw/dbs/nthw_dbs.c | 20 +++++
drivers/net/ntnic/ntnic_ethdev.c | 80 ++++++++++++++++++-
drivers/net/ntnic/ntnic_mod_reg.h | 2 +
5 files changed, 112 insertions(+), 4 deletions(-)
diff --git a/drivers/net/ntnic/dbsconfig/ntnic_dbsconfig.c b/drivers/net/ntnic/dbsconfig/ntnic_dbsconfig.c
index 94b0c97d27..0b049a8559 100644
--- a/drivers/net/ntnic/dbsconfig/ntnic_dbsconfig.c
+++ b/drivers/net/ntnic/dbsconfig/ntnic_dbsconfig.c
@@ -1115,6 +1115,16 @@ nthw_setup_mngd_tx_virt_queue(nthw_dbs_t *p_nthw_dbs,
return NULL;
}
+static int nthw_switch_rx_virt_queue(nthw_dbs_t *p_nthw_dbs, uint32_t index, uint32_t enable)
+{
+ return set_rx_am_data_enable(p_nthw_dbs, index, enable);
+}
+
+static int nthw_switch_tx_virt_queue(nthw_dbs_t *p_nthw_dbs, uint32_t index, uint32_t enable)
+{
+ return set_tx_am_data_enable(p_nthw_dbs, index, enable);
+}
+
static uint16_t nthw_get_rx_packets(struct nthw_virt_queue *rxvq,
uint16_t n,
struct nthw_received_packets *rp,
@@ -1419,6 +1429,8 @@ static struct sg_ops_s sg_ops = {
.nthw_release_mngd_rx_virt_queue = nthw_release_mngd_rx_virt_queue,
.nthw_setup_mngd_tx_virt_queue = nthw_setup_mngd_tx_virt_queue,
.nthw_release_mngd_tx_virt_queue = nthw_release_mngd_tx_virt_queue,
+ .nthw_switch_rx_virt_queue = nthw_switch_rx_virt_queue,
+ .nthw_switch_tx_virt_queue = nthw_switch_tx_virt_queue,
.nthw_get_rx_packets = nthw_get_rx_packets,
.nthw_release_rx_packets = nthw_release_rx_packets,
.nthw_get_tx_packets = nthw_get_tx_packets,
diff --git a/drivers/net/ntnic/include/ntnic_dbs.h b/drivers/net/ntnic/include/ntnic_dbs.h
index 247ae76d98..c35a7cb99b 100644
--- a/drivers/net/ntnic/include/ntnic_dbs.h
+++ b/drivers/net/ntnic/include/ntnic_dbs.h
@@ -267,6 +267,7 @@ int set_rx_am_data(nthw_dbs_t *p,
uint32_t host_id,
uint32_t packed,
uint32_t int_enable);
+int set_rx_am_data_enable(nthw_dbs_t *p, uint32_t index, uint32_t enable);
int set_tx_am_data(nthw_dbs_t *p,
uint32_t index,
uint64_t guest_physical_address,
@@ -274,6 +275,7 @@ int set_tx_am_data(nthw_dbs_t *p,
uint32_t host_id,
uint32_t packed,
uint32_t int_enable);
+int set_tx_am_data_enable(nthw_dbs_t *p, uint32_t index, uint32_t enable);
int set_rx_uw_data(nthw_dbs_t *p,
uint32_t index,
uint64_t guest_physical_address,
diff --git a/drivers/net/ntnic/nthw/dbs/nthw_dbs.c b/drivers/net/ntnic/nthw/dbs/nthw_dbs.c
index aed52f67f5..da64dbab48 100644
--- a/drivers/net/ntnic/nthw/dbs/nthw_dbs.c
+++ b/drivers/net/ntnic/nthw/dbs/nthw_dbs.c
@@ -618,6 +618,16 @@ int set_rx_am_data(nthw_dbs_t *p,
return 0;
}
+int set_rx_am_data_enable(nthw_dbs_t *p, uint32_t index, uint32_t enable)
+{
+ if (!p->mp_reg_rx_avail_monitor_data)
+ return -ENOTSUP;
+
+ nthw_dbs_set_shadow_rx_am_data_enable(p, index, enable);
+ flush_rx_am_data(p, index);
+ return 0;
+}
+
static void set_tx_am_data_index(nthw_dbs_t *p, uint32_t index)
{
nthw_field_set_val32(p->mp_fld_tx_avail_monitor_control_adr, index);
@@ -680,6 +690,16 @@ int set_tx_am_data(nthw_dbs_t *p,
return 0;
}
+int set_tx_am_data_enable(nthw_dbs_t *p, uint32_t index, uint32_t enable)
+{
+ if (!p->mp_reg_tx_avail_monitor_data)
+ return -ENOTSUP;
+
+ p->m_tx_am_shadow[index].enable = enable;
+ flush_tx_am_data(p, index);
+ return 0;
+}
+
static void set_rx_uw_data_index(nthw_dbs_t *p, uint32_t index)
{
nthw_field_set_val32(p->mp_fld_rx_used_writer_control_adr, index);
diff --git a/drivers/net/ntnic/ntnic_ethdev.c b/drivers/net/ntnic/ntnic_ethdev.c
index c2a507675c..d961edb903 100644
--- a/drivers/net/ntnic/ntnic_ethdev.c
+++ b/drivers/net/ntnic/ntnic_ethdev.c
@@ -1180,25 +1180,97 @@ static int dev_set_mtu_inline(struct rte_eth_dev *eth_dev, uint16_t mtu)
static int eth_rx_queue_start(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id)
{
+ if (sg_ops == NULL) {
+ NT_LOG_DBGX(DBG, NTNIC, "SG module is not initialized");
+ return -1;
+ }
+
+ struct pmd_internals *internals = eth_dev->data->dev_private;
+ struct drv_s *p_drv = internals->p_drv;
+ struct ntdrv_4ga_s *p_nt_drv = &p_drv->ntdrv;
+ nthw_dbs_t *p_nthw_dbs = p_nt_drv->adapter_info.fpga_info.mp_nthw_dbs;
+ struct ntnic_rx_queue *rx_q = &internals->rxq_scg[rx_queue_id];
+ int index = rx_q->queue.hw_id;
+
+ if (sg_ops->nthw_switch_rx_virt_queue(p_nthw_dbs, index, 1) != 0) {
+ NT_LOG_DBGX(DBG, NTNIC, "Failed to start Rx queue #%d", index);
+ return -1;
+ }
+
+ rx_q->enabled = 1;
eth_dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
return 0;
}
static int eth_rx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id)
{
+ if (sg_ops == NULL) {
+ NT_LOG_DBGX(DBG, NTNIC, "SG module is not initialized");
+ return -1;
+ }
+
+ struct pmd_internals *internals = eth_dev->data->dev_private;
+ struct drv_s *p_drv = internals->p_drv;
+ struct ntdrv_4ga_s *p_nt_drv = &p_drv->ntdrv;
+ nthw_dbs_t *p_nthw_dbs = p_nt_drv->adapter_info.fpga_info.mp_nthw_dbs;
+ struct ntnic_rx_queue *rx_q = &internals->rxq_scg[rx_queue_id];
+ int index = rx_q->queue.hw_id;
+
+ if (sg_ops->nthw_switch_rx_virt_queue(p_nthw_dbs, index, 0) != 0) {
+ NT_LOG_DBGX(DBG, NTNIC, "Failed to stop Rx queue #%d", index);
+ return -1;
+ }
+
+ rx_q->enabled = 0;
eth_dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
return 0;
}
-static int eth_tx_queue_start(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id)
+static int eth_tx_queue_start(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id)
{
- eth_dev->data->tx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
+ if (sg_ops == NULL) {
+ NT_LOG_DBGX(DBG, NTNIC, "SG module is not initialized");
+ return -1;
+ }
+
+ struct pmd_internals *internals = eth_dev->data->dev_private;
+ struct drv_s *p_drv = internals->p_drv;
+ struct ntdrv_4ga_s *p_nt_drv = &p_drv->ntdrv;
+ nthw_dbs_t *p_nthw_dbs = p_nt_drv->adapter_info.fpga_info.mp_nthw_dbs;
+ struct ntnic_tx_queue *tx_q = &internals->txq_scg[tx_queue_id];
+ int index = tx_q->queue.hw_id;
+
+ if (sg_ops->nthw_switch_tx_virt_queue(p_nthw_dbs, index, 1) != 0) {
+ NT_LOG_DBGX(DBG, NTNIC, "Failed to start Tx queue #%d", index);
+ return -1;
+ }
+
+ tx_q->enabled = 1;
+ eth_dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
return 0;
}
-static int eth_tx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id)
+static int eth_tx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id)
{
- eth_dev->data->tx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
+ if (sg_ops == NULL) {
+ NT_LOG_DBGX(DBG, NTNIC, "SG module is not initialized");
+ return -1;
+ }
+
+ struct pmd_internals *internals = eth_dev->data->dev_private;
+ struct drv_s *p_drv = internals->p_drv;
+ struct ntdrv_4ga_s *p_nt_drv = &p_drv->ntdrv;
+ nthw_dbs_t *p_nthw_dbs = p_nt_drv->adapter_info.fpga_info.mp_nthw_dbs;
+ struct ntnic_tx_queue *tx_q = &internals->txq_scg[tx_queue_id];
+ int index = tx_q->queue.hw_id;
+
+ if (sg_ops->nthw_switch_tx_virt_queue(p_nthw_dbs, index, 0) != 0) {
+ NT_LOG_DBGX(DBG, NTNIC, "Failed to stop Tx queue #%d", index);
+ return -1;
+ }
+
+ tx_q->enabled = 0;
+ eth_dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
return 0;
}
diff --git a/drivers/net/ntnic/ntnic_mod_reg.h b/drivers/net/ntnic/ntnic_mod_reg.h
index 9d14bebd7a..7f5bd5e0ec 100644
--- a/drivers/net/ntnic/ntnic_mod_reg.h
+++ b/drivers/net/ntnic/ntnic_mod_reg.h
@@ -94,6 +94,8 @@ struct sg_ops_s {
int irq_vector,
uint32_t in_order);
int (*nthw_release_mngd_tx_virt_queue)(struct nthw_virt_queue *txvq);
+ int (*nthw_switch_rx_virt_queue)(nthw_dbs_t *p_nthw_dbs, uint32_t index, uint32_t enable);
+ int (*nthw_switch_tx_virt_queue)(nthw_dbs_t *p_nthw_dbs, uint32_t index, uint32_t enable);
/*
* These functions handles both Split and Packed including merged buffers (jumbo)
*/
--
2.47.1
^ permalink raw reply [flat|nested] 5+ messages in thread
* [PATCH v1 2/4] net/ntnic: implement deferred start for Rx/Tx queues
2025-06-20 11:27 [PATCH v1 0/4] net/ntnic: implement start, stop and deferred start for Rx/Tx queues Oleksandr Kolomeiets
2025-06-20 11:27 ` [PATCH v1 1/4] net/ntnic: implement start/stop " Oleksandr Kolomeiets
@ 2025-06-20 11:27 ` Oleksandr Kolomeiets
2025-06-20 11:27 ` [PATCH v1 3/4] net/ntnic: unmap DMA during queue release Oleksandr Kolomeiets
2025-06-20 11:27 ` [PATCH v1 4/4] net/ntnic: add warning when sending on a stopped queue Oleksandr Kolomeiets
3 siblings, 0 replies; 5+ messages in thread
From: Oleksandr Kolomeiets @ 2025-06-20 11:27 UTC (permalink / raw)
To: dev; +Cc: mko-plv, sil-plv, ckm, stephen
Handle rx_deferred_start and tx_deferred_start flags
during configuration done in rx_queue_setup and tx_queue_setup,
so that marked queues do not start with dev_start.
Signed-off-by: Oleksandr Kolomeiets <okl-plv@napatech.com>
---
drivers/net/ntnic/dbsconfig/ntnic_dbsconfig.c | 46 +++++++++++--------
drivers/net/ntnic/include/ntos_drv.h | 3 +-
drivers/net/ntnic/ntnic_ethdev.c | 22 +++++----
drivers/net/ntnic/ntnic_mod_reg.h | 12 +++--
4 files changed, 52 insertions(+), 31 deletions(-)
diff --git a/drivers/net/ntnic/dbsconfig/ntnic_dbsconfig.c b/drivers/net/ntnic/dbsconfig/ntnic_dbsconfig.c
index 0b049a8559..107fe91394 100644
--- a/drivers/net/ntnic/dbsconfig/ntnic_dbsconfig.c
+++ b/drivers/net/ntnic/dbsconfig/ntnic_dbsconfig.c
@@ -369,7 +369,8 @@ static struct nthw_virt_queue *nthw_setup_rx_virt_queue(nthw_dbs_t *p_nthw_dbs,
uint32_t host_id,
uint32_t header,
uint32_t vq_type,
- int irq_vector)
+ int irq_vector,
+ uint8_t rx_deferred_start)
{
uint32_t qs = dbs_qsize_log2(queue_size);
uint32_t int_enable;
@@ -430,7 +431,8 @@ static struct nthw_virt_queue *nthw_setup_rx_virt_queue(nthw_dbs_t *p_nthw_dbs,
* 2. Configure the DBS.RX_AM_DATA memory and enable the queues you plan to use;
* good idea to initialize all DBS_RX_QUEUES entries.
*/
- if (set_rx_am_data(p_nthw_dbs, index, (uint64_t)avail_struct_phys_addr, RX_AM_ENABLE,
+ uint32_t enable = rx_deferred_start ? RX_AM_DISABLE : RX_AM_ENABLE;
+ if (set_rx_am_data(p_nthw_dbs, index, (uint64_t)avail_struct_phys_addr, enable,
host_id, 0, irq_vector >= 0 ? 1 : 0) != 0) {
return NULL;
}
@@ -698,7 +700,8 @@ static struct nthw_virt_queue *nthw_setup_tx_virt_queue(nthw_dbs_t *p_nthw_dbs,
uint32_t header,
uint32_t vq_type,
int irq_vector,
- uint32_t in_order)
+ uint32_t in_order,
+ uint8_t tx_deferred_start)
{
uint32_t int_enable;
uint32_t vec;
@@ -760,8 +763,9 @@ static struct nthw_virt_queue *nthw_setup_tx_virt_queue(nthw_dbs_t *p_nthw_dbs,
* kernel).
*/
if (irq_vector < 0) {
+ uint32_t enable = tx_deferred_start ? TX_AM_DISABLE : TX_AM_ENABLE;
if (set_tx_am_data(p_nthw_dbs, index, (uint64_t)avail_struct_phys_addr,
- TX_AM_ENABLE, host_id, 0, 0) != 0) {
+ enable, host_id, 0, 0) != 0) {
return NULL;
}
}
@@ -794,7 +798,8 @@ nthw_setup_mngd_rx_virt_queue_split(nthw_dbs_t *p_nthw_dbs,
uint32_t header,
struct nthw_memory_descriptor *p_virt_struct_area,
struct nthw_memory_descriptor *p_packet_buffers,
- int irq_vector)
+ int irq_vector,
+ uint8_t rx_deferred_start)
{
struct virtq_struct_layout_s virtq_struct_layout = dbs_calc_struct_layout(queue_size);
@@ -831,7 +836,7 @@ nthw_setup_mngd_rx_virt_queue_split(nthw_dbs_t *p_nthw_dbs,
virtq_struct_layout.used_offset,
(char *)p_virt_struct_area->phys_addr +
virtq_struct_layout.desc_offset,
- (uint16_t)queue_size, host_id, header, SPLIT_RING, irq_vector);
+ (uint16_t)queue_size, host_id, header, SPLIT_RING, irq_vector, rx_deferred_start);
rxvq[index].usage = NTHW_VIRTQ_MANAGED;
@@ -849,7 +854,8 @@ nthw_setup_mngd_tx_virt_queue_split(nthw_dbs_t *p_nthw_dbs,
int irq_vector,
uint32_t in_order,
struct nthw_memory_descriptor *p_virt_struct_area,
- struct nthw_memory_descriptor *p_packet_buffers)
+ struct nthw_memory_descriptor *p_packet_buffers,
+ uint8_t tx_deferred_start)
{
struct virtq_struct_layout_s virtq_struct_layout = dbs_calc_struct_layout(queue_size);
@@ -889,7 +895,7 @@ nthw_setup_mngd_tx_virt_queue_split(nthw_dbs_t *p_nthw_dbs,
(char *)p_virt_struct_area->phys_addr +
virtq_struct_layout.desc_offset,
(uint16_t)queue_size, host_id, port, virtual_port, header,
- SPLIT_RING, irq_vector, in_order);
+ SPLIT_RING, irq_vector, in_order, tx_deferred_start);
txvq[index].usage = NTHW_VIRTQ_MANAGED;
@@ -977,7 +983,8 @@ nthw_setup_managed_rx_virt_queue_packed(nthw_dbs_t *p_nthw_dbs,
uint32_t header,
struct nthw_memory_descriptor *p_virt_struct_area,
struct nthw_memory_descriptor *p_packet_buffers,
- int irq_vector)
+ int irq_vector,
+ uint8_t rx_deferred_start)
{
struct pvirtq_struct_layout_s pvirtq_layout;
struct nthw_virt_queue *vq = &rxvq[index];
@@ -996,7 +1003,7 @@ nthw_setup_managed_rx_virt_queue_packed(nthw_dbs_t *p_nthw_dbs,
(void *)((uintptr_t)p_virt_struct_area->phys_addr +
pvirtq_layout.device_event_offset),
p_virt_struct_area->phys_addr, (uint16_t)queue_size, host_id,
- header, PACKED_RING, irq_vector);
+ header, PACKED_RING, irq_vector, rx_deferred_start);
vq->usage = NTHW_VIRTQ_MANAGED;
return vq;
@@ -1013,7 +1020,8 @@ nthw_setup_managed_tx_virt_queue_packed(nthw_dbs_t *p_nthw_dbs,
int irq_vector,
uint32_t in_order,
struct nthw_memory_descriptor *p_virt_struct_area,
- struct nthw_memory_descriptor *p_packet_buffers)
+ struct nthw_memory_descriptor *p_packet_buffers,
+ uint8_t tx_deferred_start)
{
struct pvirtq_struct_layout_s pvirtq_layout;
struct nthw_virt_queue *vq = &txvq[index];
@@ -1030,7 +1038,7 @@ nthw_setup_managed_tx_virt_queue_packed(nthw_dbs_t *p_nthw_dbs,
(void *)((uintptr_t)p_virt_struct_area->phys_addr +
pvirtq_layout.device_event_offset),
p_virt_struct_area->phys_addr, (uint16_t)queue_size, host_id,
- port, virtual_port, header, PACKED_RING, irq_vector, in_order);
+ port, virtual_port, header, PACKED_RING, irq_vector, in_order, tx_deferred_start);
vq->usage = NTHW_VIRTQ_MANAGED;
return vq;
@@ -1052,18 +1060,19 @@ nthw_setup_mngd_rx_virt_queue(nthw_dbs_t *p_nthw_dbs,
struct nthw_memory_descriptor *p_virt_struct_area,
struct nthw_memory_descriptor *p_packet_buffers,
uint32_t vq_type,
- int irq_vector)
+ int irq_vector,
+ uint8_t rx_deferred_start)
{
switch (vq_type) {
case SPLIT_RING:
return nthw_setup_mngd_rx_virt_queue_split(p_nthw_dbs, index, queue_size,
host_id, header, p_virt_struct_area,
- p_packet_buffers, irq_vector);
+ p_packet_buffers, irq_vector, rx_deferred_start);
case PACKED_RING:
return nthw_setup_managed_rx_virt_queue_packed(p_nthw_dbs, index, queue_size,
host_id, header, p_virt_struct_area,
- p_packet_buffers, irq_vector);
+ p_packet_buffers, irq_vector, rx_deferred_start);
default:
break;
@@ -1091,7 +1100,8 @@ nthw_setup_mngd_tx_virt_queue(nthw_dbs_t *p_nthw_dbs,
struct nthw_memory_descriptor *p_packet_buffers,
uint32_t vq_type,
int irq_vector,
- uint32_t in_order)
+ uint32_t in_order,
+ uint8_t tx_deferred_start)
{
switch (vq_type) {
case SPLIT_RING:
@@ -1099,14 +1109,14 @@ nthw_setup_mngd_tx_virt_queue(nthw_dbs_t *p_nthw_dbs,
host_id, port, virtual_port, header,
irq_vector, in_order,
p_virt_struct_area,
- p_packet_buffers);
+ p_packet_buffers, tx_deferred_start);
case PACKED_RING:
return nthw_setup_managed_tx_virt_queue_packed(p_nthw_dbs, index, queue_size,
host_id, port, virtual_port, header,
irq_vector, in_order,
p_virt_struct_area,
- p_packet_buffers);
+ p_packet_buffers, tx_deferred_start);
default:
break;
diff --git a/drivers/net/ntnic/include/ntos_drv.h b/drivers/net/ntnic/include/ntos_drv.h
index f6ce442d17..cef3c5c277 100644
--- a/drivers/net/ntnic/include/ntos_drv.h
+++ b/drivers/net/ntnic/include/ntos_drv.h
@@ -69,7 +69,7 @@ struct __rte_cache_aligned ntnic_rx_queue {
nt_meta_port_type_t type;
uint32_t port; /* Rx port for this queue */
enum fpga_info_profile profile; /* Inline / Capture */
-
+ uint8_t rx_deferred_start;
};
struct __rte_cache_aligned ntnic_tx_queue {
@@ -89,6 +89,7 @@ struct __rte_cache_aligned ntnic_tx_queue {
unsigned long err_pkts; /* Tx error packet stat */
int enabled; /* Enabling/disabling of this queue */
enum fpga_info_profile profile; /* Inline / Capture */
+ uint8_t tx_deferred_start;
};
struct nt_mtr_profile {
diff --git a/drivers/net/ntnic/ntnic_ethdev.c b/drivers/net/ntnic/ntnic_ethdev.c
index d961edb903..d875e7c236 100644
--- a/drivers/net/ntnic/ntnic_ethdev.c
+++ b/drivers/net/ntnic/ntnic_ethdev.c
@@ -995,7 +995,7 @@ static int eth_rx_scg_queue_setup(struct rte_eth_dev *eth_dev,
uint16_t rx_queue_id,
uint16_t nb_rx_desc __rte_unused,
unsigned int socket_id __rte_unused,
- const struct rte_eth_rxconf *rx_conf __rte_unused,
+ const struct rte_eth_rxconf *rx_conf,
struct rte_mempool *mb_pool)
{
NT_LOG_DBGX(DBG, NTNIC, "Rx queue setup");
@@ -1029,7 +1029,8 @@ static int eth_rx_scg_queue_setup(struct rte_eth_dev *eth_dev,
mbp_priv = rte_mempool_get_priv(rx_q->mb_pool);
rx_q->buf_size = (uint16_t)(mbp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM);
- rx_q->enabled = 1;
+ rx_q->enabled = !rx_conf->rx_deferred_start;
+ rx_q->rx_deferred_start = rx_conf->rx_deferred_start;
if (allocate_hw_virtio_queues(eth_dev, EXCEPTION_PATH_HID, &rx_q->hwq,
SG_NB_HW_RX_DESCRIPTORS, SG_HW_RX_PKT_BUFFER_SIZE) < 0)
@@ -1048,7 +1049,8 @@ static int eth_rx_scg_queue_setup(struct rte_eth_dev *eth_dev,
&rx_q->hwq.virt_queues_ctrl,
rx_q->hwq.pkt_buffers,
SPLIT_RING,
- -1);
+ -1,
+ rx_conf->rx_deferred_start);
NT_LOG(DBG, NTNIC, "(%" PRIu32 ") NTNIC RX OVS-SW queues successfully setup",
internals->port);
@@ -1060,7 +1062,7 @@ static int eth_tx_scg_queue_setup(struct rte_eth_dev *eth_dev,
uint16_t tx_queue_id,
uint16_t nb_tx_desc __rte_unused,
unsigned int socket_id __rte_unused,
- const struct rte_eth_txconf *tx_conf __rte_unused)
+ const struct rte_eth_txconf *tx_conf)
{
const struct port_ops *port_ops = get_port_ops();
@@ -1141,9 +1143,11 @@ static int eth_tx_scg_queue_setup(struct rte_eth_dev *eth_dev,
tx_q->hwq.pkt_buffers,
SPLIT_RING,
-1,
- IN_ORDER);
+ IN_ORDER,
+ tx_conf->tx_deferred_start);
- tx_q->enabled = 1;
+ tx_q->enabled = !tx_conf->tx_deferred_start;
+ tx_q->tx_deferred_start = tx_conf->tx_deferred_start;
NT_LOG(DBG, NTNIC, "(%" PRIu32 ") NTNIC TX OVS-SW queues successfully setup",
internals->port);
@@ -1365,10 +1369,12 @@ eth_dev_start(struct rte_eth_dev *eth_dev)
uint q;
for (q = 0; q < internals->nb_rx_queues; q++)
- eth_rx_queue_start(eth_dev, q);
+ if (!internals->rxq_scg[q].rx_deferred_start)
+ eth_rx_queue_start(eth_dev, q);
for (q = 0; q < internals->nb_tx_queues; q++)
- eth_tx_queue_start(eth_dev, q);
+ if (!internals->txq_scg[q].tx_deferred_start)
+ eth_tx_queue_start(eth_dev, q);
if (internals->type == PORT_TYPE_VIRTUAL || internals->type == PORT_TYPE_OVERRIDE) {
eth_dev->data->dev_link.link_status = RTE_ETH_LINK_UP;
diff --git a/drivers/net/ntnic/ntnic_mod_reg.h b/drivers/net/ntnic/ntnic_mod_reg.h
index 7f5bd5e0ec..26efec1c65 100644
--- a/drivers/net/ntnic/ntnic_mod_reg.h
+++ b/drivers/net/ntnic/ntnic_mod_reg.h
@@ -37,7 +37,8 @@ struct sg_ops_s {
uint32_t host_id,
uint32_t header,
uint32_t vq_type,
- int irq_vector);
+ int irq_vector,
+ uint8_t rx_deferred_start);
struct nthw_virt_queue *(*nthw_setup_tx_virt_queue)(nthw_dbs_t *p_nthw_dbs,
uint32_t index,
uint16_t start_idx,
@@ -52,7 +53,8 @@ struct sg_ops_s {
uint32_t header,
uint32_t vq_type,
int irq_vector,
- uint32_t in_order);
+ uint32_t in_order,
+ uint8_t tx_deferred_start);
struct nthw_virt_queue *(*nthw_setup_mngd_rx_virt_queue)(nthw_dbs_t *p_nthw_dbs,
uint32_t index,
uint32_t queue_size,
@@ -70,7 +72,8 @@ struct sg_ops_s {
*/
struct nthw_memory_descriptor *p_packet_buffers,
uint32_t vq_type,
- int irq_vector);
+ int irq_vector,
+ uint8_t rx_deferred_start);
int (*nthw_release_mngd_rx_virt_queue)(struct nthw_virt_queue *rxvq);
struct nthw_virt_queue *(*nthw_setup_mngd_tx_virt_queue)(nthw_dbs_t *p_nthw_dbs,
uint32_t index,
@@ -92,7 +95,8 @@ struct sg_ops_s {
struct nthw_memory_descriptor *p_packet_buffers,
uint32_t vq_type,
int irq_vector,
- uint32_t in_order);
+ uint32_t in_order,
+ uint8_t tx_deferred_start);
int (*nthw_release_mngd_tx_virt_queue)(struct nthw_virt_queue *txvq);
int (*nthw_switch_rx_virt_queue)(nthw_dbs_t *p_nthw_dbs, uint32_t index, uint32_t enable);
int (*nthw_switch_tx_virt_queue)(nthw_dbs_t *p_nthw_dbs, uint32_t index, uint32_t enable);
--
2.47.1
^ permalink raw reply [flat|nested] 5+ messages in thread
* [PATCH v1 3/4] net/ntnic: unmap DMA during queue release
2025-06-20 11:27 [PATCH v1 0/4] net/ntnic: implement start, stop and deferred start for Rx/Tx queues Oleksandr Kolomeiets
2025-06-20 11:27 ` [PATCH v1 1/4] net/ntnic: implement start/stop " Oleksandr Kolomeiets
2025-06-20 11:27 ` [PATCH v1 2/4] net/ntnic: implement deferred start " Oleksandr Kolomeiets
@ 2025-06-20 11:27 ` Oleksandr Kolomeiets
2025-06-20 11:27 ` [PATCH v1 4/4] net/ntnic: add warning when sending on a stopped queue Oleksandr Kolomeiets
3 siblings, 0 replies; 5+ messages in thread
From: Oleksandr Kolomeiets @ 2025-06-20 11:27 UTC (permalink / raw)
To: dev; +Cc: mko-plv, sil-plv, ckm, stephen
Perform unmapping in a default container, which is used by queues.
Handle multiple mappings when IOMMU is unoptimized.
Signed-off-by: Oleksandr Kolomeiets <okl-plv@napatech.com>
---
drivers/net/ntnic/include/ntos_drv.h | 1 +
drivers/net/ntnic/ntnic_ethdev.c | 26 ++++++++++++++++++++++++--
drivers/net/ntnic/ntnic_vfio.c | 3 ---
3 files changed, 25 insertions(+), 5 deletions(-)
diff --git a/drivers/net/ntnic/include/ntos_drv.h b/drivers/net/ntnic/include/ntos_drv.h
index cef3c5c277..047c077057 100644
--- a/drivers/net/ntnic/include/ntos_drv.h
+++ b/drivers/net/ntnic/include/ntos_drv.h
@@ -51,6 +51,7 @@ struct nthw_memory_descriptor {
struct hwq_s {
int vf_num;
struct nthw_memory_descriptor virt_queues_ctrl;
+ struct nthw_memory_descriptor pkt_buffers_ctrl;
struct nthw_memory_descriptor *pkt_buffers;
};
diff --git a/drivers/net/ntnic/ntnic_ethdev.c b/drivers/net/ntnic/ntnic_ethdev.c
index d875e7c236..79ef9e7e7c 100644
--- a/drivers/net/ntnic/ntnic_ethdev.c
+++ b/drivers/net/ntnic/ntnic_ethdev.c
@@ -879,6 +879,10 @@ static int allocate_hw_virtio_queues(struct rte_eth_dev *eth_dev, int vf_num, st
if (res != 0)
return -1;
+ hwq->pkt_buffers_ctrl.virt_addr = virt_addr;
+ hwq->pkt_buffers_ctrl.phys_addr = (void *)iova_addr;
+ hwq->pkt_buffers_ctrl.len = size;
+
for (i = 0; i < num_descr; i++) {
hwq->pkt_buffers[i].virt_addr =
(void *)((char *)virt_addr + ((uint64_t)(i) * buf_size));
@@ -900,9 +904,13 @@ static int allocate_hw_virtio_queues(struct rte_eth_dev *eth_dev, int vf_num, st
hwq->vf_num = vf_num;
hwq->virt_queues_ctrl.virt_addr = virt;
hwq->virt_queues_ctrl.phys_addr = (void *)(iova_addr);
- hwq->virt_queues_ctrl.len = 0x100000;
+ hwq->virt_queues_ctrl.len = ONE_G_SIZE;
iova_addr += 0x100000;
+ hwq->pkt_buffers_ctrl.virt_addr = NULL;
+ hwq->pkt_buffers_ctrl.phys_addr = NULL;
+ hwq->pkt_buffers_ctrl.len = 0;
+
NT_LOG(DBG, NTNIC,
"VFIO MMAP: virt_addr=%p phys_addr=%p size=%" PRIX32 " hpa=%" PRIX64 "",
hwq->virt_queues_ctrl.virt_addr, hwq->virt_queues_ctrl.phys_addr,
@@ -948,13 +956,27 @@ static int deallocate_hw_virtio_queues(struct hwq_s *hwq)
void *virt = hwq->virt_queues_ctrl.virt_addr;
int res = nt_vfio_dma_unmap(vf_num, hwq->virt_queues_ctrl.virt_addr,
- (uint64_t)hwq->virt_queues_ctrl.phys_addr, ONE_G_SIZE);
+ (uint64_t)hwq->virt_queues_ctrl.phys_addr, hwq->virt_queues_ctrl.len);
if (res != 0) {
NT_LOG(ERR, NTNIC, "VFIO UNMMAP FAILED! res %i, vf_num %i", res, vf_num);
return -1;
}
+ if (hwq->pkt_buffers_ctrl.virt_addr != NULL &&
+ hwq->pkt_buffers_ctrl.phys_addr != NULL &&
+ hwq->pkt_buffers_ctrl.len > 0) {
+ int res = nt_vfio_dma_unmap(vf_num,
+ hwq->pkt_buffers_ctrl.virt_addr,
+ (uint64_t)hwq->pkt_buffers_ctrl.phys_addr,
+ hwq->pkt_buffers_ctrl.len);
+
+ if (res != 0) {
+ NT_LOG(ERR, NTNIC, "VFIO UNMMAP FAILED! res %i, vf_num %i", res, vf_num);
+ return -1;
+ }
+ }
+
release_hw_virtio_queues(hwq);
rte_free(hwq->pkt_buffers);
rte_free(virt);
diff --git a/drivers/net/ntnic/ntnic_vfio.c b/drivers/net/ntnic/ntnic_vfio.c
index 8d955e8342..1031b3cf67 100644
--- a/drivers/net/ntnic/ntnic_vfio.c
+++ b/drivers/net/ntnic/ntnic_vfio.c
@@ -211,9 +211,6 @@ nt_vfio_dma_unmap(int vf_num, void *virt_addr, uint64_t iova_addr, uint64_t size
return -1;
}
- if (vfio->container_fd == -1)
- return 0;
-
int res = rte_vfio_container_dma_unmap(vfio->container_fd, gp_virt_base, iova_addr, size);
if (res != 0) {
--
2.47.1
^ permalink raw reply [flat|nested] 5+ messages in thread
* [PATCH v1 4/4] net/ntnic: add warning when sending on a stopped queue
2025-06-20 11:27 [PATCH v1 0/4] net/ntnic: implement start, stop and deferred start for Rx/Tx queues Oleksandr Kolomeiets
` (2 preceding siblings ...)
2025-06-20 11:27 ` [PATCH v1 3/4] net/ntnic: unmap DMA during queue release Oleksandr Kolomeiets
@ 2025-06-20 11:27 ` Oleksandr Kolomeiets
3 siblings, 0 replies; 5+ messages in thread
From: Oleksandr Kolomeiets @ 2025-06-20 11:27 UTC (permalink / raw)
To: dev; +Cc: mko-plv, sil-plv, ckm, stephen
When sending a burst of output packets on a stopped transmit queue,
the packets are written to a memory mapped address.
On queue start the packets are processed and transmitted by the NIC.
Signed-off-by: Oleksandr Kolomeiets <okl-plv@napatech.com>
---
drivers/net/ntnic/ntnic_ethdev.c | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/drivers/net/ntnic/ntnic_ethdev.c b/drivers/net/ntnic/ntnic_ethdev.c
index 79ef9e7e7c..4145128d11 100644
--- a/drivers/net/ntnic/ntnic_ethdev.c
+++ b/drivers/net/ntnic/ntnic_ethdev.c
@@ -694,6 +694,10 @@ static uint16_t eth_dev_tx_scg(void *queue, struct rte_mbuf **bufs, uint16_t nb_
int pkts_sent = 0;
uint16_t nb_segs_arr[MAX_TX_PACKETS];
+ if (!tx_q->enabled)
+ NT_LOG(WRN, NTNIC, "Trying to send a burst of output packets "
+ "on a stopped transmit queue of an Ethernet device");
+
if (nb_pkts > MAX_TX_PACKETS)
nb_pkts = MAX_TX_PACKETS;
--
2.47.1
^ permalink raw reply [flat|nested] 5+ messages in thread
end of thread, other threads:[~2025-06-20 11:28 UTC | newest]
Thread overview: 5+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2025-06-20 11:27 [PATCH v1 0/4] net/ntnic: implement start, stop and deferred start for Rx/Tx queues Oleksandr Kolomeiets
2025-06-20 11:27 ` [PATCH v1 1/4] net/ntnic: implement start/stop " Oleksandr Kolomeiets
2025-06-20 11:27 ` [PATCH v1 2/4] net/ntnic: implement deferred start " Oleksandr Kolomeiets
2025-06-20 11:27 ` [PATCH v1 3/4] net/ntnic: unmap DMA during queue release Oleksandr Kolomeiets
2025-06-20 11:27 ` [PATCH v1 4/4] net/ntnic: add warning when sending on a stopped queue Oleksandr Kolomeiets
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).