* [dpdk-stable] [PATCH 1/3] [18.11] net/mlx5: fix recursive inclusion of header file
@ 2020-07-03 15:41 Viacheslav Ovsiienko
2020-07-03 15:41 ` [dpdk-stable] [PATCH 2/3] [18.11] net/mlx5: remove device register remap Viacheslav Ovsiienko
2020-07-03 15:41 ` [dpdk-stable] [PATCH 3/3] [18.11] net/mlx4: " Viacheslav Ovsiienko
0 siblings, 2 replies; 3+ messages in thread
From: Viacheslav Ovsiienko @ 2020-07-03 15:41 UTC (permalink / raw)
To: stable; +Cc: ktraynor, Yongseok Koh
From: Yongseok Koh <yskoh@mellanox.com>
mlx5.h includes mlx5_rxtx.h and mlx5_rxtx.h includes mlx5.h recursively.
Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
Acked-by: Shahaf Shuler <shahafs@mellanox.com>
---
drivers/net/mlx5/mlx5.h | 1 -
drivers/net/mlx5/mlx5_flow.c | 3 ++-
drivers/net/mlx5/mlx5_flow_dv.c | 3 ++-
drivers/net/mlx5/mlx5_flow_tcf.c | 1 +
drivers/net/mlx5/mlx5_flow_verbs.c | 3 ++-
drivers/net/mlx5/mlx5_vlan.c | 3 ++-
6 files changed, 9 insertions(+), 5 deletions(-)
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index c91907d..d70ab06 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -33,7 +33,6 @@
#include "mlx5_utils.h"
#include "mlx5_mr.h"
-#include "mlx5_rxtx.h"
#include "mlx5_autoconf.h"
#include "mlx5_defs.h"
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 7a67601..30d5b66 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -31,8 +31,9 @@
#include "mlx5.h"
#include "mlx5_defs.h"
#include "mlx5_prm.h"
-#include "mlx5_glue.h"
#include "mlx5_flow.h"
+#include "mlx5_glue.h"
+#include "mlx5_rxtx.h"
/* Dev ops structure defined in mlx5.c */
extern const struct eth_dev_ops mlx5_dev_ops;
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 2586728..460461a 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -29,9 +29,10 @@
#include "mlx5.h"
#include "mlx5_defs.h"
-#include "mlx5_prm.h"
#include "mlx5_glue.h"
+#include "mlx5_prm.h"
#include "mlx5_flow.h"
+#include "mlx5_rxtx.h"
#ifdef HAVE_IBV_FLOW_DV_SUPPORT
diff --git a/drivers/net/mlx5/mlx5_flow_tcf.c b/drivers/net/mlx5/mlx5_flow_tcf.c
index e9c4fe9..8456e03 100644
--- a/drivers/net/mlx5/mlx5_flow_tcf.c
+++ b/drivers/net/mlx5/mlx5_flow_tcf.c
@@ -31,6 +31,7 @@
#include <rte_cycles.h>
#include "mlx5.h"
+#include "mlx5_prm.h"
#include "mlx5_flow.h"
#include "mlx5_autoconf.h"
diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c b/drivers/net/mlx5/mlx5_flow_verbs.c
index eb74029..f4b43d5 100644
--- a/drivers/net/mlx5/mlx5_flow_verbs.c
+++ b/drivers/net/mlx5/mlx5_flow_verbs.c
@@ -30,8 +30,9 @@
#include "mlx5.h"
#include "mlx5_defs.h"
#include "mlx5_prm.h"
-#include "mlx5_glue.h"
#include "mlx5_flow.h"
+#include "mlx5_glue.h"
+#include "mlx5_rxtx.h"
#define VERBS_SPEC_INNER(item_flags) \
(!!((item_flags) & MLX5_FLOW_LAYER_TUNNEL) ? IBV_FLOW_SPEC_INNER : 0)
diff --git a/drivers/net/mlx5/mlx5_vlan.c b/drivers/net/mlx5/mlx5_vlan.c
index 6568a3a..4004930 100644
--- a/drivers/net/mlx5/mlx5_vlan.c
+++ b/drivers/net/mlx5/mlx5_vlan.c
@@ -27,10 +27,11 @@
#include <rte_ethdev_driver.h>
#include <rte_common.h>
-#include "mlx5_utils.h"
#include "mlx5.h"
#include "mlx5_autoconf.h"
#include "mlx5_glue.h"
+#include "mlx5_rxtx.h"
+#include "mlx5_utils.h"
/**
* DPDK callback to configure a VLAN filter.
--
1.8.3.1
^ permalink raw reply [flat|nested] 3+ messages in thread
* [dpdk-stable] [PATCH 2/3] [18.11] net/mlx5: remove device register remap
2020-07-03 15:41 [dpdk-stable] [PATCH 1/3] [18.11] net/mlx5: fix recursive inclusion of header file Viacheslav Ovsiienko
@ 2020-07-03 15:41 ` Viacheslav Ovsiienko
2020-07-03 15:41 ` [dpdk-stable] [PATCH 3/3] [18.11] net/mlx4: " Viacheslav Ovsiienko
1 sibling, 0 replies; 3+ messages in thread
From: Viacheslav Ovsiienko @ 2020-07-03 15:41 UTC (permalink / raw)
To: stable; +Cc: ktraynor, Yongseok Koh
From: Yongseok Koh <yskoh@mellanox.com>
UAR (User Access Region) register does not need to be remapped for
primary process but it should be remapped only for secondary process.
UAR register table is in the process private structure in
rte_eth_devices[],
(struct mlx5_proc_priv *)rte_eth_devices[port_id].process_private
The actual UAR table follows the data structure and the table is used
for both Tx and Rx.
For Tx, BlueFlame in UAR is used to ring the doorbell.
MLX5_TX_BFREG(txq) is defined to get a register for the txq. Processes
access its own private data to acquire the register from the UAR table.
For Rx, the doorbell in UAR is required in arming CQ event. However, it
is a known issue that the register isn't remapped for secondary process.
Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
---
drivers/net/mlx5/mlx5.c | 176 +++++++++++-------------------------
drivers/net/mlx5/mlx5.h | 12 +++
drivers/net/mlx5/mlx5_defs.h | 10 --
drivers/net/mlx5/mlx5_ethdev.c | 3 +
drivers/net/mlx5/mlx5_rxtx.h | 13 ++-
drivers/net/mlx5/mlx5_trigger.c | 6 --
drivers/net/mlx5/mlx5_txq.c | 196 ++++++++++++++++++++++++----------------
7 files changed, 193 insertions(+), 223 deletions(-)
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index f25f5b8..6f853dd 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -243,6 +243,54 @@
}
/**
+ * Initialize process private data structure.
+ *
+ * @param dev
+ * Pointer to Ethernet device structure.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_proc_priv_init(struct rte_eth_dev *dev)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_proc_priv *ppriv;
+ size_t ppriv_size;
+
+ /*
+ * UAR register table follows the process private structure. BlueFlame
+ * registers for Tx queues are stored in the table.
+ */
+ ppriv_size =
+ sizeof(struct mlx5_proc_priv) + priv->txqs_n * sizeof(void *);
+ ppriv = rte_malloc_socket("mlx5_proc_priv", ppriv_size,
+ RTE_CACHE_LINE_SIZE, dev->device->numa_node);
+ if (!ppriv) {
+ rte_errno = ENOMEM;
+ return -rte_errno;
+ }
+ ppriv->uar_table_sz = ppriv_size;
+ dev->process_private = ppriv;
+ return 0;
+}
+
+/**
+ * Un-initialize process private data structure.
+ *
+ * @param dev
+ * Pointer to Ethernet device structure.
+ */
+static void
+mlx5_proc_priv_uninit(struct rte_eth_dev *dev)
+{
+ if (!dev->process_private)
+ return;
+ rte_free(dev->process_private);
+ dev->process_private = NULL;
+}
+
+/**
* DPDK callback to close the device.
*
* Destroy all queues and objects, free memory.
@@ -283,6 +331,7 @@
priv->txqs_n = 0;
priv->txqs = NULL;
}
+ mlx5_proc_priv_uninit(dev);
mlx5_mprq_free_mp(dev);
mlx5_mr_release(dev);
if (priv->pd != NULL) {
@@ -600,121 +649,6 @@
static struct rte_pci_driver mlx5_driver;
-/*
- * Reserved UAR address space for TXQ UAR(hw doorbell) mapping, process
- * local resource used by both primary and secondary to avoid duplicate
- * reservation.
- * The space has to be available on both primary and secondary process,
- * TXQ UAR maps to this area using fixed mmap w/o double check.
- */
-static void *uar_base;
-
-static int
-find_lower_va_bound(const struct rte_memseg_list *msl,
- const struct rte_memseg *ms, void *arg)
-{
- void **addr = arg;
-
- if (msl->external)
- return 0;
- if (*addr == NULL)
- *addr = ms->addr;
- else
- *addr = RTE_MIN(*addr, ms->addr);
-
- return 0;
-}
-
-/**
- * Reserve UAR address space for primary process.
- *
- * @param[in] dev
- * Pointer to Ethernet device.
- *
- * @return
- * 0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_uar_init_primary(struct rte_eth_dev *dev)
-{
- struct mlx5_priv *priv = dev->data->dev_private;
- void *addr = (void *)0;
-
- if (uar_base) { /* UAR address space mapped. */
- priv->uar_base = uar_base;
- return 0;
- }
- /* find out lower bound of hugepage segments */
- rte_memseg_walk(find_lower_va_bound, &addr);
-
- /* keep distance to hugepages to minimize potential conflicts. */
- addr = RTE_PTR_SUB(addr, (uintptr_t)(MLX5_UAR_OFFSET + MLX5_UAR_SIZE));
- /* anonymous mmap, no real memory consumption. */
- addr = mmap(addr, MLX5_UAR_SIZE,
- PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
- if (addr == MAP_FAILED) {
- DRV_LOG(ERR,
- "port %u failed to reserve UAR address space, please"
- " adjust MLX5_UAR_SIZE or try --base-virtaddr",
- dev->data->port_id);
- rte_errno = ENOMEM;
- return -rte_errno;
- }
- /* Accept either same addr or a new addr returned from mmap if target
- * range occupied.
- */
- DRV_LOG(INFO, "port %u reserved UAR address space: %p",
- dev->data->port_id, addr);
- priv->uar_base = addr; /* for primary and secondary UAR re-mmap. */
- uar_base = addr; /* process local, don't reserve again. */
- return 0;
-}
-
-/**
- * Reserve UAR address space for secondary process, align with
- * primary process.
- *
- * @param[in] dev
- * Pointer to Ethernet device.
- *
- * @return
- * 0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_uar_init_secondary(struct rte_eth_dev *dev)
-{
- struct mlx5_priv *priv = dev->data->dev_private;
- void *addr;
-
- assert(priv->uar_base);
- if (uar_base) { /* already reserved. */
- assert(uar_base == priv->uar_base);
- return 0;
- }
- /* anonymous mmap, no real memory consumption. */
- addr = mmap(priv->uar_base, MLX5_UAR_SIZE,
- PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
- if (addr == MAP_FAILED) {
- DRV_LOG(ERR, "port %u UAR mmap failed: %p size: %llu",
- dev->data->port_id, priv->uar_base, MLX5_UAR_SIZE);
- rte_errno = ENXIO;
- return -rte_errno;
- }
- if (priv->uar_base != addr) {
- DRV_LOG(ERR,
- "port %u UAR address %p size %llu occupied, please"
- " adjust MLX5_UAR_OFFSET or try EAL parameter"
- " --base-virtaddr",
- dev->data->port_id, priv->uar_base, MLX5_UAR_SIZE);
- rte_errno = ENXIO;
- return -rte_errno;
- }
- uar_base = addr; /* process local, don't reserve again */
- DRV_LOG(INFO, "port %u reserved UAR address space: %p",
- dev->data->port_id, addr);
- return 0;
-}
-
/**
* Spawn an Ethernet device from Verbs information.
*
@@ -920,7 +854,7 @@
}
eth_dev->device = dpdk_dev;
eth_dev->dev_ops = &mlx5_dev_sec_ops;
- err = mlx5_uar_init_secondary(eth_dev);
+ err = mlx5_proc_priv_init(eth_dev);
if (err) {
err = rte_errno;
goto error;
@@ -932,7 +866,7 @@
goto error;
}
/* Remap UAR for Tx queues. */
- err = mlx5_tx_uar_remap(eth_dev, err);
+ err = mlx5_tx_uar_init_secondary(eth_dev, err);
if (err) {
err = rte_errno;
goto error;
@@ -1147,11 +1081,6 @@
priv->dev_data = eth_dev->data;
eth_dev->data->mac_addrs = priv->mac;
eth_dev->device = dpdk_dev;
- err = mlx5_uar_init_primary(eth_dev);
- if (err) {
- err = rte_errno;
- goto error;
- }
/* Configure the first MAC address by default. */
if (mlx5_get_mac(eth_dev, &mac.addr_bytes)) {
DRV_LOG(ERR,
@@ -1282,12 +1211,11 @@
if (own_domain_id)
claim_zero(rte_eth_switch_domain_free(priv->domain_id));
rte_free(priv);
- if (eth_dev != NULL)
- eth_dev->data->dev_private = NULL;
}
if (pd)
claim_zero(mlx5_glue->dealloc_pd(pd));
if (eth_dev != NULL) {
+ mlx5_proc_priv_uninit(eth_dev);
/* mac_addrs must not be freed alone because part of dev_private */
eth_dev->data->mac_addrs = NULL;
rte_eth_dev_release_port(eth_dev);
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index d70ab06..cf38b65 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -188,6 +188,17 @@ struct mlx5_drop {
struct mlx5_flow_tcf_context;
+/* Per-process private structure. */
+struct mlx5_proc_priv {
+ size_t uar_table_sz;
+ /* Size of UAR register table. */
+ void *uar_table[];
+ /* Table of UAR registers for each process. */
+};
+
+#define MLX5_PROC_PRIV(port_id) \
+ ((struct mlx5_proc_priv *)rte_eth_devices[port_id].process_private)
+
struct mlx5_priv {
LIST_ENTRY(mlx5_priv) mem_event_cb;
/**< Called by memory event callback. */
@@ -266,6 +277,7 @@ struct mlx5_priv {
/* mlx5.c */
int mlx5_getenv_int(const char *);
+int mlx5_proc_priv_init(struct rte_eth_dev *dev);
/* mlx5_ethdev.c */
diff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h
index 480b33c..13801a5 100644
--- a/drivers/net/mlx5/mlx5_defs.h
+++ b/drivers/net/mlx5/mlx5_defs.h
@@ -92,16 +92,6 @@
/* Timeout in seconds to get a valid link status. */
#define MLX5_LINK_STATUS_TIMEOUT 10
-/* Reserved address space for UAR mapping. */
-#define MLX5_UAR_SIZE (1ULL << (sizeof(uintptr_t) * 4))
-
-/* Offset of reserved UAR address space to hugepage memory. Offset is used here
- * to minimize possibility of address next to hugepage being used by other code
- * in either primary or secondary process, failing to map TX UAR would make TX
- * packets invisible to HW.
- */
-#define MLX5_UAR_OFFSET (1ULL << (sizeof(uintptr_t) * 4))
-
/* Maximum number of UAR pages used by a port,
* These are the size and mask for an array of mutexes used to synchronize
* the access to port's UARs on platforms that do not support 64 bit writes.
diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c
index d9ea74c..8778bf6 100644
--- a/drivers/net/mlx5/mlx5_ethdev.c
+++ b/drivers/net/mlx5/mlx5_ethdev.c
@@ -446,6 +446,9 @@ struct ethtool_link_settings {
if (++j == rxqs_n)
j = 0;
}
+ ret = mlx5_proc_priv_init(dev);
+ if (ret)
+ return ret;
return 0;
}
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index 820675b..dc2ca5e 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -202,8 +202,9 @@ struct mlx5_txq_data {
volatile void *wqes; /* Work queue (use volatile to write into). */
volatile uint32_t *qp_db; /* Work queue doorbell. */
volatile uint32_t *cq_db; /* Completion queue doorbell. */
- volatile void *bf_reg; /* Blueflame register remapped. */
struct rte_mbuf *(*elts)[]; /* TX elements. */
+ uint16_t port_id; /* Port ID of device. */
+ uint16_t idx; /* Queue index. */
struct mlx5_txq_stats stats; /* TX queue counters. */
#ifndef RTE_ARCH_64
rte_spinlock_t *uar_lock;
@@ -231,10 +232,12 @@ struct mlx5_txq_ctrl {
struct mlx5_priv *priv; /* Back pointer to private data. */
struct mlx5_txq_data txq; /* Data path structure. */
off_t uar_mmap_offset; /* UAR mmap offset for non-primary process. */
- volatile void *bf_reg_orig; /* Blueflame register from verbs. */
- uint16_t idx; /* Queue index. */
+ void *bf_reg; /* BlueFlame register from Verbs. */
};
+#define MLX5_TX_BFREG(txq) \
+ (MLX5_PROC_PRIV((txq)->port_id)->uar_table[(txq)->idx])
+
/* mlx5_rxq.c */
extern uint8_t rss_hash_default_key[];
@@ -302,7 +305,7 @@ struct mlx5_hrxq *mlx5_hrxq_get(struct rte_eth_dev *dev,
int mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
unsigned int socket, const struct rte_eth_txconf *conf);
void mlx5_tx_queue_release(void *dpdk_txq);
-int mlx5_tx_uar_remap(struct rte_eth_dev *dev, int fd);
+int mlx5_tx_uar_init_secondary(struct rte_eth_dev *dev, int fd);
struct mlx5_txq_ibv *mlx5_txq_ibv_new(struct rte_eth_dev *dev, uint16_t idx);
struct mlx5_txq_ibv *mlx5_txq_ibv_get(struct rte_eth_dev *dev, uint16_t idx);
int mlx5_txq_ibv_release(struct mlx5_txq_ibv *txq_ibv);
@@ -701,7 +704,7 @@ uint32_t mlx5_tx_update_ext_mp(struct mlx5_txq_data *txq, uintptr_t addr,
mlx5_tx_dbrec_cond_wmb(struct mlx5_txq_data *txq, volatile struct mlx5_wqe *wqe,
int cond)
{
- uint64_t *dst = (uint64_t *)((uintptr_t)txq->bf_reg);
+ uint64_t *dst = MLX5_TX_BFREG(txq);
volatile uint64_t *src = ((volatile uint64_t *)wqe);
rte_cio_wmb();
diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c
index 2137bdc..6b0f8b2 100644
--- a/drivers/net/mlx5/mlx5_trigger.c
+++ b/drivers/net/mlx5/mlx5_trigger.c
@@ -58,12 +58,6 @@
goto error;
}
}
- ret = mlx5_tx_uar_remap(dev, priv->ctx->cmd_fd);
- if (ret) {
- /* Adjust index for rollback. */
- i = priv->txqs_n - 1;
- goto error;
- }
return 0;
error:
ret = rte_errno; /* Save rte_errno before cleanup. */
diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index 7ee036f..42ab34a 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -49,7 +49,7 @@
for (i = 0; (i != elts_n); ++i)
(*txq_ctrl->txq.elts)[i] = NULL;
DRV_LOG(DEBUG, "port %u Tx queue %u allocated and configured %u WRs",
- PORT_ID(txq_ctrl->priv), txq_ctrl->idx, elts_n);
+ PORT_ID(txq_ctrl->priv), txq_ctrl->txq.idx, elts_n);
txq_ctrl->txq.elts_head = 0;
txq_ctrl->txq.elts_tail = 0;
txq_ctrl->txq.elts_comp = 0;
@@ -71,7 +71,7 @@
struct rte_mbuf *(*elts)[elts_n] = txq_ctrl->txq.elts;
DRV_LOG(DEBUG, "port %u Tx queue %u freeing WRs",
- PORT_ID(txq_ctrl->priv), txq_ctrl->idx);
+ PORT_ID(txq_ctrl->priv), txq_ctrl->txq.idx);
txq_ctrl->txq.elts_head = 0;
txq_ctrl->txq.elts_tail = 0;
txq_ctrl->txq.elts_comp = 0;
@@ -224,19 +224,106 @@
for (i = 0; (i != priv->txqs_n); ++i)
if ((*priv->txqs)[i] == txq) {
DRV_LOG(DEBUG, "port %u removing Tx queue %u from list",
- PORT_ID(priv), txq_ctrl->idx);
+ PORT_ID(priv), txq_ctrl->txq.idx);
mlx5_txq_release(ETH_DEV(priv), i);
break;
}
}
+/**
+ * Initialize Tx UAR registers for primary process.
+ *
+ * @param txq_ctrl
+ * Pointer to Tx queue control structure.
+ */
+static void
+txq_uar_init(struct mlx5_txq_ctrl *txq_ctrl)
+{
+ struct mlx5_priv *priv = txq_ctrl->priv;
+ struct mlx5_proc_priv *ppriv = MLX5_PROC_PRIV(PORT_ID(priv));
+#ifndef RTE_ARCH_64
+ unsigned int lock_idx;
+ const size_t page_size = sysconf(_SC_PAGESIZE);
+#endif
+
+ assert(rte_eal_process_type() == RTE_PROC_PRIMARY);
+ assert(ppriv);
+ ppriv->uar_table[txq_ctrl->txq.idx] = txq_ctrl->bf_reg;
+#ifndef RTE_ARCH_64
+ /* Assign an UAR lock according to UAR page number */
+ lock_idx = (txq_ctrl->uar_mmap_offset / page_size) &
+ MLX5_UAR_PAGE_NUM_MASK;
+ txq_ctrl->txq.uar_lock = &priv->uar_lock[lock_idx];
+#endif
+}
+
+/**
+ * Remap UAR register of a Tx queue for secondary process.
+ *
+ * Remapped address is stored at the table in the process private structure of
+ * the device, indexed by queue index.
+ *
+ * @param txq_ctrl
+ * Pointer to Tx queue control structure.
+ * @param fd
+ * Verbs file descriptor to map UAR pages.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+txq_uar_init_secondary(struct mlx5_txq_ctrl *txq_ctrl, int fd)
+{
+ struct mlx5_priv *priv = txq_ctrl->priv;
+ struct mlx5_proc_priv *ppriv = MLX5_PROC_PRIV(PORT_ID(priv));
+ struct mlx5_txq_data *txq = &txq_ctrl->txq;
+ void *addr;
+ uintptr_t uar_va;
+ uintptr_t offset;
+ const size_t page_size = sysconf(_SC_PAGESIZE);
+
+ assert(ppriv);
+ /*
+ * As rdma-core, UARs are mapped in size of OS page
+ * size. Ref to libmlx5 function: mlx5_init_context()
+ */
+ uar_va = (uintptr_t)txq_ctrl->bf_reg;
+ offset = uar_va & (page_size - 1); /* Offset in page. */
+ addr = mmap(NULL, page_size, PROT_WRITE, MAP_SHARED, fd,
+ txq_ctrl->uar_mmap_offset);
+ if (addr == MAP_FAILED) {
+ DRV_LOG(ERR,
+ "port %u mmap failed for BF reg of txq %u",
+ txq->port_id, txq->idx);
+ rte_errno = ENXIO;
+ return -rte_errno;
+ }
+ addr = RTE_PTR_ADD(addr, offset);
+ ppriv->uar_table[txq->idx] = addr;
+ return 0;
+}
+
+/**
+ * Unmap UAR register of a Tx queue for secondary process.
+ *
+ * @param txq_ctrl
+ * Pointer to Tx queue control structure.
+ */
+static void
+txq_uar_uninit_secondary(struct mlx5_txq_ctrl *txq_ctrl)
+{
+ struct mlx5_proc_priv *ppriv = MLX5_PROC_PRIV(PORT_ID(txq_ctrl->priv));
+ const size_t page_size = sysconf(_SC_PAGESIZE);
+ void *addr;
+
+ addr = ppriv->uar_table[txq_ctrl->txq.idx];
+ munmap(RTE_PTR_ALIGN_FLOOR(addr, page_size), page_size);
+}
/**
- * Mmap TX UAR(HW doorbell) pages into reserved UAR address space.
- * Both primary and secondary process do mmap to make UAR address
- * aligned.
+ * Initialize Tx UAR registers for secondary process.
*
- * @param[in] dev
+ * @param dev
* Pointer to Ethernet device.
* @param fd
* Verbs file descriptor to map UAR pages.
@@ -245,81 +332,36 @@
* 0 on success, a negative errno value otherwise and rte_errno is set.
*/
int
-mlx5_tx_uar_remap(struct rte_eth_dev *dev, int fd)
+mlx5_tx_uar_init_secondary(struct rte_eth_dev *dev, int fd)
{
struct mlx5_priv *priv = dev->data->dev_private;
- unsigned int i, j;
- uintptr_t pages[priv->txqs_n];
- unsigned int pages_n = 0;
- uintptr_t uar_va;
- uintptr_t off;
- void *addr;
- void *ret;
struct mlx5_txq_data *txq;
struct mlx5_txq_ctrl *txq_ctrl;
- int already_mapped;
- size_t page_size = sysconf(_SC_PAGESIZE);
-#ifndef RTE_ARCH_64
- unsigned int lock_idx;
-#endif
+ unsigned int i;
+ int ret;
- memset(pages, 0, priv->txqs_n * sizeof(uintptr_t));
- /*
- * As rdma-core, UARs are mapped in size of OS page size.
- * Use aligned address to avoid duplicate mmap.
- * Ref to libmlx5 function: mlx5_init_context()
- */
+ assert(rte_eal_process_type() == RTE_PROC_SECONDARY);
for (i = 0; i != priv->txqs_n; ++i) {
if (!(*priv->txqs)[i])
continue;
txq = (*priv->txqs)[i];
txq_ctrl = container_of(txq, struct mlx5_txq_ctrl, txq);
- assert(txq_ctrl->idx == (uint16_t)i);
- /* UAR addr form verbs used to find dup and offset in page. */
- uar_va = (uintptr_t)txq_ctrl->bf_reg_orig;
- off = uar_va & (page_size - 1); /* offset in page. */
- uar_va = RTE_ALIGN_FLOOR(uar_va, page_size); /* page addr. */
- already_mapped = 0;
- for (j = 0; j != pages_n; ++j) {
- if (pages[j] == uar_va) {
- already_mapped = 1;
- break;
- }
- }
- /* new address in reserved UAR address space. */
- addr = RTE_PTR_ADD(priv->uar_base,
- uar_va & (uintptr_t)(MLX5_UAR_SIZE - 1));
- if (!already_mapped) {
- pages[pages_n++] = uar_va;
- /* fixed mmap to specified address in reserved
- * address space.
- */
- ret = mmap(addr, page_size,
- PROT_WRITE, MAP_FIXED | MAP_SHARED, fd,
- txq_ctrl->uar_mmap_offset);
- if (ret != addr) {
- /* fixed mmap have to return same address */
- DRV_LOG(ERR,
- "port %u call to mmap failed on UAR"
- " for txq %u",
- dev->data->port_id, txq_ctrl->idx);
- rte_errno = ENXIO;
- return -rte_errno;
- }
- }
- if (rte_eal_process_type() == RTE_PROC_PRIMARY) /* save once */
- txq_ctrl->txq.bf_reg = RTE_PTR_ADD((void *)addr, off);
- else
- assert(txq_ctrl->txq.bf_reg ==
- RTE_PTR_ADD((void *)addr, off));
-#ifndef RTE_ARCH_64
- /* Assign a UAR lock according to UAR page number */
- lock_idx = (txq_ctrl->uar_mmap_offset / page_size) &
- MLX5_UAR_PAGE_NUM_MASK;
- txq->uar_lock = &priv->uar_lock[lock_idx];
-#endif
+ assert(txq->idx == (uint16_t)i);
+ ret = txq_uar_init_secondary(txq_ctrl, fd);
+ if (ret)
+ goto error;
}
return 0;
+error:
+ /* Rollback. */
+ do {
+ if (!(*priv->txqs)[i])
+ continue;
+ txq = (*priv->txqs)[i];
+ txq_ctrl = container_of(txq, struct mlx5_txq_ctrl, txq);
+ txq_uar_uninit_secondary(txq_ctrl);
+ } while (i--);
+ return -rte_errno;
}
/**
@@ -507,7 +549,6 @@ struct mlx5_txq_ibv *
txq_data->wqes = qp.sq.buf;
txq_data->wqe_n = log2above(qp.sq.wqe_cnt);
txq_data->qp_db = &qp.dbrec[MLX5_SND_DBR];
- txq_ctrl->bf_reg_orig = qp.bf.reg;
txq_data->cq_db = cq_info.dbrec;
txq_data->cqes =
(volatile struct mlx5_cqe (*)[])
@@ -521,6 +562,8 @@ struct mlx5_txq_ibv *
txq_ibv->qp = tmpl.qp;
txq_ibv->cq = tmpl.cq;
rte_atomic32_inc(&txq_ibv->refcnt);
+ txq_ctrl->bf_reg = qp.bf.reg;
+ txq_uar_init(txq_ctrl);
if (qp.comp_mask & MLX5DV_QP_MASK_UAR_MMAP_OFFSET) {
txq_ctrl->uar_mmap_offset = qp.uar_mmap_offset;
DRV_LOG(DEBUG, "port %u: uar_mmap_offset 0x%"PRIx64,
@@ -631,7 +674,7 @@ struct mlx5_txq_ibv *
LIST_FOREACH(txq_ibv, &priv->txqsibv, next) {
DRV_LOG(DEBUG, "port %u Verbs Tx queue %u still referenced",
- dev->data->port_id, txq_ibv->txq_ctrl->idx);
+ dev->data->port_id, txq_ibv->txq_ctrl->txq.idx);
++ret;
}
return ret;
@@ -801,7 +844,8 @@ struct mlx5_txq_ctrl *
tmpl->priv = priv;
tmpl->socket = socket;
tmpl->txq.elts_n = log2above(desc);
- tmpl->idx = idx;
+ tmpl->txq.port_id = dev->data->port_id;
+ tmpl->txq.idx = idx;
txq_set_params(tmpl);
if (txq_calc_wqebb_cnt(tmpl) >
priv->device_attr.orig_attr.max_qp_wr) {
@@ -866,16 +910,12 @@ struct mlx5_txq_ctrl *
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_txq_ctrl *txq;
- size_t page_size = sysconf(_SC_PAGESIZE);
if (!(*priv->txqs)[idx])
return 0;
txq = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, txq);
if (txq->ibv && !mlx5_txq_ibv_release(txq->ibv))
txq->ibv = NULL;
- if (priv->uar_base)
- munmap((void *)RTE_ALIGN_FLOOR((uintptr_t)txq->txq.bf_reg,
- page_size), page_size);
if (rte_atomic32_dec_and_test(&txq->refcnt)) {
txq_free_elts(txq);
mlx5_mr_btree_free(&txq->txq.mr_ctrl.cache_bh);
@@ -928,7 +968,7 @@ struct mlx5_txq_ctrl *
LIST_FOREACH(txq, &priv->txqsctrl, next) {
DRV_LOG(DEBUG, "port %u Tx queue %u still referenced",
- dev->data->port_id, txq->idx);
+ dev->data->port_id, txq->txq.idx);
++ret;
}
return ret;
--
1.8.3.1
^ permalink raw reply [flat|nested] 3+ messages in thread
* [dpdk-stable] [PATCH 3/3] [18.11] net/mlx4: remove device register remap
2020-07-03 15:41 [dpdk-stable] [PATCH 1/3] [18.11] net/mlx5: fix recursive inclusion of header file Viacheslav Ovsiienko
2020-07-03 15:41 ` [dpdk-stable] [PATCH 2/3] [18.11] net/mlx5: remove device register remap Viacheslav Ovsiienko
@ 2020-07-03 15:41 ` Viacheslav Ovsiienko
1 sibling, 0 replies; 3+ messages in thread
From: Viacheslav Ovsiienko @ 2020-07-03 15:41 UTC (permalink / raw)
To: stable; +Cc: ktraynor, Yongseok Koh
From: Yongseok Koh <yskoh@mellanox.com>
UAR (User Access Region) register does not need to be remapped for
primary process but it should be remapped only for secondary process.
UAR register table is in the process private structure in
rte_eth_devices[],
(struct mlx4_proc_priv *)rte_eth_devices[port_id].process_private
The actual UAR table follows the data structure and the table is used
for both Tx and Rx.
For Tx, BlueFlame in UAR is used to ring the doorbell.
MLX4_TX_BFREG(txq) is defined to get a register for the txq. Processes
access its own private data to acquire the register from the UAR table.
For Rx, the doorbell in UAR is required in arming CQ event. However, it
is a known issue that the register isn't remapped for secondary process.
Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
Acked-by: Shahaf Shuler <shahafs@mellanox.com>
---
drivers/net/mlx4/mlx4.c | 58 +++++++++++++++++++++++++++++++++++++++++++-
drivers/net/mlx4/mlx4.h | 11 +++++++++
drivers/net/mlx4/mlx4_prm.h | 3 ++-
drivers/net/mlx4/mlx4_rxtx.c | 2 +-
drivers/net/mlx4/mlx4_rxtx.h | 4 +++
drivers/net/mlx4/mlx4_txq.c | 20 +++++++++++++++
6 files changed, 95 insertions(+), 3 deletions(-)
diff --git a/drivers/net/mlx4/mlx4.c b/drivers/net/mlx4/mlx4.c
index 975df9b..522ba06 100644
--- a/drivers/net/mlx4/mlx4.c
+++ b/drivers/net/mlx4/mlx4.c
@@ -74,6 +74,53 @@ struct mlx4_conf {
static void mlx4_dev_stop(struct rte_eth_dev *dev);
/**
+ * Initialize process private data structure.
+ *
+ * @param dev
+ * Pointer to Ethernet device structure.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+mlx4_proc_priv_init(struct rte_eth_dev *dev)
+{
+ struct mlx4_proc_priv *ppriv;
+ size_t ppriv_size;
+
+ /*
+ * UAR register table follows the process private structure. BlueFlame
+ * registers for Tx queues are stored in the table.
+ */
+ ppriv_size = sizeof(struct mlx4_proc_priv) +
+ dev->data->nb_tx_queues * sizeof(void *);
+ ppriv = rte_malloc_socket("mlx4_proc_priv", ppriv_size,
+ RTE_CACHE_LINE_SIZE, dev->device->numa_node);
+ if (!ppriv) {
+ rte_errno = ENOMEM;
+ return -rte_errno;
+ }
+ ppriv->uar_table_sz = ppriv_size;
+ dev->process_private = ppriv;
+ return 0;
+}
+
+/**
+ * Un-initialize process private data structure.
+ *
+ * @param dev
+ * Pointer to Ethernet device structure.
+ */
+static void
+mlx4_proc_priv_uninit(struct rte_eth_dev *dev)
+{
+ if (!dev->process_private)
+ return;
+ rte_free(dev->process_private);
+ dev->process_private = NULL;
+}
+
+/**
* DPDK callback for Ethernet device configuration.
*
* @param dev
@@ -99,9 +146,17 @@ struct mlx4_conf {
goto exit;
}
ret = mlx4_intr_install(priv);
- if (ret)
+ if (ret) {
ERROR("%p: interrupt handler installation failed",
(void *)dev);
+ goto exit;
+ }
+ ret = mlx4_proc_priv_init(dev);
+ if (ret) {
+ ERROR("%p: process private data allocation failed",
+ (void *)dev);
+ goto exit;
+ }
exit:
return ret;
}
@@ -213,6 +268,7 @@ struct mlx4_conf {
mlx4_rx_queue_release(dev->data->rx_queues[i]);
for (i = 0; i != dev->data->nb_tx_queues; ++i)
mlx4_tx_queue_release(dev->data->tx_queues[i]);
+ mlx4_proc_priv_uninit(dev);
mlx4_mr_release(dev);
if (priv->pd != NULL) {
assert(priv->ctx != NULL);
diff --git a/drivers/net/mlx4/mlx4.h b/drivers/net/mlx4/mlx4.h
index 758b7aa..8454a5f 100644
--- a/drivers/net/mlx4/mlx4.h
+++ b/drivers/net/mlx4/mlx4.h
@@ -75,6 +75,17 @@ enum {
LIST_HEAD(mlx4_dev_list, mlx4_priv);
LIST_HEAD(mlx4_mr_list, mlx4_mr);
+/* Per-process private structure. */
+struct mlx4_proc_priv {
+ size_t uar_table_sz;
+ /* Size of UAR register table. */
+ void *uar_table[];
+ /* Table of UAR registers for each process. */
+};
+
+#define MLX4_PROC_PRIV(port_id) \
+ ((struct mlx4_proc_priv *)rte_eth_devices[port_id].process_private)
+
/** Private data structure. */
struct mlx4_priv {
LIST_ENTRY(mlx4_priv) mem_event_cb;
diff --git a/drivers/net/mlx4/mlx4_prm.h b/drivers/net/mlx4/mlx4_prm.h
index aef77ba..16ae6db 100644
--- a/drivers/net/mlx4/mlx4_prm.h
+++ b/drivers/net/mlx4/mlx4_prm.h
@@ -77,7 +77,8 @@ struct mlx4_sq {
uint32_t owner_opcode;
/**< Default owner opcode with HW valid owner bit. */
uint32_t stamp; /**< Stamp value with an invalid HW owner bit. */
- volatile uint32_t *db; /**< Pointer to the doorbell. */
+ uint32_t *db; /**< Pointer to the doorbell. */
+ off_t uar_mmap_offset; /* UAR mmap offset for non-primary process. */
uint32_t doorbell_qpn; /**< qp number to write to the doorbell. */
};
diff --git a/drivers/net/mlx4/mlx4_rxtx.c b/drivers/net/mlx4/mlx4_rxtx.c
index 8c88eff..d5290162 100644
--- a/drivers/net/mlx4/mlx4_rxtx.c
+++ b/drivers/net/mlx4/mlx4_rxtx.c
@@ -1048,7 +1048,7 @@ struct tso_info {
/* Make sure that descriptors are written before doorbell record. */
rte_wmb();
/* Ring QP doorbell. */
- rte_write32(txq->msq.doorbell_qpn, txq->msq.db);
+ rte_write32(txq->msq.doorbell_qpn, MLX4_TX_BFREG(txq));
txq->elts_head += i;
return i;
}
diff --git a/drivers/net/mlx4/mlx4_rxtx.h b/drivers/net/mlx4/mlx4_rxtx.h
index baf0cd7..7a7cc36 100644
--- a/drivers/net/mlx4/mlx4_rxtx.h
+++ b/drivers/net/mlx4/mlx4_rxtx.h
@@ -97,6 +97,7 @@ struct mlx4_txq_stats {
struct txq {
struct mlx4_sq msq; /**< Info for directly manipulating the SQ. */
struct mlx4_cq mcq; /**< Info for directly manipulating the CQ. */
+ uint16_t port_id; /**< Port ID of device. */
unsigned int elts_head; /**< Current index in (*elts)[]. */
unsigned int elts_tail; /**< First element awaiting completion. */
int elts_comp_cd; /**< Countdown for next completion. */
@@ -118,6 +119,9 @@ struct txq {
uint8_t data[]; /**< Remaining queue resources. */
};
+#define MLX4_TX_BFREG(txq) \
+ (MLX4_PROC_PRIV((txq)->port_id)->uar_table[(txq)->stats.idx])
+
/* mlx4_rxq.c */
extern uint8_t mlx4_rss_hash_key_default[MLX4_RSS_HASH_KEY_SIZE];
diff --git a/drivers/net/mlx4/mlx4_txq.c b/drivers/net/mlx4/mlx4_txq.c
index 3527008..92c9c03 100644
--- a/drivers/net/mlx4/mlx4_txq.c
+++ b/drivers/net/mlx4/mlx4_txq.c
@@ -38,6 +38,23 @@
#include "mlx4_utils.h"
/**
+ * Initialize Tx UAR registers for primary process.
+ *
+ * @param txq
+ * Pointer to Tx queue structure.
+ */
+static void
+txq_uar_init(struct txq *txq)
+{
+ struct mlx4_priv *priv = txq->priv;
+ struct mlx4_proc_priv *ppriv = MLX4_PROC_PRIV(PORT_ID(priv));
+
+ assert(rte_eal_process_type() == RTE_PROC_PRIMARY);
+ assert(ppriv);
+ ppriv->uar_table[txq->stats.idx] = txq->msq.db;
+}
+
+/**
* Free Tx queue elements.
*
* @param txq
@@ -89,6 +106,7 @@
sq->owner_opcode = MLX4_OPCODE_SEND | (0u << MLX4_SQ_OWNER_BIT);
sq->stamp = rte_cpu_to_be_32(MLX4_SQ_STAMP_VAL |
(0u << MLX4_SQ_OWNER_BIT));
+ sq->uar_mmap_offset = -1; /* Make mmap() fail. */
sq->db = dqp->sdb;
sq->doorbell_qpn = dqp->doorbell_qpn;
cq->buf = dcq->buf.buf;
@@ -214,6 +232,7 @@
}
*txq = (struct txq){
.priv = priv,
+ .port_id = dev->data->port_id,
.stats = {
.idx = idx,
},
@@ -319,6 +338,7 @@
goto error;
}
mlx4_txq_fill_dv_obj_info(txq, &mlxdv);
+ txq_uar_init(txq);
/* Save first wqe pointer in the first element. */
(&(*txq->elts)[0])->wqe =
(volatile struct mlx4_wqe_ctrl_seg *)txq->msq.buf;
--
1.8.3.1
^ permalink raw reply [flat|nested] 3+ messages in thread
end of thread, other threads:[~2020-07-03 15:41 UTC | newest]
Thread overview: 3+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2020-07-03 15:41 [dpdk-stable] [PATCH 1/3] [18.11] net/mlx5: fix recursive inclusion of header file Viacheslav Ovsiienko
2020-07-03 15:41 ` [dpdk-stable] [PATCH 2/3] [18.11] net/mlx5: remove device register remap Viacheslav Ovsiienko
2020-07-03 15:41 ` [dpdk-stable] [PATCH 3/3] [18.11] net/mlx4: " Viacheslav Ovsiienko
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).