* [dpdk-dev] [PATCH 1/5] net/mlx5: trim debug messages for reference counters
2018-05-02 23:16 [dpdk-dev] [PATCH 0/5] net/mlx: add new Memory Region support Yongseok Koh
@ 2018-05-02 23:16 ` Yongseok Koh
2018-05-06 6:37 ` Shahaf Shuler
2018-05-02 23:16 ` [dpdk-dev] [PATCH 2/5] net/mlx5: remove Memory Region support Yongseok Koh
` (4 subsequent siblings)
5 siblings, 1 reply; 23+ messages in thread
From: Yongseok Koh @ 2018-05-02 23:16 UTC (permalink / raw)
To: adrien.mazarguil, nelio.laranjeiro; +Cc: dev, Yongseok Koh
Remove debug messages when getting an object. When releasing an object,
debug message will be printed only if the object is really freed.
Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
---
drivers/net/mlx5/mlx5_mr.c | 7 ++-----
drivers/net/mlx5/mlx5_rxq.c | 36 +++++++++++++-----------------------
drivers/net/mlx5/mlx5_txq.c | 21 ++++++++-------------
3 files changed, 23 insertions(+), 41 deletions(-)
diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c
index 7a337d0c3..6613bd6b9 100644
--- a/drivers/net/mlx5/mlx5_mr.c
+++ b/drivers/net/mlx5/mlx5_mr.c
@@ -308,9 +308,6 @@ mlx5_mr_get(struct rte_eth_dev *dev, struct rte_mempool *mp)
LIST_FOREACH(mr, &priv->mr, next) {
if (mr->mp == mp) {
rte_atomic32_inc(&mr->refcnt);
- DRV_LOG(DEBUG, "port %u memory region %p refcnt: %d",
- dev->data->port_id, (void *)mr,
- rte_atomic32_read(&mr->refcnt));
return mr;
}
}
@@ -330,9 +327,9 @@ int
mlx5_mr_release(struct mlx5_mr *mr)
{
assert(mr);
- DRV_LOG(DEBUG, "memory region %p refcnt: %d", (void *)mr,
- rte_atomic32_read(&mr->refcnt));
if (rte_atomic32_dec_and_test(&mr->refcnt)) {
+ DRV_LOG(DEBUG, "memory region %p refcnt: %d", (void *)mr,
+ rte_atomic32_read(&mr->refcnt));
claim_zero(mlx5_glue->dereg_mr(mr->mr));
LIST_REMOVE(mr, next);
rte_free(mr);
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index a85b628fe..d993e3846 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -868,9 +868,6 @@ mlx5_rxq_ibv_get(struct rte_eth_dev *dev, uint16_t idx)
if (rxq_ctrl->ibv) {
mlx5_mr_get(dev, rxq_data->mp);
rte_atomic32_inc(&rxq_ctrl->ibv->refcnt);
- DRV_LOG(DEBUG, "port %u Verbs Rx queue %u: refcnt %d",
- dev->data->port_id, rxq_ctrl->idx,
- rte_atomic32_read(&rxq_ctrl->ibv->refcnt));
}
return rxq_ctrl->ibv;
}
@@ -896,10 +893,11 @@ mlx5_rxq_ibv_release(struct mlx5_rxq_ibv *rxq_ibv)
ret = mlx5_mr_release(rxq_ibv->mr);
if (!ret)
rxq_ibv->mr = NULL;
- DRV_LOG(DEBUG, "port %u Verbs Rx queue %u: refcnt %d",
- port_id(rxq_ibv->rxq_ctrl->priv),
- rxq_ibv->rxq_ctrl->idx, rte_atomic32_read(&rxq_ibv->refcnt));
if (rte_atomic32_dec_and_test(&rxq_ibv->refcnt)) {
+ DRV_LOG(DEBUG, "port %u Verbs Rx queue %u: refcnt %d",
+ port_id(rxq_ibv->rxq_ctrl->priv),
+ rxq_ibv->rxq_ctrl->idx,
+ rte_atomic32_read(&rxq_ibv->refcnt));
rxq_free_elts(rxq_ibv->rxq_ctrl);
claim_zero(mlx5_glue->destroy_wq(rxq_ibv->wq));
claim_zero(mlx5_glue->destroy_cq(rxq_ibv->cq));
@@ -1111,9 +1109,6 @@ mlx5_rxq_get(struct rte_eth_dev *dev, uint16_t idx)
rxq);
mlx5_rxq_ibv_get(dev, idx);
rte_atomic32_inc(&rxq_ctrl->refcnt);
- DRV_LOG(DEBUG, "port %u Rx queue %u: refcnt %d",
- dev->data->port_id, rxq_ctrl->idx,
- rte_atomic32_read(&rxq_ctrl->refcnt));
}
return rxq_ctrl;
}
@@ -1141,9 +1136,10 @@ mlx5_rxq_release(struct rte_eth_dev *dev, uint16_t idx)
assert(rxq_ctrl->priv);
if (rxq_ctrl->ibv && !mlx5_rxq_ibv_release(rxq_ctrl->ibv))
rxq_ctrl->ibv = NULL;
- DRV_LOG(DEBUG, "port %u Rx queue %u: refcnt %d", dev->data->port_id,
- rxq_ctrl->idx, rte_atomic32_read(&rxq_ctrl->refcnt));
if (rte_atomic32_dec_and_test(&rxq_ctrl->refcnt)) {
+ DRV_LOG(DEBUG, "port %u Rx queue %u: refcnt %d",
+ dev->data->port_id, rxq_ctrl->idx,
+ rte_atomic32_read(&rxq_ctrl->refcnt));
LIST_REMOVE(rxq_ctrl, next);
rte_free(rxq_ctrl);
(*priv->rxqs)[idx] = NULL;
@@ -1301,9 +1297,6 @@ mlx5_ind_table_ibv_get(struct rte_eth_dev *dev, const uint16_t *queues,
unsigned int i;
rte_atomic32_inc(&ind_tbl->refcnt);
- DRV_LOG(DEBUG, "port %u indirection table %p: refcnt %d",
- dev->data->port_id, (void *)ind_tbl,
- rte_atomic32_read(&ind_tbl->refcnt));
for (i = 0; i != ind_tbl->queues_n; ++i)
mlx5_rxq_get(dev, ind_tbl->queues[i]);
}
@@ -1327,9 +1320,6 @@ mlx5_ind_table_ibv_release(struct rte_eth_dev *dev,
{
unsigned int i;
- DRV_LOG(DEBUG, "port %u indirection table %p: refcnt %d",
- ((struct priv *)dev->data->dev_private)->port,
- (void *)ind_tbl, rte_atomic32_read(&ind_tbl->refcnt));
if (rte_atomic32_dec_and_test(&ind_tbl->refcnt)) {
claim_zero(mlx5_glue->destroy_rwq_ind_table
(ind_tbl->ind_table));
@@ -1339,6 +1329,9 @@ mlx5_ind_table_ibv_release(struct rte_eth_dev *dev,
for (i = 0; i != ind_tbl->queues_n; ++i)
claim_nonzero(mlx5_rxq_release(dev, ind_tbl->queues[i]));
if (!rte_atomic32_read(&ind_tbl->refcnt)) {
+ DRV_LOG(DEBUG, "port %u indirection table %p: refcnt %d",
+ ((struct priv *)dev->data->dev_private)->port,
+ (void *)ind_tbl, rte_atomic32_read(&ind_tbl->refcnt));
LIST_REMOVE(ind_tbl, next);
rte_free(ind_tbl);
return 0;
@@ -1566,9 +1559,6 @@ mlx5_hrxq_get(struct rte_eth_dev *dev,
continue;
}
rte_atomic32_inc(&hrxq->refcnt);
- DRV_LOG(DEBUG, "port %u hash Rx queue %p: refcnt %d",
- dev->data->port_id, (void *)hrxq,
- rte_atomic32_read(&hrxq->refcnt));
return hrxq;
}
return NULL;
@@ -1588,10 +1578,10 @@ mlx5_hrxq_get(struct rte_eth_dev *dev,
int
mlx5_hrxq_release(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq)
{
- DRV_LOG(DEBUG, "port %u hash Rx queue %p: refcnt %d",
- ((struct priv *)dev->data->dev_private)->port,
- (void *)hrxq, rte_atomic32_read(&hrxq->refcnt));
if (rte_atomic32_dec_and_test(&hrxq->refcnt)) {
+ DRV_LOG(DEBUG, "port %u hash Rx queue %p: refcnt %d",
+ ((struct priv *)dev->data->dev_private)->port,
+ (void *)hrxq, rte_atomic32_read(&hrxq->refcnt));
claim_zero(mlx5_glue->destroy_qp(hrxq->qp));
DEBUG("port %u delete QP %p: hash: 0x%" PRIx64 ", tunnel:"
" 0x%x, level: %u",
diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index 29959b4c7..3f4b5fea5 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -595,12 +595,8 @@ mlx5_txq_ibv_get(struct rte_eth_dev *dev, uint16_t idx)
if (!(*priv->txqs)[idx])
return NULL;
txq_ctrl = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, txq);
- if (txq_ctrl->ibv) {
+ if (txq_ctrl->ibv)
rte_atomic32_inc(&txq_ctrl->ibv->refcnt);
- DRV_LOG(DEBUG, "port %u Verbs Tx queue %u: refcnt %d",
- dev->data->port_id, txq_ctrl->idx,
- rte_atomic32_read(&txq_ctrl->ibv->refcnt));
- }
return txq_ctrl->ibv;
}
@@ -617,10 +613,11 @@ int
mlx5_txq_ibv_release(struct mlx5_txq_ibv *txq_ibv)
{
assert(txq_ibv);
- DRV_LOG(DEBUG, "port %u Verbs Tx queue %u: refcnt %d",
- port_id(txq_ibv->txq_ctrl->priv),
- txq_ibv->txq_ctrl->idx, rte_atomic32_read(&txq_ibv->refcnt));
if (rte_atomic32_dec_and_test(&txq_ibv->refcnt)) {
+ DRV_LOG(DEBUG, "port %u Verbs Tx queue %u: refcnt %d",
+ port_id(txq_ibv->txq_ctrl->priv),
+ txq_ibv->txq_ctrl->idx,
+ rte_atomic32_read(&txq_ibv->refcnt));
claim_zero(mlx5_glue->destroy_qp(txq_ibv->qp));
claim_zero(mlx5_glue->destroy_cq(txq_ibv->cq));
LIST_REMOVE(txq_ibv, next);
@@ -860,9 +857,6 @@ mlx5_txq_get(struct rte_eth_dev *dev, uint16_t idx)
ctrl->txq.mp2mr[i]->mp));
}
rte_atomic32_inc(&ctrl->refcnt);
- DRV_LOG(DEBUG, "port %u Tx queue %u refcnt %d",
- dev->data->port_id,
- ctrl->idx, rte_atomic32_read(&ctrl->refcnt));
}
return ctrl;
}
@@ -889,8 +883,6 @@ mlx5_txq_release(struct rte_eth_dev *dev, uint16_t idx)
if (!(*priv->txqs)[idx])
return 0;
txq = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, txq);
- DRV_LOG(DEBUG, "port %u Tx queue %u: refcnt %d", dev->data->port_id,
- txq->idx, rte_atomic32_read(&txq->refcnt));
if (txq->ibv && !mlx5_txq_ibv_release(txq->ibv))
txq->ibv = NULL;
for (i = 0; i != MLX5_PMD_TX_MP_CACHE; ++i) {
@@ -903,6 +895,9 @@ mlx5_txq_release(struct rte_eth_dev *dev, uint16_t idx)
munmap((void *)RTE_ALIGN_FLOOR((uintptr_t)txq->txq.bf_reg,
page_size), page_size);
if (rte_atomic32_dec_and_test(&txq->refcnt)) {
+ DRV_LOG(DEBUG, "port %u Tx queue %u: refcnt %d",
+ dev->data->port_id, txq->idx,
+ rte_atomic32_read(&txq->refcnt));
txq_free_elts(txq);
LIST_REMOVE(txq, next);
rte_free(txq);
--
2.11.0
^ permalink raw reply [flat|nested] 23+ messages in thread* Re: [dpdk-dev] [PATCH 1/5] net/mlx5: trim debug messages for reference counters
2018-05-02 23:16 ` [dpdk-dev] [PATCH 1/5] net/mlx5: trim debug messages for reference counters Yongseok Koh
@ 2018-05-06 6:37 ` Shahaf Shuler
2018-05-07 21:37 ` Yongseok Koh
0 siblings, 1 reply; 23+ messages in thread
From: Shahaf Shuler @ 2018-05-06 6:37 UTC (permalink / raw)
To: Yongseok Koh, Adrien Mazarguil, Nélio Laranjeiro; +Cc: dev, Yongseok Koh
Thursday, May 3, 2018 2:17 AM, Yongseok Koh:
> Subject: [dpdk-dev] [PATCH 1/5] net/mlx5: trim debug messages for
> reference counters
>
> Remove debug messages when getting an object. When releasing an object,
> debug message will be printed only if the object is really freed.
>
> Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
> ---
Only one general comment as you are making some order here.
I think it will be better to be explicit in the logging. Saying when new object is created:
"port %u new <obj type> ...<pointer> .. for queue ... "
No need for the refcnt as it is obviously 0.
And when object is destroyed:
"port %u <obj type>.. <pointer> ... for queue... is destroyed"
> drivers/net/mlx5/mlx5_mr.c | 7 ++----- drivers/net/mlx5/mlx5_rxq.c | 36
> +++++++++++++-----------------------
> drivers/net/mlx5/mlx5_txq.c | 21 ++++++++-------------
> 3 files changed, 23 insertions(+), 41 deletions(-)
>
> diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c index
> 7a337d0c3..6613bd6b9 100644
> --- a/drivers/net/mlx5/mlx5_mr.c
> +++ b/drivers/net/mlx5/mlx5_mr.c
> @@ -308,9 +308,6 @@ mlx5_mr_get(struct rte_eth_dev *dev, struct
> rte_mempool *mp)
> LIST_FOREACH(mr, &priv->mr, next) {
> if (mr->mp == mp) {
> rte_atomic32_inc(&mr->refcnt);
> - DRV_LOG(DEBUG, "port %u memory region %p
> refcnt: %d",
> - dev->data->port_id, (void *)mr,
> - rte_atomic32_read(&mr->refcnt));
> return mr;
> }
> }
> @@ -330,9 +327,9 @@ int
> mlx5_mr_release(struct mlx5_mr *mr)
> {
> assert(mr);
> - DRV_LOG(DEBUG, "memory region %p refcnt: %d", (void *)mr,
> - rte_atomic32_read(&mr->refcnt));
> if (rte_atomic32_dec_and_test(&mr->refcnt)) {
> + DRV_LOG(DEBUG, "memory region %p refcnt: %d", (void
> *)mr,
> + rte_atomic32_read(&mr->refcnt));
> claim_zero(mlx5_glue->dereg_mr(mr->mr));
> LIST_REMOVE(mr, next);
> rte_free(mr);
> diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
> index a85b628fe..d993e3846 100644
> --- a/drivers/net/mlx5/mlx5_rxq.c
> +++ b/drivers/net/mlx5/mlx5_rxq.c
> @@ -868,9 +868,6 @@ mlx5_rxq_ibv_get(struct rte_eth_dev *dev, uint16_t
> idx)
> if (rxq_ctrl->ibv) {
> mlx5_mr_get(dev, rxq_data->mp);
> rte_atomic32_inc(&rxq_ctrl->ibv->refcnt);
> - DRV_LOG(DEBUG, "port %u Verbs Rx queue %u: refcnt %d",
> - dev->data->port_id, rxq_ctrl->idx,
> - rte_atomic32_read(&rxq_ctrl->ibv->refcnt));
> }
> return rxq_ctrl->ibv;
> }
> @@ -896,10 +893,11 @@ mlx5_rxq_ibv_release(struct mlx5_rxq_ibv
> *rxq_ibv)
> ret = mlx5_mr_release(rxq_ibv->mr);
> if (!ret)
> rxq_ibv->mr = NULL;
> - DRV_LOG(DEBUG, "port %u Verbs Rx queue %u: refcnt %d",
> - port_id(rxq_ibv->rxq_ctrl->priv),
> - rxq_ibv->rxq_ctrl->idx, rte_atomic32_read(&rxq_ibv-
> >refcnt));
> if (rte_atomic32_dec_and_test(&rxq_ibv->refcnt)) {
> + DRV_LOG(DEBUG, "port %u Verbs Rx queue %u: refcnt %d",
> + port_id(rxq_ibv->rxq_ctrl->priv),
> + rxq_ibv->rxq_ctrl->idx,
> + rte_atomic32_read(&rxq_ibv->refcnt));
> rxq_free_elts(rxq_ibv->rxq_ctrl);
> claim_zero(mlx5_glue->destroy_wq(rxq_ibv->wq));
> claim_zero(mlx5_glue->destroy_cq(rxq_ibv->cq));
> @@ -1111,9 +1109,6 @@ mlx5_rxq_get(struct rte_eth_dev *dev, uint16_t
> idx)
> rxq);
> mlx5_rxq_ibv_get(dev, idx);
> rte_atomic32_inc(&rxq_ctrl->refcnt);
> - DRV_LOG(DEBUG, "port %u Rx queue %u: refcnt %d",
> - dev->data->port_id, rxq_ctrl->idx,
> - rte_atomic32_read(&rxq_ctrl->refcnt));
> }
> return rxq_ctrl;
> }
> @@ -1141,9 +1136,10 @@ mlx5_rxq_release(struct rte_eth_dev *dev,
> uint16_t idx)
> assert(rxq_ctrl->priv);
> if (rxq_ctrl->ibv && !mlx5_rxq_ibv_release(rxq_ctrl->ibv))
> rxq_ctrl->ibv = NULL;
> - DRV_LOG(DEBUG, "port %u Rx queue %u: refcnt %d", dev->data-
> >port_id,
> - rxq_ctrl->idx, rte_atomic32_read(&rxq_ctrl->refcnt));
> if (rte_atomic32_dec_and_test(&rxq_ctrl->refcnt)) {
> + DRV_LOG(DEBUG, "port %u Rx queue %u: refcnt %d",
> + dev->data->port_id, rxq_ctrl->idx,
> + rte_atomic32_read(&rxq_ctrl->refcnt));
> LIST_REMOVE(rxq_ctrl, next);
> rte_free(rxq_ctrl);
> (*priv->rxqs)[idx] = NULL;
> @@ -1301,9 +1297,6 @@ mlx5_ind_table_ibv_get(struct rte_eth_dev *dev,
> const uint16_t *queues,
> unsigned int i;
>
> rte_atomic32_inc(&ind_tbl->refcnt);
> - DRV_LOG(DEBUG, "port %u indirection table %p: refcnt %d",
> - dev->data->port_id, (void *)ind_tbl,
> - rte_atomic32_read(&ind_tbl->refcnt));
> for (i = 0; i != ind_tbl->queues_n; ++i)
> mlx5_rxq_get(dev, ind_tbl->queues[i]);
> }
> @@ -1327,9 +1320,6 @@ mlx5_ind_table_ibv_release(struct rte_eth_dev
> *dev, {
> unsigned int i;
>
> - DRV_LOG(DEBUG, "port %u indirection table %p: refcnt %d",
> - ((struct priv *)dev->data->dev_private)->port,
> - (void *)ind_tbl, rte_atomic32_read(&ind_tbl->refcnt));
> if (rte_atomic32_dec_and_test(&ind_tbl->refcnt)) {
> claim_zero(mlx5_glue->destroy_rwq_ind_table
> (ind_tbl->ind_table));
> @@ -1339,6 +1329,9 @@ mlx5_ind_table_ibv_release(struct rte_eth_dev
> *dev,
> for (i = 0; i != ind_tbl->queues_n; ++i)
> claim_nonzero(mlx5_rxq_release(dev, ind_tbl->queues[i]));
> if (!rte_atomic32_read(&ind_tbl->refcnt)) {
> + DRV_LOG(DEBUG, "port %u indirection table %p: refcnt %d",
> + ((struct priv *)dev->data->dev_private)->port,
> + (void *)ind_tbl, rte_atomic32_read(&ind_tbl-
> >refcnt));
> LIST_REMOVE(ind_tbl, next);
> rte_free(ind_tbl);
> return 0;
> @@ -1566,9 +1559,6 @@ mlx5_hrxq_get(struct rte_eth_dev *dev,
> continue;
> }
> rte_atomic32_inc(&hrxq->refcnt);
> - DRV_LOG(DEBUG, "port %u hash Rx queue %p: refcnt %d",
> - dev->data->port_id, (void *)hrxq,
> - rte_atomic32_read(&hrxq->refcnt));
> return hrxq;
> }
> return NULL;
> @@ -1588,10 +1578,10 @@ mlx5_hrxq_get(struct rte_eth_dev *dev, int
> mlx5_hrxq_release(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq) {
> - DRV_LOG(DEBUG, "port %u hash Rx queue %p: refcnt %d",
> - ((struct priv *)dev->data->dev_private)->port,
> - (void *)hrxq, rte_atomic32_read(&hrxq->refcnt));
> if (rte_atomic32_dec_and_test(&hrxq->refcnt)) {
> + DRV_LOG(DEBUG, "port %u hash Rx queue %p: refcnt %d",
> + ((struct priv *)dev->data->dev_private)->port,
> + (void *)hrxq, rte_atomic32_read(&hrxq->refcnt));
> claim_zero(mlx5_glue->destroy_qp(hrxq->qp));
> DEBUG("port %u delete QP %p: hash: 0x%" PRIx64 ", tunnel:"
> " 0x%x, level: %u",
> diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
> index 29959b4c7..3f4b5fea5 100644
> --- a/drivers/net/mlx5/mlx5_txq.c
> +++ b/drivers/net/mlx5/mlx5_txq.c
> @@ -595,12 +595,8 @@ mlx5_txq_ibv_get(struct rte_eth_dev *dev,
> uint16_t idx)
> if (!(*priv->txqs)[idx])
> return NULL;
> txq_ctrl = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, txq);
> - if (txq_ctrl->ibv) {
> + if (txq_ctrl->ibv)
> rte_atomic32_inc(&txq_ctrl->ibv->refcnt);
> - DRV_LOG(DEBUG, "port %u Verbs Tx queue %u: refcnt %d",
> - dev->data->port_id, txq_ctrl->idx,
> - rte_atomic32_read(&txq_ctrl->ibv->refcnt));
> - }
> return txq_ctrl->ibv;
> }
>
> @@ -617,10 +613,11 @@ int
> mlx5_txq_ibv_release(struct mlx5_txq_ibv *txq_ibv) {
> assert(txq_ibv);
> - DRV_LOG(DEBUG, "port %u Verbs Tx queue %u: refcnt %d",
> - port_id(txq_ibv->txq_ctrl->priv),
> - txq_ibv->txq_ctrl->idx, rte_atomic32_read(&txq_ibv-
> >refcnt));
> if (rte_atomic32_dec_and_test(&txq_ibv->refcnt)) {
> + DRV_LOG(DEBUG, "port %u Verbs Tx queue %u: refcnt %d",
> + port_id(txq_ibv->txq_ctrl->priv),
> + txq_ibv->txq_ctrl->idx,
> + rte_atomic32_read(&txq_ibv->refcnt));
> claim_zero(mlx5_glue->destroy_qp(txq_ibv->qp));
> claim_zero(mlx5_glue->destroy_cq(txq_ibv->cq));
> LIST_REMOVE(txq_ibv, next);
> @@ -860,9 +857,6 @@ mlx5_txq_get(struct rte_eth_dev *dev, uint16_t idx)
> ctrl->txq.mp2mr[i]->mp));
> }
> rte_atomic32_inc(&ctrl->refcnt);
> - DRV_LOG(DEBUG, "port %u Tx queue %u refcnt %d",
> - dev->data->port_id,
> - ctrl->idx, rte_atomic32_read(&ctrl->refcnt));
> }
> return ctrl;
> }
> @@ -889,8 +883,6 @@ mlx5_txq_release(struct rte_eth_dev *dev, uint16_t
> idx)
> if (!(*priv->txqs)[idx])
> return 0;
> txq = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, txq);
> - DRV_LOG(DEBUG, "port %u Tx queue %u: refcnt %d", dev->data-
> >port_id,
> - txq->idx, rte_atomic32_read(&txq->refcnt));
> if (txq->ibv && !mlx5_txq_ibv_release(txq->ibv))
> txq->ibv = NULL;
> for (i = 0; i != MLX5_PMD_TX_MP_CACHE; ++i) { @@ -903,6 +895,9
> @@ mlx5_txq_release(struct rte_eth_dev *dev, uint16_t idx)
> munmap((void *)RTE_ALIGN_FLOOR((uintptr_t)txq-
> >txq.bf_reg,
> page_size), page_size);
> if (rte_atomic32_dec_and_test(&txq->refcnt)) {
> + DRV_LOG(DEBUG, "port %u Tx queue %u: refcnt %d",
> + dev->data->port_id, txq->idx,
> + rte_atomic32_read(&txq->refcnt));
> txq_free_elts(txq);
> LIST_REMOVE(txq, next);
> rte_free(txq);
> --
> 2.11.0
^ permalink raw reply [flat|nested] 23+ messages in thread* Re: [dpdk-dev] [PATCH 1/5] net/mlx5: trim debug messages for reference counters
2018-05-06 6:37 ` Shahaf Shuler
@ 2018-05-07 21:37 ` Yongseok Koh
0 siblings, 0 replies; 23+ messages in thread
From: Yongseok Koh @ 2018-05-07 21:37 UTC (permalink / raw)
To: Shahaf Shuler; +Cc: Adrien Mazarguil, Nélio Laranjeiro, dev
> On May 5, 2018, at 11:37 PM, Shahaf Shuler <shahafs@mellanox.com> wrote:
>
> Thursday, May 3, 2018 2:17 AM, Yongseok Koh:
>> Subject: [dpdk-dev] [PATCH 1/5] net/mlx5: trim debug messages for
>> reference counters
>>
>> Remove debug messages when getting an object. When releasing an object,
>> debug message will be printed only if the object is really freed.
>>
>> Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
>> ---
>
> Only one general comment as you are making some order here.
>
> I think it will be better to be explicit in the logging. Saying when new object is created:
> "port %u new <obj type> ...<pointer> .. for queue ... "
>
> No need for the refcnt as it is obviously 0.
>
> And when object is destroyed:
> "port %u <obj type>.. <pointer> ... for queue... is destroyed"
I'd rather drop this patch. This is Nelio's code and he might still want to
keep it. As he is now OOO, we can't hear from him.
This patch is nothing but cleanup.
Thanks,
Yongseok
^ permalink raw reply [flat|nested] 23+ messages in thread
* [dpdk-dev] [PATCH 2/5] net/mlx5: remove Memory Region support
2018-05-02 23:16 [dpdk-dev] [PATCH 0/5] net/mlx: add new Memory Region support Yongseok Koh
2018-05-02 23:16 ` [dpdk-dev] [PATCH 1/5] net/mlx5: trim debug messages for reference counters Yongseok Koh
@ 2018-05-02 23:16 ` Yongseok Koh
2018-05-06 6:41 ` Shahaf Shuler
2018-05-02 23:16 ` [dpdk-dev] [PATCH 3/5] net/mlx5: add new " Yongseok Koh
` (3 subsequent siblings)
5 siblings, 1 reply; 23+ messages in thread
From: Yongseok Koh @ 2018-05-02 23:16 UTC (permalink / raw)
To: adrien.mazarguil, nelio.laranjeiro; +Cc: dev, Yongseok Koh
This patch removes current support of Memory Region (MR) in order to
accommodate the dynamic memory hotplug patch. This patch can be compiled
but traffic can't flow and HW will raise faults. Subsequent patches will
add new MR support.
Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
---
config/common_base | 1 -
doc/guides/nics/mlx5.rst | 8 -
drivers/net/mlx5/Makefile | 4 -
drivers/net/mlx5/mlx5.c | 4 -
drivers/net/mlx5/mlx5.h | 10 --
drivers/net/mlx5/mlx5_defs.h | 11 --
drivers/net/mlx5/mlx5_mr.c | 343 ----------------------------------------
drivers/net/mlx5/mlx5_rxq.c | 24 +--
drivers/net/mlx5/mlx5_rxtx.h | 90 +----------
drivers/net/mlx5/mlx5_trigger.c | 14 --
drivers/net/mlx5/mlx5_txq.c | 17 --
11 files changed, 5 insertions(+), 521 deletions(-)
diff --git a/config/common_base b/config/common_base
index 03a8688b5..bf7d5e785 100644
--- a/config/common_base
+++ b/config/common_base
@@ -296,7 +296,6 @@ CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE=8
CONFIG_RTE_LIBRTE_MLX5_PMD=n
CONFIG_RTE_LIBRTE_MLX5_DEBUG=n
CONFIG_RTE_LIBRTE_MLX5_DLOPEN_DEPS=n
-CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE=8
#
# Compile burst-oriented Netronome NFP PMD driver
diff --git a/doc/guides/nics/mlx5.rst b/doc/guides/nics/mlx5.rst
index 853c48f81..0fe6e1835 100644
--- a/doc/guides/nics/mlx5.rst
+++ b/doc/guides/nics/mlx5.rst
@@ -167,14 +167,6 @@ These options can be modified in the ``.config`` file.
adds additional run-time checks and debugging messages at the cost of
lower performance.
-- ``CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE`` (default **8**)
-
- Maximum number of cached memory pools (MPs) per TX queue. Each MP from
- which buffers are to be transmitted must be associated to memory regions
- (MRs). This is a slow operation that must be cached.
-
- This value is always 1 for RX queues since they use a single MP.
-
Environment variables
~~~~~~~~~~~~~~~~~~~~~
diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile
index 3c5b4943a..13f079334 100644
--- a/drivers/net/mlx5/Makefile
+++ b/drivers/net/mlx5/Makefile
@@ -82,10 +82,6 @@ else
CFLAGS += -DNDEBUG -UPEDANTIC
endif
-ifdef CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE
-CFLAGS += -DMLX5_PMD_TX_MP_CACHE=$(CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE)
-endif
-
include $(RTE_SDK)/mk/rte.lib.mk
# Generate and clean-up mlx5_autoconf.h.
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 6c4a571ab..01d554758 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -245,10 +245,6 @@ mlx5_dev_close(struct rte_eth_dev *dev)
if (ret)
DRV_LOG(WARNING, "port %u some flows still remain",
dev->data->port_id);
- ret = mlx5_mr_verify(dev);
- if (ret)
- DRV_LOG(WARNING, "port %u some memory region still remain",
- dev->data->port_id);
memset(priv, 0, sizeof(*priv));
}
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 3ab16bfa2..47d266c90 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -26,7 +26,6 @@
#include <rte_pci.h>
#include <rte_ether.h>
#include <rte_ethdev_driver.h>
-#include <rte_spinlock.h>
#include <rte_interrupts.h>
#include <rte_errno.h>
#include <rte_flow.h>
@@ -147,7 +146,6 @@ struct priv {
struct mlx5_hrxq_drop *flow_drop_queue; /* Flow drop queue. */
struct mlx5_flows flows; /* RTE Flow rules. */
struct mlx5_flows ctrl_flows; /* Control flow rules. */
- LIST_HEAD(mr, mlx5_mr) mr; /* Memory region. */
LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */
LIST_HEAD(rxqibv, mlx5_rxq_ibv) rxqsibv; /* Verbs Rx queues. */
LIST_HEAD(hrxq, mlx5_hrxq) hrxqs; /* Verbs Hash Rx queues. */
@@ -157,7 +155,6 @@ struct priv {
LIST_HEAD(ind_tables, mlx5_ind_table_ibv) ind_tbls;
uint32_t link_speed_capa; /* Link speed capabilities. */
struct mlx5_xstats_ctrl xstats_ctrl; /* Extended stats control. */
- rte_spinlock_t mr_lock; /* MR Lock. */
int primary_socket; /* Unix socket for primary process. */
void *uar_base; /* Reserved address space for UAR mapping */
struct rte_intr_handle intr_handle_socket; /* Interrupt handler. */
@@ -309,13 +306,6 @@ void mlx5_socket_uninit(struct rte_eth_dev *priv);
void mlx5_socket_handle(struct rte_eth_dev *priv);
int mlx5_socket_connect(struct rte_eth_dev *priv);
-/* mlx5_mr.c */
-
-struct mlx5_mr *mlx5_mr_new(struct rte_eth_dev *dev, struct rte_mempool *mp);
-struct mlx5_mr *mlx5_mr_get(struct rte_eth_dev *dev, struct rte_mempool *mp);
-int mlx5_mr_release(struct mlx5_mr *mr);
-int mlx5_mr_verify(struct rte_eth_dev *dev);
-
/* mlx5_nl.c */
int mlx5_nl_init(uint32_t nlgroups);
diff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h
index 55a86957d..f9093777d 100644
--- a/drivers/net/mlx5/mlx5_defs.h
+++ b/drivers/net/mlx5/mlx5_defs.h
@@ -38,17 +38,6 @@
#define MLX5_TX_COMP_THRESH_INLINE_DIV (1 << 3)
/*
- * Maximum number of cached Memory Pools (MPs) per TX queue. Each RTE MP
- * from which buffers are to be transmitted will have to be mapped by this
- * driver to their own Memory Region (MR). This is a slow operation.
- *
- * This value is always 1 for RX queues.
- */
-#ifndef MLX5_PMD_TX_MP_CACHE
-#define MLX5_PMD_TX_MP_CACHE 8
-#endif
-
-/*
* If defined, only use software counters. The PMD will never ask the hardware
* for these, and many of them won't be available.
*/
diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c
index 6613bd6b9..736c40ae4 100644
--- a/drivers/net/mlx5/mlx5_mr.c
+++ b/drivers/net/mlx5/mlx5_mr.c
@@ -18,346 +18,3 @@
#include "mlx5_rxtx.h"
#include "mlx5_glue.h"
-struct mlx5_check_mempool_data {
- int ret;
- char *start;
- char *end;
-};
-
-/* Called by mlx5_check_mempool() when iterating the memory chunks. */
-static void
-mlx5_check_mempool_cb(struct rte_mempool *mp __rte_unused,
- void *opaque, struct rte_mempool_memhdr *memhdr,
- unsigned int mem_idx __rte_unused)
-{
- struct mlx5_check_mempool_data *data = opaque;
-
- /* It already failed, skip the next chunks. */
- if (data->ret != 0)
- return;
- /* It is the first chunk. */
- if (data->start == NULL && data->end == NULL) {
- data->start = memhdr->addr;
- data->end = data->start + memhdr->len;
- return;
- }
- if (data->end == memhdr->addr) {
- data->end += memhdr->len;
- return;
- }
- if (data->start == (char *)memhdr->addr + memhdr->len) {
- data->start -= memhdr->len;
- return;
- }
- /* Error, mempool is not virtually contiguous. */
- data->ret = -1;
-}
-
-/**
- * Check if a mempool can be used: it must be virtually contiguous.
- *
- * @param[in] mp
- * Pointer to memory pool.
- * @param[out] start
- * Pointer to the start address of the mempool virtual memory area
- * @param[out] end
- * Pointer to the end address of the mempool virtual memory area
- *
- * @return
- * 0 on success (mempool is virtually contiguous), -1 on error.
- */
-static int
-mlx5_check_mempool(struct rte_mempool *mp, uintptr_t *start,
- uintptr_t *end)
-{
- struct mlx5_check_mempool_data data;
-
- memset(&data, 0, sizeof(data));
- rte_mempool_mem_iter(mp, mlx5_check_mempool_cb, &data);
- *start = (uintptr_t)data.start;
- *end = (uintptr_t)data.end;
- return data.ret;
-}
-
-/**
- * Register a Memory Region (MR) <-> Memory Pool (MP) association in
- * txq->mp2mr[]. If mp2mr[] is full, remove an entry first.
- *
- * @param txq
- * Pointer to TX queue structure.
- * @param[in] mp
- * Memory Pool for which a Memory Region lkey must be returned.
- * @param idx
- * Index of the next available entry.
- *
- * @return
- * mr on success, NULL on failure and rte_errno is set.
- */
-struct mlx5_mr *
-mlx5_txq_mp2mr_reg(struct mlx5_txq_data *txq, struct rte_mempool *mp,
- unsigned int idx)
-{
- struct mlx5_txq_ctrl *txq_ctrl =
- container_of(txq, struct mlx5_txq_ctrl, txq);
- struct rte_eth_dev *dev;
- struct mlx5_mr *mr;
-
- rte_spinlock_lock(&txq_ctrl->priv->mr_lock);
- /* Add a new entry, register MR first. */
- DRV_LOG(DEBUG, "port %u discovered new memory pool \"%s\" (%p)",
- port_id(txq_ctrl->priv), mp->name, (void *)mp);
- dev = eth_dev(txq_ctrl->priv);
- mr = mlx5_mr_get(dev, mp);
- if (mr == NULL) {
- if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
- DRV_LOG(DEBUG,
- "port %u using unregistered mempool 0x%p(%s)"
- " in secondary process, please create mempool"
- " before rte_eth_dev_start()",
- port_id(txq_ctrl->priv), (void *)mp, mp->name);
- rte_spinlock_unlock(&txq_ctrl->priv->mr_lock);
- rte_errno = ENOTSUP;
- return NULL;
- }
- mr = mlx5_mr_new(dev, mp);
- }
- if (unlikely(mr == NULL)) {
- DRV_LOG(DEBUG,
- "port %u unable to configure memory region,"
- " ibv_reg_mr() failed.",
- port_id(txq_ctrl->priv));
- rte_spinlock_unlock(&txq_ctrl->priv->mr_lock);
- return NULL;
- }
- if (unlikely(idx == RTE_DIM(txq->mp2mr))) {
- /* Table is full, remove oldest entry. */
- DRV_LOG(DEBUG,
- "port %u memory region <-> memory pool table full, "
- " dropping oldest entry",
- port_id(txq_ctrl->priv));
- --idx;
- mlx5_mr_release(txq->mp2mr[0]);
- memmove(&txq->mp2mr[0], &txq->mp2mr[1],
- (sizeof(txq->mp2mr) - sizeof(txq->mp2mr[0])));
- }
- /* Store the new entry. */
- txq_ctrl->txq.mp2mr[idx] = mr;
- DRV_LOG(DEBUG,
- "port %u new memory region lkey for MP \"%s\" (%p): 0x%08"
- PRIu32,
- port_id(txq_ctrl->priv), mp->name, (void *)mp,
- txq_ctrl->txq.mp2mr[idx]->lkey);
- rte_spinlock_unlock(&txq_ctrl->priv->mr_lock);
- return mr;
-}
-
-struct mlx5_mp2mr_mbuf_check_data {
- int ret;
-};
-
-/**
- * Callback function for rte_mempool_obj_iter() to check whether a given
- * mempool object looks like a mbuf.
- *
- * @param[in] mp
- * The mempool pointer
- * @param[in] arg
- * Context data (struct txq_mp2mr_mbuf_check_data). Contains the
- * return value.
- * @param[in] obj
- * Object address.
- * @param index
- * Object index, unused.
- */
-static void
-txq_mp2mr_mbuf_check(struct rte_mempool *mp, void *arg, void *obj,
- uint32_t index __rte_unused)
-{
- struct mlx5_mp2mr_mbuf_check_data *data = arg;
- struct rte_mbuf *buf = obj;
-
- /*
- * Check whether mbuf structure fits element size and whether mempool
- * pointer is valid.
- */
- if (sizeof(*buf) > mp->elt_size || buf->pool != mp)
- data->ret = -1;
-}
-
-/**
- * Iterator function for rte_mempool_walk() to register existing mempools and
- * fill the MP to MR cache of a TX queue.
- *
- * @param[in] mp
- * Memory Pool to register.
- * @param *arg
- * Pointer to TX queue structure.
- */
-void
-mlx5_mp2mr_iter(struct rte_mempool *mp, void *arg)
-{
- struct priv *priv = (struct priv *)arg;
- struct mlx5_mp2mr_mbuf_check_data data = {
- .ret = 0,
- };
- struct mlx5_mr *mr;
-
- /* Register mempool only if the first element looks like a mbuf. */
- if (rte_mempool_obj_iter(mp, txq_mp2mr_mbuf_check, &data) == 0 ||
- data.ret == -1)
- return;
- mr = mlx5_mr_get(eth_dev(priv), mp);
- if (mr) {
- mlx5_mr_release(mr);
- return;
- }
- mr = mlx5_mr_new(eth_dev(priv), mp);
- if (!mr)
- DRV_LOG(ERR, "port %u cannot create memory region: %s",
- port_id(priv), strerror(rte_errno));
-}
-
-/**
- * Register a new memory region from the mempool and store it in the memory
- * region list.
- *
- * @param dev
- * Pointer to Ethernet device.
- * @param mp
- * Pointer to the memory pool to register.
- *
- * @return
- * The memory region on success, NULL on failure and rte_errno is set.
- */
-struct mlx5_mr *
-mlx5_mr_new(struct rte_eth_dev *dev, struct rte_mempool *mp)
-{
- struct priv *priv = dev->data->dev_private;
- const struct rte_memseg *ms;
- uintptr_t start;
- uintptr_t end;
- struct mlx5_mr *mr;
-
- mr = rte_zmalloc_socket(__func__, sizeof(*mr), 0, mp->socket_id);
- if (!mr) {
- DRV_LOG(DEBUG,
- "port %u unable to configure memory region,"
- " ibv_reg_mr() failed.",
- dev->data->port_id);
- rte_errno = ENOMEM;
- return NULL;
- }
- if (mlx5_check_mempool(mp, &start, &end) != 0) {
- DRV_LOG(ERR, "port %u mempool %p: not virtually contiguous",
- dev->data->port_id, (void *)mp);
- rte_errno = ENOMEM;
- return NULL;
- }
- DRV_LOG(DEBUG, "port %u mempool %p area start=%p end=%p size=%zu",
- dev->data->port_id, (void *)mp, (void *)start, (void *)end,
- (size_t)(end - start));
- /* Save original addresses for exact MR lookup. */
- mr->start = start;
- mr->end = end;
-
- /* Round start and end to page boundary if found in memory segments. */
- ms = rte_mem_virt2memseg((void *)start, NULL);
- if (ms != NULL)
- start = RTE_ALIGN_FLOOR(start, ms->hugepage_sz);
- end = RTE_ALIGN_CEIL(end, ms->hugepage_sz);
- DRV_LOG(DEBUG,
- "port %u mempool %p using start=%p end=%p size=%zu for memory"
- " region",
- dev->data->port_id, (void *)mp, (void *)start, (void *)end,
- (size_t)(end - start));
- mr->mr = mlx5_glue->reg_mr(priv->pd, (void *)start, end - start,
- IBV_ACCESS_LOCAL_WRITE);
- if (!mr->mr) {
- rte_errno = ENOMEM;
- return NULL;
- }
- mr->mp = mp;
- mr->lkey = rte_cpu_to_be_32(mr->mr->lkey);
- rte_atomic32_inc(&mr->refcnt);
- DRV_LOG(DEBUG, "port %u new memory Region %p refcnt: %d",
- dev->data->port_id, (void *)mr, rte_atomic32_read(&mr->refcnt));
- LIST_INSERT_HEAD(&priv->mr, mr, next);
- return mr;
-}
-
-/**
- * Search the memory region object in the memory region list.
- *
- * @param dev
- * Pointer to Ethernet device.
- * @param mp
- * Pointer to the memory pool to register.
- *
- * @return
- * The memory region on success.
- */
-struct mlx5_mr *
-mlx5_mr_get(struct rte_eth_dev *dev, struct rte_mempool *mp)
-{
- struct priv *priv = dev->data->dev_private;
- struct mlx5_mr *mr;
-
- assert(mp);
- if (LIST_EMPTY(&priv->mr))
- return NULL;
- LIST_FOREACH(mr, &priv->mr, next) {
- if (mr->mp == mp) {
- rte_atomic32_inc(&mr->refcnt);
- return mr;
- }
- }
- return NULL;
-}
-
-/**
- * Release the memory region object.
- *
- * @param mr
- * Pointer to memory region to release.
- *
- * @return
- * 1 while a reference on it exists, 0 when freed.
- */
-int
-mlx5_mr_release(struct mlx5_mr *mr)
-{
- assert(mr);
- if (rte_atomic32_dec_and_test(&mr->refcnt)) {
- DRV_LOG(DEBUG, "memory region %p refcnt: %d", (void *)mr,
- rte_atomic32_read(&mr->refcnt));
- claim_zero(mlx5_glue->dereg_mr(mr->mr));
- LIST_REMOVE(mr, next);
- rte_free(mr);
- return 0;
- }
- return 1;
-}
-
-/**
- * Verify the flow list is empty
- *
- * @param dev
- * Pointer to Ethernet device.
- *
- * @return
- * The number of object not released.
- */
-int
-mlx5_mr_verify(struct rte_eth_dev *dev)
-{
- struct priv *priv = dev->data->dev_private;
- int ret = 0;
- struct mlx5_mr *mr;
-
- LIST_FOREACH(mr, &priv->mr, next) {
- DRV_LOG(DEBUG, "port %u memory region %p still referenced",
- dev->data->port_id, (void *)mr);
- ++ret;
- }
- return ret;
-}
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index d993e3846..d4fe1fed7 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -649,16 +649,6 @@ mlx5_rxq_ibv_new(struct rte_eth_dev *dev, uint16_t idx)
goto error;
}
tmpl->rxq_ctrl = rxq_ctrl;
- /* Use the entire RX mempool as the memory region. */
- tmpl->mr = mlx5_mr_get(dev, rxq_data->mp);
- if (!tmpl->mr) {
- tmpl->mr = mlx5_mr_new(dev, rxq_data->mp);
- if (!tmpl->mr) {
- DRV_LOG(ERR, "port %u: memeroy region creation failure",
- dev->data->port_id);
- goto error;
- }
- }
if (rxq_ctrl->irq) {
tmpl->channel = mlx5_glue->create_comp_channel(priv->ctx);
if (!tmpl->channel) {
@@ -799,7 +789,7 @@ mlx5_rxq_ibv_new(struct rte_eth_dev *dev, uint16_t idx)
.addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(buf,
uintptr_t)),
.byte_count = rte_cpu_to_be_32(DATA_LEN(buf)),
- .lkey = tmpl->mr->lkey,
+ .lkey = UINT32_MAX,
};
}
rxq_data->rq_db = rwq.dbrec;
@@ -835,8 +825,6 @@ mlx5_rxq_ibv_new(struct rte_eth_dev *dev, uint16_t idx)
claim_zero(mlx5_glue->destroy_cq(tmpl->cq));
if (tmpl->channel)
claim_zero(mlx5_glue->destroy_comp_channel(tmpl->channel));
- if (tmpl->mr)
- mlx5_mr_release(tmpl->mr);
priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_NONE;
rte_errno = ret; /* Restore rte_errno. */
return NULL;
@@ -865,10 +853,8 @@ mlx5_rxq_ibv_get(struct rte_eth_dev *dev, uint16_t idx)
if (!rxq_data)
return NULL;
rxq_ctrl = container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
- if (rxq_ctrl->ibv) {
- mlx5_mr_get(dev, rxq_data->mp);
+ if (rxq_ctrl->ibv)
rte_atomic32_inc(&rxq_ctrl->ibv->refcnt);
- }
return rxq_ctrl->ibv;
}
@@ -884,15 +870,9 @@ mlx5_rxq_ibv_get(struct rte_eth_dev *dev, uint16_t idx)
int
mlx5_rxq_ibv_release(struct mlx5_rxq_ibv *rxq_ibv)
{
- int ret;
-
assert(rxq_ibv);
assert(rxq_ibv->wq);
assert(rxq_ibv->cq);
- assert(rxq_ibv->mr);
- ret = mlx5_mr_release(rxq_ibv->mr);
- if (!ret)
- rxq_ibv->mr = NULL;
if (rte_atomic32_dec_and_test(&rxq_ibv->refcnt)) {
DRV_LOG(DEBUG, "port %u Verbs Rx queue %u: refcnt %d",
port_id(rxq_ibv->rxq_ctrl->priv),
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index 2fc12a186..e8cad51aa 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -54,17 +54,6 @@ struct mlx5_txq_stats {
struct priv;
-/* Memory region queue object. */
-struct mlx5_mr {
- LIST_ENTRY(mlx5_mr) next; /**< Pointer to the next element. */
- rte_atomic32_t refcnt; /*<< Reference counter. */
- uint32_t lkey; /*<< rte_cpu_to_be_32(mr->lkey) */
- uintptr_t start; /* Start address of MR */
- uintptr_t end; /* End address of MR */
- struct ibv_mr *mr; /*<< Memory Region. */
- struct rte_mempool *mp; /*<< Memory Pool. */
-};
-
/* Compressed CQE context. */
struct rxq_zip {
uint16_t ai; /* Array index. */
@@ -114,7 +103,6 @@ struct mlx5_rxq_ibv {
struct ibv_cq *cq; /* Completion Queue. */
struct ibv_wq *wq; /* Work Queue. */
struct ibv_comp_channel *channel;
- struct mlx5_mr *mr; /* Memory Region (for mp). */
};
/* RX queue control descriptor. */
@@ -175,7 +163,6 @@ struct mlx5_txq_data {
uint16_t mpw_hdr_dseg:1; /* Enable DSEGs in the title WQEBB. */
uint16_t max_inline; /* Multiple of RTE_CACHE_LINE_SIZE to inline. */
uint16_t inline_max_packet_sz; /* Max packet size for inlining. */
- uint16_t mr_cache_idx; /* Index of last hit entry. */
uint32_t qp_num_8s; /* QP number shifted by 8. */
uint64_t offloads; /* Offloads for Tx Queue. */
volatile struct mlx5_cqe (*cqes)[]; /* Completion queue. */
@@ -183,7 +170,6 @@ struct mlx5_txq_data {
volatile uint32_t *qp_db; /* Work queue doorbell. */
volatile uint32_t *cq_db; /* Completion queue doorbell. */
volatile void *bf_reg; /* Blueflame register remapped. */
- struct mlx5_mr *mp2mr[MLX5_PMD_TX_MP_CACHE]; /* MR translation table. */
struct rte_mbuf *(*elts)[]; /* TX elements. */
struct mlx5_txq_stats stats; /* TX queue counters. */
} __rte_cache_aligned;
@@ -322,12 +308,6 @@ uint16_t mlx5_tx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts,
uint16_t mlx5_rx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts,
uint16_t pkts_n);
-/* mlx5_mr.c */
-
-void mlx5_mp2mr_iter(struct rte_mempool *mp, void *arg);
-struct mlx5_mr *mlx5_txq_mp2mr_reg(struct mlx5_txq_data *txq,
- struct rte_mempool *mp, unsigned int idx);
-
#ifndef NDEBUG
/**
* Verify or set magic value in CQE.
@@ -513,76 +493,12 @@ mlx5_tx_complete(struct mlx5_txq_data *txq)
*txq->cq_db = rte_cpu_to_be_32(cq_ci);
}
-/**
- * Get Memory Pool (MP) from mbuf. If mbuf is indirect, the pool from which
- * the cloned mbuf is allocated is returned instead.
- *
- * @param buf
- * Pointer to mbuf.
- *
- * @return
- * Memory pool where data is located for given mbuf.
- */
-static struct rte_mempool *
-mlx5_tx_mb2mp(struct rte_mbuf *buf)
-{
- if (unlikely(RTE_MBUF_INDIRECT(buf)))
- return rte_mbuf_from_indirect(buf)->pool;
- return buf->pool;
-}
-
-/**
- * Get Memory Region (MR) <-> rte_mbuf association from txq->mp2mr[].
- * Add MP to txq->mp2mr[] if it's not registered yet. If mp2mr[] is full,
- * remove an entry first.
- *
- * @param txq
- * Pointer to TX queue structure.
- * @param[in] mp
- * Memory Pool for which a Memory Region lkey must be returned.
- *
- * @return
- * mr->lkey on success, (uint32_t)-1 on failure.
- */
static __rte_always_inline uint32_t
mlx5_tx_mb2mr(struct mlx5_txq_data *txq, struct rte_mbuf *mb)
{
- uint16_t i = txq->mr_cache_idx;
- uintptr_t addr = rte_pktmbuf_mtod(mb, uintptr_t);
- struct mlx5_mr *mr;
-
- assert(i < RTE_DIM(txq->mp2mr));
- if (likely(txq->mp2mr[i]->start <= addr && txq->mp2mr[i]->end > addr))
- return txq->mp2mr[i]->lkey;
- for (i = 0; (i != RTE_DIM(txq->mp2mr)); ++i) {
- if (unlikely(txq->mp2mr[i] == NULL ||
- txq->mp2mr[i]->mr == NULL)) {
- /* Unknown MP, add a new MR for it. */
- break;
- }
- if (txq->mp2mr[i]->start <= addr &&
- txq->mp2mr[i]->end > addr) {
- assert(txq->mp2mr[i]->lkey != (uint32_t)-1);
- txq->mr_cache_idx = i;
- return txq->mp2mr[i]->lkey;
- }
- }
- mr = mlx5_txq_mp2mr_reg(txq, mlx5_tx_mb2mp(mb), i);
- /*
- * Request the reference to use in this queue, the original one is
- * kept by the control plane.
- */
- if (mr) {
- rte_atomic32_inc(&mr->refcnt);
- txq->mr_cache_idx = i >= RTE_DIM(txq->mp2mr) ? i - 1 : i;
- return mr->lkey;
- } else {
- struct rte_mempool *mp = mlx5_tx_mb2mp(mb);
-
- DRV_LOG(WARNING, "failed to register mempool 0x%p(%s)",
- (void *)mp, mp->name);
- }
- return (uint32_t)-1;
+ (void)txq;
+ (void)mb;
+ return UINT32_MAX;
}
/**
diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c
index fc56d1ee8..3db6c3f35 100644
--- a/drivers/net/mlx5/mlx5_trigger.c
+++ b/drivers/net/mlx5/mlx5_trigger.c
@@ -48,17 +48,10 @@ mlx5_txq_start(struct rte_eth_dev *dev)
/* Add memory regions to Tx queues. */
for (i = 0; i != priv->txqs_n; ++i) {
- unsigned int idx = 0;
- struct mlx5_mr *mr;
struct mlx5_txq_ctrl *txq_ctrl = mlx5_txq_get(dev, i);
if (!txq_ctrl)
continue;
- LIST_FOREACH(mr, &priv->mr, next) {
- mlx5_txq_mp2mr_reg(&txq_ctrl->txq, mr->mp, idx++);
- if (idx == MLX5_PMD_TX_MP_CACHE)
- break;
- }
txq_alloc_elts(txq_ctrl);
txq_ctrl->ibv = mlx5_txq_ibv_new(dev, i);
if (!txq_ctrl->ibv) {
@@ -144,13 +137,11 @@ int
mlx5_dev_start(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
- struct mlx5_mr *mr = NULL;
int ret;
dev->data->dev_started = 1;
DRV_LOG(DEBUG, "port %u allocating and configuring hash Rx queues",
dev->data->port_id);
- rte_mempool_walk(mlx5_mp2mr_iter, priv);
ret = mlx5_txq_start(dev);
if (ret) {
DRV_LOG(ERR, "port %u Tx queue allocation failed: %s",
@@ -190,8 +181,6 @@ mlx5_dev_start(struct rte_eth_dev *dev)
ret = rte_errno; /* Save rte_errno before cleanup. */
/* Rollback. */
dev->data->dev_started = 0;
- for (mr = LIST_FIRST(&priv->mr); mr; mr = LIST_FIRST(&priv->mr))
- mlx5_mr_release(mr);
mlx5_flow_stop(dev, &priv->flows);
mlx5_traffic_disable(dev);
mlx5_txq_stop(dev);
@@ -212,7 +201,6 @@ void
mlx5_dev_stop(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
- struct mlx5_mr *mr;
dev->data->dev_started = 0;
/* Prevent crashes when queues are still in use. */
@@ -228,8 +216,6 @@ mlx5_dev_stop(struct rte_eth_dev *dev)
mlx5_dev_interrupt_handler_uninstall(dev);
mlx5_txq_stop(dev);
mlx5_rxq_stop(dev);
- for (mr = LIST_FIRST(&priv->mr); mr; mr = LIST_FIRST(&priv->mr))
- mlx5_mr_release(mr);
}
/**
diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index 3f4b5fea5..a71f3d0f0 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -409,7 +409,6 @@ mlx5_txq_ibv_new(struct rte_eth_dev *dev, uint16_t idx)
return NULL;
}
memset(&tmpl, 0, sizeof(struct mlx5_txq_ibv));
- /* MRs will be registered in mp2mr[] later. */
attr.cq = (struct ibv_cq_init_attr_ex){
.comp_mask = 0,
};
@@ -812,7 +811,6 @@ mlx5_txq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
tmpl->txq.elts_n = log2above(desc);
tmpl->idx = idx;
txq_set_params(tmpl);
- /* MRs will be registered in mp2mr[] later. */
DRV_LOG(DEBUG, "port %u priv->device_attr.max_qp_wr is %d",
dev->data->port_id, priv->device_attr.orig_attr.max_qp_wr);
DRV_LOG(DEBUG, "port %u priv->device_attr.max_sge is %d",
@@ -847,15 +845,7 @@ mlx5_txq_get(struct rte_eth_dev *dev, uint16_t idx)
if ((*priv->txqs)[idx]) {
ctrl = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl,
txq);
- unsigned int i;
-
mlx5_txq_ibv_get(dev, idx);
- for (i = 0; i != MLX5_PMD_TX_MP_CACHE; ++i) {
- if (ctrl->txq.mp2mr[i])
- claim_nonzero
- (mlx5_mr_get(dev,
- ctrl->txq.mp2mr[i]->mp));
- }
rte_atomic32_inc(&ctrl->refcnt);
}
return ctrl;
@@ -876,7 +866,6 @@ int
mlx5_txq_release(struct rte_eth_dev *dev, uint16_t idx)
{
struct priv *priv = dev->data->dev_private;
- unsigned int i;
struct mlx5_txq_ctrl *txq;
size_t page_size = sysconf(_SC_PAGESIZE);
@@ -885,12 +874,6 @@ mlx5_txq_release(struct rte_eth_dev *dev, uint16_t idx)
txq = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, txq);
if (txq->ibv && !mlx5_txq_ibv_release(txq->ibv))
txq->ibv = NULL;
- for (i = 0; i != MLX5_PMD_TX_MP_CACHE; ++i) {
- if (txq->txq.mp2mr[i]) {
- mlx5_mr_release(txq->txq.mp2mr[i]);
- txq->txq.mp2mr[i] = NULL;
- }
- }
if (priv->uar_base)
munmap((void *)RTE_ALIGN_FLOOR((uintptr_t)txq->txq.bf_reg,
page_size), page_size);
--
2.11.0
^ permalink raw reply [flat|nested] 23+ messages in thread* Re: [dpdk-dev] [PATCH 2/5] net/mlx5: remove Memory Region support
2018-05-02 23:16 ` [dpdk-dev] [PATCH 2/5] net/mlx5: remove Memory Region support Yongseok Koh
@ 2018-05-06 6:41 ` Shahaf Shuler
0 siblings, 0 replies; 23+ messages in thread
From: Shahaf Shuler @ 2018-05-06 6:41 UTC (permalink / raw)
To: Yongseok Koh, Adrien Mazarguil, Nélio Laranjeiro; +Cc: dev, Yongseok Koh
Thursday, May 3, 2018 2:17 AM, Yongseok Koh:
> Subject: [dpdk-dev] [PATCH 2/5] net/mlx5: remove Memory Region support
>
> This patch removes current support of Memory Region (MR) in order to
> accommodate the dynamic memory hotplug patch. This patch can be
> compiled but traffic can't flow and HW will raise faults. Subsequent patches
> will add new MR support.
>
> Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
Acked-by: Shahaf Shuler <shahafs@mellanox.com>
> ---
> config/common_base | 1 -
> doc/guides/nics/mlx5.rst | 8 -
> drivers/net/mlx5/Makefile | 4 -
> drivers/net/mlx5/mlx5.c | 4 -
> drivers/net/mlx5/mlx5.h | 10 --
> drivers/net/mlx5/mlx5_defs.h | 11 --
> drivers/net/mlx5/mlx5_mr.c | 343 ----------------------------------------
> drivers/net/mlx5/mlx5_rxq.c | 24 +--
> drivers/net/mlx5/mlx5_rxtx.h | 90 +----------
> drivers/net/mlx5/mlx5_trigger.c | 14 --
> drivers/net/mlx5/mlx5_txq.c | 17 --
> 11 files changed, 5 insertions(+), 521 deletions(-)
>
> diff --git a/config/common_base b/config/common_base index
> 03a8688b5..bf7d5e785 100644
> --- a/config/common_base
> +++ b/config/common_base
> @@ -296,7 +296,6 @@ CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE=8
> CONFIG_RTE_LIBRTE_MLX5_PMD=n
> CONFIG_RTE_LIBRTE_MLX5_DEBUG=n
> CONFIG_RTE_LIBRTE_MLX5_DLOPEN_DEPS=n
> -CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE=8
>
> #
> # Compile burst-oriented Netronome NFP PMD driver diff --git
> a/doc/guides/nics/mlx5.rst b/doc/guides/nics/mlx5.rst index
> 853c48f81..0fe6e1835 100644
> --- a/doc/guides/nics/mlx5.rst
> +++ b/doc/guides/nics/mlx5.rst
> @@ -167,14 +167,6 @@ These options can be modified in the ``.config`` file.
> adds additional run-time checks and debugging messages at the cost of
> lower performance.
>
> -- ``CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE`` (default **8**)
> -
> - Maximum number of cached memory pools (MPs) per TX queue. Each MP
> from
> - which buffers are to be transmitted must be associated to memory regions
> - (MRs). This is a slow operation that must be cached.
> -
> - This value is always 1 for RX queues since they use a single MP.
> -
> Environment variables
> ~~~~~~~~~~~~~~~~~~~~~
>
> diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile index
> 3c5b4943a..13f079334 100644
> --- a/drivers/net/mlx5/Makefile
> +++ b/drivers/net/mlx5/Makefile
> @@ -82,10 +82,6 @@ else
> CFLAGS += -DNDEBUG -UPEDANTIC
> endif
>
> -ifdef CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE
> -CFLAGS += -
> DMLX5_PMD_TX_MP_CACHE=$(CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACH
> E)
> -endif
> -
> include $(RTE_SDK)/mk/rte.lib.mk
>
> # Generate and clean-up mlx5_autoconf.h.
> diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index
> 6c4a571ab..01d554758 100644
> --- a/drivers/net/mlx5/mlx5.c
> +++ b/drivers/net/mlx5/mlx5.c
> @@ -245,10 +245,6 @@ mlx5_dev_close(struct rte_eth_dev *dev)
> if (ret)
> DRV_LOG(WARNING, "port %u some flows still remain",
> dev->data->port_id);
> - ret = mlx5_mr_verify(dev);
> - if (ret)
> - DRV_LOG(WARNING, "port %u some memory region still
> remain",
> - dev->data->port_id);
> memset(priv, 0, sizeof(*priv));
> }
>
> diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index
> 3ab16bfa2..47d266c90 100644
> --- a/drivers/net/mlx5/mlx5.h
> +++ b/drivers/net/mlx5/mlx5.h
> @@ -26,7 +26,6 @@
> #include <rte_pci.h>
> #include <rte_ether.h>
> #include <rte_ethdev_driver.h>
> -#include <rte_spinlock.h>
> #include <rte_interrupts.h>
> #include <rte_errno.h>
> #include <rte_flow.h>
> @@ -147,7 +146,6 @@ struct priv {
> struct mlx5_hrxq_drop *flow_drop_queue; /* Flow drop queue. */
> struct mlx5_flows flows; /* RTE Flow rules. */
> struct mlx5_flows ctrl_flows; /* Control flow rules. */
> - LIST_HEAD(mr, mlx5_mr) mr; /* Memory region. */
> LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */
> LIST_HEAD(rxqibv, mlx5_rxq_ibv) rxqsibv; /* Verbs Rx queues. */
> LIST_HEAD(hrxq, mlx5_hrxq) hrxqs; /* Verbs Hash Rx queues. */
> @@ -157,7 +155,6 @@ struct priv {
> LIST_HEAD(ind_tables, mlx5_ind_table_ibv) ind_tbls;
> uint32_t link_speed_capa; /* Link speed capabilities. */
> struct mlx5_xstats_ctrl xstats_ctrl; /* Extended stats control. */
> - rte_spinlock_t mr_lock; /* MR Lock. */
> int primary_socket; /* Unix socket for primary process. */
> void *uar_base; /* Reserved address space for UAR mapping */
> struct rte_intr_handle intr_handle_socket; /* Interrupt handler. */
> @@ -309,13 +306,6 @@ void mlx5_socket_uninit(struct rte_eth_dev *priv);
> void mlx5_socket_handle(struct rte_eth_dev *priv); int
> mlx5_socket_connect(struct rte_eth_dev *priv);
>
> -/* mlx5_mr.c */
> -
> -struct mlx5_mr *mlx5_mr_new(struct rte_eth_dev *dev, struct
> rte_mempool *mp); -struct mlx5_mr *mlx5_mr_get(struct rte_eth_dev
> *dev, struct rte_mempool *mp); -int mlx5_mr_release(struct mlx5_mr *mr);
> -int mlx5_mr_verify(struct rte_eth_dev *dev);
> -
> /* mlx5_nl.c */
>
> int mlx5_nl_init(uint32_t nlgroups);
> diff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h
> index 55a86957d..f9093777d 100644
> --- a/drivers/net/mlx5/mlx5_defs.h
> +++ b/drivers/net/mlx5/mlx5_defs.h
> @@ -38,17 +38,6 @@
> #define MLX5_TX_COMP_THRESH_INLINE_DIV (1 << 3)
>
> /*
> - * Maximum number of cached Memory Pools (MPs) per TX queue. Each
> RTE MP
> - * from which buffers are to be transmitted will have to be mapped by this
> - * driver to their own Memory Region (MR). This is a slow operation.
> - *
> - * This value is always 1 for RX queues.
> - */
> -#ifndef MLX5_PMD_TX_MP_CACHE
> -#define MLX5_PMD_TX_MP_CACHE 8
> -#endif
> -
> -/*
> * If defined, only use software counters. The PMD will never ask the
> hardware
> * for these, and many of them won't be available.
> */
> diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c index
> 6613bd6b9..736c40ae4 100644
> --- a/drivers/net/mlx5/mlx5_mr.c
> +++ b/drivers/net/mlx5/mlx5_mr.c
> @@ -18,346 +18,3 @@
> #include "mlx5_rxtx.h"
> #include "mlx5_glue.h"
>
> -struct mlx5_check_mempool_data {
> - int ret;
> - char *start;
> - char *end;
> -};
> -
> -/* Called by mlx5_check_mempool() when iterating the memory chunks. */
> -static void -mlx5_check_mempool_cb(struct rte_mempool *mp
> __rte_unused,
> - void *opaque, struct rte_mempool_memhdr *memhdr,
> - unsigned int mem_idx __rte_unused)
> -{
> - struct mlx5_check_mempool_data *data = opaque;
> -
> - /* It already failed, skip the next chunks. */
> - if (data->ret != 0)
> - return;
> - /* It is the first chunk. */
> - if (data->start == NULL && data->end == NULL) {
> - data->start = memhdr->addr;
> - data->end = data->start + memhdr->len;
> - return;
> - }
> - if (data->end == memhdr->addr) {
> - data->end += memhdr->len;
> - return;
> - }
> - if (data->start == (char *)memhdr->addr + memhdr->len) {
> - data->start -= memhdr->len;
> - return;
> - }
> - /* Error, mempool is not virtually contiguous. */
> - data->ret = -1;
> -}
> -
> -/**
> - * Check if a mempool can be used: it must be virtually contiguous.
> - *
> - * @param[in] mp
> - * Pointer to memory pool.
> - * @param[out] start
> - * Pointer to the start address of the mempool virtual memory area
> - * @param[out] end
> - * Pointer to the end address of the mempool virtual memory area
> - *
> - * @return
> - * 0 on success (mempool is virtually contiguous), -1 on error.
> - */
> -static int
> -mlx5_check_mempool(struct rte_mempool *mp, uintptr_t *start,
> - uintptr_t *end)
> -{
> - struct mlx5_check_mempool_data data;
> -
> - memset(&data, 0, sizeof(data));
> - rte_mempool_mem_iter(mp, mlx5_check_mempool_cb, &data);
> - *start = (uintptr_t)data.start;
> - *end = (uintptr_t)data.end;
> - return data.ret;
> -}
> -
> -/**
> - * Register a Memory Region (MR) <-> Memory Pool (MP) association in
> - * txq->mp2mr[]. If mp2mr[] is full, remove an entry first.
> - *
> - * @param txq
> - * Pointer to TX queue structure.
> - * @param[in] mp
> - * Memory Pool for which a Memory Region lkey must be returned.
> - * @param idx
> - * Index of the next available entry.
> - *
> - * @return
> - * mr on success, NULL on failure and rte_errno is set.
> - */
> -struct mlx5_mr *
> -mlx5_txq_mp2mr_reg(struct mlx5_txq_data *txq, struct rte_mempool
> *mp,
> - unsigned int idx)
> -{
> - struct mlx5_txq_ctrl *txq_ctrl =
> - container_of(txq, struct mlx5_txq_ctrl, txq);
> - struct rte_eth_dev *dev;
> - struct mlx5_mr *mr;
> -
> - rte_spinlock_lock(&txq_ctrl->priv->mr_lock);
> - /* Add a new entry, register MR first. */
> - DRV_LOG(DEBUG, "port %u discovered new memory pool \"%s\"
> (%p)",
> - port_id(txq_ctrl->priv), mp->name, (void *)mp);
> - dev = eth_dev(txq_ctrl->priv);
> - mr = mlx5_mr_get(dev, mp);
> - if (mr == NULL) {
> - if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
> - DRV_LOG(DEBUG,
> - "port %u using unregistered mempool
> 0x%p(%s)"
> - " in secondary process, please create
> mempool"
> - " before rte_eth_dev_start()",
> - port_id(txq_ctrl->priv), (void *)mp, mp-
> >name);
> - rte_spinlock_unlock(&txq_ctrl->priv->mr_lock);
> - rte_errno = ENOTSUP;
> - return NULL;
> - }
> - mr = mlx5_mr_new(dev, mp);
> - }
> - if (unlikely(mr == NULL)) {
> - DRV_LOG(DEBUG,
> - "port %u unable to configure memory region,"
> - " ibv_reg_mr() failed.",
> - port_id(txq_ctrl->priv));
> - rte_spinlock_unlock(&txq_ctrl->priv->mr_lock);
> - return NULL;
> - }
> - if (unlikely(idx == RTE_DIM(txq->mp2mr))) {
> - /* Table is full, remove oldest entry. */
> - DRV_LOG(DEBUG,
> - "port %u memory region <-> memory pool table full,
> "
> - " dropping oldest entry",
> - port_id(txq_ctrl->priv));
> - --idx;
> - mlx5_mr_release(txq->mp2mr[0]);
> - memmove(&txq->mp2mr[0], &txq->mp2mr[1],
> - (sizeof(txq->mp2mr) - sizeof(txq->mp2mr[0])));
> - }
> - /* Store the new entry. */
> - txq_ctrl->txq.mp2mr[idx] = mr;
> - DRV_LOG(DEBUG,
> - "port %u new memory region lkey for MP \"%s\" (%p):
> 0x%08"
> - PRIu32,
> - port_id(txq_ctrl->priv), mp->name, (void *)mp,
> - txq_ctrl->txq.mp2mr[idx]->lkey);
> - rte_spinlock_unlock(&txq_ctrl->priv->mr_lock);
> - return mr;
> -}
> -
> -struct mlx5_mp2mr_mbuf_check_data {
> - int ret;
> -};
> -
> -/**
> - * Callback function for rte_mempool_obj_iter() to check whether a given
> - * mempool object looks like a mbuf.
> - *
> - * @param[in] mp
> - * The mempool pointer
> - * @param[in] arg
> - * Context data (struct txq_mp2mr_mbuf_check_data). Contains the
> - * return value.
> - * @param[in] obj
> - * Object address.
> - * @param index
> - * Object index, unused.
> - */
> -static void
> -txq_mp2mr_mbuf_check(struct rte_mempool *mp, void *arg, void *obj,
> - uint32_t index __rte_unused)
> -{
> - struct mlx5_mp2mr_mbuf_check_data *data = arg;
> - struct rte_mbuf *buf = obj;
> -
> - /*
> - * Check whether mbuf structure fits element size and whether
> mempool
> - * pointer is valid.
> - */
> - if (sizeof(*buf) > mp->elt_size || buf->pool != mp)
> - data->ret = -1;
> -}
> -
> -/**
> - * Iterator function for rte_mempool_walk() to register existing mempools
> and
> - * fill the MP to MR cache of a TX queue.
> - *
> - * @param[in] mp
> - * Memory Pool to register.
> - * @param *arg
> - * Pointer to TX queue structure.
> - */
> -void
> -mlx5_mp2mr_iter(struct rte_mempool *mp, void *arg) -{
> - struct priv *priv = (struct priv *)arg;
> - struct mlx5_mp2mr_mbuf_check_data data = {
> - .ret = 0,
> - };
> - struct mlx5_mr *mr;
> -
> - /* Register mempool only if the first element looks like a mbuf. */
> - if (rte_mempool_obj_iter(mp, txq_mp2mr_mbuf_check, &data) ==
> 0 ||
> - data.ret == -1)
> - return;
> - mr = mlx5_mr_get(eth_dev(priv), mp);
> - if (mr) {
> - mlx5_mr_release(mr);
> - return;
> - }
> - mr = mlx5_mr_new(eth_dev(priv), mp);
> - if (!mr)
> - DRV_LOG(ERR, "port %u cannot create memory region: %s",
> - port_id(priv), strerror(rte_errno));
> -}
> -
> -/**
> - * Register a new memory region from the mempool and store it in the
> memory
> - * region list.
> - *
> - * @param dev
> - * Pointer to Ethernet device.
> - * @param mp
> - * Pointer to the memory pool to register.
> - *
> - * @return
> - * The memory region on success, NULL on failure and rte_errno is set.
> - */
> -struct mlx5_mr *
> -mlx5_mr_new(struct rte_eth_dev *dev, struct rte_mempool *mp) -{
> - struct priv *priv = dev->data->dev_private;
> - const struct rte_memseg *ms;
> - uintptr_t start;
> - uintptr_t end;
> - struct mlx5_mr *mr;
> -
> - mr = rte_zmalloc_socket(__func__, sizeof(*mr), 0, mp->socket_id);
> - if (!mr) {
> - DRV_LOG(DEBUG,
> - "port %u unable to configure memory region,"
> - " ibv_reg_mr() failed.",
> - dev->data->port_id);
> - rte_errno = ENOMEM;
> - return NULL;
> - }
> - if (mlx5_check_mempool(mp, &start, &end) != 0) {
> - DRV_LOG(ERR, "port %u mempool %p: not virtually
> contiguous",
> - dev->data->port_id, (void *)mp);
> - rte_errno = ENOMEM;
> - return NULL;
> - }
> - DRV_LOG(DEBUG, "port %u mempool %p area start=%p end=%p
> size=%zu",
> - dev->data->port_id, (void *)mp, (void *)start, (void *)end,
> - (size_t)(end - start));
> - /* Save original addresses for exact MR lookup. */
> - mr->start = start;
> - mr->end = end;
> -
> - /* Round start and end to page boundary if found in memory
> segments. */
> - ms = rte_mem_virt2memseg((void *)start, NULL);
> - if (ms != NULL)
> - start = RTE_ALIGN_FLOOR(start, ms->hugepage_sz);
> - end = RTE_ALIGN_CEIL(end, ms->hugepage_sz);
> - DRV_LOG(DEBUG,
> - "port %u mempool %p using start=%p end=%p size=%zu for
> memory"
> - " region",
> - dev->data->port_id, (void *)mp, (void *)start, (void *)end,
> - (size_t)(end - start));
> - mr->mr = mlx5_glue->reg_mr(priv->pd, (void *)start, end - start,
> - IBV_ACCESS_LOCAL_WRITE);
> - if (!mr->mr) {
> - rte_errno = ENOMEM;
> - return NULL;
> - }
> - mr->mp = mp;
> - mr->lkey = rte_cpu_to_be_32(mr->mr->lkey);
> - rte_atomic32_inc(&mr->refcnt);
> - DRV_LOG(DEBUG, "port %u new memory Region %p refcnt: %d",
> - dev->data->port_id, (void *)mr, rte_atomic32_read(&mr-
> >refcnt));
> - LIST_INSERT_HEAD(&priv->mr, mr, next);
> - return mr;
> -}
> -
> -/**
> - * Search the memory region object in the memory region list.
> - *
> - * @param dev
> - * Pointer to Ethernet device.
> - * @param mp
> - * Pointer to the memory pool to register.
> - *
> - * @return
> - * The memory region on success.
> - */
> -struct mlx5_mr *
> -mlx5_mr_get(struct rte_eth_dev *dev, struct rte_mempool *mp) -{
> - struct priv *priv = dev->data->dev_private;
> - struct mlx5_mr *mr;
> -
> - assert(mp);
> - if (LIST_EMPTY(&priv->mr))
> - return NULL;
> - LIST_FOREACH(mr, &priv->mr, next) {
> - if (mr->mp == mp) {
> - rte_atomic32_inc(&mr->refcnt);
> - return mr;
> - }
> - }
> - return NULL;
> -}
> -
> -/**
> - * Release the memory region object.
> - *
> - * @param mr
> - * Pointer to memory region to release.
> - *
> - * @return
> - * 1 while a reference on it exists, 0 when freed.
> - */
> -int
> -mlx5_mr_release(struct mlx5_mr *mr)
> -{
> - assert(mr);
> - if (rte_atomic32_dec_and_test(&mr->refcnt)) {
> - DRV_LOG(DEBUG, "memory region %p refcnt: %d", (void
> *)mr,
> - rte_atomic32_read(&mr->refcnt));
> - claim_zero(mlx5_glue->dereg_mr(mr->mr));
> - LIST_REMOVE(mr, next);
> - rte_free(mr);
> - return 0;
> - }
> - return 1;
> -}
> -
> -/**
> - * Verify the flow list is empty
> - *
> - * @param dev
> - * Pointer to Ethernet device.
> - *
> - * @return
> - * The number of object not released.
> - */
> -int
> -mlx5_mr_verify(struct rte_eth_dev *dev) -{
> - struct priv *priv = dev->data->dev_private;
> - int ret = 0;
> - struct mlx5_mr *mr;
> -
> - LIST_FOREACH(mr, &priv->mr, next) {
> - DRV_LOG(DEBUG, "port %u memory region %p still
> referenced",
> - dev->data->port_id, (void *)mr);
> - ++ret;
> - }
> - return ret;
> -}
> diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
> index d993e3846..d4fe1fed7 100644
> --- a/drivers/net/mlx5/mlx5_rxq.c
> +++ b/drivers/net/mlx5/mlx5_rxq.c
> @@ -649,16 +649,6 @@ mlx5_rxq_ibv_new(struct rte_eth_dev *dev,
> uint16_t idx)
> goto error;
> }
> tmpl->rxq_ctrl = rxq_ctrl;
> - /* Use the entire RX mempool as the memory region. */
> - tmpl->mr = mlx5_mr_get(dev, rxq_data->mp);
> - if (!tmpl->mr) {
> - tmpl->mr = mlx5_mr_new(dev, rxq_data->mp);
> - if (!tmpl->mr) {
> - DRV_LOG(ERR, "port %u: memeroy region creation
> failure",
> - dev->data->port_id);
> - goto error;
> - }
> - }
> if (rxq_ctrl->irq) {
> tmpl->channel = mlx5_glue->create_comp_channel(priv-
> >ctx);
> if (!tmpl->channel) {
> @@ -799,7 +789,7 @@ mlx5_rxq_ibv_new(struct rte_eth_dev *dev,
> uint16_t idx)
> .addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(buf,
> uintptr_t)),
> .byte_count = rte_cpu_to_be_32(DATA_LEN(buf)),
> - .lkey = tmpl->mr->lkey,
> + .lkey = UINT32_MAX,
> };
> }
> rxq_data->rq_db = rwq.dbrec;
> @@ -835,8 +825,6 @@ mlx5_rxq_ibv_new(struct rte_eth_dev *dev,
> uint16_t idx)
> claim_zero(mlx5_glue->destroy_cq(tmpl->cq));
> if (tmpl->channel)
> claim_zero(mlx5_glue->destroy_comp_channel(tmpl-
> >channel));
> - if (tmpl->mr)
> - mlx5_mr_release(tmpl->mr);
> priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_NONE;
> rte_errno = ret; /* Restore rte_errno. */
> return NULL;
> @@ -865,10 +853,8 @@ mlx5_rxq_ibv_get(struct rte_eth_dev *dev,
> uint16_t idx)
> if (!rxq_data)
> return NULL;
> rxq_ctrl = container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
> - if (rxq_ctrl->ibv) {
> - mlx5_mr_get(dev, rxq_data->mp);
> + if (rxq_ctrl->ibv)
> rte_atomic32_inc(&rxq_ctrl->ibv->refcnt);
> - }
> return rxq_ctrl->ibv;
> }
>
> @@ -884,15 +870,9 @@ mlx5_rxq_ibv_get(struct rte_eth_dev *dev,
> uint16_t idx) int mlx5_rxq_ibv_release(struct mlx5_rxq_ibv *rxq_ibv) {
> - int ret;
> -
> assert(rxq_ibv);
> assert(rxq_ibv->wq);
> assert(rxq_ibv->cq);
> - assert(rxq_ibv->mr);
> - ret = mlx5_mr_release(rxq_ibv->mr);
> - if (!ret)
> - rxq_ibv->mr = NULL;
> if (rte_atomic32_dec_and_test(&rxq_ibv->refcnt)) {
> DRV_LOG(DEBUG, "port %u Verbs Rx queue %u: refcnt %d",
> port_id(rxq_ibv->rxq_ctrl->priv),
> diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
> index 2fc12a186..e8cad51aa 100644
> --- a/drivers/net/mlx5/mlx5_rxtx.h
> +++ b/drivers/net/mlx5/mlx5_rxtx.h
> @@ -54,17 +54,6 @@ struct mlx5_txq_stats {
>
> struct priv;
>
> -/* Memory region queue object. */
> -struct mlx5_mr {
> - LIST_ENTRY(mlx5_mr) next; /**< Pointer to the next element. */
> - rte_atomic32_t refcnt; /*<< Reference counter. */
> - uint32_t lkey; /*<< rte_cpu_to_be_32(mr->lkey) */
> - uintptr_t start; /* Start address of MR */
> - uintptr_t end; /* End address of MR */
> - struct ibv_mr *mr; /*<< Memory Region. */
> - struct rte_mempool *mp; /*<< Memory Pool. */
> -};
> -
> /* Compressed CQE context. */
> struct rxq_zip {
> uint16_t ai; /* Array index. */
> @@ -114,7 +103,6 @@ struct mlx5_rxq_ibv {
> struct ibv_cq *cq; /* Completion Queue. */
> struct ibv_wq *wq; /* Work Queue. */
> struct ibv_comp_channel *channel;
> - struct mlx5_mr *mr; /* Memory Region (for mp). */
> };
>
> /* RX queue control descriptor. */
> @@ -175,7 +163,6 @@ struct mlx5_txq_data {
> uint16_t mpw_hdr_dseg:1; /* Enable DSEGs in the title WQEBB. */
> uint16_t max_inline; /* Multiple of RTE_CACHE_LINE_SIZE to inline.
> */
> uint16_t inline_max_packet_sz; /* Max packet size for inlining. */
> - uint16_t mr_cache_idx; /* Index of last hit entry. */
> uint32_t qp_num_8s; /* QP number shifted by 8. */
> uint64_t offloads; /* Offloads for Tx Queue. */
> volatile struct mlx5_cqe (*cqes)[]; /* Completion queue. */ @@ -
> 183,7 +170,6 @@ struct mlx5_txq_data {
> volatile uint32_t *qp_db; /* Work queue doorbell. */
> volatile uint32_t *cq_db; /* Completion queue doorbell. */
> volatile void *bf_reg; /* Blueflame register remapped. */
> - struct mlx5_mr *mp2mr[MLX5_PMD_TX_MP_CACHE]; /* MR
> translation table. */
> struct rte_mbuf *(*elts)[]; /* TX elements. */
> struct mlx5_txq_stats stats; /* TX queue counters. */ }
> __rte_cache_aligned; @@ -322,12 +308,6 @@ uint16_t
> mlx5_tx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t
> mlx5_rx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts,
> uint16_t pkts_n);
>
> -/* mlx5_mr.c */
> -
> -void mlx5_mp2mr_iter(struct rte_mempool *mp, void *arg); -struct
> mlx5_mr *mlx5_txq_mp2mr_reg(struct mlx5_txq_data *txq,
> - struct rte_mempool *mp, unsigned int idx);
> -
> #ifndef NDEBUG
> /**
> * Verify or set magic value in CQE.
> @@ -513,76 +493,12 @@ mlx5_tx_complete(struct mlx5_txq_data *txq)
> *txq->cq_db = rte_cpu_to_be_32(cq_ci); }
>
> -/**
> - * Get Memory Pool (MP) from mbuf. If mbuf is indirect, the pool from
> which
> - * the cloned mbuf is allocated is returned instead.
> - *
> - * @param buf
> - * Pointer to mbuf.
> - *
> - * @return
> - * Memory pool where data is located for given mbuf.
> - */
> -static struct rte_mempool *
> -mlx5_tx_mb2mp(struct rte_mbuf *buf)
> -{
> - if (unlikely(RTE_MBUF_INDIRECT(buf)))
> - return rte_mbuf_from_indirect(buf)->pool;
> - return buf->pool;
> -}
> -
> -/**
> - * Get Memory Region (MR) <-> rte_mbuf association from txq->mp2mr[].
> - * Add MP to txq->mp2mr[] if it's not registered yet. If mp2mr[] is full,
> - * remove an entry first.
> - *
> - * @param txq
> - * Pointer to TX queue structure.
> - * @param[in] mp
> - * Memory Pool for which a Memory Region lkey must be returned.
> - *
> - * @return
> - * mr->lkey on success, (uint32_t)-1 on failure.
> - */
> static __rte_always_inline uint32_t
> mlx5_tx_mb2mr(struct mlx5_txq_data *txq, struct rte_mbuf *mb) {
> - uint16_t i = txq->mr_cache_idx;
> - uintptr_t addr = rte_pktmbuf_mtod(mb, uintptr_t);
> - struct mlx5_mr *mr;
> -
> - assert(i < RTE_DIM(txq->mp2mr));
> - if (likely(txq->mp2mr[i]->start <= addr && txq->mp2mr[i]->end >
> addr))
> - return txq->mp2mr[i]->lkey;
> - for (i = 0; (i != RTE_DIM(txq->mp2mr)); ++i) {
> - if (unlikely(txq->mp2mr[i] == NULL ||
> - txq->mp2mr[i]->mr == NULL)) {
> - /* Unknown MP, add a new MR for it. */
> - break;
> - }
> - if (txq->mp2mr[i]->start <= addr &&
> - txq->mp2mr[i]->end > addr) {
> - assert(txq->mp2mr[i]->lkey != (uint32_t)-1);
> - txq->mr_cache_idx = i;
> - return txq->mp2mr[i]->lkey;
> - }
> - }
> - mr = mlx5_txq_mp2mr_reg(txq, mlx5_tx_mb2mp(mb), i);
> - /*
> - * Request the reference to use in this queue, the original one is
> - * kept by the control plane.
> - */
> - if (mr) {
> - rte_atomic32_inc(&mr->refcnt);
> - txq->mr_cache_idx = i >= RTE_DIM(txq->mp2mr) ? i - 1 : i;
> - return mr->lkey;
> - } else {
> - struct rte_mempool *mp = mlx5_tx_mb2mp(mb);
> -
> - DRV_LOG(WARNING, "failed to register mempool
> 0x%p(%s)",
> - (void *)mp, mp->name);
> - }
> - return (uint32_t)-1;
> + (void)txq;
> + (void)mb;
> + return UINT32_MAX;
> }
>
> /**
> diff --git a/drivers/net/mlx5/mlx5_trigger.c
> b/drivers/net/mlx5/mlx5_trigger.c index fc56d1ee8..3db6c3f35 100644
> --- a/drivers/net/mlx5/mlx5_trigger.c
> +++ b/drivers/net/mlx5/mlx5_trigger.c
> @@ -48,17 +48,10 @@ mlx5_txq_start(struct rte_eth_dev *dev)
>
> /* Add memory regions to Tx queues. */
> for (i = 0; i != priv->txqs_n; ++i) {
> - unsigned int idx = 0;
> - struct mlx5_mr *mr;
> struct mlx5_txq_ctrl *txq_ctrl = mlx5_txq_get(dev, i);
>
> if (!txq_ctrl)
> continue;
> - LIST_FOREACH(mr, &priv->mr, next) {
> - mlx5_txq_mp2mr_reg(&txq_ctrl->txq, mr->mp,
> idx++);
> - if (idx == MLX5_PMD_TX_MP_CACHE)
> - break;
> - }
> txq_alloc_elts(txq_ctrl);
> txq_ctrl->ibv = mlx5_txq_ibv_new(dev, i);
> if (!txq_ctrl->ibv) {
> @@ -144,13 +137,11 @@ int
> mlx5_dev_start(struct rte_eth_dev *dev) {
> struct priv *priv = dev->data->dev_private;
> - struct mlx5_mr *mr = NULL;
> int ret;
>
> dev->data->dev_started = 1;
> DRV_LOG(DEBUG, "port %u allocating and configuring hash Rx
> queues",
> dev->data->port_id);
> - rte_mempool_walk(mlx5_mp2mr_iter, priv);
> ret = mlx5_txq_start(dev);
> if (ret) {
> DRV_LOG(ERR, "port %u Tx queue allocation failed: %s", @@
> -190,8 +181,6 @@ mlx5_dev_start(struct rte_eth_dev *dev)
> ret = rte_errno; /* Save rte_errno before cleanup. */
> /* Rollback. */
> dev->data->dev_started = 0;
> - for (mr = LIST_FIRST(&priv->mr); mr; mr = LIST_FIRST(&priv->mr))
> - mlx5_mr_release(mr);
> mlx5_flow_stop(dev, &priv->flows);
> mlx5_traffic_disable(dev);
> mlx5_txq_stop(dev);
> @@ -212,7 +201,6 @@ void
> mlx5_dev_stop(struct rte_eth_dev *dev)
> {
> struct priv *priv = dev->data->dev_private;
> - struct mlx5_mr *mr;
>
> dev->data->dev_started = 0;
> /* Prevent crashes when queues are still in use. */ @@ -228,8 +216,6
> @@ mlx5_dev_stop(struct rte_eth_dev *dev)
> mlx5_dev_interrupt_handler_uninstall(dev);
> mlx5_txq_stop(dev);
> mlx5_rxq_stop(dev);
> - for (mr = LIST_FIRST(&priv->mr); mr; mr = LIST_FIRST(&priv->mr))
> - mlx5_mr_release(mr);
> }
>
> /**
> diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
> index 3f4b5fea5..a71f3d0f0 100644
> --- a/drivers/net/mlx5/mlx5_txq.c
> +++ b/drivers/net/mlx5/mlx5_txq.c
> @@ -409,7 +409,6 @@ mlx5_txq_ibv_new(struct rte_eth_dev *dev,
> uint16_t idx)
> return NULL;
> }
> memset(&tmpl, 0, sizeof(struct mlx5_txq_ibv));
> - /* MRs will be registered in mp2mr[] later. */
> attr.cq = (struct ibv_cq_init_attr_ex){
> .comp_mask = 0,
> };
> @@ -812,7 +811,6 @@ mlx5_txq_new(struct rte_eth_dev *dev, uint16_t
> idx, uint16_t desc,
> tmpl->txq.elts_n = log2above(desc);
> tmpl->idx = idx;
> txq_set_params(tmpl);
> - /* MRs will be registered in mp2mr[] later. */
> DRV_LOG(DEBUG, "port %u priv->device_attr.max_qp_wr is %d",
> dev->data->port_id, priv-
> >device_attr.orig_attr.max_qp_wr);
> DRV_LOG(DEBUG, "port %u priv->device_attr.max_sge is %d", @@ -
> 847,15 +845,7 @@ mlx5_txq_get(struct rte_eth_dev *dev, uint16_t idx)
> if ((*priv->txqs)[idx]) {
> ctrl = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl,
> txq);
> - unsigned int i;
> -
> mlx5_txq_ibv_get(dev, idx);
> - for (i = 0; i != MLX5_PMD_TX_MP_CACHE; ++i) {
> - if (ctrl->txq.mp2mr[i])
> - claim_nonzero
> - (mlx5_mr_get(dev,
> - ctrl->txq.mp2mr[i]->mp));
> - }
> rte_atomic32_inc(&ctrl->refcnt);
> }
> return ctrl;
> @@ -876,7 +866,6 @@ int
> mlx5_txq_release(struct rte_eth_dev *dev, uint16_t idx) {
> struct priv *priv = dev->data->dev_private;
> - unsigned int i;
> struct mlx5_txq_ctrl *txq;
> size_t page_size = sysconf(_SC_PAGESIZE);
>
> @@ -885,12 +874,6 @@ mlx5_txq_release(struct rte_eth_dev *dev,
> uint16_t idx)
> txq = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, txq);
> if (txq->ibv && !mlx5_txq_ibv_release(txq->ibv))
> txq->ibv = NULL;
> - for (i = 0; i != MLX5_PMD_TX_MP_CACHE; ++i) {
> - if (txq->txq.mp2mr[i]) {
> - mlx5_mr_release(txq->txq.mp2mr[i]);
> - txq->txq.mp2mr[i] = NULL;
> - }
> - }
> if (priv->uar_base)
> munmap((void *)RTE_ALIGN_FLOOR((uintptr_t)txq-
> >txq.bf_reg,
> page_size), page_size);
> --
> 2.11.0
^ permalink raw reply [flat|nested] 23+ messages in thread
* [dpdk-dev] [PATCH 3/5] net/mlx5: add new Memory Region support
2018-05-02 23:16 [dpdk-dev] [PATCH 0/5] net/mlx: add new Memory Region support Yongseok Koh
2018-05-02 23:16 ` [dpdk-dev] [PATCH 1/5] net/mlx5: trim debug messages for reference counters Yongseok Koh
2018-05-02 23:16 ` [dpdk-dev] [PATCH 2/5] net/mlx5: remove Memory Region support Yongseok Koh
@ 2018-05-02 23:16 ` Yongseok Koh
2018-05-03 8:21 ` Burakov, Anatoly
2018-05-06 12:53 ` Shahaf Shuler
2018-05-02 23:16 ` [dpdk-dev] [PATCH 4/5] net/mlx4: remove " Yongseok Koh
` (2 subsequent siblings)
5 siblings, 2 replies; 23+ messages in thread
From: Yongseok Koh @ 2018-05-02 23:16 UTC (permalink / raw)
To: adrien.mazarguil, nelio.laranjeiro; +Cc: dev, Yongseok Koh
This is the new design of Memory Region (MR) for mlx PMD, in order to:
- Accommodate the new memory hotplug model.
- Support non-contiguous Mempool.
Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
---
drivers/net/mlx5/mlx5.c | 45 ++
drivers/net/mlx5/mlx5.h | 22 +
drivers/net/mlx5/mlx5_defs.h | 6 +
drivers/net/mlx5/mlx5_ethdev.c | 16 +
drivers/net/mlx5/mlx5_mr.c | 1194 ++++++++++++++++++++++++++++++++++++++
drivers/net/mlx5/mlx5_mr.h | 121 ++++
drivers/net/mlx5/mlx5_rxq.c | 8 +-
drivers/net/mlx5/mlx5_rxtx.c | 3 +
drivers/net/mlx5/mlx5_rxtx.h | 73 ++-
drivers/net/mlx5/mlx5_rxtx_vec.h | 6 +-
drivers/net/mlx5/mlx5_trigger.c | 11 +
drivers/net/mlx5/mlx5_txq.c | 11 +
12 files changed, 1508 insertions(+), 8 deletions(-)
create mode 100644 drivers/net/mlx5/mlx5_mr.h
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 01d554758..2883f20af 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -41,6 +41,7 @@
#include "mlx5_autoconf.h"
#include "mlx5_defs.h"
#include "mlx5_glue.h"
+#include "mlx5_mr.h"
/* Device parameter to enable RX completion queue compression. */
#define MLX5_RXQ_CQE_COMP_EN "rxq_cqe_comp_en"
@@ -84,10 +85,49 @@
#define MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP (1 << 4)
#endif
+static const char *MZ_MLX5_PMD_SHARED_DATA = "mlx5_pmd_shared_data";
+
+/* Shared memory between primary and secondary processes. */
+struct mlx5_shared_data *mlx5_shared_data;
+
+/* Spinlock for mlx5_shared_data allocation. */
+static rte_spinlock_t mlx5_shared_data_lock = RTE_SPINLOCK_INITIALIZER;
+
/** Driver-specific log messages type. */
int mlx5_logtype;
/**
+ * Prepare shared data between primary and secondary process.
+ */
+static void
+mlx5_prepare_shared_data(void)
+{
+ const struct rte_memzone *mz;
+
+ rte_spinlock_lock(&mlx5_shared_data_lock);
+ if (mlx5_shared_data == NULL) {
+ if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+ /* Allocate shared memory. */
+ mz = rte_memzone_reserve(MZ_MLX5_PMD_SHARED_DATA,
+ sizeof(*mlx5_shared_data),
+ SOCKET_ID_ANY, 0);
+ } else {
+ /* Lookup allocated shared memory. */
+ mz = rte_memzone_lookup(MZ_MLX5_PMD_SHARED_DATA);
+ }
+ if (mz == NULL)
+ rte_panic("Cannot allocate mlx5 shared data\n");
+ mlx5_shared_data = mz->addr;
+ /* Initialize shared data. */
+ if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+ LIST_INIT(&mlx5_shared_data->mem_event_cb_list);
+ rte_rwlock_init(&mlx5_shared_data->mem_event_rwlock);
+ }
+ }
+ rte_spinlock_unlock(&mlx5_shared_data_lock);
+}
+
+/**
* Retrieve integer value from environment variable.
*
* @param[in] name
@@ -201,6 +241,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
priv->txqs = NULL;
}
mlx5_flow_delete_drop_queue(dev);
+ mlx5_mr_release(dev);
if (priv->pd != NULL) {
assert(priv->ctx != NULL);
claim_zero(mlx5_glue->dealloc_pd(priv->pd));
@@ -633,6 +674,8 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
struct ibv_counter_set_description cs_desc;
#endif
+ /* Prepare shared data between primary and secondary process. */
+ mlx5_prepare_shared_data();
assert(pci_drv == &mlx5_driver);
/* Get mlx5_dev[] index. */
idx = mlx5_dev_idx(&pci_dev->addr);
@@ -1293,6 +1336,8 @@ rte_mlx5_pmd_init(void)
}
mlx5_glue->fork_init();
rte_pci_register(&mlx5_driver);
+ rte_mem_event_callback_register("MLX5_MEM_EVENT_CB",
+ mlx5_mr_mem_event_cb);
}
RTE_PMD_EXPORT_NAME(net_mlx5, __COUNTER__);
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 47d266c90..d3fc74dc1 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -26,11 +26,13 @@
#include <rte_pci.h>
#include <rte_ether.h>
#include <rte_ethdev_driver.h>
+#include <rte_rwlock.h>
#include <rte_interrupts.h>
#include <rte_errno.h>
#include <rte_flow.h>
#include "mlx5_utils.h"
+#include "mlx5_mr.h"
#include "mlx5_rxtx.h"
#include "mlx5_autoconf.h"
#include "mlx5_defs.h"
@@ -50,6 +52,16 @@ enum {
PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF = 0x101a,
};
+LIST_HEAD(mlx5_dev_list, priv);
+
+/* Shared memory between primary and secondary processes. */
+struct mlx5_shared_data {
+ struct mlx5_dev_list mem_event_cb_list;
+ rte_rwlock_t mem_event_rwlock;
+};
+
+extern struct mlx5_shared_data *mlx5_shared_data;
+
struct mlx5_xstats_ctrl {
/* Number of device stats. */
uint16_t stats_n;
@@ -119,7 +131,10 @@ struct mlx5_verbs_alloc_ctx {
const void *obj; /* Pointer to the DPDK object. */
};
+LIST_HEAD(mlx5_mr_list, mlx5_mr);
+
struct priv {
+ LIST_ENTRY(priv) mem_event_cb; /* Called by memory event callback. */
struct rte_eth_dev_data *dev_data; /* Pointer to device data. */
struct ibv_context *ctx; /* Verbs context. */
struct ibv_device_attr_ex device_attr; /* Device properties. */
@@ -146,6 +161,13 @@ struct priv {
struct mlx5_hrxq_drop *flow_drop_queue; /* Flow drop queue. */
struct mlx5_flows flows; /* RTE Flow rules. */
struct mlx5_flows ctrl_flows; /* Control flow rules. */
+ struct {
+ uint32_t dev_gen; /* Generation number to flush local caches. */
+ rte_rwlock_t rwlock; /* MR Lock. */
+ struct mlx5_mr_btree cache; /* Global MR cache table. */
+ struct mlx5_mr_list mr_list; /* Registered MR list. */
+ struct mlx5_mr_list mr_free_list; /* Freed MR list. */
+ } mr;
LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */
LIST_HEAD(rxqibv, mlx5_rxq_ibv) rxqsibv; /* Verbs Rx queues. */
LIST_HEAD(hrxq, mlx5_hrxq) hrxqs; /* Verbs Hash Rx queues. */
diff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h
index f9093777d..72e80af26 100644
--- a/drivers/net/mlx5/mlx5_defs.h
+++ b/drivers/net/mlx5/mlx5_defs.h
@@ -37,6 +37,12 @@
*/
#define MLX5_TX_COMP_THRESH_INLINE_DIV (1 << 3)
+/* Size of per-queue MR cache array for linear search. */
+#define MLX5_MR_CACHE_N 8
+
+/* Size of MR cache table for binary search. */
+#define MLX5_MR_BTREE_CACHE_N 256
+
/*
* If defined, only use software counters. The PMD will never ask the hardware
* for these, and many of them won't be available.
diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c
index 746b94f73..6bb43cf4e 100644
--- a/drivers/net/mlx5/mlx5_ethdev.c
+++ b/drivers/net/mlx5/mlx5_ethdev.c
@@ -34,6 +34,7 @@
#include <rte_interrupts.h>
#include <rte_malloc.h>
#include <rte_string_fns.h>
+#include <rte_rwlock.h>
#include "mlx5.h"
#include "mlx5_glue.h"
@@ -413,6 +414,21 @@ mlx5_dev_configure(struct rte_eth_dev *dev)
if (++j == rxqs_n)
j = 0;
}
+ /*
+ * Once the device is added to the list of memory event callback, its
+ * global MR cache table cannot be expanded on the fly because of
+ * deadlock. If it overflows, lookup should be done by searching MR list
+ * linearly, which is slow.
+ */
+ if (mlx5_mr_btree_init(&priv->mr.cache, MLX5_MR_BTREE_CACHE_N * 2,
+ dev->device->numa_node)) {
+ /* rte_errno is already set. */
+ return -rte_errno;
+ }
+ rte_rwlock_write_lock(&mlx5_shared_data->mem_event_rwlock);
+ LIST_INSERT_HEAD(&mlx5_shared_data->mem_event_cb_list,
+ priv, mem_event_cb);
+ rte_rwlock_write_unlock(&mlx5_shared_data->mem_event_rwlock);
return 0;
}
diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c
index 736c40ae4..e964912bb 100644
--- a/drivers/net/mlx5/mlx5_mr.c
+++ b/drivers/net/mlx5/mlx5_mr.c
@@ -13,8 +13,1202 @@
#include <rte_mempool.h>
#include <rte_malloc.h>
+#include <rte_rwlock.h>
#include "mlx5.h"
+#include "mlx5_mr.h"
#include "mlx5_rxtx.h"
#include "mlx5_glue.h"
+struct mr_find_contig_memsegs_data {
+ uintptr_t addr;
+ uintptr_t start;
+ uintptr_t end;
+ const struct rte_memseg_list *msl;
+};
+
+struct mr_update_mp_data {
+ struct rte_eth_dev *dev;
+ struct mlx5_mr_ctrl *mr_ctrl;
+ int ret;
+};
+
+/**
+ * Expand B-tree table to a given size. Can't be called with holding
+ * memory_hotplug_lock or priv->mr.rwlock due to rte_realloc().
+ *
+ * @param bt
+ * Pointer to B-tree structure.
+ * @param n
+ * Number of entries for expansion.
+ *
+ * @return
+ * 0 on success, -1 on failure.
+ */
+static int
+mr_btree_expand(struct mlx5_mr_btree *bt, int n)
+{
+ void *mem;
+ int ret = 0;
+
+ if (n <= bt->size)
+ return ret;
+ /*
+ * Downside of directly using rte_realloc() is that SOCKET_ID_ANY is
+ * used inside if there's no room to expand. Because this is a quite
+ * rare case and a part of very slow path, it is very acceptable.
+ * Initially cache_bh[] will be given practically enough space and once
+ * it is expanded, expansion wouldn't be needed again ever.
+ */
+ mem = rte_realloc(bt->table, n * sizeof(struct mlx5_mr_cache), 0);
+ if (mem == NULL) {
+ /* Not an error, B-tree search will be skipped. */
+ DRV_LOG(WARNING, "failed to expand MR B-tree (%p) table",
+ (void *)bt);
+ ret = -1;
+ } else {
+ DRV_LOG(DEBUG, "expanded MR B-tree table (size=%u)", n);
+ bt->table = mem;
+ bt->size = n;
+ }
+ return ret;
+}
+
+/**
+ * Look up LKey from given B-tree lookup table, store the last index and return
+ * searched LKey.
+ *
+ * @param bt
+ * Pointer to B-tree structure.
+ * @param[out] idx
+ * Pointer to index. Even on searh failure, returns index where it stops
+ * searching so that index can be used when inserting a new entry.
+ * @param addr
+ * Search key.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on no match.
+ */
+static uint32_t
+mr_btree_lookup(struct mlx5_mr_btree *bt, uint16_t *idx, uintptr_t addr)
+{
+ struct mlx5_mr_cache *lkp_tbl;
+ uint16_t n;
+ uint16_t base = 0;
+
+ assert(bt != NULL);
+ lkp_tbl = *bt->table;
+ n = bt->len;
+ /* First entry must be NULL for comparison. */
+ assert(bt->len > 0 || (lkp_tbl[0].start == 0 &&
+ lkp_tbl[0].lkey == UINT32_MAX));
+ /* Binary search. */
+ do {
+ register uint16_t delta = n >> 1;
+
+ if (addr < lkp_tbl[base + delta].start) {
+ n = delta;
+ } else {
+ base += delta;
+ n -= delta;
+ }
+ } while (n > 1);
+ assert(addr >= lkp_tbl[base].start);
+ *idx = base;
+ if (addr < lkp_tbl[base].end)
+ return lkp_tbl[base].lkey;
+ /* Not found. */
+ return UINT32_MAX;
+}
+
+/**
+ * Insert an entry to B-tree lookup table.
+ *
+ * @param bt
+ * Pointer to B-tree structure.
+ * @param entry
+ * Pointer to new entry to insert.
+ *
+ * @return
+ * 0 on success, -1 on failure.
+ */
+static int
+mr_btree_insert(struct mlx5_mr_btree *bt, struct mlx5_mr_cache *entry)
+{
+ struct mlx5_mr_cache *lkp_tbl;
+ uint16_t idx = 0;
+ size_t shift;
+
+ assert(bt != NULL);
+ assert(bt->len <= bt->size);
+ assert(bt->len > 0);
+ lkp_tbl = *bt->table;
+ /* Find out the slot for insertion. */
+ if (mr_btree_lookup(bt, &idx, entry->start) != UINT32_MAX) {
+ DRV_LOG(DEBUG,
+ "abort insertion to B-tree(%p):"
+ " already exist at idx=%u [0x%lx, 0x%lx) lkey=0x%x",
+ (void *)bt, idx, entry->start, entry->end, entry->lkey);
+ /* Already exist, return. */
+ return 0;
+ }
+ /* If table is full, return error. */
+ if (unlikely(bt->len == bt->size)) {
+ bt->overflow = 1;
+ return -1;
+ }
+ /* Insert entry. */
+ ++idx;
+ shift = (bt->len - idx) * sizeof(struct mlx5_mr_cache);
+ if (shift)
+ memmove(&lkp_tbl[idx + 1], &lkp_tbl[idx], shift);
+ lkp_tbl[idx] = *entry;
+ bt->len++;
+ DRV_LOG(DEBUG,
+ "inserted B-tree(%p)[%u], [0x%lx, 0x%lx) lkey=0x%x",
+ (void *)bt, idx, entry->start, entry->end, entry->lkey);
+ return 0;
+}
+
+/**
+ * Initialize B-tree and allocate memory for lookup table.
+ *
+ * @param bt
+ * Pointer to B-tree structure.
+ * @param n
+ * Number of entries to allocate.
+ * @param socket
+ * NUMA socket on which memory must be allocated.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_mr_btree_init(struct mlx5_mr_btree *bt, int n, int socket)
+{
+ if (bt == NULL) {
+ rte_errno = EINVAL;
+ return -rte_errno;
+ }
+ memset(bt, 0, sizeof(*bt));
+ bt->table = rte_calloc_socket("B-tree table",
+ n, sizeof(struct mlx5_mr_cache),
+ 0, socket);
+ if (bt->table == NULL) {
+ rte_errno = ENOMEM;
+ DRV_LOG(ERR,
+ "failed to allocate memory for btree cache on socket %d",
+ socket);
+ return -rte_errno;
+ }
+ bt->size = n;
+ /* First entry must be NULL for binary search. */
+ (*bt->table)[bt->len++] = (struct mlx5_mr_cache) {
+ .lkey = UINT32_MAX,
+ };
+ DRV_LOG(DEBUG, "initialized B-tree %p with table %p",
+ (void *)bt, (void *)bt->table);
+ return 0;
+}
+
+/**
+ * Free B-tree resources.
+ *
+ * @param bt
+ * Pointer to B-tree structure.
+ */
+void
+mlx5_mr_btree_free(struct mlx5_mr_btree *bt)
+{
+ if (bt == NULL)
+ return;
+ DRV_LOG(DEBUG, "freeing B-tree %p with table %p",
+ (void *)bt, (void *)bt->table);
+ rte_free(bt->table);
+ memset(bt, 0, sizeof(*bt));
+}
+
+/**
+ * Dump all the entries in a B-tree
+ *
+ * @param bt
+ * Pointer to B-tree structure.
+ */
+static void
+mlx5_mr_btree_dump(struct mlx5_mr_btree *bt)
+{
+ int idx;
+ struct mlx5_mr_cache *lkp_tbl;
+
+ if (bt == NULL)
+ return;
+ lkp_tbl = *bt->table;
+ for (idx = 0; idx < bt->len; ++idx) {
+ struct mlx5_mr_cache *entry = &lkp_tbl[idx];
+
+ DRV_LOG(DEBUG,
+ "B-tree(%p)[%u], [0x%lx, 0x%lx) lkey=0x%x",
+ (void *)bt, idx, entry->start, entry->end, entry->lkey);
+ }
+}
+
+/**
+ * Find virtually contiguous memory chunk in a given MR.
+ *
+ * @param dev
+ * Pointer to MR structure.
+ * @param[out] entry
+ * Pointer to returning MR cache entry. If not found, this will not be
+ * updated.
+ * @param start_idx
+ * Start index of the memseg bitmap.
+ *
+ * @return
+ * Next index to go on lookup.
+ */
+static int
+mr_find_next_chunk(struct mlx5_mr *mr, struct mlx5_mr_cache *entry,
+ int base_idx)
+{
+ uintptr_t start = 0;
+ uintptr_t end = 0;
+ uint32_t idx = 0;
+
+ for (idx = base_idx; idx < mr->ms_bmp_n; ++idx) {
+ if (rte_bitmap_get(mr->ms_bmp, idx)) {
+ const struct rte_memseg_list *msl;
+ const struct rte_memseg *ms;
+
+ msl = mr->msl;
+ ms = rte_fbarray_get(&msl->memseg_arr,
+ mr->ms_base_idx + idx);
+ assert(msl->page_sz == ms->hugepage_sz);
+ if (!start)
+ start = ms->addr_64;
+ end = ms->addr_64 + ms->hugepage_sz;
+ } else if (start) {
+ /* Passed the end of a fragment. */
+ break;
+ }
+ }
+ if (start) {
+ /* Found one chunk. */
+ entry->start = start;
+ entry->end = end;
+ entry->lkey = rte_cpu_to_be_32(mr->ibv_mr->lkey);
+ }
+ return idx;
+}
+
+/**
+ * Insert a MR to the global B-tree cache. It may fail due to low-on-memory.
+ * Then, this entry will have to be searched by mr_lookup_dev_list() in
+ * mlx5_mr_create() on miss.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param mr
+ * Pointer to MR to insert.
+ *
+ * @return
+ * 0 on success, -1 on failure.
+ */
+static int
+mr_insert_dev_cache(struct rte_eth_dev *dev, struct mlx5_mr *mr)
+{
+ struct priv *priv = dev->data->dev_private;
+ unsigned int n;
+
+ DRV_LOG(DEBUG, "port %u inserting MR(%p) to global cache",
+ dev->data->port_id, (void *)mr);
+ for (n = 0; n < mr->ms_bmp_n; ) {
+ struct mlx5_mr_cache entry = { 0, };
+
+ /* Find a contiguous chunk and advance the index. */
+ n = mr_find_next_chunk(mr, &entry, n);
+ if (!entry.end)
+ break;
+ if (mr_btree_insert(&priv->mr.cache, &entry) < 0) {
+ /*
+ * Overflowed, but the global table cannot be expanded
+ * because of deadlock.
+ */
+ return -1;
+ }
+ }
+ return 0;
+}
+
+/**
+ * Look up address in the original global MR list.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param[out] entry
+ * Pointer to returning MR cache entry. If no match, this will not be updated.
+ * @param addr
+ * Search key.
+ *
+ * @return
+ * Found MR on match, NULL otherwise.
+ */
+static struct mlx5_mr *
+mr_lookup_dev_list(struct rte_eth_dev *dev, struct mlx5_mr_cache *entry,
+ uintptr_t addr)
+{
+ struct priv *priv = dev->data->dev_private;
+ struct mlx5_mr *mr;
+
+ /* Iterate all the existing MRs. */
+ LIST_FOREACH(mr, &priv->mr.mr_list, mr) {
+ unsigned int n;
+
+ if (mr->ms_n == 0)
+ continue;
+ for (n = 0; n < mr->ms_bmp_n; ) {
+ struct mlx5_mr_cache ret = { 0, };
+
+ n = mr_find_next_chunk(mr, &ret, n);
+ if (addr >= ret.start && addr < ret.end) {
+ /* Found. */
+ *entry = ret;
+ return mr;
+ }
+ }
+ }
+ return NULL;
+}
+
+/**
+ * Look up address on device.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param[out] entry
+ * Pointer to returning MR cache entry. If no match, this will not be updated.
+ * @param addr
+ * Search key.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on failure and rte_errno is set.
+ */
+static uint32_t
+mr_lookup_dev(struct rte_eth_dev *dev, struct mlx5_mr_cache *entry,
+ uintptr_t addr)
+{
+ struct priv *priv = dev->data->dev_private;
+ uint16_t idx;
+ uint32_t lkey = UINT32_MAX;
+ struct mlx5_mr *mr;
+
+ /*
+ * If the global cache has overflowed since it failed to expand the
+ * B-tree table, it can't have all the exisitng MRs. Then, the address
+ * has to be searched by traversing the original MR list instead, which
+ * is very slow path. Otherwise, the global cache is all inclusive.
+ */
+ if (!unlikely(priv->mr.cache.overflow)) {
+ lkey = mr_btree_lookup(&priv->mr.cache, &idx, addr);
+ if (lkey != UINT32_MAX)
+ *entry = (*priv->mr.cache.table)[idx];
+ } else {
+ /* Falling back to the slowest path. */
+ mr = mr_lookup_dev_list(dev, entry, addr);
+ if (mr != NULL)
+ lkey = entry->lkey;
+ }
+ assert(lkey == UINT32_MAX || (addr >= entry->start &&
+ addr < entry->end));
+ return lkey;
+}
+
+/**
+ * Free MR resources. MR lock must not be held to avoid a deadlock. rte_free()
+ * can raise memory free event and the callback function will spin on the lock.
+ *
+ * @param mr
+ * Pointer to MR to free.
+ */
+static void
+mr_free(struct mlx5_mr *mr)
+{
+ if (mr == NULL)
+ return;
+ DRV_LOG(DEBUG, "freeing MR(%p):", (void *)mr);
+ if (mr->ibv_mr != NULL)
+ claim_zero(mlx5_glue->dereg_mr(mr->ibv_mr));
+ if (mr->ms_bmp != NULL)
+ rte_bitmap_free(mr->ms_bmp);
+ rte_free(mr);
+}
+
+/**
+ * Free Memory Region (MR).
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param mr
+ * Pointer to MR to free.
+ */
+void
+mlx5_mr_free(struct rte_eth_dev *dev, struct mlx5_mr *mr)
+{
+ struct priv *priv = dev->data->dev_private;
+
+ /* Detach from the list and free resources later. */
+ rte_rwlock_write_lock(&priv->mr.rwlock);
+ LIST_REMOVE(mr, mr);
+ rte_rwlock_write_unlock(&priv->mr.rwlock);
+ /*
+ * rte_free() inside can't be called with holding the lock. This could
+ * cause deadlock when calling free callback.
+ */
+ mr_free(mr);
+ DRV_LOG(DEBUG, "port %u MR(%p) freed", dev->data->port_id, (void *)mr);
+}
+
+/**
+ * Releass resources of detached MR having no online entry.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ */
+static void
+mlx5_mr_garbage_collect(struct rte_eth_dev *dev)
+{
+ struct priv *priv = dev->data->dev_private;
+ struct mlx5_mr *mr_next;
+ struct mlx5_mr_list free_list = LIST_HEAD_INITIALIZER(free_list);
+
+ /* Must be called from the primary process. */
+ assert(rte_eal_process_type() == RTE_PROC_PRIMARY);
+ /*
+ * MR can't be freed with holding the lock because rte_free() could call
+ * memory free callback function. This will be a deadlock situation.
+ */
+ rte_rwlock_write_lock(&priv->mr.rwlock);
+ /* Detach the whole free list and release it after unlocking. */
+ free_list = priv->mr.mr_free_list;
+ LIST_INIT(&priv->mr.mr_free_list);
+ rte_rwlock_write_unlock(&priv->mr.rwlock);
+ /* Release resources. */
+ mr_next = LIST_FIRST(&free_list);
+ while (mr_next != NULL) {
+ struct mlx5_mr *mr = mr_next;
+
+ mr_next = LIST_NEXT(mr, mr);
+ mr_free(mr);
+ }
+}
+
+/* Called during rte_memseg_contig_walk() by mlx5_mr_create(). */
+static int
+mr_find_contig_memsegs_cb(const struct rte_memseg_list *msl,
+ const struct rte_memseg *ms, size_t len, void *arg)
+{
+ struct mr_find_contig_memsegs_data *data = arg;
+
+ if (data->addr < ms->addr_64 || data->addr >= ms->addr_64 + len)
+ return 0;
+ /* Found, save it and stop walking. */
+ data->start = ms->addr_64;
+ data->end = ms->addr_64 + len;
+ data->msl = msl;
+ return 1;
+}
+
+/**
+ * Create a new global Memroy Region (MR) for a missing virtual address.
+ * Register entire virtually contiguous memory chunk around the address.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param[out] entry
+ * Pointer to returning MR cache entry, found in the global cache or newly
+ * created. If failed to create one, this will not be updated.
+ * @param addr
+ * Target virtual address to register.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on failure and rte_errno is set.
+ */
+static uint32_t
+mlx5_mr_create(struct rte_eth_dev *dev, struct mlx5_mr_cache *entry,
+ uintptr_t addr)
+{
+ struct priv *priv = dev->data->dev_private;
+ struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+ const struct rte_memseg_list *msl;
+ const struct rte_memseg *ms;
+ struct mlx5_mr *mr = NULL;
+ size_t len;
+ uint32_t ms_n;
+ uint32_t bmp_size;
+ void *bmp_mem;
+ int ms_idx_shift = -1;
+ unsigned int n;
+ struct mr_find_contig_memsegs_data data = {
+ .addr = addr,
+ };
+ struct mr_find_contig_memsegs_data data_re;
+
+ DRV_LOG(DEBUG, "port %u creating a MR using address (%p)",
+ dev->data->port_id, (void *)addr);
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+ DRV_LOG(WARNING,
+ "port %u using address (%p) of unregistered mempool"
+ " in secondary process, please create mempool"
+ " before rte_eth_dev_start()",
+ dev->data->port_id, (void *)addr);
+ rte_errno = EPERM;
+ goto err_nolock;
+ }
+ /*
+ * Release detached MRs if any. This can't be called with holding either
+ * memory_hotplug_lock or priv->mr.rwlock. MRs on the free list have
+ * been detached by the memory free event but it couldn't be released
+ * inside the callback due to deadlock. As a result, releasing resources
+ * is quite opportunistic.
+ */
+ mlx5_mr_garbage_collect(dev);
+ /*
+ * Find out a contiguous virtual address chunk in use, to which the
+ * given address belongs, in order to register maximum range. In the
+ * best case where mempools are not dynamically recreated and
+ * '--socket-mem' is speicified as an EAL option, it is very likely to
+ * have only one MR(LKey) per a socket and per a hugepage-size even
+ * though the system memory is highly fragmented.
+ */
+ if (!rte_memseg_contig_walk(mr_find_contig_memsegs_cb, &data)) {
+ DRV_LOG(WARNING,
+ "port %u unable to find virtually contigous"
+ " chunk for address (%p)."
+ " rte_memseg_contig_walk() failed.",
+ dev->data->port_id, (void *)addr);
+ rte_errno = ENXIO;
+ goto err_nolock;
+ }
+alloc_resources:
+ /* Addresses must be page-aligned. */
+ assert(rte_is_aligned((void *)data.start, data.msl->page_sz));
+ assert(rte_is_aligned((void *)data.end, data.msl->page_sz));
+ msl = data.msl;
+ ms = rte_mem_virt2memseg((void *)data.start, msl);
+ len = data.end - data.start;
+ assert(msl->page_sz == ms->hugepage_sz);
+ /* Number of memsegs in the range. */
+ ms_n = len / msl->page_sz;
+ DRV_LOG(DEBUG,
+ "port %u extending %p to [0x%lx, 0x%lx), page_sz=0x%lx, ms_n=%u",
+ dev->data->port_id, (void *)addr,
+ data.start, data.end, msl->page_sz, ms_n);
+ /* Size of memory for bitmap. */
+ bmp_size = rte_bitmap_get_memory_footprint(ms_n);
+ mr = rte_zmalloc_socket(NULL,
+ RTE_ALIGN_CEIL(sizeof(*mr),
+ RTE_CACHE_LINE_SIZE) +
+ bmp_size,
+ RTE_CACHE_LINE_SIZE, msl->socket_id);
+ if (mr == NULL) {
+ DRV_LOG(WARNING,
+ "port %u unable to allocate memory for a new MR of"
+ " address (%p).",
+ dev->data->port_id, (void *)addr);
+ rte_errno = ENOMEM;
+ goto err_nolock;
+ }
+ mr->msl = msl;
+ /*
+ * Save the index of the first memseg and initialize memseg bitmap. To
+ * see if a memseg of ms_idx in the memseg-list is still valid, check:
+ * rte_bitmap_get(mr->bmp, ms_idx - mr->ms_base_idx)
+ */
+ mr->ms_base_idx = rte_fbarray_find_idx(&msl->memseg_arr, ms);
+ bmp_mem = RTE_PTR_ALIGN_CEIL(mr + 1, RTE_CACHE_LINE_SIZE);
+ mr->ms_bmp = rte_bitmap_init(ms_n, bmp_mem, bmp_size);
+ if (mr->ms_bmp == NULL) {
+ DRV_LOG(WARNING,
+ "port %u unable to initialize bitamp for a new MR of"
+ " address (%p).",
+ dev->data->port_id, (void *)addr);
+ rte_errno = EINVAL;
+ goto err_nolock;
+ }
+ /*
+ * Should recheck whether the extended contiguous chunk is still valid.
+ * Because memory_hotplug_lock can't be held if there's any memory
+ * related calls in a critical path, resource allocation above can't be
+ * locked. If the memory has been changed at this point, try again with
+ * just single page. If not, go on with the big chunk atomically from
+ * here.
+ */
+ rte_rwlock_read_lock(&mcfg->memory_hotplug_lock);
+ data_re = data;
+ if (len > msl->page_sz &&
+ !rte_memseg_contig_walk(mr_find_contig_memsegs_cb, &data_re)) {
+ DRV_LOG(WARNING,
+ "port %u unable to find virtually contigous"
+ " chunk for address (%p)."
+ " rte_memseg_contig_walk() failed.",
+ dev->data->port_id, (void *)addr);
+ rte_errno = ENXIO;
+ goto err_memlock;
+ }
+ if (data.start != data_re.start || data.end != data_re.end) {
+ /*
+ * The extended contiguous chunk has been changed. Try again
+ * with single memseg instead.
+ */
+ data.start = RTE_ALIGN_FLOOR(addr, msl->page_sz);
+ data.end = data.start + msl->page_sz;
+ rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
+ mr_free(mr);
+ goto alloc_resources;
+ }
+ assert(data.msl == data_re.msl);
+ rte_rwlock_write_lock(&priv->mr.rwlock);
+ /*
+ * Check the address is really missing. If other thread already created
+ * one or it is not found due to overflow, abort and return.
+ */
+ if (mr_lookup_dev(dev, entry, addr) != UINT32_MAX) {
+ /*
+ * Insert to the global cache table. It may fail due to
+ * low-on-memory. Then, this entry will have to be searched
+ * here again.
+ */
+ mr_btree_insert(&priv->mr.cache, entry);
+ DRV_LOG(DEBUG,
+ "port %u found MR for %p on final lookup, abort",
+ dev->data->port_id, (void *)addr);
+ rte_rwlock_write_unlock(&priv->mr.rwlock);
+ rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
+ /*
+ * Must be unlocked before calling rte_free() because
+ * mlx5_mr_mem_event_free_cb() can be called inside.
+ */
+ mr_free(mr);
+ return entry->lkey;
+ }
+ /*
+ * Trim start and end addresses for verbs MR. Set bits for registering
+ * memsegs but exclude already registered ones. Bitmap can be
+ * fragmented.
+ */
+ for (n = 0; n < ms_n; ++n) {
+ uintptr_t start;
+ struct mlx5_mr_cache ret = { 0, };
+
+ start = data_re.start + n * msl->page_sz;
+ /* Exclude memsegs already registered by other MRs. */
+ if (mr_lookup_dev(dev, &ret, start) == UINT32_MAX) {
+ /*
+ * Start from the first unregistered memseg in the
+ * extended range.
+ */
+ if (ms_idx_shift == -1) {
+ mr->ms_base_idx += n;
+ data.start = start;
+ ms_idx_shift = n;
+ }
+ data.end = start + msl->page_sz;
+ rte_bitmap_set(mr->ms_bmp, n - ms_idx_shift);
+ ++mr->ms_n;
+ }
+ }
+ len = data.end - data.start;
+ mr->ms_bmp_n = len / msl->page_sz;
+ assert(ms_idx_shift + mr->ms_bmp_n <= ms_n);
+ /*
+ * Finally create a verbs MR for the memory chunk. ibv_reg_mr() can be
+ * called with holding the memory lock because it doesn't use
+ * mlx5_alloc_buf_extern() which eventually calls rte_malloc_socket()
+ * through mlx5_alloc_verbs_buf().
+ */
+ mr->ibv_mr = mlx5_glue->reg_mr(priv->pd, (void *)data.start, len,
+ IBV_ACCESS_LOCAL_WRITE);
+ if (mr->ibv_mr == NULL) {
+ DRV_LOG(WARNING,
+ "port %u fail to create a verbs MR for address (%p)",
+ dev->data->port_id, (void *)addr);
+ rte_errno = EINVAL;
+ goto err_mrlock;
+ }
+ assert((uintptr_t)mr->ibv_mr->addr == data.start);
+ assert(mr->ibv_mr->length == len);
+ LIST_INSERT_HEAD(&priv->mr.mr_list, mr, mr);
+ DRV_LOG(DEBUG,
+ "port %u MR CREATED (%p) for %p:\n"
+ " [0x%lx, 0x%lx), lkey=0x%x base_idx=%u ms_n=%u, ms_bmp_n=%u",
+ dev->data->port_id, (void *)mr, (void *)addr,
+ data.start, data.end, rte_cpu_to_be_32(mr->ibv_mr->lkey),
+ mr->ms_base_idx, mr->ms_n, mr->ms_bmp_n);
+ /* Insert to the global cache table. */
+ mr_insert_dev_cache(dev, mr);
+ /* Fill in output data. */
+ mr_lookup_dev(dev, entry, addr);
+ /* Lookup can't fail. */
+ assert(entry->lkey != UINT32_MAX);
+ rte_rwlock_write_unlock(&priv->mr.rwlock);
+ rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
+ return entry->lkey;
+err_mrlock:
+ rte_rwlock_write_unlock(&priv->mr.rwlock);
+err_memlock:
+ rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
+err_nolock:
+ /*
+ * In case of error, as this can be called in a datapath, a warning
+ * message per an error is preferable instead. Must be unlocked before
+ * calling rte_free() because mlx5_mr_mem_event_free_cb() can be called
+ * inside.
+ */
+ mr_free(mr);
+ return UINT32_MAX;
+}
+
+/**
+ * Rebuild the global B-tree cache of device from the original MR list.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ */
+static void
+mr_rebuild_dev_cache(struct rte_eth_dev *dev)
+{
+ struct priv *priv = dev->data->dev_private;
+ struct mlx5_mr *mr;
+
+ DRV_LOG(DEBUG, "port %u rebuild dev cache[]", dev->data->port_id);
+ /* Flush cache to rebuild. */
+ priv->mr.cache.len = 1;
+ priv->mr.cache.overflow = 0;
+ /* Iterate all the existing MRs. */
+ LIST_FOREACH(mr, &priv->mr.mr_list, mr)
+ if (mr_insert_dev_cache(dev, mr) < 0)
+ return;
+}
+
+/**
+ * Callback for memory free event. Iterate freed memsegs and check whether it
+ * belongs to an existing MR. If found, clear the bit from bitmap of MR. As a
+ * result, the MR would be fragmented. If it becomes empty, the MR will be freed
+ * later by mlx5_mr_garbage_collect(). Even if this callback is called from a
+ * secondary process, the garbage collector will be called in primary process
+ * as the secondary process can't call mlx5_mr_create().
+ *
+ * The global cache must be rebuilt if there's any change and this event has to
+ * be propagated to dataplane threads to flush the local caches.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param addr
+ * Address of freed memory.
+ * @param len
+ * Size of freed memory.
+ */
+static void
+mlx5_mr_mem_event_free_cb(struct rte_eth_dev *dev, const void *addr, size_t len)
+{
+ struct priv *priv = dev->data->dev_private;
+ const struct rte_memseg_list *msl;
+ struct mlx5_mr *mr;
+ int ms_n;
+ int i;
+ int rebuild = 0;
+
+ DRV_LOG(DEBUG, "port %u free callback: addr=%p, len=%lu",
+ dev->data->port_id, addr, len);
+ msl = rte_mem_virt2memseg_list(addr);
+ /* addr and len must be page-aligned. */
+ assert((uintptr_t)addr == RTE_ALIGN((uintptr_t)addr, msl->page_sz));
+ assert(len == RTE_ALIGN(len, msl->page_sz));
+ ms_n = len / msl->page_sz;
+ rte_rwlock_write_lock(&priv->mr.rwlock);
+ /* Clear bits of freed memsegs from MR. */
+ for (i = 0; i < ms_n; ++i) {
+ const struct rte_memseg *ms;
+ struct mlx5_mr_cache entry;
+ uintptr_t start;
+ int ms_idx;
+ uint32_t pos;
+
+ /* Find MR having this memseg. */
+ start = (uintptr_t)addr + i * msl->page_sz;
+ mr = mr_lookup_dev_list(dev, &entry, start);
+ if (mr == NULL)
+ continue;
+ ms = rte_mem_virt2memseg((void *)start, msl);
+ assert(ms != NULL);
+ assert(msl->page_sz == ms->hugepage_sz);
+ ms_idx = rte_fbarray_find_idx(&msl->memseg_arr, ms);
+ pos = ms_idx - mr->ms_base_idx;
+ assert(rte_bitmap_get(mr->ms_bmp, pos));
+ assert(pos < mr->ms_bmp_n);
+ DRV_LOG(DEBUG, "port %u MR(%p): clear bitmap[%u] for addr %p",
+ dev->data->port_id, (void *)mr, pos, (void *)start);
+ rte_bitmap_clear(mr->ms_bmp, pos);
+ if (--mr->ms_n == 0) {
+ LIST_REMOVE(mr, mr);
+ LIST_INSERT_HEAD(&priv->mr.mr_free_list, mr, mr);
+ DRV_LOG(DEBUG, "port %u remove MR(%p) from list",
+ dev->data->port_id, (void *)mr);
+ }
+ /*
+ * MR is fragmented or will be freed. the global cache must be
+ * rebuilt.
+ */
+ rebuild = 1;
+ }
+ if (rebuild) {
+ mr_rebuild_dev_cache(dev);
+ /*
+ * Flush local caches by propagating invalidation across cores.
+ * rte_smp_wmb() is enough to synchronize this event. If one of
+ * freed memsegs is seen by other core, that means the memseg
+ * has been allocated by allocator, which will come after this
+ * free call. Therefore, this store instruction (incrementing
+ * generation below) will be guaranteed to be seen by other core
+ * before the core sees the newly allocated memory.
+ */
+ ++priv->mr.dev_gen;
+ DRV_LOG(DEBUG, "broadcasting local cache flush, gen=%d",
+ priv->mr.dev_gen);
+ rte_smp_wmb();
+ }
+ rte_rwlock_write_unlock(&priv->mr.rwlock);
+ if (rebuild && rte_log_get_level(mlx5_logtype) == RTE_LOG_DEBUG)
+ mlx5_mr_dump_dev(dev);
+}
+
+/**
+ * Callback for memory event. This can be called from both primary and secondary
+ * process.
+ *
+ * @param event_type
+ * Memory event type.
+ * @param addr
+ * Address of memory.
+ * @param len
+ * Size of memory.
+ */
+void
+mlx5_mr_mem_event_cb(enum rte_mem_event event_type, const void *addr,
+ size_t len)
+{
+ struct priv *priv;
+ struct mlx5_dev_list *dev_list = &mlx5_shared_data->mem_event_cb_list;
+
+ switch (event_type) {
+ case RTE_MEM_EVENT_FREE:
+ rte_rwlock_write_lock(&mlx5_shared_data->mem_event_rwlock);
+ /* Iterate all the existing mlx5 devices. */
+ LIST_FOREACH(priv, dev_list, mem_event_cb)
+ mlx5_mr_mem_event_free_cb(eth_dev(priv), addr, len);
+ rte_rwlock_write_unlock(&mlx5_shared_data->mem_event_rwlock);
+ break;
+ case RTE_MEM_EVENT_ALLOC:
+ default:
+ break;
+ }
+}
+
+/**
+ * Look up address in the global MR cache table. If not found, create a new MR.
+ * Insert the found/created entry to local bottom-half cache table.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param mr_ctrl
+ * Pointer to per-queue MR control structure.
+ * @param[out] entry
+ * Pointer to returning MR cache entry, found in the global cache or newly
+ * created. If failed to create one, this is not written.
+ * @param addr
+ * Search key.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on no match.
+ */
+static uint32_t
+mlx5_mr_lookup_dev(struct rte_eth_dev *dev, struct mlx5_mr_ctrl *mr_ctrl,
+ struct mlx5_mr_cache *entry, uintptr_t addr)
+{
+ struct priv *priv = dev->data->dev_private;
+ struct mlx5_mr_btree *bt = &mr_ctrl->cache_bh;
+ uint16_t idx;
+ uint32_t lkey;
+
+ /* If local cache table is full, try to double it. */
+ if (unlikely(bt->len == bt->size))
+ mr_btree_expand(bt, bt->size << 1);
+ /* Look up in the global cache. */
+ rte_rwlock_read_lock(&priv->mr.rwlock);
+ lkey = mr_btree_lookup(&priv->mr.cache, &idx, addr);
+ if (lkey != UINT32_MAX) {
+ /* Found. */
+ *entry = (*priv->mr.cache.table)[idx];
+ rte_rwlock_read_unlock(&priv->mr.rwlock);
+ /*
+ * Update local cache. Even if it fails, return the found entry
+ * to update top-half cache. Next time, this entry will be found
+ * in the global cache.
+ */
+ mr_btree_insert(bt, entry);
+ return lkey;
+ }
+ rte_rwlock_read_unlock(&priv->mr.rwlock);
+ /* First time to see the address? Create a new MR. */
+ lkey = mlx5_mr_create(dev, entry, addr);
+ /*
+ * Update the local cache if successfully created a new global MR. Even
+ * if failed to create one, there's no action to take in this datapath
+ * code. As returning LKey is invalid, this will eventually make HW
+ * fail.
+ */
+ if (lkey != UINT32_MAX)
+ mr_btree_insert(bt, entry);
+ return lkey;
+}
+
+/**
+ * Bottom-half of LKey search on datapath. Firstly search in cache_bh[] and if
+ * misses, search in the global MR cache table and update the new entry to
+ * per-queue local caches.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param mr_ctrl
+ * Pointer to per-queue MR control structure.
+ * @param addr
+ * Search key.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on no match.
+ */
+static uint32_t
+mlx5_mr_addr2mr_bh(struct rte_eth_dev *dev, struct mlx5_mr_ctrl *mr_ctrl,
+ uintptr_t addr)
+{
+ uint32_t lkey;
+ uint16_t bh_idx = 0;
+ /* Victim in top-half cache to replace with new entry. */
+ struct mlx5_mr_cache *repl = &mr_ctrl->cache[mr_ctrl->head];
+
+ /* Binary-search MR translation table. */
+ lkey = mr_btree_lookup(&mr_ctrl->cache_bh, &bh_idx, addr);
+ /* Update top-half cache. */
+ if (likely(lkey != UINT32_MAX)) {
+ *repl = (*mr_ctrl->cache_bh.table)[bh_idx];
+ } else {
+ /*
+ * If missed in local lookup table, search in the global cache
+ * and local cache_bh[] will be updated inside if possible.
+ * Top-half cache entry will also be updated.
+ */
+ lkey = mlx5_mr_lookup_dev(dev, mr_ctrl, repl, addr);
+ if (unlikely(lkey == UINT32_MAX))
+ return UINT32_MAX;
+ }
+ /* Update the most recently used entry. */
+ mr_ctrl->mru = mr_ctrl->head;
+ /* Point to the next victim, the oldest. */
+ mr_ctrl->head = (mr_ctrl->head + 1) % MLX5_MR_CACHE_N;
+ return lkey;
+}
+
+/**
+ * Bottom-half of LKey search on Rx.
+ *
+ * @param rxq
+ * Pointer to Rx queue structure.
+ * @param addr
+ * Search key.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on no match.
+ */
+uint32_t
+mlx5_rx_addr2mr_bh(struct mlx5_rxq_data *rxq, uintptr_t addr)
+{
+ struct mlx5_rxq_ctrl *rxq_ctrl =
+ container_of(rxq, struct mlx5_rxq_ctrl, rxq);
+ struct mlx5_mr_ctrl *mr_ctrl = &rxq->mr_ctrl;
+ struct priv *priv = rxq_ctrl->priv;
+
+ DRV_LOG(DEBUG,
+ "Rx queue %u: miss on top-half, mru=%u, head=%u, addr=%p",
+ rxq_ctrl->idx, mr_ctrl->mru, mr_ctrl->head, (void *)addr);
+ return mlx5_mr_addr2mr_bh(eth_dev(priv), mr_ctrl, addr);
+}
+
+/**
+ * Bottom-half of LKey search on Tx.
+ *
+ * @param txq
+ * Pointer to Tx queue structure.
+ * @param addr
+ * Search key.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on no match.
+ */
+uint32_t
+mlx5_tx_addr2mr_bh(struct mlx5_txq_data *txq, uintptr_t addr)
+{
+ struct mlx5_txq_ctrl *txq_ctrl =
+ container_of(txq, struct mlx5_txq_ctrl, txq);
+ struct mlx5_mr_ctrl *mr_ctrl = &txq->mr_ctrl;
+ struct priv *priv = txq_ctrl->priv;
+
+ DRV_LOG(DEBUG,
+ "Tx queue %u: miss on top-half, mru=%u, head=%u, addr=%p",
+ txq_ctrl->idx, mr_ctrl->mru, mr_ctrl->head, (void *)addr);
+ return mlx5_mr_addr2mr_bh(eth_dev(priv), mr_ctrl, addr);
+}
+
+/**
+ * Flush all of the local cache entries.
+ *
+ * @param mr_ctrl
+ * Pointer to per-queue MR control structure.
+ */
+void
+mlx5_mr_flush_local_cache(struct mlx5_mr_ctrl *mr_ctrl)
+{
+ /* Reset the most-recently-used index. */
+ mr_ctrl->mru = 0;
+ /* Reset the linear search array. */
+ mr_ctrl->head = 0;
+ memset(mr_ctrl->cache, 0, sizeof(mr_ctrl->cache));
+ /* Reset the B-tree table. */
+ mr_ctrl->cache_bh.len = 1;
+ mr_ctrl->cache_bh.overflow = 0;
+ /* Update the generation number. */
+ mr_ctrl->cur_gen = *mr_ctrl->dev_gen_ptr;
+ DRV_LOG(DEBUG, "mr_ctrl(%p): flushed, cur_gen=%d",
+ (void *)mr_ctrl, mr_ctrl->cur_gen);
+}
+
+/* Called during rte_mempool_mem_iter() by mlx5_mr_update_mp(). */
+static void
+mlx5_mr_update_mp_cb(struct rte_mempool *mp __rte_unused, void *opaque,
+ struct rte_mempool_memhdr *memhdr,
+ unsigned mem_idx __rte_unused)
+{
+ struct mr_update_mp_data *data = opaque;
+ uint32_t lkey;
+
+ /* Stop iteration if failed in the previous walk. */
+ if (data->ret < 0)
+ return;
+ /* Register address of the chunk and update local caches. */
+ lkey = mlx5_mr_addr2mr_bh(data->dev, data->mr_ctrl,
+ (uintptr_t)memhdr->addr);
+ if (lkey == UINT32_MAX)
+ data->ret = -1;
+}
+
+/**
+ * Register entire memory chunks in a Mempool.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param mr_ctrl
+ * Pointer to per-queue MR control structure.
+ * @param mp
+ * Pointer to registering Mempool.
+ *
+ * @return
+ * 0 on success, -1 on failure.
+ */
+int
+mlx5_mr_update_mp(struct rte_eth_dev *dev, struct mlx5_mr_ctrl *mr_ctrl,
+ struct rte_mempool *mp)
+{
+ struct mr_update_mp_data data = {
+ .dev = dev,
+ .mr_ctrl = mr_ctrl,
+ .ret = 0,
+ };
+
+ rte_mempool_mem_iter(mp, mlx5_mr_update_mp_cb, &data);
+ return data.ret;
+}
+
+/**
+ * Dump all the created MRs and the global cache entries.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ */
+void
+mlx5_mr_dump_dev(struct rte_eth_dev *dev)
+{
+ struct priv *priv = dev->data->dev_private;
+ struct mlx5_mr *mr;
+ int mr_n = 0;
+ int chunk_n = 0;
+
+ rte_rwlock_read_lock(&priv->mr.rwlock);
+ /* Iterate all the existing MRs. */
+ LIST_FOREACH(mr, &priv->mr.mr_list, mr) {
+ unsigned int n;
+
+ DRV_LOG(DEBUG,
+ "port %u MR[%u], LKey = 0x%x, ms_n = %u, ms_bmp_n = %u",
+ dev->data->port_id, mr_n++,
+ rte_cpu_to_be_32(mr->ibv_mr->lkey),
+ mr->ms_n, mr->ms_bmp_n);
+ if (mr->ms_n == 0)
+ continue;
+ for (n = 0; n < mr->ms_bmp_n; ) {
+ struct mlx5_mr_cache ret = { 0, };
+
+ n = mr_find_next_chunk(mr, &ret, n);
+ if (!ret.end)
+ break;
+ DRV_LOG(DEBUG, " chunk[%u], [0x%lx, 0x%lx)",
+ chunk_n++, ret.start, ret.end);
+ }
+ }
+ DRV_LOG(DEBUG, "port %u dumping global cache", dev->data->port_id);
+ mlx5_mr_btree_dump(&priv->mr.cache);
+ rte_rwlock_read_unlock(&priv->mr.rwlock);
+}
+
+/**
+ * Release all the created MRs and resources. Remove device from memory callback
+ * list.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ */
+void
+mlx5_mr_release(struct rte_eth_dev *dev)
+{
+ struct priv *priv = dev->data->dev_private;
+ struct mlx5_mr *mr_next = LIST_FIRST(&priv->mr.mr_list);
+
+ /* Remove from memory callback device list. */
+ rte_rwlock_write_lock(&mlx5_shared_data->mem_event_rwlock);
+ LIST_REMOVE(priv, mem_event_cb);
+ rte_rwlock_write_unlock(&mlx5_shared_data->mem_event_rwlock);
+ if (rte_log_get_level(mlx5_logtype) == RTE_LOG_DEBUG)
+ mlx5_mr_dump_dev(dev);
+ rte_rwlock_write_lock(&priv->mr.rwlock);
+ /* Detach from MR list and move to free list. */
+ while (mr_next != NULL) {
+ struct mlx5_mr *mr = mr_next;
+
+ mr_next = LIST_NEXT(mr, mr);
+ LIST_REMOVE(mr, mr);
+ LIST_INSERT_HEAD(&priv->mr.mr_free_list, mr, mr);
+ }
+ LIST_INIT(&priv->mr.mr_list);
+ /* Free global cache. */
+ mlx5_mr_btree_free(&priv->mr.cache);
+ rte_rwlock_write_unlock(&priv->mr.rwlock);
+ /* Free all remaining MRs. */
+ mlx5_mr_garbage_collect(dev);
+}
diff --git a/drivers/net/mlx5/mlx5_mr.h b/drivers/net/mlx5/mlx5_mr.h
new file mode 100644
index 000000000..a0a0ef755
--- /dev/null
+++ b/drivers/net/mlx5/mlx5_mr.h
@@ -0,0 +1,121 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2018 6WIND S.A.
+ * Copyright 2018 Mellanox Technologies, Ltd
+ */
+
+#ifndef RTE_PMD_MLX5_MR_H_
+#define RTE_PMD_MLX5_MR_H_
+
+#include <stddef.h>
+#include <stdint.h>
+#include <sys/queue.h>
+
+/* Verbs header. */
+/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+#include <infiniband/verbs.h>
+#include <infiniband/mlx5dv.h>
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
+#include <rte_eal_memconfig.h>
+#include <rte_ethdev.h>
+#include <rte_rwlock.h>
+#include <rte_bitmap.h>
+
+/* Memory Region object. */
+struct mlx5_mr {
+ LIST_ENTRY(mlx5_mr) mr; /**< Pointer to the prev/next entry. */
+ struct ibv_mr *ibv_mr; /* Verbs Memory Region. */
+ const struct rte_memseg_list *msl;
+ int ms_base_idx; /* Start index of msl->memseg_arr[]. */
+ int ms_n; /* Number of memsegs in use. */
+ uint32_t ms_bmp_n; /* Number of bits in memsegs bit-mask. */
+ struct rte_bitmap *ms_bmp; /* Bit-mask of memsegs belonged to MR. */
+};
+
+/* Cache entry for Memory Region. */
+struct mlx5_mr_cache {
+ uintptr_t start; /* Start address of MR. */
+ uintptr_t end; /* End address of MR. */
+ uint32_t lkey; /* rte_cpu_to_be_32(ibv_mr->lkey). */
+} __rte_packed;
+
+/* MR Cache table for Binary search. */
+struct mlx5_mr_btree {
+ uint16_t len; /* Number of entries. */
+ uint16_t size; /* Total number of entries. */
+ int overflow; /* Mark failure of table expansion. */
+ struct mlx5_mr_cache (*table)[];
+} __rte_packed;
+
+/* Per-queue MR control descriptor. */
+struct mlx5_mr_ctrl {
+ uint32_t *dev_gen_ptr; /* Generation number of device to poll. */
+ uint32_t cur_gen; /* Generation number saved to flush caches. */
+ uint16_t mru; /* Index of last hit entry in top-half cache. */
+ uint16_t head; /* Index of the oldest entry in top-half cache. */
+ struct mlx5_mr_cache cache[MLX5_MR_CACHE_N]; /* Cache for top-half. */
+ struct mlx5_mr_btree cache_bh; /* Cache for bottom-half. */
+} __rte_packed;
+
+/* First entry must be NULL for comparison. */
+#define MR_N(n) ((n) - 1)
+
+/* Whether there's only one entry in MR lookup table. */
+#define IS_SINGLE_MR(n) (MR_N(n) == 1)
+
+extern struct mlx5_dev_list mlx5_mem_event_cb_list;
+extern rte_rwlock_t mlx5_mem_event_rwlock;
+
+void mlx5_mr_free(struct rte_eth_dev *dev, struct mlx5_mr *mr);
+int mlx5_mr_btree_init(struct mlx5_mr_btree *bt, int n, int socket);
+void mlx5_mr_btree_free(struct mlx5_mr_btree *bt);
+void mlx5_mr_mem_event_cb(enum rte_mem_event event_type, const void *addr,
+ size_t len);
+int mlx5_mr_update_mp(struct rte_eth_dev *dev, struct mlx5_mr_ctrl *mr_ctrl,
+ struct rte_mempool *mp);
+void mlx5_mr_dump_dev(struct rte_eth_dev *dev);
+void mlx5_mr_release(struct rte_eth_dev *dev);
+
+/**
+ * Look up LKey from given lookup table by linear search. Firstly look up the
+ * last-hit entry. If miss, the entire array is searched. If found, update the
+ * last-hit index and return LKey.
+ *
+ * @param lkp_tbl
+ * Pointer to lookup table.
+ * @param[in,out] cached_idx
+ * Pointer to last-hit index.
+ * @param n
+ * Size of lookup table.
+ * @param addr
+ * Search key.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on no match.
+ */
+static __rte_always_inline uint32_t
+mlx5_mr_lookup_cache(struct mlx5_mr_cache *lkp_tbl, uint16_t *cached_idx,
+ uint16_t n, uintptr_t addr)
+{
+ uint16_t idx;
+
+ if (likely(addr >= lkp_tbl[*cached_idx].start &&
+ addr < lkp_tbl[*cached_idx].end))
+ return lkp_tbl[*cached_idx].lkey;
+ for (idx = 0; idx < n && lkp_tbl[idx].start != 0; ++idx) {
+ if (addr >= lkp_tbl[idx].start &&
+ addr < lkp_tbl[idx].end) {
+ /* Found. */
+ *cached_idx = idx;
+ return lkp_tbl[idx].lkey;
+ }
+ }
+ return UINT32_MAX;
+}
+
+#endif /* RTE_PMD_MLX5_MR_H_ */
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index d4fe1fed7..22e2f9673 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -789,7 +789,7 @@ mlx5_rxq_ibv_new(struct rte_eth_dev *dev, uint16_t idx)
.addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(buf,
uintptr_t)),
.byte_count = rte_cpu_to_be_32(DATA_LEN(buf)),
- .lkey = UINT32_MAX,
+ .lkey = mlx5_rx_mb2mr(rxq_data, buf),
};
}
rxq_data->rq_db = rwq.dbrec;
@@ -967,6 +967,11 @@ mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
rte_errno = ENOMEM;
return NULL;
}
+ if (mlx5_mr_btree_init(&tmpl->rxq.mr_ctrl.cache_bh,
+ MLX5_MR_BTREE_CACHE_N, socket)) {
+ /* rte_errno is already set. */
+ goto error;
+ }
tmpl->socket = socket;
if (dev->data->dev_conf.intr_conf.rxq)
tmpl->irq = 1;
@@ -1120,6 +1125,7 @@ mlx5_rxq_release(struct rte_eth_dev *dev, uint16_t idx)
DRV_LOG(DEBUG, "port %u Rx queue %u: refcnt %d",
dev->data->port_id, rxq_ctrl->idx,
rte_atomic32_read(&rxq_ctrl->refcnt));
+ mlx5_mr_btree_free(&rxq_ctrl->rxq.mr_ctrl.cache_bh);
LIST_REMOVE(rxq_ctrl, next);
rte_free(rxq_ctrl);
(*priv->rxqs)[idx] = NULL;
diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 56c243495..8a863c157 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -1965,6 +1965,9 @@ mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
* changes.
*/
wqe->addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(rep, uintptr_t));
+ /* If there's only one MR, no need to replace LKey in WQE. */
+ if (unlikely(!IS_SINGLE_MR(rxq->mr_ctrl.cache_bh.len)))
+ wqe->lkey = mlx5_rx_mb2mr(rxq, rep);
if (len > DATA_LEN(seg)) {
len -= DATA_LEN(seg);
++NB_SEGS(pkt);
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index e8cad51aa..74581cf9b 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -29,6 +29,7 @@
#include "mlx5_utils.h"
#include "mlx5.h"
+#include "mlx5_mr.h"
#include "mlx5_autoconf.h"
#include "mlx5_defs.h"
#include "mlx5_prm.h"
@@ -81,6 +82,7 @@ struct mlx5_rxq_data {
uint16_t rq_ci;
uint16_t rq_pi;
uint16_t cq_ci;
+ struct mlx5_mr_ctrl mr_ctrl; /* MR control descriptor. */
volatile struct mlx5_wqe_data_seg(*wqes)[];
volatile struct mlx5_cqe(*cqes)[];
struct rxq_zip zip; /* Compressed context. */
@@ -109,8 +111,8 @@ struct mlx5_rxq_ibv {
struct mlx5_rxq_ctrl {
LIST_ENTRY(mlx5_rxq_ctrl) next; /* Pointer to the next element. */
rte_atomic32_t refcnt; /* Reference counter. */
- struct priv *priv; /* Back pointer to private data. */
struct mlx5_rxq_ibv *ibv; /* Verbs elements. */
+ struct priv *priv; /* Back pointer to private data. */
struct mlx5_rxq_data rxq; /* Data path structure. */
unsigned int socket; /* CPU socket ID for allocations. */
uint32_t tunnel_types[16]; /* Tunnel type counter. */
@@ -165,6 +167,7 @@ struct mlx5_txq_data {
uint16_t inline_max_packet_sz; /* Max packet size for inlining. */
uint32_t qp_num_8s; /* QP number shifted by 8. */
uint64_t offloads; /* Offloads for Tx Queue. */
+ struct mlx5_mr_ctrl mr_ctrl; /* MR control descriptor. */
volatile struct mlx5_cqe (*cqes)[]; /* Completion queue. */
volatile void *wqes; /* Work queue (use volatile to write into). */
volatile uint32_t *qp_db; /* Work queue doorbell. */
@@ -187,11 +190,11 @@ struct mlx5_txq_ibv {
struct mlx5_txq_ctrl {
LIST_ENTRY(mlx5_txq_ctrl) next; /* Pointer to the next element. */
rte_atomic32_t refcnt; /* Reference counter. */
- struct priv *priv; /* Back pointer to private data. */
unsigned int socket; /* CPU socket ID for allocations. */
unsigned int max_inline_data; /* Max inline data. */
unsigned int max_tso_header; /* Max TSO header size. */
struct mlx5_txq_ibv *ibv; /* Verbs queue object. */
+ struct priv *priv; /* Back pointer to private data. */
struct mlx5_txq_data txq; /* Data path structure. */
off_t uar_mmap_offset; /* UAR mmap offset for non-primary process. */
volatile void *bf_reg_orig; /* Blueflame register from verbs. */
@@ -308,6 +311,12 @@ uint16_t mlx5_tx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts,
uint16_t mlx5_rx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts,
uint16_t pkts_n);
+/* mlx5_mr.c */
+
+void mlx5_mr_flush_local_cache(struct mlx5_mr_ctrl *mr_ctrl);
+uint32_t mlx5_rx_addr2mr_bh(struct mlx5_rxq_data *rxq, uintptr_t addr);
+uint32_t mlx5_tx_addr2mr_bh(struct mlx5_txq_data *txq, uintptr_t addr);
+
#ifndef NDEBUG
/**
* Verify or set magic value in CQE.
@@ -493,14 +502,66 @@ mlx5_tx_complete(struct mlx5_txq_data *txq)
*txq->cq_db = rte_cpu_to_be_32(cq_ci);
}
+/**
+ * Query LKey from a packet buffer for Rx. No need to flush local caches for Rx
+ * as mempool is pre-configured and static.
+ *
+ * @param rxq
+ * Pointer to Rx queue structure.
+ * @param addr
+ * Address to search.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on no match.
+ */
static __rte_always_inline uint32_t
-mlx5_tx_mb2mr(struct mlx5_txq_data *txq, struct rte_mbuf *mb)
+mlx5_rx_addr2mr(struct mlx5_rxq_data *rxq, uintptr_t addr)
{
- (void)txq;
- (void)mb;
- return UINT32_MAX;
+ struct mlx5_mr_ctrl *mr_ctrl = &rxq->mr_ctrl;
+ uint32_t lkey;
+
+ /* Linear search on MR cache array. */
+ lkey = mlx5_mr_lookup_cache(mr_ctrl->cache, &mr_ctrl->mru,
+ MLX5_MR_CACHE_N, addr);
+ if (likely(lkey != UINT32_MAX))
+ return lkey;
+ /* Take slower bottom-half (Binary Search) on miss. */
+ return mlx5_rx_addr2mr_bh(rxq, addr);
}
+#define mlx5_rx_mb2mr(rxq, mb) mlx5_rx_addr2mr(rxq, (uintptr_t)((mb)->buf_addr))
+
+/**
+ * Query LKey from a packet buffer for Tx. If not found, add the mempool.
+ *
+ * @param txq
+ * Pointer to Tx queue structure.
+ * @param addr
+ * Address to search.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on no match.
+ */
+static __rte_always_inline uint32_t
+mlx5_tx_addr2mr(struct mlx5_txq_data *txq, uintptr_t addr)
+{
+ struct mlx5_mr_ctrl *mr_ctrl = &txq->mr_ctrl;
+ uint32_t lkey;
+
+ /* Check generation bit to see if there's any change on existing MRs. */
+ if (unlikely(*mr_ctrl->dev_gen_ptr != mr_ctrl->cur_gen))
+ mlx5_mr_flush_local_cache(mr_ctrl);
+ /* Linear search on MR cache array. */
+ lkey = mlx5_mr_lookup_cache(mr_ctrl->cache, &mr_ctrl->mru,
+ MLX5_MR_CACHE_N, addr);
+ if (likely(lkey != UINT32_MAX))
+ return lkey;
+ /* Take slower bottom-half (binary search) on miss. */
+ return mlx5_tx_addr2mr_bh(txq, addr);
+}
+
+#define mlx5_tx_mb2mr(rxq, mb) mlx5_tx_addr2mr(rxq, (uintptr_t)((mb)->buf_addr))
+
/**
* Ring TX queue doorbell and flush the update if requested.
*
diff --git a/drivers/net/mlx5/mlx5_rxtx_vec.h b/drivers/net/mlx5/mlx5_rxtx_vec.h
index 56c5a1b0c..76678a820 100644
--- a/drivers/net/mlx5/mlx5_rxtx_vec.h
+++ b/drivers/net/mlx5/mlx5_rxtx_vec.h
@@ -99,9 +99,13 @@ mlx5_rx_replenish_bulk_mbuf(struct mlx5_rxq_data *rxq, uint16_t n)
rxq->stats.rx_nombuf += n;
return;
}
- for (i = 0; i < n; ++i)
+ for (i = 0; i < n; ++i) {
wq[i].addr = rte_cpu_to_be_64((uintptr_t)elts[i]->buf_addr +
RTE_PKTMBUF_HEADROOM);
+ /* If there's only one MR, no need to replace LKey in WQE. */
+ if (unlikely(!IS_SINGLE_MR(rxq->mr_ctrl.cache_bh.len)))
+ wq[i].lkey = mlx5_rx_mb2mr(rxq, elts[i]);
+ }
rxq->rq_ci += n;
/* Prevent overflowing into consumed mbufs. */
elts_idx = rxq->rq_ci & q_mask;
diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c
index 3db6c3f35..36b7c9e2f 100644
--- a/drivers/net/mlx5/mlx5_trigger.c
+++ b/drivers/net/mlx5/mlx5_trigger.c
@@ -104,9 +104,18 @@ mlx5_rxq_start(struct rte_eth_dev *dev)
for (i = 0; i != priv->rxqs_n; ++i) {
struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_get(dev, i);
+ struct rte_mempool *mp;
if (!rxq_ctrl)
continue;
+ /* Pre-register Rx mempool. */
+ mp = rxq_ctrl->rxq.mp;
+ DRV_LOG(DEBUG,
+ "port %u Rx queue %u registering"
+ " mp %s having %u chunks",
+ dev->data->port_id, rxq_ctrl->idx,
+ mp->name, mp->nb_mem_chunks);
+ mlx5_mr_update_mp(dev, &rxq_ctrl->rxq.mr_ctrl, mp);
ret = rxq_alloc_elts(rxq_ctrl);
if (ret)
goto error;
@@ -154,6 +163,8 @@ mlx5_dev_start(struct rte_eth_dev *dev)
dev->data->port_id, strerror(rte_errno));
goto error;
}
+ if (rte_log_get_level(mlx5_logtype) == RTE_LOG_DEBUG)
+ mlx5_mr_dump_dev(dev);
ret = mlx5_rx_intr_vec_enable(dev);
if (ret) {
DRV_LOG(ERR, "port %u Rx interrupt vector creation failed",
diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index a71f3d0f0..9ce6f2098 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -804,6 +804,13 @@ mlx5_txq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
rte_errno = ENOMEM;
return NULL;
}
+ if (mlx5_mr_btree_init(&tmpl->txq.mr_ctrl.cache_bh,
+ MLX5_MR_BTREE_CACHE_N, socket)) {
+ /* rte_errno is already set. */
+ goto error;
+ }
+ /* Save pointer of global generation number to check memory event. */
+ tmpl->txq.mr_ctrl.dev_gen_ptr = &priv->mr.dev_gen;
assert(desc > MLX5_TX_COMP_THRESH);
tmpl->txq.offloads = conf->offloads;
tmpl->priv = priv;
@@ -823,6 +830,9 @@ mlx5_txq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
idx, rte_atomic32_read(&tmpl->refcnt));
LIST_INSERT_HEAD(&priv->txqsctrl, tmpl, next);
return tmpl;
+error:
+ rte_free(tmpl);
+ return NULL;
}
/**
@@ -882,6 +892,7 @@ mlx5_txq_release(struct rte_eth_dev *dev, uint16_t idx)
dev->data->port_id, txq->idx,
rte_atomic32_read(&txq->refcnt));
txq_free_elts(txq);
+ mlx5_mr_btree_free(&txq->txq.mr_ctrl.cache_bh);
LIST_REMOVE(txq, next);
rte_free(txq);
(*priv->txqs)[idx] = NULL;
--
2.11.0
^ permalink raw reply [flat|nested] 23+ messages in thread* Re: [dpdk-dev] [PATCH 3/5] net/mlx5: add new Memory Region support
2018-05-02 23:16 ` [dpdk-dev] [PATCH 3/5] net/mlx5: add new " Yongseok Koh
@ 2018-05-03 8:21 ` Burakov, Anatoly
2018-05-06 12:53 ` Shahaf Shuler
1 sibling, 0 replies; 23+ messages in thread
From: Burakov, Anatoly @ 2018-05-03 8:21 UTC (permalink / raw)
To: Yongseok Koh, adrien.mazarguil, nelio.laranjeiro; +Cc: dev
On 03-May-18 12:16 AM, Yongseok Koh wrote:
> This is the new design of Memory Region (MR) for mlx PMD, in order to:
> - Accommodate the new memory hotplug model.
> - Support non-contiguous Mempool.
>
> Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
> ---
> mlx5_flow_delete_drop_queue(dev);
> + mlx5_mr_release(dev);
> if (priv->pd != NULL) {
> assert(priv->ctx != NULL);
> claim_zero(mlx5_glue->dealloc_pd(priv->pd));
> @@ -633,6 +674,8 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
> struct ibv_counter_set_description cs_desc;
> #endif
>
> + /* Prepare shared data between primary and secondary process. */
> + mlx5_prepare_shared_data();
> assert(pci_drv == &mlx5_driver);
> /* Get mlx5_dev[] index. */
> idx = mlx5_dev_idx(&pci_dev->addr);
> @@ -1293,6 +1336,8 @@ rte_mlx5_pmd_init(void)
> }
> mlx5_glue->fork_init();
> rte_pci_register(&mlx5_driver);
> + rte_mem_event_callback_register("MLX5_MEM_EVENT_CB",
> + mlx5_mr_mem_event_cb);
Heads up - this API may/will change slightly, see:
http://dpdk.org/dev/patchwork/patch/39280/
--
Thanks,
Anatoly
^ permalink raw reply [flat|nested] 23+ messages in thread* Re: [dpdk-dev] [PATCH 3/5] net/mlx5: add new Memory Region support
2018-05-02 23:16 ` [dpdk-dev] [PATCH 3/5] net/mlx5: add new " Yongseok Koh
2018-05-03 8:21 ` Burakov, Anatoly
@ 2018-05-06 12:53 ` Shahaf Shuler
2018-05-08 1:52 ` Yongseok Koh
1 sibling, 1 reply; 23+ messages in thread
From: Shahaf Shuler @ 2018-05-06 12:53 UTC (permalink / raw)
To: Yongseok Koh, Adrien Mazarguil, Nélio Laranjeiro; +Cc: dev, Yongseok Koh
Hi Koh,
Huge work. It takes (and will take) me some time to process.
In the meanwhile find some small comments.
As this design heavily relies on synchronization (cache flush) between the control thread and the data path thread along with possible deadlocks from the memory hotplug events the documentation is critical.
Otherwise future work will introduce heavy bugs.
Thursday, May 3, 2018 2:17 AM, Yongseok Koh:
> Subject: [dpdk-dev] [PATCH 3/5] net/mlx5: add new Memory Region support
>
> This is the new design of Memory Region (MR) for mlx PMD, in order to:
> - Accommodate the new memory hotplug model.
> - Support non-contiguous Mempool.
This commit log is missing a lot of details about the design that you did. You must make it clear for every Mellanox PMD developer.
Just to make sure I understand all the details:
We have
1. Cache (L0) per rxq/txq in size of MLX5_MR_CACHE_N. searching in it starting from mru and fallback to linear search
2. btree (L1) per rxq/txq in dynamic size. searching using binary search. This is what you refer as the bottom half right?
3. global mr cache (L2) per device in dynamic size (?)
4. list of all MRs (L3) per device.
>
> Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
> ---
> drivers/net/mlx5/mlx5.c | 45 ++
> drivers/net/mlx5/mlx5.h | 22 +
> drivers/net/mlx5/mlx5_defs.h | 6 +
> drivers/net/mlx5/mlx5_ethdev.c | 16 +
> drivers/net/mlx5/mlx5_mr.c | 1194
> ++++++++++++++++++++++++++++++++++++++
> drivers/net/mlx5/mlx5_mr.h | 121 ++++
> drivers/net/mlx5/mlx5_rxq.c | 8 +-
> drivers/net/mlx5/mlx5_rxtx.c | 3 +
> drivers/net/mlx5/mlx5_rxtx.h | 73 ++-
> drivers/net/mlx5/mlx5_rxtx_vec.h | 6 +-
> drivers/net/mlx5/mlx5_trigger.c | 11 +
> drivers/net/mlx5/mlx5_txq.c | 11 +
> 12 files changed, 1508 insertions(+), 8 deletions(-)
> create mode 100644 drivers/net/mlx5/mlx5_mr.h
>
> diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
> index 01d554758..2883f20af 100644
> --- a/drivers/net/mlx5/mlx5.c
> +++ b/drivers/net/mlx5/mlx5.c
> @@ -41,6 +41,7 @@
> #include "mlx5_autoconf.h"
> #include "mlx5_defs.h"
> #include "mlx5_glue.h"
> +#include "mlx5_mr.h"
>
> /* Device parameter to enable RX completion queue compression. */
> #define MLX5_RXQ_CQE_COMP_EN "rxq_cqe_comp_en"
> @@ -84,10 +85,49 @@
> #define MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP (1 << 4)
> #endif
>
> +static const char *MZ_MLX5_PMD_SHARED_DATA =
> "mlx5_pmd_shared_data";
> +
> +/* Shared memory between primary and secondary processes. */
> +struct mlx5_shared_data *mlx5_shared_data;
> +
> +/* Spinlock for mlx5_shared_data allocation. */
> +static rte_spinlock_t mlx5_shared_data_lock = RTE_SPINLOCK_INITIALIZER;
> +
> /** Driver-specific log messages type. */
> int mlx5_logtype;
>
> /**
> + * Prepare shared data between primary and secondary process.
> + */
> +static void
> +mlx5_prepare_shared_data(void)
> +{
> + const struct rte_memzone *mz;
> +
> + rte_spinlock_lock(&mlx5_shared_data_lock);
> + if (mlx5_shared_data == NULL) {
> + if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
> + /* Allocate shared memory. */
> + mz =
> rte_memzone_reserve(MZ_MLX5_PMD_SHARED_DATA,
> + sizeof(*mlx5_shared_data),
> + SOCKET_ID_ANY, 0);
> + } else {
> + /* Lookup allocated shared memory. */
> + mz =
> rte_memzone_lookup(MZ_MLX5_PMD_SHARED_DATA);
> + }
> + if (mz == NULL)
> + rte_panic("Cannot allocate mlx5 shared data\n");
> + mlx5_shared_data = mz->addr;
> + /* Initialize shared data. */
> + if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
> + LIST_INIT(&mlx5_shared_data-
> >mem_event_cb_list);
> + rte_rwlock_init(&mlx5_shared_data-
> >mem_event_rwlock);
> + }
> + }
> + rte_spinlock_unlock(&mlx5_shared_data_lock);
> +}
> +
Can you elaborate why mlx5_shared_data can't be part of priv?
Priv is already allocated on the shared memory and rte_eth_dev layer enforce the secondary process creation as part of the rte_eth_dev_data allocation.
> +/**
> * Retrieve integer value from environment variable.
> *
> * @param[in] name
> @@ -201,6 +241,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
> priv->txqs = NULL;
> }
> mlx5_flow_delete_drop_queue(dev);
> + mlx5_mr_release(dev);
> if (priv->pd != NULL) {
> assert(priv->ctx != NULL);
> claim_zero(mlx5_glue->dealloc_pd(priv->pd));
> @@ -633,6 +674,8 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv
> __rte_unused,
> struct ibv_counter_set_description cs_desc;
> #endif
>
> + /* Prepare shared data between primary and secondary process. */
> + mlx5_prepare_shared_data();
> assert(pci_drv == &mlx5_driver);
> /* Get mlx5_dev[] index. */
> idx = mlx5_dev_idx(&pci_dev->addr);
> @@ -1293,6 +1336,8 @@ rte_mlx5_pmd_init(void)
> }
> mlx5_glue->fork_init();
> rte_pci_register(&mlx5_driver);
> + rte_mem_event_callback_register("MLX5_MEM_EVENT_CB",
> + mlx5_mr_mem_event_cb);
mlx5_mr_mem_event_cb requires PMD private structure. Is registering for the cb on the init makes sense? It looks like a better place is the PCI probe, after the eth_dev allocation.
> }
>
> RTE_PMD_EXPORT_NAME(net_mlx5, __COUNTER__);
> diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
> index 47d266c90..d3fc74dc1 100644
> --- a/drivers/net/mlx5/mlx5.h
> +++ b/drivers/net/mlx5/mlx5.h
> @@ -26,11 +26,13 @@
> #include <rte_pci.h>
> #include <rte_ether.h>
> #include <rte_ethdev_driver.h>
> +#include <rte_rwlock.h>
> #include <rte_interrupts.h>
> #include <rte_errno.h>
> #include <rte_flow.h>
>
> #include "mlx5_utils.h"
> +#include "mlx5_mr.h"
> #include "mlx5_rxtx.h"
> #include "mlx5_autoconf.h"
> #include "mlx5_defs.h"
> @@ -50,6 +52,16 @@ enum {
> PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF = 0x101a,
> };
>
> +LIST_HEAD(mlx5_dev_list, priv);
> +
> +/* Shared memory between primary and secondary processes. */
> +struct mlx5_shared_data {
> + struct mlx5_dev_list mem_event_cb_list;
> + rte_rwlock_t mem_event_rwlock;
> +};
> +
> +extern struct mlx5_shared_data *mlx5_shared_data;
> +
> struct mlx5_xstats_ctrl {
> /* Number of device stats. */
> uint16_t stats_n;
> @@ -119,7 +131,10 @@ struct mlx5_verbs_alloc_ctx {
> const void *obj; /* Pointer to the DPDK object. */
> };
>
> +LIST_HEAD(mlx5_mr_list, mlx5_mr);
> +
> struct priv {
> + LIST_ENTRY(priv) mem_event_cb; /* Called by memory event
> callback. */
> struct rte_eth_dev_data *dev_data; /* Pointer to device data. */
> struct ibv_context *ctx; /* Verbs context. */
> struct ibv_device_attr_ex device_attr; /* Device properties. */
> @@ -146,6 +161,13 @@ struct priv {
> struct mlx5_hrxq_drop *flow_drop_queue; /* Flow drop queue. */
> struct mlx5_flows flows; /* RTE Flow rules. */
> struct mlx5_flows ctrl_flows; /* Control flow rules. */
> + struct {
> + uint32_t dev_gen; /* Generation number to flush local
> caches. */
> + rte_rwlock_t rwlock; /* MR Lock. */
> + struct mlx5_mr_btree cache; /* Global MR cache table. */
> + struct mlx5_mr_list mr_list; /* Registered MR list. */
> + struct mlx5_mr_list mr_free_list; /* Freed MR list. */
> + } mr;
> LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */
> LIST_HEAD(rxqibv, mlx5_rxq_ibv) rxqsibv; /* Verbs Rx queues. */
> LIST_HEAD(hrxq, mlx5_hrxq) hrxqs; /* Verbs Hash Rx queues. */
> diff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h
> index f9093777d..72e80af26 100644
> --- a/drivers/net/mlx5/mlx5_defs.h
> +++ b/drivers/net/mlx5/mlx5_defs.h
> @@ -37,6 +37,12 @@
> */
> #define MLX5_TX_COMP_THRESH_INLINE_DIV (1 << 3)
>
> +/* Size of per-queue MR cache array for linear search. */
> +#define MLX5_MR_CACHE_N 8
> +
> +/* Size of MR cache table for binary search. */
> +#define MLX5_MR_BTREE_CACHE_N 256
> +
> /*
> * If defined, only use software counters. The PMD will never ask the
> hardware
> * for these, and many of them won't be available.
> diff --git a/drivers/net/mlx5/mlx5_ethdev.c
> b/drivers/net/mlx5/mlx5_ethdev.c
> index 746b94f73..6bb43cf4e 100644
> --- a/drivers/net/mlx5/mlx5_ethdev.c
> +++ b/drivers/net/mlx5/mlx5_ethdev.c
> @@ -34,6 +34,7 @@
> #include <rte_interrupts.h>
> #include <rte_malloc.h>
> #include <rte_string_fns.h>
> +#include <rte_rwlock.h>
>
> #include "mlx5.h"
> #include "mlx5_glue.h"
> @@ -413,6 +414,21 @@ mlx5_dev_configure(struct rte_eth_dev *dev)
> if (++j == rxqs_n)
> j = 0;
> }
> + /*
> + * Once the device is added to the list of memory event callback, its
> + * global MR cache table cannot be expanded on the fly because of
> + * deadlock. If it overflows, lookup should be done by searching MR
> list
> + * linearly, which is slow.
> + */
> + if (mlx5_mr_btree_init(&priv->mr.cache,
> MLX5_MR_BTREE_CACHE_N * 2,
Why multiple by 2? Because it holds all the rxq/txq mrs?
> + dev->device->numa_node)) {
> + /* rte_errno is already set. */
> + return -rte_errno;
> + }
> + rte_rwlock_write_lock(&mlx5_shared_data->mem_event_rwlock);
> + LIST_INSERT_HEAD(&mlx5_shared_data->mem_event_cb_list,
> + priv, mem_event_cb);
> + rte_rwlock_write_unlock(&mlx5_shared_data-
> >mem_event_rwlock);
> return 0;
> }
Why registration is done only on configure and not on probe after priv initialization?
>
> diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c
> index 736c40ae4..e964912bb 100644
> --- a/drivers/net/mlx5/mlx5_mr.c
> +++ b/drivers/net/mlx5/mlx5_mr.c
> @@ -13,8 +13,1202 @@
>
> #include <rte_mempool.h>
> #include <rte_malloc.h>
> +#include <rte_rwlock.h>
>
> #include "mlx5.h"
> +#include "mlx5_mr.h"
> #include "mlx5_rxtx.h"
> #include "mlx5_glue.h"
>
> +struct mr_find_contig_memsegs_data {
> + uintptr_t addr;
> + uintptr_t start;
> + uintptr_t end;
> + const struct rte_memseg_list *msl;
> +};
> +
> +struct mr_update_mp_data {
> + struct rte_eth_dev *dev;
> + struct mlx5_mr_ctrl *mr_ctrl;
> + int ret;
> +};
> +
> +/**
> + * Expand B-tree table to a given size. Can't be called with holding
> + * memory_hotplug_lock or priv->mr.rwlock due to rte_realloc().
> + *
> + * @param bt
> + * Pointer to B-tree structure.
> + * @param n
> + * Number of entries for expansion.
> + *
> + * @return
> + * 0 on success, -1 on failure.
> + */
> +static int
> +mr_btree_expand(struct mlx5_mr_btree *bt, int n)
> +{
> + void *mem;
> + int ret = 0;
> +
> + if (n <= bt->size)
> + return ret;
> + /*
> + * Downside of directly using rte_realloc() is that SOCKET_ID_ANY is
> + * used inside if there's no room to expand. Because this is a quite
> + * rare case and a part of very slow path, it is very acceptable.
> + * Initially cache_bh[] will be given practically enough space and once
> + * it is expanded, expansion wouldn't be needed again ever.
> + */
> + mem = rte_realloc(bt->table, n * sizeof(struct mlx5_mr_cache), 0);
> + if (mem == NULL) {
> + /* Not an error, B-tree search will be skipped. */
> + DRV_LOG(WARNING, "failed to expand MR B-tree (%p)
> table",
> + (void *)bt);
DRV_LOG should have the port id of the device. For all of the DRV_LOG instances in the patch.
Per my understating it falls back to the old bt in case the expansion failed, right? Bt searches will still happen.
> + ret = -1;
> + } else {
> + DRV_LOG(DEBUG, "expanded MR B-tree table (size=%u)",
> n);
> + bt->table = mem;
> + bt->size = n;
> + }
> + return ret;
> +}
> +
> +/**
> + * Look up LKey from given B-tree lookup table, store the last index and
> return
> + * searched LKey.
> + *
> + * @param bt
> + * Pointer to B-tree structure.
> + * @param[out] idx
> + * Pointer to index. Even on searh failure, returns index where it stops
Searh->search
> + * searching so that index can be used when inserting a new entry.
> + * @param addr
> + * Search key.
> + *
> + * @return
> + * Searched LKey on success, UINT32_MAX on no match.
> + */
> +static uint32_t
> +mr_btree_lookup(struct mlx5_mr_btree *bt, uint16_t *idx, uintptr_t addr)
> +{
> + struct mlx5_mr_cache *lkp_tbl;
> + uint16_t n;
> + uint16_t base = 0;
> +
> + assert(bt != NULL);
> + lkp_tbl = *bt->table;
> + n = bt->len;
> + /* First entry must be NULL for comparison. */
> + assert(bt->len > 0 || (lkp_tbl[0].start == 0 &&
> + lkp_tbl[0].lkey == UINT32_MAX));
> + /* Binary search. */
> + do {
> + register uint16_t delta = n >> 1;
> +
> + if (addr < lkp_tbl[base + delta].start) {
> + n = delta;
> + } else {
> + base += delta;
> + n -= delta;
> + }
> + } while (n > 1);
> + assert(addr >= lkp_tbl[base].start);
> + *idx = base;
> + if (addr < lkp_tbl[base].end)
> + return lkp_tbl[base].lkey;
> + /* Not found. */
> + return UINT32_MAX;
> +}
> +
> +/**
> + * Insert an entry to B-tree lookup table.
> + *
> + * @param bt
> + * Pointer to B-tree structure.
> + * @param entry
> + * Pointer to new entry to insert.
> + *
> + * @return
> + * 0 on success, -1 on failure.
> + */
> +static int
> +mr_btree_insert(struct mlx5_mr_btree *bt, struct mlx5_mr_cache *entry)
> +{
> + struct mlx5_mr_cache *lkp_tbl;
> + uint16_t idx = 0;
> + size_t shift;
> +
> + assert(bt != NULL);
> + assert(bt->len <= bt->size);
> + assert(bt->len > 0);
> + lkp_tbl = *bt->table;
> + /* Find out the slot for insertion. */
> + if (mr_btree_lookup(bt, &idx, entry->start) != UINT32_MAX) {
> + DRV_LOG(DEBUG,
> + "abort insertion to B-tree(%p):"
> + " already exist at idx=%u [0x%lx, 0x%lx) lkey=0x%x",
> + (void *)bt, idx, entry->start, entry->end, entry-
> >lkey);
> + /* Already exist, return. */
> + return 0;
> + }
> + /* If table is full, return error. */
> + if (unlikely(bt->len == bt->size)) {
> + bt->overflow = 1;
> + return -1;
> + }
> + /* Insert entry. */
> + ++idx;
> + shift = (bt->len - idx) * sizeof(struct mlx5_mr_cache);
> + if (shift)
> + memmove(&lkp_tbl[idx + 1], &lkp_tbl[idx], shift);
> + lkp_tbl[idx] = *entry;
> + bt->len++;
> + DRV_LOG(DEBUG,
> + "inserted B-tree(%p)[%u], [0x%lx, 0x%lx) lkey=0x%x",
> + (void *)bt, idx, entry->start, entry->end, entry->lkey);
> + return 0;
> +}
Can you elaborate on how you make sure the btree is always sorted based on the start addr?
> +
> +/**
> + * Initialize B-tree and allocate memory for lookup table.
> + *
> + * @param bt
> + * Pointer to B-tree structure.
> + * @param n
> + * Number of entries to allocate.
> + * @param socket
> + * NUMA socket on which memory must be allocated.
> + *
> + * @return
> + * 0 on success, a negative errno value otherwise and rte_errno is set.
> + */
> +int
> +mlx5_mr_btree_init(struct mlx5_mr_btree *bt, int n, int socket)
> +{
> + if (bt == NULL) {
> + rte_errno = EINVAL;
> + return -rte_errno;
> + }
> + memset(bt, 0, sizeof(*bt));
> + bt->table = rte_calloc_socket("B-tree table",
> + n, sizeof(struct mlx5_mr_cache),
> + 0, socket);
> + if (bt->table == NULL) {
> + rte_errno = ENOMEM;
> + DRV_LOG(ERR,
> + "failed to allocate memory for btree cache on socket
> %d",
> + socket);
> + return -rte_errno;
> + }
> + bt->size = n;
> + /* First entry must be NULL for binary search. */
> + (*bt->table)[bt->len++] = (struct mlx5_mr_cache) {
> + .lkey = UINT32_MAX,
> + };
> + DRV_LOG(DEBUG, "initialized B-tree %p with table %p",
> + (void *)bt, (void *)bt->table);
> + return 0;
> +}
> +
> +/**
> + * Free B-tree resources.
> + *
> + * @param bt
> + * Pointer to B-tree structure.
> + */
> +void
> +mlx5_mr_btree_free(struct mlx5_mr_btree *bt)
> +{
> + if (bt == NULL)
> + return;
> + DRV_LOG(DEBUG, "freeing B-tree %p with table %p",
> + (void *)bt, (void *)bt->table);
> + rte_free(bt->table);
> + memset(bt, 0, sizeof(*bt));
> +}
> +
> +/**
> + * Dump all the entries in a B-tree
> + *
> + * @param bt
> + * Pointer to B-tree structure.
> + */
> +static void
> +mlx5_mr_btree_dump(struct mlx5_mr_btree *bt)
> +{
> + int idx;
> + struct mlx5_mr_cache *lkp_tbl;
> +
> + if (bt == NULL)
> + return;
> + lkp_tbl = *bt->table;
> + for (idx = 0; idx < bt->len; ++idx) {
> + struct mlx5_mr_cache *entry = &lkp_tbl[idx];
> +
> + DRV_LOG(DEBUG,
> + "B-tree(%p)[%u], [0x%lx, 0x%lx) lkey=0x%x",
> + (void *)bt, idx, entry->start, entry->end, entry-
> >lkey);
> + }
> +}
> +
> +/**
> + * Find virtually contiguous memory chunk in a given MR.
> + *
> + * @param dev
> + * Pointer to MR structure.
> + * @param[out] entry
> + * Pointer to returning MR cache entry. If not found, this will not be
> + * updated.
> + * @param start_idx
> + * Start index of the memseg bitmap.
> + *
> + * @return
> + * Next index to go on lookup.
> + */
> +static int
> +mr_find_next_chunk(struct mlx5_mr *mr, struct mlx5_mr_cache *entry,
> + int base_idx)
> +{
> + uintptr_t start = 0;
> + uintptr_t end = 0;
> + uint32_t idx = 0;
> +
> + for (idx = base_idx; idx < mr->ms_bmp_n; ++idx) {
> + if (rte_bitmap_get(mr->ms_bmp, idx)) {
> + const struct rte_memseg_list *msl;
> + const struct rte_memseg *ms;
> +
> + msl = mr->msl;
> + ms = rte_fbarray_get(&msl->memseg_arr,
> + mr->ms_base_idx + idx);
> + assert(msl->page_sz == ms->hugepage_sz);
> + if (!start)
> + start = ms->addr_64;
> + end = ms->addr_64 + ms->hugepage_sz;
> + } else if (start) {
> + /* Passed the end of a fragment. */
> + break;
> + }
> + }
> + if (start) {
> + /* Found one chunk. */
> + entry->start = start;
> + entry->end = end;
> + entry->lkey = rte_cpu_to_be_32(mr->ibv_mr->lkey);
> + }
> + return idx;
> +}
> +
> +/**
> + * Insert a MR to the global B-tree cache. It may fail due to low-on-memory.
> + * Then, this entry will have to be searched by mr_lookup_dev_list() in
> + * mlx5_mr_create() on miss.
> + *
> + * @param dev
> + * Pointer to Ethernet device.
> + * @param mr
> + * Pointer to MR to insert.
> + *
> + * @return
> + * 0 on success, -1 on failure.
> + */
> +static int
> +mr_insert_dev_cache(struct rte_eth_dev *dev, struct mlx5_mr *mr)
> +{
> + struct priv *priv = dev->data->dev_private;
> + unsigned int n;
> +
> + DRV_LOG(DEBUG, "port %u inserting MR(%p) to global cache",
> + dev->data->port_id, (void *)mr);
> + for (n = 0; n < mr->ms_bmp_n; ) {
> + struct mlx5_mr_cache entry = { 0, };
> +
> + /* Find a contiguous chunk and advance the index. */
> + n = mr_find_next_chunk(mr, &entry, n);
> + if (!entry.end)
> + break;
> + if (mr_btree_insert(&priv->mr.cache, &entry) < 0) {
> + /*
> + * Overflowed, but the global table cannot be
> expanded
> + * because of deadlock.
> + */
> + return -1;
> + }
> + }
> + return 0;
> +}
> +
> +/**
> + * Look up address in the original global MR list.
> + *
> + * @param dev
> + * Pointer to Ethernet device.
> + * @param[out] entry
> + * Pointer to returning MR cache entry. If no match, this will not be
> updated.
> + * @param addr
> + * Search key.
> + *
> + * @return
> + * Found MR on match, NULL otherwise.
> + */
> +static struct mlx5_mr *
> +mr_lookup_dev_list(struct rte_eth_dev *dev, struct mlx5_mr_cache
> *entry,
> + uintptr_t addr)
> +{
> + struct priv *priv = dev->data->dev_private;
> + struct mlx5_mr *mr;
> +
> + /* Iterate all the existing MRs. */
> + LIST_FOREACH(mr, &priv->mr.mr_list, mr) {
> + unsigned int n;
> +
> + if (mr->ms_n == 0)
> + continue;
> + for (n = 0; n < mr->ms_bmp_n; ) {
> + struct mlx5_mr_cache ret = { 0, };
> +
> + n = mr_find_next_chunk(mr, &ret, n);
> + if (addr >= ret.start && addr < ret.end) {
> + /* Found. */
> + *entry = ret;
> + return mr;
> + }
> + }
> + }
> + return NULL;
> +}
> +
> +/**
> + * Look up address on device.
> + *
> + * @param dev
> + * Pointer to Ethernet device.
> + * @param[out] entry
> + * Pointer to returning MR cache entry. If no match, this will not be
> updated.
> + * @param addr
> + * Search key.
> + *
> + * @return
> + * Searched LKey on success, UINT32_MAX on failure and rte_errno is set.
> + */
> +static uint32_t
> +mr_lookup_dev(struct rte_eth_dev *dev, struct mlx5_mr_cache *entry,
> + uintptr_t addr)
> +{
> + struct priv *priv = dev->data->dev_private;
> + uint16_t idx;
> + uint32_t lkey = UINT32_MAX;
> + struct mlx5_mr *mr;
> +
> + /*
> + * If the global cache has overflowed since it failed to expand the
> + * B-tree table, it can't have all the exisitng MRs. Then, the address
> + * has to be searched by traversing the original MR list instead, which
> + * is very slow path. Otherwise, the global cache is all inclusive.
> + */
> + if (!unlikely(priv->mr.cache.overflow)) {
> + lkey = mr_btree_lookup(&priv->mr.cache, &idx, addr);
> + if (lkey != UINT32_MAX)
> + *entry = (*priv->mr.cache.table)[idx];
> + } else {
> + /* Falling back to the slowest path. */
> + mr = mr_lookup_dev_list(dev, entry, addr);
> + if (mr != NULL)
> + lkey = entry->lkey;
> + }
> + assert(lkey == UINT32_MAX || (addr >= entry->start &&
> + addr < entry->end));
> + return lkey;
> +}
> +
> +/**
> + * Free MR resources. MR lock must not be held to avoid a deadlock.
> rte_free()
> + * can raise memory free event and the callback function will spin on the
> lock.
> + *
> + * @param mr
> + * Pointer to MR to free.
> + */
> +static void
> +mr_free(struct mlx5_mr *mr)
> +{
> + if (mr == NULL)
> + return;
> + DRV_LOG(DEBUG, "freeing MR(%p):", (void *)mr);
> + if (mr->ibv_mr != NULL)
> + claim_zero(mlx5_glue->dereg_mr(mr->ibv_mr));
> + if (mr->ms_bmp != NULL)
> + rte_bitmap_free(mr->ms_bmp);
> + rte_free(mr);
> +}
> +
> +/**
> + * Free Memory Region (MR).
> + *
> + * @param dev
> + * Pointer to Ethernet device.
> + * @param mr
> + * Pointer to MR to free.
> + */
> +void
> +mlx5_mr_free(struct rte_eth_dev *dev, struct mlx5_mr *mr)
> +{
Who calls this function? I didn't saw any.
> + struct priv *priv = dev->data->dev_private;
> +
> + /* Detach from the list and free resources later. */
> + rte_rwlock_write_lock(&priv->mr.rwlock);
> + LIST_REMOVE(mr, mr);
> + rte_rwlock_write_unlock(&priv->mr.rwlock);
> + /*
> + * rte_free() inside can't be called with holding the lock. This could
> + * cause deadlock when calling free callback.
> + */
> + mr_free(mr);
> + DRV_LOG(DEBUG, "port %u MR(%p) freed", dev->data->port_id,
> (void *)mr);
> +}
> +
> +/**
> + * Releass resources of detached MR having no online entry.
> + *
> + * @param dev
> + * Pointer to Ethernet device.
> + */
> +static void
> +mlx5_mr_garbage_collect(struct rte_eth_dev *dev)
> +{
> + struct priv *priv = dev->data->dev_private;
> + struct mlx5_mr *mr_next;
> + struct mlx5_mr_list free_list = LIST_HEAD_INITIALIZER(free_list);
> +
> + /* Must be called from the primary process. */
> + assert(rte_eal_process_type() == RTE_PROC_PRIMARY);
Perhaps it is better to have this check not under assert?
> + /*
> + * MR can't be freed with holding the lock because rte_free() could
> call
> + * memory free callback function. This will be a deadlock situation.
> + */
> + rte_rwlock_write_lock(&priv->mr.rwlock);
> + /* Detach the whole free list and release it after unlocking. */
> + free_list = priv->mr.mr_free_list;
> + LIST_INIT(&priv->mr.mr_free_list);
> + rte_rwlock_write_unlock(&priv->mr.rwlock);
> + /* Release resources. */
> + mr_next = LIST_FIRST(&free_list);
> + while (mr_next != NULL) {
> + struct mlx5_mr *mr = mr_next;
> +
> + mr_next = LIST_NEXT(mr, mr);
> + mr_free(mr);
> + }
> +}
> +
> +/* Called during rte_memseg_contig_walk() by mlx5_mr_create(). */
> +static int
> +mr_find_contig_memsegs_cb(const struct rte_memseg_list *msl,
> + const struct rte_memseg *ms, size_t len, void *arg)
> +{
> + struct mr_find_contig_memsegs_data *data = arg;
> +
> + if (data->addr < ms->addr_64 || data->addr >= ms->addr_64 + len)
> + return 0;
> + /* Found, save it and stop walking. */
> + data->start = ms->addr_64;
> + data->end = ms->addr_64 + len;
> + data->msl = msl;
> + return 1;
> +}
> +
> +/**
> + * Create a new global Memroy Region (MR) for a missing virtual address.
> + * Register entire virtually contiguous memory chunk around the address.
> + *
> + * @param dev
> + * Pointer to Ethernet device.
> + * @param[out] entry
> + * Pointer to returning MR cache entry, found in the global cache or newly
> + * created. If failed to create one, this will not be updated.
> + * @param addr
> + * Target virtual address to register.
> + *
> + * @return
> + * Searched LKey on success, UINT32_MAX on failure and rte_errno is set.
> + */
> +static uint32_t
> +mlx5_mr_create(struct rte_eth_dev *dev, struct mlx5_mr_cache *entry,
> + uintptr_t addr)
> +{
> + struct priv *priv = dev->data->dev_private;
> + struct rte_mem_config *mcfg = rte_eal_get_configuration()-
> >mem_config;
> + const struct rte_memseg_list *msl;
> + const struct rte_memseg *ms;
> + struct mlx5_mr *mr = NULL;
> + size_t len;
> + uint32_t ms_n;
> + uint32_t bmp_size;
> + void *bmp_mem;
> + int ms_idx_shift = -1;
> + unsigned int n;
> + struct mr_find_contig_memsegs_data data = {
> + .addr = addr,
> + };
> + struct mr_find_contig_memsegs_data data_re;
> +
> + DRV_LOG(DEBUG, "port %u creating a MR using address (%p)",
> + dev->data->port_id, (void *)addr);
> + if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
> + DRV_LOG(WARNING,
> + "port %u using address (%p) of unregistered
> mempool"
> + " in secondary process, please create mempool"
> + " before rte_eth_dev_start()",
> + dev->data->port_id, (void *)addr);
> + rte_errno = EPERM;
> + goto err_nolock;
> + }
> + /*
> + * Release detached MRs if any. This can't be called with holding
> either
> + * memory_hotplug_lock or priv->mr.rwlock. MRs on the free list
> have
> + * been detached by the memory free event but it couldn't be
> released
> + * inside the callback due to deadlock. As a result, releasing resources
> + * is quite opportunistic.
> + */
> + mlx5_mr_garbage_collect(dev);
> + /*
> + * Find out a contiguous virtual address chunk in use, to which the
> + * given address belongs, in order to register maximum range. In the
> + * best case where mempools are not dynamically recreated and
> + * '--socket-mem' is speicified as an EAL option, it is very likely to
> + * have only one MR(LKey) per a socket and per a hugepage-size
> even
> + * though the system memory is highly fragmented.
> + */
> + if (!rte_memseg_contig_walk(mr_find_contig_memsegs_cb,
> &data)) {
> + DRV_LOG(WARNING,
> + "port %u unable to find virtually contigous"
> + " chunk for address (%p)."
> + " rte_memseg_contig_walk() failed.",
> + dev->data->port_id, (void *)addr);
> + rte_errno = ENXIO;
> + goto err_nolock;
> + }
> +alloc_resources:
> + /* Addresses must be page-aligned. */
> + assert(rte_is_aligned((void *)data.start, data.msl->page_sz));
> + assert(rte_is_aligned((void *)data.end, data.msl->page_sz));
Better to have this check outsize of assert.
> + msl = data.msl;
> + ms = rte_mem_virt2memseg((void *)data.start, msl);
> + len = data.end - data.start;
> + assert(msl->page_sz == ms->hugepage_sz);
> + /* Number of memsegs in the range. */
> + ms_n = len / msl->page_sz;
> + DRV_LOG(DEBUG,
> + "port %u extending %p to [0x%lx, 0x%lx), page_sz=0x%lx,
> ms_n=%u",
> + dev->data->port_id, (void *)addr,
> + data.start, data.end, msl->page_sz, ms_n);
> + /* Size of memory for bitmap. */
> + bmp_size = rte_bitmap_get_memory_footprint(ms_n);
> + mr = rte_zmalloc_socket(NULL,
> + RTE_ALIGN_CEIL(sizeof(*mr),
> + RTE_CACHE_LINE_SIZE) +
> + bmp_size,
> + RTE_CACHE_LINE_SIZE, msl->socket_id);
> + if (mr == NULL) {
> + DRV_LOG(WARNING,
> + "port %u unable to allocate memory for a new MR
> of"
> + " address (%p).",
> + dev->data->port_id, (void *)addr);
> + rte_errno = ENOMEM;
> + goto err_nolock;
> + }
> + mr->msl = msl;
> + /*
> + * Save the index of the first memseg and initialize memseg bitmap.
> To
> + * see if a memseg of ms_idx in the memseg-list is still valid, check:
> + * rte_bitmap_get(mr->bmp, ms_idx - mr->ms_base_idx)
> + */
> + mr->ms_base_idx = rte_fbarray_find_idx(&msl->memseg_arr, ms);
> + bmp_mem = RTE_PTR_ALIGN_CEIL(mr + 1, RTE_CACHE_LINE_SIZE);
> + mr->ms_bmp = rte_bitmap_init(ms_n, bmp_mem, bmp_size);
> + if (mr->ms_bmp == NULL) {
> + DRV_LOG(WARNING,
> + "port %u unable to initialize bitamp for a new MR of"
> + " address (%p).",
> + dev->data->port_id, (void *)addr);
> + rte_errno = EINVAL;
> + goto err_nolock;
> + }
> + /*
> + * Should recheck whether the extended contiguous chunk is still
> valid.
> + * Because memory_hotplug_lock can't be held if there's any
> memory
> + * related calls in a critical path, resource allocation above can't be
> + * locked. If the memory has been changed at this point, try again
> with
> + * just single page. If not, go on with the big chunk atomically from
> + * here.
> + */
> + rte_rwlock_read_lock(&mcfg->memory_hotplug_lock);
> + data_re = data;
> + if (len > msl->page_sz &&
> + !rte_memseg_contig_walk(mr_find_contig_memsegs_cb,
> &data_re)) {
> + DRV_LOG(WARNING,
> + "port %u unable to find virtually contigous"
> + " chunk for address (%p)."
> + " rte_memseg_contig_walk() failed.",
> + dev->data->port_id, (void *)addr);
> + rte_errno = ENXIO;
> + goto err_memlock;
> + }
> + if (data.start != data_re.start || data.end != data_re.end) {
> + /*
> + * The extended contiguous chunk has been changed. Try
> again
> + * with single memseg instead.
> + */
> + data.start = RTE_ALIGN_FLOOR(addr, msl->page_sz);
> + data.end = data.start + msl->page_sz;
> + rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
> + mr_free(mr);
> + goto alloc_resources;
> + }
> + assert(data.msl == data_re.msl);
> + rte_rwlock_write_lock(&priv->mr.rwlock);
> + /*
> + * Check the address is really missing. If other thread already created
> + * one or it is not found due to overflow, abort and return.
> + */
> + if (mr_lookup_dev(dev, entry, addr) != UINT32_MAX) {
> + /*
> + * Insert to the global cache table. It may fail due to
> + * low-on-memory. Then, this entry will have to be searched
> + * here again.
> + */
> + mr_btree_insert(&priv->mr.cache, entry);
> + DRV_LOG(DEBUG,
> + "port %u found MR for %p on final lookup, abort",
> + dev->data->port_id, (void *)addr);
> + rte_rwlock_write_unlock(&priv->mr.rwlock);
> + rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
> + /*
> + * Must be unlocked before calling rte_free() because
> + * mlx5_mr_mem_event_free_cb() can be called inside.
> + */
> + mr_free(mr);
> + return entry->lkey;
> + }
> + /*
> + * Trim start and end addresses for verbs MR. Set bits for registering
> + * memsegs but exclude already registered ones. Bitmap can be
> + * fragmented.
> + */
> + for (n = 0; n < ms_n; ++n) {
> + uintptr_t start;
> + struct mlx5_mr_cache ret = { 0, };
> +
> + start = data_re.start + n * msl->page_sz;
> + /* Exclude memsegs already registered by other MRs. */
> + if (mr_lookup_dev(dev, &ret, start) == UINT32_MAX) {
> + /*
> + * Start from the first unregistered memseg in the
> + * extended range.
> + */
> + if (ms_idx_shift == -1) {
> + mr->ms_base_idx += n;
> + data.start = start;
> + ms_idx_shift = n;
> + }
> + data.end = start + msl->page_sz;
> + rte_bitmap_set(mr->ms_bmp, n - ms_idx_shift);
> + ++mr->ms_n;
> + }
> + }
> + len = data.end - data.start;
> + mr->ms_bmp_n = len / msl->page_sz;
> + assert(ms_idx_shift + mr->ms_bmp_n <= ms_n);
> + /*
> + * Finally create a verbs MR for the memory chunk. ibv_reg_mr() can
> be
> + * called with holding the memory lock because it doesn't use
> + * mlx5_alloc_buf_extern() which eventually calls
> rte_malloc_socket()
> + * through mlx5_alloc_verbs_buf().
> + */
> + mr->ibv_mr = mlx5_glue->reg_mr(priv->pd, (void *)data.start, len,
> + IBV_ACCESS_LOCAL_WRITE);
> + if (mr->ibv_mr == NULL) {
> + DRV_LOG(WARNING,
> + "port %u fail to create a verbs MR for address (%p)",
> + dev->data->port_id, (void *)addr);
> + rte_errno = EINVAL;
> + goto err_mrlock;
> + }
> + assert((uintptr_t)mr->ibv_mr->addr == data.start);
> + assert(mr->ibv_mr->length == len);
> + LIST_INSERT_HEAD(&priv->mr.mr_list, mr, mr);
> + DRV_LOG(DEBUG,
> + "port %u MR CREATED (%p) for %p:\n"
> + " [0x%lx, 0x%lx), lkey=0x%x base_idx=%u ms_n=%u,
> ms_bmp_n=%u",
> + dev->data->port_id, (void *)mr, (void *)addr,
> + data.start, data.end, rte_cpu_to_be_32(mr->ibv_mr->lkey),
> + mr->ms_base_idx, mr->ms_n, mr->ms_bmp_n);
> + /* Insert to the global cache table. */
> + mr_insert_dev_cache(dev, mr);
> + /* Fill in output data. */
> + mr_lookup_dev(dev, entry, addr);
> + /* Lookup can't fail. */
> + assert(entry->lkey != UINT32_MAX);
> + rte_rwlock_write_unlock(&priv->mr.rwlock);
> + rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
> + return entry->lkey;
> +err_mrlock:
> + rte_rwlock_write_unlock(&priv->mr.rwlock);
> +err_memlock:
> + rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
> +err_nolock:
> + /*
> + * In case of error, as this can be called in a datapath, a warning
> + * message per an error is preferable instead. Must be unlocked
> before
> + * calling rte_free() because mlx5_mr_mem_event_free_cb() can be
> called
> + * inside.
> + */
> + mr_free(mr);
> + return UINT32_MAX;
> +}
> +
> +/**
> + * Rebuild the global B-tree cache of device from the original MR list.
> + *
> + * @param dev
> + * Pointer to Ethernet device.
> + */
> +static void
> +mr_rebuild_dev_cache(struct rte_eth_dev *dev)
> +{
> + struct priv *priv = dev->data->dev_private;
> + struct mlx5_mr *mr;
> +
> + DRV_LOG(DEBUG, "port %u rebuild dev cache[]", dev->data-
> >port_id);
> + /* Flush cache to rebuild. */
> + priv->mr.cache.len = 1;
> + priv->mr.cache.overflow = 0;
> + /* Iterate all the existing MRs. */
> + LIST_FOREACH(mr, &priv->mr.mr_list, mr)
> + if (mr_insert_dev_cache(dev, mr) < 0)
> + return;
> +}
> +
> +/**
> + * Callback for memory free event. Iterate freed memsegs and check
> whether it
> + * belongs to an existing MR. If found, clear the bit from bitmap of MR. As a
> + * result, the MR would be fragmented. If it becomes empty, the MR will be
> freed
> + * later by mlx5_mr_garbage_collect(). Even if this callback is called from a
> + * secondary process, the garbage collector will be called in primary process
> + * as the secondary process can't call mlx5_mr_create().
> + *
> + * The global cache must be rebuilt if there's any change and this event has
> to
> + * be propagated to dataplane threads to flush the local caches.
> + *
> + * @param dev
> + * Pointer to Ethernet device.
> + * @param addr
> + * Address of freed memory.
> + * @param len
> + * Size of freed memory.
> + */
> +static void
> +mlx5_mr_mem_event_free_cb(struct rte_eth_dev *dev, const void *addr,
> size_t len)
> +{
> + struct priv *priv = dev->data->dev_private;
> + const struct rte_memseg_list *msl;
> + struct mlx5_mr *mr;
> + int ms_n;
> + int i;
> + int rebuild = 0;
> +
> + DRV_LOG(DEBUG, "port %u free callback: addr=%p, len=%lu",
> + dev->data->port_id, addr, len);
> + msl = rte_mem_virt2memseg_list(addr);
> + /* addr and len must be page-aligned. */
> + assert((uintptr_t)addr == RTE_ALIGN((uintptr_t)addr, msl-
> >page_sz));
> + assert(len == RTE_ALIGN(len, msl->page_sz));
> + ms_n = len / msl->page_sz;
> + rte_rwlock_write_lock(&priv->mr.rwlock);
> + /* Clear bits of freed memsegs from MR. */
> + for (i = 0; i < ms_n; ++i) {
> + const struct rte_memseg *ms;
> + struct mlx5_mr_cache entry;
> + uintptr_t start;
> + int ms_idx;
> + uint32_t pos;
> +
> + /* Find MR having this memseg. */
> + start = (uintptr_t)addr + i * msl->page_sz;
> + mr = mr_lookup_dev_list(dev, &entry, start);
> + if (mr == NULL)
> + continue;
> + ms = rte_mem_virt2memseg((void *)start, msl);
> + assert(ms != NULL);
> + assert(msl->page_sz == ms->hugepage_sz);
> + ms_idx = rte_fbarray_find_idx(&msl->memseg_arr, ms);
> + pos = ms_idx - mr->ms_base_idx;
> + assert(rte_bitmap_get(mr->ms_bmp, pos));
> + assert(pos < mr->ms_bmp_n);
> + DRV_LOG(DEBUG, "port %u MR(%p): clear bitmap[%u] for
> addr %p",
> + dev->data->port_id, (void *)mr, pos, (void *)start);
> + rte_bitmap_clear(mr->ms_bmp, pos);
> + if (--mr->ms_n == 0) {
> + LIST_REMOVE(mr, mr);
> + LIST_INSERT_HEAD(&priv->mr.mr_free_list, mr, mr);
> + DRV_LOG(DEBUG, "port %u remove MR(%p) from
> list",
> + dev->data->port_id, (void *)mr);
> + }
> + /*
> + * MR is fragmented or will be freed. the global cache must
> be
> + * rebuilt.
> + */
> + rebuild = 1;
> + }
> + if (rebuild) {
> + mr_rebuild_dev_cache(dev);
> + /*
> + * Flush local caches by propagating invalidation across cores.
> + * rte_smp_wmb() is enough to synchronize this event. If
> one of
> + * freed memsegs is seen by other core, that means the
> memseg
> + * has been allocated by allocator, which will come after this
> + * free call. Therefore, this store instruction (incrementing
> + * generation below) will be guaranteed to be seen by other
> core
> + * before the core sees the newly allocated memory.
> + */
> + ++priv->mr.dev_gen;
> + DRV_LOG(DEBUG, "broadcasting local cache flush, gen=%d",
> + priv->mr.dev_gen);
> + rte_smp_wmb();
> + }
> + rte_rwlock_write_unlock(&priv->mr.rwlock);
> + if (rebuild && rte_log_get_level(mlx5_logtype) == RTE_LOG_DEBUG)
> + mlx5_mr_dump_dev(dev);
> +}
> +
> +/**
> + * Callback for memory event. This can be called from both primary and
> secondary
> + * process.
> + *
> + * @param event_type
> + * Memory event type.
> + * @param addr
> + * Address of memory.
> + * @param len
> + * Size of memory.
> + */
> +void
> +mlx5_mr_mem_event_cb(enum rte_mem_event event_type, const void
> *addr,
> + size_t len)
> +{
> + struct priv *priv;
> + struct mlx5_dev_list *dev_list = &mlx5_shared_data-
> >mem_event_cb_list;
> +
> + switch (event_type) {
> + case RTE_MEM_EVENT_FREE:
> + rte_rwlock_write_lock(&mlx5_shared_data-
> >mem_event_rwlock);
> + /* Iterate all the existing mlx5 devices. */
> + LIST_FOREACH(priv, dev_list, mem_event_cb)
> + mlx5_mr_mem_event_free_cb(eth_dev(priv), addr,
> len);
> + rte_rwlock_write_unlock(&mlx5_shared_data-
> >mem_event_rwlock);
> + break;
> + case RTE_MEM_EVENT_ALLOC:
> + default:
> + break;
> + }
> +}
> +
> +/**
> + * Look up address in the global MR cache table. If not found, create a new
> MR.
> + * Insert the found/created entry to local bottom-half cache table.
> + *
> + * @param dev
> + * Pointer to Ethernet device.
> + * @param mr_ctrl
> + * Pointer to per-queue MR control structure.
> + * @param[out] entry
> + * Pointer to returning MR cache entry, found in the global cache or newly
> + * created. If failed to create one, this is not written.
> + * @param addr
> + * Search key.
> + *
> + * @return
> + * Searched LKey on success, UINT32_MAX on no match.
> + */
> +static uint32_t
> +mlx5_mr_lookup_dev(struct rte_eth_dev *dev, struct mlx5_mr_ctrl
> *mr_ctrl,
> + struct mlx5_mr_cache *entry, uintptr_t addr)
> +{
> + struct priv *priv = dev->data->dev_private;
> + struct mlx5_mr_btree *bt = &mr_ctrl->cache_bh;
> + uint16_t idx;
> + uint32_t lkey;
> +
> + /* If local cache table is full, try to double it. */
> + if (unlikely(bt->len == bt->size))
> + mr_btree_expand(bt, bt->size << 1);
> + /* Look up in the global cache. */
> + rte_rwlock_read_lock(&priv->mr.rwlock);
> + lkey = mr_btree_lookup(&priv->mr.cache, &idx, addr);
> + if (lkey != UINT32_MAX) {
> + /* Found. */
> + *entry = (*priv->mr.cache.table)[idx];
> + rte_rwlock_read_unlock(&priv->mr.rwlock);
> + /*
> + * Update local cache. Even if it fails, return the found entry
> + * to update top-half cache. Next time, this entry will be
> found
> + * in the global cache.
> + */
> + mr_btree_insert(bt, entry);
> + return lkey;
> + }
> + rte_rwlock_read_unlock(&priv->mr.rwlock);
> + /* First time to see the address? Create a new MR. */
> + lkey = mlx5_mr_create(dev, entry, addr);
Shouldn't we check if the add is not in the global mr list? For the case the global cache overflows?
> + /*
> + * Update the local cache if successfully created a new global MR.
> Even
> + * if failed to create one, there's no action to take in this datapath
> + * code. As returning LKey is invalid, this will eventually make HW
> + * fail.
> + */
> + if (lkey != UINT32_MAX)
> + mr_btree_insert(bt, entry);
> + return lkey;
> +}
> +
> +/**
> + * Bottom-half of LKey search on datapath. Firstly search in cache_bh[] and
> if
> + * misses, search in the global MR cache table and update the new entry to
> + * per-queue local caches.
> + *
> + * @param dev
> + * Pointer to Ethernet device.
> + * @param mr_ctrl
> + * Pointer to per-queue MR control structure.
> + * @param addr
> + * Search key.
> + *
> + * @return
> + * Searched LKey on success, UINT32_MAX on no match.
> + */
> +static uint32_t
> +mlx5_mr_addr2mr_bh(struct rte_eth_dev *dev, struct mlx5_mr_ctrl
> *mr_ctrl,
> + uintptr_t addr)
> +{
> + uint32_t lkey;
> + uint16_t bh_idx = 0;
> + /* Victim in top-half cache to replace with new entry. */
> + struct mlx5_mr_cache *repl = &mr_ctrl->cache[mr_ctrl->head];
> +
> + /* Binary-search MR translation table. */
> + lkey = mr_btree_lookup(&mr_ctrl->cache_bh, &bh_idx, addr);
> + /* Update top-half cache. */
> + if (likely(lkey != UINT32_MAX)) {
> + *repl = (*mr_ctrl->cache_bh.table)[bh_idx];
> + } else {
> + /*
> + * If missed in local lookup table, search in the global cache
> + * and local cache_bh[] will be updated inside if possible.
> + * Top-half cache entry will also be updated.
> + */
> + lkey = mlx5_mr_lookup_dev(dev, mr_ctrl, repl, addr);
> + if (unlikely(lkey == UINT32_MAX))
> + return UINT32_MAX;
> + }
> + /* Update the most recently used entry. */
> + mr_ctrl->mru = mr_ctrl->head;
> + /* Point to the next victim, the oldest. */
> + mr_ctrl->head = (mr_ctrl->head + 1) % MLX5_MR_CACHE_N;
> + return lkey;
> +}
> +
> +/**
> + * Bottom-half of LKey search on Rx.
> + *
> + * @param rxq
> + * Pointer to Rx queue structure.
> + * @param addr
> + * Search key.
> + *
> + * @return
> + * Searched LKey on success, UINT32_MAX on no match.
> + */
> +uint32_t
> +mlx5_rx_addr2mr_bh(struct mlx5_rxq_data *rxq, uintptr_t addr)
> +{
> + struct mlx5_rxq_ctrl *rxq_ctrl =
> + container_of(rxq, struct mlx5_rxq_ctrl, rxq);
> + struct mlx5_mr_ctrl *mr_ctrl = &rxq->mr_ctrl;
> + struct priv *priv = rxq_ctrl->priv;
> +
> + DRV_LOG(DEBUG,
> + "Rx queue %u: miss on top-half, mru=%u, head=%u,
> addr=%p",
> + rxq_ctrl->idx, mr_ctrl->mru, mr_ctrl->head, (void *)addr);
> + return mlx5_mr_addr2mr_bh(eth_dev(priv), mr_ctrl, addr);
> +}
Shouldn't this code path be in the mlxx5_rxq?
> +
> +/**
> + * Bottom-half of LKey search on Tx.
> + *
> + * @param txq
> + * Pointer to Tx queue structure.
> + * @param addr
> + * Search key.
> + *
> + * @return
> + * Searched LKey on success, UINT32_MAX on no match.
> + */
> +uint32_t
> +mlx5_tx_addr2mr_bh(struct mlx5_txq_data *txq, uintptr_t addr)
> +{
> + struct mlx5_txq_ctrl *txq_ctrl =
> + container_of(txq, struct mlx5_txq_ctrl, txq);
> + struct mlx5_mr_ctrl *mr_ctrl = &txq->mr_ctrl;
> + struct priv *priv = txq_ctrl->priv;
> +
> + DRV_LOG(DEBUG,
> + "Tx queue %u: miss on top-half, mru=%u, head=%u,
> addr=%p",
> + txq_ctrl->idx, mr_ctrl->mru, mr_ctrl->head, (void *)addr);
> + return mlx5_mr_addr2mr_bh(eth_dev(priv), mr_ctrl, addr);
> +}
> +
Same for txq.
> +/**
> + * Flush all of the local cache entries.
> + *
> + * @param mr_ctrl
> + * Pointer to per-queue MR control structure.
> + */
> +void
> +mlx5_mr_flush_local_cache(struct mlx5_mr_ctrl *mr_ctrl)
> +{
> + /* Reset the most-recently-used index. */
> + mr_ctrl->mru = 0;
> + /* Reset the linear search array. */
> + mr_ctrl->head = 0;
> + memset(mr_ctrl->cache, 0, sizeof(mr_ctrl->cache));
> + /* Reset the B-tree table. */
> + mr_ctrl->cache_bh.len = 1;
> + mr_ctrl->cache_bh.overflow = 0;
> + /* Update the generation number. */
> + mr_ctrl->cur_gen = *mr_ctrl->dev_gen_ptr;
> + DRV_LOG(DEBUG, "mr_ctrl(%p): flushed, cur_gen=%d",
> + (void *)mr_ctrl, mr_ctrl->cur_gen);
> +}
> +
> +/* Called during rte_mempool_mem_iter() by mlx5_mr_update_mp(). */
> +static void
> +mlx5_mr_update_mp_cb(struct rte_mempool *mp __rte_unused, void
> *opaque,
> + struct rte_mempool_memhdr *memhdr,
> + unsigned mem_idx __rte_unused)
> +{
> + struct mr_update_mp_data *data = opaque;
> + uint32_t lkey;
> +
> + /* Stop iteration if failed in the previous walk. */
> + if (data->ret < 0)
> + return;
> + /* Register address of the chunk and update local caches. */
> + lkey = mlx5_mr_addr2mr_bh(data->dev, data->mr_ctrl,
> + (uintptr_t)memhdr->addr);
> + if (lkey == UINT32_MAX)
> + data->ret = -1;
> +}
> +
> +/**
> + * Register entire memory chunks in a Mempool.
> + *
> + * @param dev
> + * Pointer to Ethernet device.
> + * @param mr_ctrl
> + * Pointer to per-queue MR control structure.
> + * @param mp
> + * Pointer to registering Mempool.
> + *
> + * @return
> + * 0 on success, -1 on failure.
> + */
> +int
> +mlx5_mr_update_mp(struct rte_eth_dev *dev, struct mlx5_mr_ctrl
> *mr_ctrl,
> + struct rte_mempool *mp)
> +{
> + struct mr_update_mp_data data = {
> + .dev = dev,
> + .mr_ctrl = mr_ctrl,
> + .ret = 0,
> + };
> +
> + rte_mempool_mem_iter(mp, mlx5_mr_update_mp_cb, &data);
> + return data.ret;
> +}
> +
> +/**
> + * Dump all the created MRs and the global cache entries.
> + *
> + * @param dev
> + * Pointer to Ethernet device.
> + */
> +void
> +mlx5_mr_dump_dev(struct rte_eth_dev *dev)
> +{
> + struct priv *priv = dev->data->dev_private;
> + struct mlx5_mr *mr;
> + int mr_n = 0;
> + int chunk_n = 0;
> +
> + rte_rwlock_read_lock(&priv->mr.rwlock);
> + /* Iterate all the existing MRs. */
> + LIST_FOREACH(mr, &priv->mr.mr_list, mr) {
> + unsigned int n;
> +
> + DRV_LOG(DEBUG,
> + "port %u MR[%u], LKey = 0x%x, ms_n = %u,
> ms_bmp_n = %u",
> + dev->data->port_id, mr_n++,
> + rte_cpu_to_be_32(mr->ibv_mr->lkey),
> + mr->ms_n, mr->ms_bmp_n);
> + if (mr->ms_n == 0)
> + continue;
> + for (n = 0; n < mr->ms_bmp_n; ) {
> + struct mlx5_mr_cache ret = { 0, };
> +
> + n = mr_find_next_chunk(mr, &ret, n);
> + if (!ret.end)
> + break;
> + DRV_LOG(DEBUG, " chunk[%u], [0x%lx, 0x%lx)",
> + chunk_n++, ret.start, ret.end);
> + }
> + }
> + DRV_LOG(DEBUG, "port %u dumping global cache", dev->data-
> >port_id);
> + mlx5_mr_btree_dump(&priv->mr.cache);
> + rte_rwlock_read_unlock(&priv->mr.rwlock);
> +}
> +
> +/**
> + * Release all the created MRs and resources. Remove device from memory
> callback
> + * list.
> + *
> + * @param dev
> + * Pointer to Ethernet device.
> + */
> +void
> +mlx5_mr_release(struct rte_eth_dev *dev)
> +{
> + struct priv *priv = dev->data->dev_private;
> + struct mlx5_mr *mr_next = LIST_FIRST(&priv->mr.mr_list);
> +
> + /* Remove from memory callback device list. */
> + rte_rwlock_write_lock(&mlx5_shared_data->mem_event_rwlock);
> + LIST_REMOVE(priv, mem_event_cb);
> + rte_rwlock_write_unlock(&mlx5_shared_data-
> >mem_event_rwlock);
> + if (rte_log_get_level(mlx5_logtype) == RTE_LOG_DEBUG)
> + mlx5_mr_dump_dev(dev);
> + rte_rwlock_write_lock(&priv->mr.rwlock);
> + /* Detach from MR list and move to free list. */
> + while (mr_next != NULL) {
> + struct mlx5_mr *mr = mr_next;
> +
> + mr_next = LIST_NEXT(mr, mr);
> + LIST_REMOVE(mr, mr);
> + LIST_INSERT_HEAD(&priv->mr.mr_free_list, mr, mr);
> + }
> + LIST_INIT(&priv->mr.mr_list);
> + /* Free global cache. */
> + mlx5_mr_btree_free(&priv->mr.cache);
> + rte_rwlock_write_unlock(&priv->mr.rwlock);
> + /* Free all remaining MRs. */
> + mlx5_mr_garbage_collect(dev);
> +}
> diff --git a/drivers/net/mlx5/mlx5_mr.h b/drivers/net/mlx5/mlx5_mr.h
> new file mode 100644
> index 000000000..a0a0ef755
> --- /dev/null
> +++ b/drivers/net/mlx5/mlx5_mr.h
> @@ -0,0 +1,121 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright 2018 6WIND S.A.
> + * Copyright 2018 Mellanox Technologies, Ltd
> + */
> +
> +#ifndef RTE_PMD_MLX5_MR_H_
> +#define RTE_PMD_MLX5_MR_H_
> +
> +#include <stddef.h>
> +#include <stdint.h>
> +#include <sys/queue.h>
> +
> +/* Verbs header. */
> +/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
> +#ifdef PEDANTIC
> +#pragma GCC diagnostic ignored "-Wpedantic"
> +#endif
> +#include <infiniband/verbs.h>
> +#include <infiniband/mlx5dv.h>
> +#ifdef PEDANTIC
> +#pragma GCC diagnostic error "-Wpedantic"
> +#endif
> +
> +#include <rte_eal_memconfig.h>
> +#include <rte_ethdev.h>
> +#include <rte_rwlock.h>
> +#include <rte_bitmap.h>
> +
> +/* Memory Region object. */
> +struct mlx5_mr {
> + LIST_ENTRY(mlx5_mr) mr; /**< Pointer to the prev/next entry. */
> + struct ibv_mr *ibv_mr; /* Verbs Memory Region. */
> + const struct rte_memseg_list *msl;
> + int ms_base_idx; /* Start index of msl->memseg_arr[]. */
> + int ms_n; /* Number of memsegs in use. */
> + uint32_t ms_bmp_n; /* Number of bits in memsegs bit-mask. */
> + struct rte_bitmap *ms_bmp; /* Bit-mask of memsegs belonged to
> MR. */
> +};
> +
> +/* Cache entry for Memory Region. */
> +struct mlx5_mr_cache {
> + uintptr_t start; /* Start address of MR. */
> + uintptr_t end; /* End address of MR. */
> + uint32_t lkey; /* rte_cpu_to_be_32(ibv_mr->lkey). */
> +} __rte_packed;
> +
> +/* MR Cache table for Binary search. */
> +struct mlx5_mr_btree {
> + uint16_t len; /* Number of entries. */
> + uint16_t size; /* Total number of entries. */
> + int overflow; /* Mark failure of table expansion. */
> + struct mlx5_mr_cache (*table)[];
> +} __rte_packed;
> +
> +/* Per-queue MR control descriptor. */
> +struct mlx5_mr_ctrl {
> + uint32_t *dev_gen_ptr; /* Generation number of device to poll. */
> + uint32_t cur_gen; /* Generation number saved to flush caches. */
> + uint16_t mru; /* Index of last hit entry in top-half cache. */
> + uint16_t head; /* Index of the oldest entry in top-half cache. */
> + struct mlx5_mr_cache cache[MLX5_MR_CACHE_N]; /* Cache for
> top-half. */
> + struct mlx5_mr_btree cache_bh; /* Cache for bottom-half. */
> +} __rte_packed;
> +
> +/* First entry must be NULL for comparison. */
> +#define MR_N(n) ((n) - 1)
> +
> +/* Whether there's only one entry in MR lookup table. */
> +#define IS_SINGLE_MR(n) (MR_N(n) == 1)
MLX5_IS_SINGLE_MR
> +
> +extern struct mlx5_dev_list mlx5_mem_event_cb_list;
> +extern rte_rwlock_t mlx5_mem_event_rwlock;
> +
> +void mlx5_mr_free(struct rte_eth_dev *dev, struct mlx5_mr *mr);
> +int mlx5_mr_btree_init(struct mlx5_mr_btree *bt, int n, int socket);
> +void mlx5_mr_btree_free(struct mlx5_mr_btree *bt);
> +void mlx5_mr_mem_event_cb(enum rte_mem_event event_type, const
> void *addr,
> + size_t len);
> +int mlx5_mr_update_mp(struct rte_eth_dev *dev, struct mlx5_mr_ctrl
> *mr_ctrl,
> + struct rte_mempool *mp);
> +void mlx5_mr_dump_dev(struct rte_eth_dev *dev);
> +void mlx5_mr_release(struct rte_eth_dev *dev);
> +
> +/**
> + * Look up LKey from given lookup table by linear search. Firstly look up the
> + * last-hit entry. If miss, the entire array is searched. If found, update the
> + * last-hit index and return LKey.
> + *
> + * @param lkp_tbl
> + * Pointer to lookup table.
> + * @param[in,out] cached_idx
> + * Pointer to last-hit index.
> + * @param n
> + * Size of lookup table.
> + * @param addr
> + * Search key.
> + *
> + * @return
> + * Searched LKey on success, UINT32_MAX on no match.
> + */
> +static __rte_always_inline uint32_t
> +mlx5_mr_lookup_cache(struct mlx5_mr_cache *lkp_tbl, uint16_t
> *cached_idx,
> + uint16_t n, uintptr_t addr)
> +{
> + uint16_t idx;
> +
> + if (likely(addr >= lkp_tbl[*cached_idx].start &&
> + addr < lkp_tbl[*cached_idx].end))
> + return lkp_tbl[*cached_idx].lkey;
> + for (idx = 0; idx < n && lkp_tbl[idx].start != 0; ++idx) {
> + if (addr >= lkp_tbl[idx].start &&
> + addr < lkp_tbl[idx].end) {
> + /* Found. */
> + *cached_idx = idx;
> + return lkp_tbl[idx].lkey;
> + }
> + }
> + return UINT32_MAX;
> +}
> +
> +#endif /* RTE_PMD_MLX5_MR_H_ */
> diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
> index d4fe1fed7..22e2f9673 100644
> --- a/drivers/net/mlx5/mlx5_rxq.c
> +++ b/drivers/net/mlx5/mlx5_rxq.c
> @@ -789,7 +789,7 @@ mlx5_rxq_ibv_new(struct rte_eth_dev *dev,
> uint16_t idx)
> .addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(buf,
> uintptr_t)),
> .byte_count = rte_cpu_to_be_32(DATA_LEN(buf)),
> - .lkey = UINT32_MAX,
> + .lkey = mlx5_rx_mb2mr(rxq_data, buf),
> };
> }
> rxq_data->rq_db = rwq.dbrec;
> @@ -967,6 +967,11 @@ mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t
> idx, uint16_t desc,
> rte_errno = ENOMEM;
> return NULL;
> }
> + if (mlx5_mr_btree_init(&tmpl->rxq.mr_ctrl.cache_bh,
> + MLX5_MR_BTREE_CACHE_N, socket)) {
> + /* rte_errno is already set. */
> + goto error;
> + }
> tmpl->socket = socket;
> if (dev->data->dev_conf.intr_conf.rxq)
> tmpl->irq = 1;
> @@ -1120,6 +1125,7 @@ mlx5_rxq_release(struct rte_eth_dev *dev,
> uint16_t idx)
> DRV_LOG(DEBUG, "port %u Rx queue %u: refcnt %d",
> dev->data->port_id, rxq_ctrl->idx,
> rte_atomic32_read(&rxq_ctrl->refcnt));
> + mlx5_mr_btree_free(&rxq_ctrl->rxq.mr_ctrl.cache_bh);
> LIST_REMOVE(rxq_ctrl, next);
> rte_free(rxq_ctrl);
> (*priv->rxqs)[idx] = NULL;
> diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
> index 56c243495..8a863c157 100644
> --- a/drivers/net/mlx5/mlx5_rxtx.c
> +++ b/drivers/net/mlx5/mlx5_rxtx.c
> @@ -1965,6 +1965,9 @@ mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf
> **pkts, uint16_t pkts_n)
> * changes.
> */
> wqe->addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(rep,
> uintptr_t));
> + /* If there's only one MR, no need to replace LKey in WQE. */
> + if (unlikely(!IS_SINGLE_MR(rxq->mr_ctrl.cache_bh.len)))
> + wqe->lkey = mlx5_rx_mb2mr(rxq, rep);
> if (len > DATA_LEN(seg)) {
> len -= DATA_LEN(seg);
> ++NB_SEGS(pkt);
> diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
> index e8cad51aa..74581cf9b 100644
> --- a/drivers/net/mlx5/mlx5_rxtx.h
> +++ b/drivers/net/mlx5/mlx5_rxtx.h
> @@ -29,6 +29,7 @@
>
> #include "mlx5_utils.h"
> #include "mlx5.h"
> +#include "mlx5_mr.h"
> #include "mlx5_autoconf.h"
> #include "mlx5_defs.h"
> #include "mlx5_prm.h"
> @@ -81,6 +82,7 @@ struct mlx5_rxq_data {
> uint16_t rq_ci;
> uint16_t rq_pi;
> uint16_t cq_ci;
> + struct mlx5_mr_ctrl mr_ctrl; /* MR control descriptor. */
> volatile struct mlx5_wqe_data_seg(*wqes)[];
> volatile struct mlx5_cqe(*cqes)[];
> struct rxq_zip zip; /* Compressed context. */
> @@ -109,8 +111,8 @@ struct mlx5_rxq_ibv {
> struct mlx5_rxq_ctrl {
> LIST_ENTRY(mlx5_rxq_ctrl) next; /* Pointer to the next element. */
> rte_atomic32_t refcnt; /* Reference counter. */
> - struct priv *priv; /* Back pointer to private data. */
> struct mlx5_rxq_ibv *ibv; /* Verbs elements. */
> + struct priv *priv; /* Back pointer to private data. */
> struct mlx5_rxq_data rxq; /* Data path structure. */
> unsigned int socket; /* CPU socket ID for allocations. */
> uint32_t tunnel_types[16]; /* Tunnel type counter. */
> @@ -165,6 +167,7 @@ struct mlx5_txq_data {
> uint16_t inline_max_packet_sz; /* Max packet size for inlining. */
> uint32_t qp_num_8s; /* QP number shifted by 8. */
> uint64_t offloads; /* Offloads for Tx Queue. */
> + struct mlx5_mr_ctrl mr_ctrl; /* MR control descriptor. */
> volatile struct mlx5_cqe (*cqes)[]; /* Completion queue. */
> volatile void *wqes; /* Work queue (use volatile to write into). */
> volatile uint32_t *qp_db; /* Work queue doorbell. */
> @@ -187,11 +190,11 @@ struct mlx5_txq_ibv {
> struct mlx5_txq_ctrl {
> LIST_ENTRY(mlx5_txq_ctrl) next; /* Pointer to the next element. */
> rte_atomic32_t refcnt; /* Reference counter. */
> - struct priv *priv; /* Back pointer to private data. */
> unsigned int socket; /* CPU socket ID for allocations. */
> unsigned int max_inline_data; /* Max inline data. */
> unsigned int max_tso_header; /* Max TSO header size. */
> struct mlx5_txq_ibv *ibv; /* Verbs queue object. */
> + struct priv *priv; /* Back pointer to private data. */
> struct mlx5_txq_data txq; /* Data path structure. */
> off_t uar_mmap_offset; /* UAR mmap offset for non-primary
> process. */
> volatile void *bf_reg_orig; /* Blueflame register from verbs. */
> @@ -308,6 +311,12 @@ uint16_t mlx5_tx_burst_vec(void *dpdk_txq, struct
> rte_mbuf **pkts,
> uint16_t mlx5_rx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts,
> uint16_t pkts_n);
>
> +/* mlx5_mr.c */
> +
> +void mlx5_mr_flush_local_cache(struct mlx5_mr_ctrl *mr_ctrl);
> +uint32_t mlx5_rx_addr2mr_bh(struct mlx5_rxq_data *rxq, uintptr_t addr);
> +uint32_t mlx5_tx_addr2mr_bh(struct mlx5_txq_data *txq, uintptr_t addr);
> +
> #ifndef NDEBUG
> /**
> * Verify or set magic value in CQE.
> @@ -493,14 +502,66 @@ mlx5_tx_complete(struct mlx5_txq_data *txq)
> *txq->cq_db = rte_cpu_to_be_32(cq_ci);
> }
>
> +/**
> + * Query LKey from a packet buffer for Rx. No need to flush local caches for
> Rx
> + * as mempool is pre-configured and static.
> + *
> + * @param rxq
> + * Pointer to Rx queue structure.
> + * @param addr
> + * Address to search.
> + *
> + * @return
> + * Searched LKey on success, UINT32_MAX on no match.
> + */
> static __rte_always_inline uint32_t
> -mlx5_tx_mb2mr(struct mlx5_txq_data *txq, struct rte_mbuf *mb)
> +mlx5_rx_addr2mr(struct mlx5_rxq_data *rxq, uintptr_t addr)
> {
> - (void)txq;
> - (void)mb;
> - return UINT32_MAX;
> + struct mlx5_mr_ctrl *mr_ctrl = &rxq->mr_ctrl;
> + uint32_t lkey;
> +
> + /* Linear search on MR cache array. */
> + lkey = mlx5_mr_lookup_cache(mr_ctrl->cache, &mr_ctrl->mru,
> + MLX5_MR_CACHE_N, addr);
> + if (likely(lkey != UINT32_MAX))
> + return lkey;
> + /* Take slower bottom-half (Binary Search) on miss. */
> + return mlx5_rx_addr2mr_bh(rxq, addr);
> }
>
> +#define mlx5_rx_mb2mr(rxq, mb) mlx5_rx_addr2mr(rxq, (uintptr_t)((mb)-
> >buf_addr))
> +
> +/**
> + * Query LKey from a packet buffer for Tx. If not found, add the mempool.
> + *
> + * @param txq
> + * Pointer to Tx queue structure.
> + * @param addr
> + * Address to search.
> + *
> + * @return
> + * Searched LKey on success, UINT32_MAX on no match.
> + */
> +static __rte_always_inline uint32_t
> +mlx5_tx_addr2mr(struct mlx5_txq_data *txq, uintptr_t addr)
> +{
> + struct mlx5_mr_ctrl *mr_ctrl = &txq->mr_ctrl;
> + uint32_t lkey;
> +
> + /* Check generation bit to see if there's any change on existing MRs.
> */
> + if (unlikely(*mr_ctrl->dev_gen_ptr != mr_ctrl->cur_gen))
> + mlx5_mr_flush_local_cache(mr_ctrl);
> + /* Linear search on MR cache array. */
> + lkey = mlx5_mr_lookup_cache(mr_ctrl->cache, &mr_ctrl->mru,
> + MLX5_MR_CACHE_N, addr);
> + if (likely(lkey != UINT32_MAX))
> + return lkey;
> + /* Take slower bottom-half (binary search) on miss. */
> + return mlx5_tx_addr2mr_bh(txq, addr);
> +}
> +
> +#define mlx5_tx_mb2mr(rxq, mb) mlx5_tx_addr2mr(rxq, (uintptr_t)((mb)-
> >buf_addr))
> +
> /**
> * Ring TX queue doorbell and flush the update if requested.
> *
> diff --git a/drivers/net/mlx5/mlx5_rxtx_vec.h
> b/drivers/net/mlx5/mlx5_rxtx_vec.h
> index 56c5a1b0c..76678a820 100644
> --- a/drivers/net/mlx5/mlx5_rxtx_vec.h
> +++ b/drivers/net/mlx5/mlx5_rxtx_vec.h
> @@ -99,9 +99,13 @@ mlx5_rx_replenish_bulk_mbuf(struct mlx5_rxq_data
> *rxq, uint16_t n)
> rxq->stats.rx_nombuf += n;
> return;
> }
> - for (i = 0; i < n; ++i)
> + for (i = 0; i < n; ++i) {
> wq[i].addr = rte_cpu_to_be_64((uintptr_t)elts[i]->buf_addr
> +
> RTE_PKTMBUF_HEADROOM);
> + /* If there's only one MR, no need to replace LKey in WQE. */
> + if (unlikely(!IS_SINGLE_MR(rxq->mr_ctrl.cache_bh.len)))
> + wq[i].lkey = mlx5_rx_mb2mr(rxq, elts[i]);
> + }
> rxq->rq_ci += n;
> /* Prevent overflowing into consumed mbufs. */
> elts_idx = rxq->rq_ci & q_mask;
> diff --git a/drivers/net/mlx5/mlx5_trigger.c
> b/drivers/net/mlx5/mlx5_trigger.c
> index 3db6c3f35..36b7c9e2f 100644
> --- a/drivers/net/mlx5/mlx5_trigger.c
> +++ b/drivers/net/mlx5/mlx5_trigger.c
> @@ -104,9 +104,18 @@ mlx5_rxq_start(struct rte_eth_dev *dev)
>
> for (i = 0; i != priv->rxqs_n; ++i) {
> struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_get(dev, i);
> + struct rte_mempool *mp;
>
> if (!rxq_ctrl)
> continue;
> + /* Pre-register Rx mempool. */
> + mp = rxq_ctrl->rxq.mp;
> + DRV_LOG(DEBUG,
> + "port %u Rx queue %u registering"
> + " mp %s having %u chunks",
> + dev->data->port_id, rxq_ctrl->idx,
> + mp->name, mp->nb_mem_chunks);
> + mlx5_mr_update_mp(dev, &rxq_ctrl->rxq.mr_ctrl, mp);
> ret = rxq_alloc_elts(rxq_ctrl);
> if (ret)
> goto error;
> @@ -154,6 +163,8 @@ mlx5_dev_start(struct rte_eth_dev *dev)
> dev->data->port_id, strerror(rte_errno));
> goto error;
> }
> + if (rte_log_get_level(mlx5_logtype) == RTE_LOG_DEBUG)
> + mlx5_mr_dump_dev(dev);
> ret = mlx5_rx_intr_vec_enable(dev);
> if (ret) {
> DRV_LOG(ERR, "port %u Rx interrupt vector creation failed",
> diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
> index a71f3d0f0..9ce6f2098 100644
> --- a/drivers/net/mlx5/mlx5_txq.c
> +++ b/drivers/net/mlx5/mlx5_txq.c
> @@ -804,6 +804,13 @@ mlx5_txq_new(struct rte_eth_dev *dev, uint16_t
> idx, uint16_t desc,
> rte_errno = ENOMEM;
> return NULL;
> }
> + if (mlx5_mr_btree_init(&tmpl->txq.mr_ctrl.cache_bh,
> + MLX5_MR_BTREE_CACHE_N, socket)) {
> + /* rte_errno is already set. */
> + goto error;
> + }
> + /* Save pointer of global generation number to check memory
> event. */
> + tmpl->txq.mr_ctrl.dev_gen_ptr = &priv->mr.dev_gen;
> assert(desc > MLX5_TX_COMP_THRESH);
> tmpl->txq.offloads = conf->offloads;
> tmpl->priv = priv;
> @@ -823,6 +830,9 @@ mlx5_txq_new(struct rte_eth_dev *dev, uint16_t
> idx, uint16_t desc,
> idx, rte_atomic32_read(&tmpl->refcnt));
> LIST_INSERT_HEAD(&priv->txqsctrl, tmpl, next);
> return tmpl;
> +error:
> + rte_free(tmpl);
> + return NULL;
> }
>
> /**
> @@ -882,6 +892,7 @@ mlx5_txq_release(struct rte_eth_dev *dev, uint16_t
> idx)
> dev->data->port_id, txq->idx,
> rte_atomic32_read(&txq->refcnt));
> txq_free_elts(txq);
> + mlx5_mr_btree_free(&txq->txq.mr_ctrl.cache_bh);
> LIST_REMOVE(txq, next);
> rte_free(txq);
> (*priv->txqs)[idx] = NULL;
> --
> 2.11.0
^ permalink raw reply [flat|nested] 23+ messages in thread* Re: [dpdk-dev] [PATCH 3/5] net/mlx5: add new Memory Region support
2018-05-06 12:53 ` Shahaf Shuler
@ 2018-05-08 1:52 ` Yongseok Koh
0 siblings, 0 replies; 23+ messages in thread
From: Yongseok Koh @ 2018-05-08 1:52 UTC (permalink / raw)
To: Shahaf Shuler; +Cc: Adrien Mazarguil, Nélio Laranjeiro, dev
On Sun, May 06, 2018 at 05:53:18AM -0700, Shahaf Shuler wrote:
> Hi Koh,
>
> Huge work. It takes (and will take) me some time to process.
> In the meanwhile find some small comments.
>
> As this design heavily relies on synchronization (cache flush) between the
> control thread and the data path thread along with possible deadlocks from
> the memory hotplug events the documentation is critical. Otherwise future
> work will introduce heavy bugs.
Right. Even though I put lots of comments in the code, I'll try to write an
abstract in the commit message.
> Thursday, May 3, 2018 2:17 AM, Yongseok Koh:
> > Subject: [dpdk-dev] [PATCH 3/5] net/mlx5: add new Memory Region support
> >
> > This is the new design of Memory Region (MR) for mlx PMD, in order to:
> > - Accommodate the new memory hotplug model.
> > - Support non-contiguous Mempool.
>
> This commit log is missing a lot of details about the design that you did. You
> must make it clear for every Mellanox PMD developer.
>
> Just to make sure I understand all the details:
> We have
> 1. Cache (L0) per rxq/txq in size of MLX5_MR_CACHE_N. searching in it starting from mru and fallback to linear search
> 2. btree (L1) per rxq/txq in dynamic size. searching using binary search. This is what you refer as the bottom half right?
Right.
> 3. global mr cache (L2) per device in dynamic size (?)
The size of the global btree table isn't dynamically increased. To make it
possible, there should be quite complex code work due to avoid deadlock. The
table can only be changed with locking and the rte_malloc() to expand the table
may raise memory hotplug event. Because it is very rare case, I decided to not
implement it. Instead, I made the global cache table twice bigger than the local
table.
> 4. list of all MRs (L3) per device.
This is the original data. Even if the global cache gets overflowed, the overall
process will be working fine but slow because it has to directly access this
list and the search will be done linearly by traversing the entry one by one.
[...]
> > diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
> > index 01d554758..2883f20af 100644
> > --- a/drivers/net/mlx5/mlx5.c
> > +++ b/drivers/net/mlx5/mlx5.c
> > @@ -41,6 +41,7 @@
> > #include "mlx5_autoconf.h"
> > #include "mlx5_defs.h"
> > #include "mlx5_glue.h"
> > +#include "mlx5_mr.h"
> >
> > /* Device parameter to enable RX completion queue compression. */
> > #define MLX5_RXQ_CQE_COMP_EN "rxq_cqe_comp_en"
> > @@ -84,10 +85,49 @@
> > #define MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP (1 << 4)
> > #endif
> >
> > +static const char *MZ_MLX5_PMD_SHARED_DATA =
> > "mlx5_pmd_shared_data";
> > +
> > +/* Shared memory between primary and secondary processes. */
> > +struct mlx5_shared_data *mlx5_shared_data;
> > +
> > +/* Spinlock for mlx5_shared_data allocation. */
> > +static rte_spinlock_t mlx5_shared_data_lock = RTE_SPINLOCK_INITIALIZER;
> > +
> > /** Driver-specific log messages type. */
> > int mlx5_logtype;
> >
> > /**
> > + * Prepare shared data between primary and secondary process.
> > + */
> > +static void
> > +mlx5_prepare_shared_data(void)
> > +{
> > + const struct rte_memzone *mz;
> > +
> > + rte_spinlock_lock(&mlx5_shared_data_lock);
> > + if (mlx5_shared_data == NULL) {
> > + if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
> > + /* Allocate shared memory. */
> > + mz =
> > rte_memzone_reserve(MZ_MLX5_PMD_SHARED_DATA,
> > + sizeof(*mlx5_shared_data),
> > + SOCKET_ID_ANY, 0);
> > + } else {
> > + /* Lookup allocated shared memory. */
> > + mz =
> > rte_memzone_lookup(MZ_MLX5_PMD_SHARED_DATA);
> > + }
> > + if (mz == NULL)
> > + rte_panic("Cannot allocate mlx5 shared data\n");
> > + mlx5_shared_data = mz->addr;
> > + /* Initialize shared data. */
> > + if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
> > + LIST_INIT(&mlx5_shared_data-
> > >mem_event_cb_list);
> > + rte_rwlock_init(&mlx5_shared_data-
> > >mem_event_rwlock);
> > + }
> > + }
> > + rte_spinlock_unlock(&mlx5_shared_data_lock);
> > +}
> > +
>
> Can you elaborate why mlx5_shared_data can't be part of priv? Priv is already
> allocated on the shared memory and rte_eth_dev layer enforce the secondary
> process creation as part of the rte_eth_dev_data allocation.
Good question. The priv is a per-device structure but the callback mechanism is
not device-specific but global. The callback func has to iterate all the
registered devices. So, there has to be a place to locate the list head and its
lock, which are dpdk-global.
> > +/**
> > * Retrieve integer value from environment variable.
> > *
> > * @param[in] name
> > @@ -201,6 +241,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
> > priv->txqs = NULL;
> > }
> > mlx5_flow_delete_drop_queue(dev);
> > + mlx5_mr_release(dev);
> > if (priv->pd != NULL) {
> > assert(priv->ctx != NULL);
> > claim_zero(mlx5_glue->dealloc_pd(priv->pd));
> > @@ -633,6 +674,8 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv
> > __rte_unused,
> > struct ibv_counter_set_description cs_desc;
> > #endif
> >
> > + /* Prepare shared data between primary and secondary process. */
> > + mlx5_prepare_shared_data();
> > assert(pci_drv == &mlx5_driver);
> > /* Get mlx5_dev[] index. */
> > idx = mlx5_dev_idx(&pci_dev->addr);
> > @@ -1293,6 +1336,8 @@ rte_mlx5_pmd_init(void)
> > }
> > mlx5_glue->fork_init();
> > rte_pci_register(&mlx5_driver);
> > + rte_mem_event_callback_register("MLX5_MEM_EVENT_CB",
> > + mlx5_mr_mem_event_cb);
>
> mlx5_mr_mem_event_cb requires PMD private structure. Is registering for the cb
> on the init makes sense? It looks like a better place is the PCI probe, after
> the eth_dev allocation.
The callback func has to be registered once per a PMD. But the probe func is
called per a device.
[...]
> > diff --git a/drivers/net/mlx5/mlx5_ethdev.c
> > b/drivers/net/mlx5/mlx5_ethdev.c
> > index 746b94f73..6bb43cf4e 100644
> > --- a/drivers/net/mlx5/mlx5_ethdev.c
> > +++ b/drivers/net/mlx5/mlx5_ethdev.c
> > @@ -34,6 +34,7 @@
> > #include <rte_interrupts.h>
> > #include <rte_malloc.h>
> > #include <rte_string_fns.h>
> > +#include <rte_rwlock.h>
> >
> > #include "mlx5.h"
> > #include "mlx5_glue.h"
> > @@ -413,6 +414,21 @@ mlx5_dev_configure(struct rte_eth_dev *dev)
> > if (++j == rxqs_n)
> > j = 0;
> > }
> > + /*
> > + * Once the device is added to the list of memory event callback, its
> > + * global MR cache table cannot be expanded on the fly because of
> > + * deadlock. If it overflows, lookup should be done by searching MR
> > list
> > + * linearly, which is slow.
> > + */
> > + if (mlx5_mr_btree_init(&priv->mr.cache,
> > MLX5_MR_BTREE_CACHE_N * 2,
>
> Why multiple by 2? Because it holds all the rxq/txq mrs?
Like I mentioned, it can't be dynamically expanded due to deadlock. So, I wanted
to give enough space for the global table. However, there's no theory to
calculate what size is the best. I just doubled it by my gut feeling :-). The
size of a cache entry is 20B so the size of the table is 256 * 2 * 20B = 20KB.
Even though it can't be expanded, it will work fine by linearly accessing the
original MR list. It is slow but not a frequently accessed code.
> > + dev->device->numa_node)) {
> > + /* rte_errno is already set. */
> > + return -rte_errno;
> > + }
> > + rte_rwlock_write_lock(&mlx5_shared_data->mem_event_rwlock);
> > + LIST_INSERT_HEAD(&mlx5_shared_data->mem_event_cb_list,
> > + priv, mem_event_cb);
> > + rte_rwlock_write_unlock(&mlx5_shared_data-
> > >mem_event_rwlock);
> > return 0;
> > }
>
> Why registration is done only on configure and not on probe after priv
> initialization?
Once a device is added to this list, the device will start to be searched by the
callback func. It would be better to defer the unnecessary calls until it is
really configured.
> > diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c
> > index 736c40ae4..e964912bb 100644
> > --- a/drivers/net/mlx5/mlx5_mr.c
> > +++ b/drivers/net/mlx5/mlx5_mr.c
> > @@ -13,8 +13,1202 @@
> >
> > #include <rte_mempool.h>
> > #include <rte_malloc.h>
> > +#include <rte_rwlock.h>
> >
> > #include "mlx5.h"
> > +#include "mlx5_mr.h"
> > #include "mlx5_rxtx.h"
> > #include "mlx5_glue.h"
> >
> > +struct mr_find_contig_memsegs_data {
> > + uintptr_t addr;
> > + uintptr_t start;
> > + uintptr_t end;
> > + const struct rte_memseg_list *msl;
> > +};
> > +
> > +struct mr_update_mp_data {
> > + struct rte_eth_dev *dev;
> > + struct mlx5_mr_ctrl *mr_ctrl;
> > + int ret;
> > +};
> > +
> > +/**
> > + * Expand B-tree table to a given size. Can't be called with holding
> > + * memory_hotplug_lock or priv->mr.rwlock due to rte_realloc().
> > + *
> > + * @param bt
> > + * Pointer to B-tree structure.
> > + * @param n
> > + * Number of entries for expansion.
> > + *
> > + * @return
> > + * 0 on success, -1 on failure.
> > + */
> > +static int
> > +mr_btree_expand(struct mlx5_mr_btree *bt, int n)
> > +{
> > + void *mem;
> > + int ret = 0;
> > +
> > + if (n <= bt->size)
> > + return ret;
> > + /*
> > + * Downside of directly using rte_realloc() is that SOCKET_ID_ANY is
> > + * used inside if there's no room to expand. Because this is a quite
> > + * rare case and a part of very slow path, it is very acceptable.
> > + * Initially cache_bh[] will be given practically enough space and once
> > + * it is expanded, expansion wouldn't be needed again ever.
> > + */
> > + mem = rte_realloc(bt->table, n * sizeof(struct mlx5_mr_cache), 0);
> > + if (mem == NULL) {
> > + /* Not an error, B-tree search will be skipped. */
> > + DRV_LOG(WARNING, "failed to expand MR B-tree (%p)
> > table",
> > + (void *)bt);
>
>
> DRV_LOG should have the port id of the device. For all of the DRV_LOG
> instances in the patch.
This func isn't a device-specific one. I don't want to get port_id as an
argument only for debug message.
> Per my understating it falls back to the old bt in case the expansion failed,
> right? Bt searches will still happen.
Right. If a datapath fails to expand its local cache, then it will keep having
misses on some entries and it will eventually be found on device cache by
mr_lookup_dev() in mlx5_mr_create(). In the mr_lookup_dev(), if even the global
cache is overflowed, the addr will have to be searched against the MR list.
[...]
> > +/**
> > + * Insert an entry to B-tree lookup table.
> > + *
> > + * @param bt
> > + * Pointer to B-tree structure.
> > + * @param entry
> > + * Pointer to new entry to insert.
> > + *
> > + * @return
> > + * 0 on success, -1 on failure.
> > + */
> > +static int
> > +mr_btree_insert(struct mlx5_mr_btree *bt, struct mlx5_mr_cache *entry)
> > +{
> > + struct mlx5_mr_cache *lkp_tbl;
> > + uint16_t idx = 0;
> > + size_t shift;
> > +
> > + assert(bt != NULL);
> > + assert(bt->len <= bt->size);
> > + assert(bt->len > 0);
> > + lkp_tbl = *bt->table;
> > + /* Find out the slot for insertion. */
> > + if (mr_btree_lookup(bt, &idx, entry->start) != UINT32_MAX) {
> > + DRV_LOG(DEBUG,
> > + "abort insertion to B-tree(%p):"
> > + " already exist at idx=%u [0x%lx, 0x%lx) lkey=0x%x",
> > + (void *)bt, idx, entry->start, entry->end, entry-
> > >lkey);
> > + /* Already exist, return. */
> > + return 0;
> > + }
> > + /* If table is full, return error. */
> > + if (unlikely(bt->len == bt->size)) {
> > + bt->overflow = 1;
> > + return -1;
> > + }
> > + /* Insert entry. */
> > + ++idx;
> > + shift = (bt->len - idx) * sizeof(struct mlx5_mr_cache);
> > + if (shift)
> > + memmove(&lkp_tbl[idx + 1], &lkp_tbl[idx], shift);
> > + lkp_tbl[idx] = *entry;
> > + bt->len++;
> > + DRV_LOG(DEBUG,
> > + "inserted B-tree(%p)[%u], [0x%lx, 0x%lx) lkey=0x%x",
> > + (void *)bt, idx, entry->start, entry->end, entry->lkey);
> > + return 0;
> > +}
>
> Can you elaborate on how you make sure the btree is always sorted based on the start addr?
mr_btree_lookup() above will return the appropriate index to insert the new
entry to.
[...]
> > +/**
> > + * Free Memory Region (MR).
> > + *
> > + * @param dev
> > + * Pointer to Ethernet device.
> > + * @param mr
> > + * Pointer to MR to free.
> > + */
> > +void
> > +mlx5_mr_free(struct rte_eth_dev *dev, struct mlx5_mr *mr)
> > +{
>
> Who calls this function? I didn't saw any.
Right. When I wrote mlx5_mr_garbage_collect(), I should've deleted it.
Will remove.
> > + struct priv *priv = dev->data->dev_private;
> > +
> > + /* Detach from the list and free resources later. */
> > + rte_rwlock_write_lock(&priv->mr.rwlock);
> > + LIST_REMOVE(mr, mr);
> > + rte_rwlock_write_unlock(&priv->mr.rwlock);
> > + /*
> > + * rte_free() inside can't be called with holding the lock. This could
> > + * cause deadlock when calling free callback.
> > + */
> > + mr_free(mr);
> > + DRV_LOG(DEBUG, "port %u MR(%p) freed", dev->data->port_id,
> > (void *)mr);
> > +}
> > +
> > +/**
> > + * Releass resources of detached MR having no online entry.
> > + *
> > + * @param dev
> > + * Pointer to Ethernet device.
> > + */
> > +static void
> > +mlx5_mr_garbage_collect(struct rte_eth_dev *dev)
> > +{
> > + struct priv *priv = dev->data->dev_private;
> > + struct mlx5_mr *mr_next;
> > + struct mlx5_mr_list free_list = LIST_HEAD_INITIALIZER(free_list);
> > +
> > + /* Must be called from the primary process. */
> > + assert(rte_eal_process_type() == RTE_PROC_PRIMARY);
>
> Perhaps it is better to have this check not under assert?
It is a static func called by mlx5_mr_create(), which prints out the error
message when sec proc attempts to create it. Looks okay to leave it as is.
[...]
> > +/**
> > + * Create a new global Memroy Region (MR) for a missing virtual address.
> > + * Register entire virtually contiguous memory chunk around the address.
> > + *
> > + * @param dev
> > + * Pointer to Ethernet device.
> > + * @param[out] entry
> > + * Pointer to returning MR cache entry, found in the global cache or newly
> > + * created. If failed to create one, this will not be updated.
> > + * @param addr
> > + * Target virtual address to register.
> > + *
> > + * @return
> > + * Searched LKey on success, UINT32_MAX on failure and rte_errno is set.
> > + */
> > +static uint32_t
> > +mlx5_mr_create(struct rte_eth_dev *dev, struct mlx5_mr_cache *entry,
> > + uintptr_t addr)
> > +{
> > + struct priv *priv = dev->data->dev_private;
> > + struct rte_mem_config *mcfg = rte_eal_get_configuration()-
> > >mem_config;
> > + const struct rte_memseg_list *msl;
> > + const struct rte_memseg *ms;
> > + struct mlx5_mr *mr = NULL;
> > + size_t len;
> > + uint32_t ms_n;
> > + uint32_t bmp_size;
> > + void *bmp_mem;
> > + int ms_idx_shift = -1;
> > + unsigned int n;
> > + struct mr_find_contig_memsegs_data data = {
> > + .addr = addr,
> > + };
> > + struct mr_find_contig_memsegs_data data_re;
> > +
> > + DRV_LOG(DEBUG, "port %u creating a MR using address (%p)",
> > + dev->data->port_id, (void *)addr);
> > + if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
> > + DRV_LOG(WARNING,
> > + "port %u using address (%p) of unregistered
> > mempool"
> > + " in secondary process, please create mempool"
> > + " before rte_eth_dev_start()",
> > + dev->data->port_id, (void *)addr);
> > + rte_errno = EPERM;
> > + goto err_nolock;
> > + }
> > + /*
> > + * Release detached MRs if any. This can't be called with holding
> > either
> > + * memory_hotplug_lock or priv->mr.rwlock. MRs on the free list
> > have
> > + * been detached by the memory free event but it couldn't be
> > released
> > + * inside the callback due to deadlock. As a result, releasing resources
> > + * is quite opportunistic.
> > + */
> > + mlx5_mr_garbage_collect(dev);
> > + /*
> > + * Find out a contiguous virtual address chunk in use, to which the
> > + * given address belongs, in order to register maximum range. In the
> > + * best case where mempools are not dynamically recreated and
> > + * '--socket-mem' is speicified as an EAL option, it is very likely to
> > + * have only one MR(LKey) per a socket and per a hugepage-size
> > even
> > + * though the system memory is highly fragmented.
> > + */
> > + if (!rte_memseg_contig_walk(mr_find_contig_memsegs_cb,
> > &data)) {
> > + DRV_LOG(WARNING,
> > + "port %u unable to find virtually contigous"
> > + " chunk for address (%p)."
> > + " rte_memseg_contig_walk() failed.",
> > + dev->data->port_id, (void *)addr);
> > + rte_errno = ENXIO;
> > + goto err_nolock;
> > + }
> > +alloc_resources:
> > + /* Addresses must be page-aligned. */
> > + assert(rte_is_aligned((void *)data.start, data.msl->page_sz));
> > + assert(rte_is_aligned((void *)data.end, data.msl->page_sz));
>
> Better to have this check outsize of assert.
rte_memseg_contig_walk() guarantees it but it is just my paranoid as the API is
still experimental.
[...]
> > +/**
> > + * Look up address in the global MR cache table. If not found, create a new
> > MR.
> > + * Insert the found/created entry to local bottom-half cache table.
> > + *
> > + * @param dev
> > + * Pointer to Ethernet device.
> > + * @param mr_ctrl
> > + * Pointer to per-queue MR control structure.
> > + * @param[out] entry
> > + * Pointer to returning MR cache entry, found in the global cache or newly
> > + * created. If failed to create one, this is not written.
> > + * @param addr
> > + * Search key.
> > + *
> > + * @return
> > + * Searched LKey on success, UINT32_MAX on no match.
> > + */
> > +static uint32_t
> > +mlx5_mr_lookup_dev(struct rte_eth_dev *dev, struct mlx5_mr_ctrl
> > *mr_ctrl,
> > + struct mlx5_mr_cache *entry, uintptr_t addr)
> > +{
> > + struct priv *priv = dev->data->dev_private;
> > + struct mlx5_mr_btree *bt = &mr_ctrl->cache_bh;
> > + uint16_t idx;
> > + uint32_t lkey;
> > +
> > + /* If local cache table is full, try to double it. */
> > + if (unlikely(bt->len == bt->size))
> > + mr_btree_expand(bt, bt->size << 1);
> > + /* Look up in the global cache. */
> > + rte_rwlock_read_lock(&priv->mr.rwlock);
> > + lkey = mr_btree_lookup(&priv->mr.cache, &idx, addr);
> > + if (lkey != UINT32_MAX) {
> > + /* Found. */
> > + *entry = (*priv->mr.cache.table)[idx];
> > + rte_rwlock_read_unlock(&priv->mr.rwlock);
> > + /*
> > + * Update local cache. Even if it fails, return the found entry
> > + * to update top-half cache. Next time, this entry will be
> > found
> > + * in the global cache.
> > + */
> > + mr_btree_insert(bt, entry);
> > + return lkey;
> > + }
> > + rte_rwlock_read_unlock(&priv->mr.rwlock);
> > + /* First time to see the address? Create a new MR. */
> > + lkey = mlx5_mr_create(dev, entry, addr);
>
> Shouldn't we check if the add is not in the global mr list? For the case the global cache overflows?
That is checked inside mlx5_mr_create() by mr_lookup_dev().
[...]
> > +/**
> > + * Bottom-half of LKey search on Rx.
> > + *
> > + * @param rxq
> > + * Pointer to Rx queue structure.
> > + * @param addr
> > + * Search key.
> > + *
> > + * @return
> > + * Searched LKey on success, UINT32_MAX on no match.
> > + */
> > +uint32_t
> > +mlx5_rx_addr2mr_bh(struct mlx5_rxq_data *rxq, uintptr_t addr)
> > +{
> > + struct mlx5_rxq_ctrl *rxq_ctrl =
> > + container_of(rxq, struct mlx5_rxq_ctrl, rxq);
> > + struct mlx5_mr_ctrl *mr_ctrl = &rxq->mr_ctrl;
> > + struct priv *priv = rxq_ctrl->priv;
> > +
> > + DRV_LOG(DEBUG,
> > + "Rx queue %u: miss on top-half, mru=%u, head=%u,
> > addr=%p",
> > + rxq_ctrl->idx, mr_ctrl->mru, mr_ctrl->head, (void *)addr);
> > + return mlx5_mr_addr2mr_bh(eth_dev(priv), mr_ctrl, addr);
> > +}
>
> Shouldn't this code path be in the mlxx5_rxq?
It is an interface between rxq and mr. I prefer to keep here for better
maintenance.
[...]
> > diff --git a/drivers/net/mlx5/mlx5_mr.h b/drivers/net/mlx5/mlx5_mr.h
> > new file mode 100644
> > index 000000000..a0a0ef755
> > --- /dev/null
> > +++ b/drivers/net/mlx5/mlx5_mr.h
> > @@ -0,0 +1,121 @@
> > +/* SPDX-License-Identifier: BSD-3-Clause
> > + * Copyright 2018 6WIND S.A.
> > + * Copyright 2018 Mellanox Technologies, Ltd
> > + */
> > +
> > +#ifndef RTE_PMD_MLX5_MR_H_
> > +#define RTE_PMD_MLX5_MR_H_
> > +
> > +#include <stddef.h>
> > +#include <stdint.h>
> > +#include <sys/queue.h>
> > +
> > +/* Verbs header. */
> > +/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
> > +#ifdef PEDANTIC
> > +#pragma GCC diagnostic ignored "-Wpedantic"
> > +#endif
> > +#include <infiniband/verbs.h>
> > +#include <infiniband/mlx5dv.h>
> > +#ifdef PEDANTIC
> > +#pragma GCC diagnostic error "-Wpedantic"
> > +#endif
> > +
> > +#include <rte_eal_memconfig.h>
> > +#include <rte_ethdev.h>
> > +#include <rte_rwlock.h>
> > +#include <rte_bitmap.h>
> > +
> > +/* Memory Region object. */
> > +struct mlx5_mr {
> > + LIST_ENTRY(mlx5_mr) mr; /**< Pointer to the prev/next entry. */
> > + struct ibv_mr *ibv_mr; /* Verbs Memory Region. */
> > + const struct rte_memseg_list *msl;
> > + int ms_base_idx; /* Start index of msl->memseg_arr[]. */
> > + int ms_n; /* Number of memsegs in use. */
> > + uint32_t ms_bmp_n; /* Number of bits in memsegs bit-mask. */
> > + struct rte_bitmap *ms_bmp; /* Bit-mask of memsegs belonged to
> > MR. */
> > +};
> > +
> > +/* Cache entry for Memory Region. */
> > +struct mlx5_mr_cache {
> > + uintptr_t start; /* Start address of MR. */
> > + uintptr_t end; /* End address of MR. */
> > + uint32_t lkey; /* rte_cpu_to_be_32(ibv_mr->lkey). */
> > +} __rte_packed;
> > +
> > +/* MR Cache table for Binary search. */
> > +struct mlx5_mr_btree {
> > + uint16_t len; /* Number of entries. */
> > + uint16_t size; /* Total number of entries. */
> > + int overflow; /* Mark failure of table expansion. */
> > + struct mlx5_mr_cache (*table)[];
> > +} __rte_packed;
> > +
> > +/* Per-queue MR control descriptor. */
> > +struct mlx5_mr_ctrl {
> > + uint32_t *dev_gen_ptr; /* Generation number of device to poll. */
> > + uint32_t cur_gen; /* Generation number saved to flush caches. */
> > + uint16_t mru; /* Index of last hit entry in top-half cache. */
> > + uint16_t head; /* Index of the oldest entry in top-half cache. */
> > + struct mlx5_mr_cache cache[MLX5_MR_CACHE_N]; /* Cache for
> > top-half. */
> > + struct mlx5_mr_btree cache_bh; /* Cache for bottom-half. */
> > +} __rte_packed;
> > +
> > +/* First entry must be NULL for comparison. */
> > +#define MR_N(n) ((n) - 1)
> > +
> > +/* Whether there's only one entry in MR lookup table. */
> > +#define IS_SINGLE_MR(n) (MR_N(n) == 1)
>
> MLX5_IS_SINGLE_MR
Replaced it with mlx5_mr_btree_len().
Thanks,
Yongseok
^ permalink raw reply [flat|nested] 23+ messages in thread
* [dpdk-dev] [PATCH 4/5] net/mlx4: remove Memory Region support
2018-05-02 23:16 [dpdk-dev] [PATCH 0/5] net/mlx: add new Memory Region support Yongseok Koh
` (2 preceding siblings ...)
2018-05-02 23:16 ` [dpdk-dev] [PATCH 3/5] net/mlx5: add new " Yongseok Koh
@ 2018-05-02 23:16 ` Yongseok Koh
2018-05-02 23:16 ` [dpdk-dev] [PATCH 5/5] net/mlx4: add new " Yongseok Koh
2018-05-09 11:09 ` [dpdk-dev] [PATCH v2 0/4] net/mlx: " Yongseok Koh
5 siblings, 0 replies; 23+ messages in thread
From: Yongseok Koh @ 2018-05-02 23:16 UTC (permalink / raw)
To: adrien.mazarguil, nelio.laranjeiro; +Cc: dev, Yongseok Koh
This patch removes current support of Memory Region (MR) in order to
accommodate the dynamic memory hotplug patch. This patch can be compiled
but traffic can't flow and HW will raise faults. Subsequent patches will
add new MR support.
Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
---
config/common_base | 1 -
doc/guides/nics/mlx4.rst | 8 --
drivers/net/mlx4/Makefile | 4 -
drivers/net/mlx4/mlx4.h | 33 -------
drivers/net/mlx4/mlx4_mr.c | 222 -------------------------------------------
drivers/net/mlx4/mlx4_rxq.c | 11 +--
drivers/net/mlx4/mlx4_rxtx.h | 34 +------
drivers/net/mlx4/mlx4_txq.c | 66 -------------
8 files changed, 4 insertions(+), 375 deletions(-)
diff --git a/config/common_base b/config/common_base
index bf7d5e785..b0ae631f5 100644
--- a/config/common_base
+++ b/config/common_base
@@ -288,7 +288,6 @@ CONFIG_RTE_LIBRTE_AVF_16BYTE_RX_DESC=n
CONFIG_RTE_LIBRTE_MLX4_PMD=n
CONFIG_RTE_LIBRTE_MLX4_DEBUG=n
CONFIG_RTE_LIBRTE_MLX4_DLOPEN_DEPS=n
-CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE=8
#
# Compile burst-oriented Mellanox ConnectX-4 & ConnectX-5 (MLX5) PMD
diff --git a/doc/guides/nics/mlx4.rst b/doc/guides/nics/mlx4.rst
index 9564f890a..25d339d94 100644
--- a/doc/guides/nics/mlx4.rst
+++ b/doc/guides/nics/mlx4.rst
@@ -85,14 +85,6 @@ These options can be modified in the ``.config`` file.
adds additional run-time checks and debugging messages at the cost of
lower performance.
-- ``CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE`` (default **8**)
-
- Maximum number of cached memory pools (MPs) per TX queue. Each MP from
- which buffers are to be transmitted must be associated to memory regions
- (MRs). This is a slow operation that must be cached.
-
- This value is always 1 for RX queues since they use a single MP.
-
Environment variables
~~~~~~~~~~~~~~~~~~~~~
diff --git a/drivers/net/mlx4/Makefile b/drivers/net/mlx4/Makefile
index ac5b67f10..73f9d4056 100644
--- a/drivers/net/mlx4/Makefile
+++ b/drivers/net/mlx4/Makefile
@@ -69,10 +69,6 @@ else
CFLAGS += -DNDEBUG -UPEDANTIC
endif
-ifdef CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE
-CFLAGS += -DMLX4_PMD_TX_MP_CACHE=$(CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE)
-endif
-
include $(RTE_SDK)/mk/rte.lib.mk
# Generate and clean-up mlx4_autoconf.h.
diff --git a/drivers/net/mlx4/mlx4.h b/drivers/net/mlx4/mlx4.h
index 415b7d40f..e0e1b5d4c 100644
--- a/drivers/net/mlx4/mlx4.h
+++ b/drivers/net/mlx4/mlx4.h
@@ -23,7 +23,6 @@
#include <rte_ether.h>
#include <rte_interrupts.h>
#include <rte_mempool.h>
-#include <rte_spinlock.h>
#ifndef IBV_RX_HASH_INNER
/** This is not necessarily defined by supported RDMA core versions. */
@@ -42,17 +41,6 @@
/** Fixed RSS hash key size in bytes. Cannot be modified. */
#define MLX4_RSS_HASH_KEY_SIZE 40
-/**
- * Maximum number of cached Memory Pools (MPs) per TX queue. Each RTE MP
- * from which buffers are to be transmitted will have to be mapped by this
- * driver to their own Memory Region (MR). This is a slow operation.
- *
- * This value is always 1 for RX queues.
- */
-#ifndef MLX4_PMD_TX_MP_CACHE
-#define MLX4_PMD_TX_MP_CACHE 8
-#endif
-
/** Interrupt alarm timeout value in microseconds. */
#define MLX4_INTR_ALARM_TIMEOUT 100000
@@ -78,18 +66,6 @@ struct rxq;
struct txq;
struct rte_flow;
-/** Memory region descriptor. */
-struct mlx4_mr {
- LIST_ENTRY(mlx4_mr) next; /**< Next entry in list. */
- uintptr_t start; /**< Base address for memory region. */
- uintptr_t end; /**< End address for memory region. */
- uint32_t lkey; /**< L_Key extracted from @p mr. */
- uint32_t refcnt; /**< Reference count for this object. */
- struct priv *priv; /**< Back pointer to private data. */
- struct ibv_mr *mr; /**< Memory region associated with @p mp. */
- struct rte_mempool *mp; /**< Target memory pool (mempool). */
-};
-
/** Private data structure. */
struct priv {
struct rte_eth_dev *dev; /**< Ethernet device. */
@@ -112,8 +88,6 @@ struct priv {
struct mlx4_drop *drop; /**< Shared resources for drop flow rules. */
LIST_HEAD(, mlx4_rss) rss; /**< Shared targets for Rx flow rules. */
LIST_HEAD(, rte_flow) flows; /**< Configured flow rule handles. */
- LIST_HEAD(, mlx4_mr) mr; /**< Registered memory regions. */
- rte_spinlock_t mr_lock; /**< Lock for @p mr access. */
struct ether_addr mac[MLX4_MAX_MAC_ADDRESSES];
/**< Configured MAC addresses. Unused entries are zeroed. */
};
@@ -156,11 +130,4 @@ void mlx4_rxq_intr_disable(struct priv *priv);
int mlx4_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx);
int mlx4_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx);
-/* mlx4_mr.c */
-
-struct mlx4_mr *mlx4_mr_get(struct priv *priv, struct rte_mempool *mp);
-void mlx4_mr_put(struct mlx4_mr *mr);
-uint32_t mlx4_txq_add_mr(struct txq *txq, struct rte_mempool *mp,
- uint32_t i);
-
#endif /* RTE_PMD_MLX4_H_ */
diff --git a/drivers/net/mlx4/mlx4_mr.c b/drivers/net/mlx4/mlx4_mr.c
index 8d5a6741d..3c87f6849 100644
--- a/drivers/net/mlx4/mlx4_mr.c
+++ b/drivers/net/mlx4/mlx4_mr.c
@@ -30,230 +30,8 @@
#include <rte_malloc.h>
#include <rte_memory.h>
#include <rte_mempool.h>
-#include <rte_spinlock.h>
#include "mlx4_glue.h"
#include "mlx4_rxtx.h"
#include "mlx4_utils.h"
-struct mlx4_check_mempool_data {
- int ret;
- char *start;
- char *end;
-};
-
-/**
- * Called by mlx4_check_mempool() when iterating the memory chunks.
- *
- * @param[in] mp
- * Pointer to memory pool (unused).
- * @param[in, out] data
- * Pointer to shared buffer with mlx4_check_mempool().
- * @param[in] memhdr
- * Pointer to mempool chunk header.
- * @param mem_idx
- * Mempool element index (unused).
- */
-static void
-mlx4_check_mempool_cb(struct rte_mempool *mp, void *opaque,
- struct rte_mempool_memhdr *memhdr,
- unsigned int mem_idx)
-{
- struct mlx4_check_mempool_data *data = opaque;
-
- (void)mp;
- (void)mem_idx;
- /* It already failed, skip the next chunks. */
- if (data->ret != 0)
- return;
- /* It is the first chunk. */
- if (data->start == NULL && data->end == NULL) {
- data->start = memhdr->addr;
- data->end = data->start + memhdr->len;
- return;
- }
- if (data->end == memhdr->addr) {
- data->end += memhdr->len;
- return;
- }
- if (data->start == (char *)memhdr->addr + memhdr->len) {
- data->start -= memhdr->len;
- return;
- }
- /* Error, mempool is not virtually contiguous. */
- data->ret = -1;
-}
-
-/**
- * Check if a mempool can be used: it must be virtually contiguous.
- *
- * @param[in] mp
- * Pointer to memory pool.
- * @param[out] start
- * Pointer to the start address of the mempool virtual memory area.
- * @param[out] end
- * Pointer to the end address of the mempool virtual memory area.
- *
- * @return
- * 0 on success (mempool is virtually contiguous), -1 on error.
- */
-static int
-mlx4_check_mempool(struct rte_mempool *mp, uintptr_t *start, uintptr_t *end)
-{
- struct mlx4_check_mempool_data data;
-
- memset(&data, 0, sizeof(data));
- rte_mempool_mem_iter(mp, mlx4_check_mempool_cb, &data);
- *start = (uintptr_t)data.start;
- *end = (uintptr_t)data.end;
- return data.ret;
-}
-
-/**
- * Obtain a memory region from a memory pool.
- *
- * If a matching memory region already exists, it is returned with its
- * reference count incremented, otherwise a new one is registered.
- *
- * @param priv
- * Pointer to private structure.
- * @param mp
- * Pointer to memory pool.
- *
- * @return
- * Memory region pointer, NULL in case of error and rte_errno is set.
- */
-struct mlx4_mr *
-mlx4_mr_get(struct priv *priv, struct rte_mempool *mp)
-{
- const struct rte_memseg *ms;
- uintptr_t start;
- uintptr_t end;
- struct mlx4_mr *mr;
-
- if (mlx4_check_mempool(mp, &start, &end) != 0) {
- rte_errno = EINVAL;
- ERROR("mempool %p: not virtually contiguous",
- (void *)mp);
- return NULL;
- }
- DEBUG("mempool %p area start=%p end=%p size=%zu",
- (void *)mp, (void *)start, (void *)end,
- (size_t)(end - start));
- /* Round start and end to page boundary if found in memory segments. */
- ms = rte_mem_virt2memseg((void *)start, NULL);
- if (ms != NULL)
- start = RTE_ALIGN_FLOOR(start, ms->hugepage_sz);
- end = RTE_ALIGN_CEIL(end, ms->hugepage_sz);
- DEBUG("mempool %p using start=%p end=%p size=%zu for MR",
- (void *)mp, (void *)start, (void *)end,
- (size_t)(end - start));
- rte_spinlock_lock(&priv->mr_lock);
- LIST_FOREACH(mr, &priv->mr, next)
- if (mp == mr->mp && start >= mr->start && end <= mr->end)
- break;
- if (mr) {
- ++mr->refcnt;
- goto release;
- }
- mr = rte_malloc(__func__, sizeof(*mr), 0);
- if (!mr) {
- rte_errno = ENOMEM;
- goto release;
- }
- *mr = (struct mlx4_mr){
- .start = start,
- .end = end,
- .refcnt = 1,
- .priv = priv,
- .mr = mlx4_glue->reg_mr(priv->pd, (void *)start, end - start,
- IBV_ACCESS_LOCAL_WRITE),
- .mp = mp,
- };
- if (mr->mr) {
- mr->lkey = mr->mr->lkey;
- LIST_INSERT_HEAD(&priv->mr, mr, next);
- } else {
- rte_free(mr);
- mr = NULL;
- rte_errno = errno ? errno : EINVAL;
- }
-release:
- rte_spinlock_unlock(&priv->mr_lock);
- return mr;
-}
-
-/**
- * Release a memory region.
- *
- * This function decrements its reference count and destroys it after
- * reaching 0.
- *
- * Note to avoid race conditions given this function may be used from the
- * data plane, it's extremely important that each user holds its own
- * reference.
- *
- * @param mr
- * Memory region to release.
- */
-void
-mlx4_mr_put(struct mlx4_mr *mr)
-{
- struct priv *priv = mr->priv;
-
- rte_spinlock_lock(&priv->mr_lock);
- assert(mr->refcnt);
- if (--mr->refcnt)
- goto release;
- LIST_REMOVE(mr, next);
- claim_zero(mlx4_glue->dereg_mr(mr->mr));
- rte_free(mr);
-release:
- rte_spinlock_unlock(&priv->mr_lock);
-}
-
-/**
- * Add memory region (MR) <-> memory pool (MP) association to txq->mp2mr[].
- * If mp2mr[] is full, remove an entry first.
- *
- * @param txq
- * Pointer to Tx queue structure.
- * @param[in] mp
- * Memory pool for which a memory region lkey must be added.
- * @param[in] i
- * Index in memory pool (MP) where to add memory region (MR).
- *
- * @return
- * Added mr->lkey on success, (uint32_t)-1 on failure.
- */
-uint32_t
-mlx4_txq_add_mr(struct txq *txq, struct rte_mempool *mp, uint32_t i)
-{
- struct mlx4_mr *mr;
-
- /* Add a new entry, register MR first. */
- DEBUG("%p: discovered new memory pool \"%s\" (%p)",
- (void *)txq, mp->name, (void *)mp);
- mr = mlx4_mr_get(txq->priv, mp);
- if (unlikely(mr == NULL)) {
- DEBUG("%p: unable to configure MR, mlx4_mr_get() failed",
- (void *)txq);
- return (uint32_t)-1;
- }
- if (unlikely(i == RTE_DIM(txq->mp2mr))) {
- /* Table is full, remove oldest entry. */
- DEBUG("%p: MR <-> MP table full, dropping oldest entry.",
- (void *)txq);
- --i;
- mlx4_mr_put(txq->mp2mr[0].mr);
- memmove(&txq->mp2mr[0], &txq->mp2mr[1],
- (sizeof(txq->mp2mr) - sizeof(txq->mp2mr[0])));
- }
- /* Store the new entry. */
- txq->mp2mr[i].mp = mp;
- txq->mp2mr[i].mr = mr;
- txq->mp2mr[i].lkey = mr->lkey;
- DEBUG("%p: new MR lkey for MP \"%s\" (%p): 0x%08" PRIu32,
- (void *)txq, mp->name, (void *)mp, txq->mp2mr[i].lkey);
- return txq->mp2mr[i].lkey;
-}
diff --git a/drivers/net/mlx4/mlx4_rxq.c b/drivers/net/mlx4/mlx4_rxq.c
index 65f099423..5621d5bd4 100644
--- a/drivers/net/mlx4/mlx4_rxq.c
+++ b/drivers/net/mlx4/mlx4_rxq.c
@@ -583,7 +583,7 @@ mlx4_rxq_attach(struct rxq *rxq)
.addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(buf,
uintptr_t)),
.byte_count = rte_cpu_to_be_32(buf->data_len),
- .lkey = rte_cpu_to_be_32(rxq->mr->lkey),
+ .lkey = UINT32_MAX,
};
(*elts)[i] = buf;
}
@@ -883,13 +883,6 @@ mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1 << rxq->sges_n);
goto error;
}
- /* Use the entire Rx mempool as the memory region. */
- rxq->mr = mlx4_mr_get(priv, mp);
- if (!rxq->mr) {
- ERROR("%p: MR creation failure: %s",
- (void *)dev, strerror(rte_errno));
- goto error;
- }
if (dev->data->dev_conf.intr_conf.rxq) {
rxq->channel = mlx4_glue->create_comp_channel(priv->ctx);
if (rxq->channel == NULL) {
@@ -947,7 +940,5 @@ mlx4_rx_queue_release(void *dpdk_rxq)
assert(!rxq->rq_db);
if (rxq->channel)
claim_zero(mlx4_glue->destroy_comp_channel(rxq->channel));
- if (rxq->mr)
- mlx4_mr_put(rxq->mr);
rte_free(rxq);
}
diff --git a/drivers/net/mlx4/mlx4_rxtx.h b/drivers/net/mlx4/mlx4_rxtx.h
index 2dfee957f..2f9d3798b 100644
--- a/drivers/net/mlx4/mlx4_rxtx.h
+++ b/drivers/net/mlx4/mlx4_rxtx.h
@@ -39,7 +39,6 @@ struct mlx4_rxq_stats {
struct rxq {
struct priv *priv; /**< Back pointer to private data. */
struct rte_mempool *mp; /**< Memory pool for allocations. */
- struct mlx4_mr *mr; /**< Memory region. */
struct ibv_cq *cq; /**< Completion queue. */
struct ibv_wq *wq; /**< Work queue. */
struct ibv_comp_channel *channel; /**< Rx completion channel. */
@@ -109,11 +108,6 @@ struct txq {
uint32_t lb:1; /**< Whether packets should be looped back by eSwitch. */
uint8_t *bounce_buf;
/**< Memory used for storing the first DWORD of data TXBBs. */
- struct {
- const struct rte_mempool *mp; /**< Cached memory pool. */
- struct mlx4_mr *mr; /**< Memory region (for mp). */
- uint32_t lkey; /**< mr->lkey copy. */
- } mp2mr[MLX4_PMD_TX_MP_CACHE]; /**< MP to MR translation table. */
struct priv *priv; /**< Back pointer to private data. */
unsigned int socket; /**< CPU socket ID for allocations. */
struct ibv_cq *cq; /**< Completion queue. */
@@ -161,34 +155,12 @@ int mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx,
const struct rte_eth_txconf *conf);
void mlx4_tx_queue_release(void *dpdk_txq);
-/**
- * Get memory region (MR) <-> memory pool (MP) association from txq->mp2mr[].
- * Call mlx4_txq_add_mr() if MP is not registered yet.
- *
- * @param txq
- * Pointer to Tx queue structure.
- * @param[in] mp
- * Memory pool for which a memory region lkey must be returned.
- *
- * @return
- * mr->lkey on success, (uint32_t)-1 on failure.
- */
static inline uint32_t
mlx4_txq_mp2mr(struct txq *txq, struct rte_mempool *mp)
{
- unsigned int i;
-
- for (i = 0; (i != RTE_DIM(txq->mp2mr)); ++i) {
- if (unlikely(txq->mp2mr[i].mp == NULL)) {
- /* Unknown MP, add a new MR for it. */
- break;
- }
- if (txq->mp2mr[i].mp == mp) {
- /* MP found MP. */
- return txq->mp2mr[i].lkey;
- }
- }
- return mlx4_txq_add_mr(txq, mp, i);
+ (void)txq;
+ (void)mp;
+ return UINT32_MAX;
}
#endif /* MLX4_RXTX_H_ */
diff --git a/drivers/net/mlx4/mlx4_txq.c b/drivers/net/mlx4/mlx4_txq.c
index fe6a8e07e..5ea09b0b0 100644
--- a/drivers/net/mlx4/mlx4_txq.c
+++ b/drivers/net/mlx4/mlx4_txq.c
@@ -63,64 +63,6 @@ mlx4_txq_free_elts(struct txq *txq)
txq->elts_tail = txq->elts_head;
}
-struct txq_mp2mr_mbuf_check_data {
- int ret;
-};
-
-/**
- * Callback function for rte_mempool_obj_iter() to check whether a given
- * mempool object looks like a mbuf.
- *
- * @param[in] mp
- * The mempool pointer
- * @param[in] arg
- * Context data (struct mlx4_txq_mp2mr_mbuf_check_data). Contains the
- * return value.
- * @param[in] obj
- * Object address.
- * @param index
- * Object index, unused.
- */
-static void
-mlx4_txq_mp2mr_mbuf_check(struct rte_mempool *mp, void *arg, void *obj,
- uint32_t index)
-{
- struct txq_mp2mr_mbuf_check_data *data = arg;
- struct rte_mbuf *buf = obj;
-
- (void)index;
- /*
- * Check whether mbuf structure fits element size and whether mempool
- * pointer is valid.
- */
- if (sizeof(*buf) > mp->elt_size || buf->pool != mp)
- data->ret = -1;
-}
-
-/**
- * Iterator function for rte_mempool_walk() to register existing mempools and
- * fill the MP to MR cache of a Tx queue.
- *
- * @param[in] mp
- * Memory Pool to register.
- * @param *arg
- * Pointer to Tx queue structure.
- */
-static void
-mlx4_txq_mp2mr_iter(struct rte_mempool *mp, void *arg)
-{
- struct txq *txq = arg;
- struct txq_mp2mr_mbuf_check_data data = {
- .ret = 0,
- };
-
- /* Register mempool only if the first element looks like a mbuf. */
- if (rte_mempool_obj_iter(mp, mlx4_txq_mp2mr_mbuf_check, &data) == 0 ||
- data.ret == -1)
- return;
- mlx4_txq_mp2mr(txq, mp);
-}
-
/**
* Retrieves information needed in order to directly access the Tx queue.
*
@@ -404,8 +346,6 @@ mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
/* Save first wqe pointer in the first element. */
(&(*txq->elts)[0])->wqe =
(volatile struct mlx4_wqe_ctrl_seg *)txq->msq.buf;
- /* Pre-register known mempools. */
- rte_mempool_walk(mlx4_txq_mp2mr_iter, txq);
DEBUG("%p: adding Tx queue %p to list", (void *)dev, (void *)txq);
dev->data->tx_queues[idx] = txq;
return 0;
@@ -446,11 +386,5 @@ mlx4_tx_queue_release(void *dpdk_txq)
claim_zero(mlx4_glue->destroy_qp(txq->qp));
if (txq->cq)
claim_zero(mlx4_glue->destroy_cq(txq->cq));
- for (i = 0; i != RTE_DIM(txq->mp2mr); ++i) {
- if (!txq->mp2mr[i].mp)
- break;
- assert(txq->mp2mr[i].mr);
- mlx4_mr_put(txq->mp2mr[i].mr);
- }
rte_free(txq);
}
--
2.11.0
^ permalink raw reply [flat|nested] 23+ messages in thread* [dpdk-dev] [PATCH 5/5] net/mlx4: add new Memory Region support
2018-05-02 23:16 [dpdk-dev] [PATCH 0/5] net/mlx: add new Memory Region support Yongseok Koh
` (3 preceding siblings ...)
2018-05-02 23:16 ` [dpdk-dev] [PATCH 4/5] net/mlx4: remove " Yongseok Koh
@ 2018-05-02 23:16 ` Yongseok Koh
2018-05-09 11:09 ` [dpdk-dev] [PATCH v2 0/4] net/mlx: " Yongseok Koh
5 siblings, 0 replies; 23+ messages in thread
From: Yongseok Koh @ 2018-05-02 23:16 UTC (permalink / raw)
To: adrien.mazarguil, nelio.laranjeiro; +Cc: dev, Yongseok Koh
This is the new design of Memory Region (MR) for mlx PMD, in order to:
- Accommodate the new memory hotplug model.
- Support non-contiguous Mempool.
Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
---
drivers/net/mlx4/mlx4.c | 26 +
drivers/net/mlx4/mlx4.h | 14 +
drivers/net/mlx4/mlx4_mr.c | 1165 ++++++++++++++++++++++++++++++++++++++++++
drivers/net/mlx4/mlx4_mr.h | 126 +++++
drivers/net/mlx4/mlx4_rxq.c | 14 +-
drivers/net/mlx4/mlx4_rxtx.c | 35 +-
drivers/net/mlx4/mlx4_rxtx.h | 71 ++-
drivers/net/mlx4/mlx4_txq.c | 8 +
8 files changed, 1428 insertions(+), 31 deletions(-)
create mode 100644 drivers/net/mlx4/mlx4_mr.h
diff --git a/drivers/net/mlx4/mlx4.c b/drivers/net/mlx4/mlx4.c
index 47451b651..e6c701d13 100644
--- a/drivers/net/mlx4/mlx4.c
+++ b/drivers/net/mlx4/mlx4.c
@@ -44,9 +44,15 @@
#include "mlx4.h"
#include "mlx4_glue.h"
#include "mlx4_flow.h"
+#include "mlx4_mr.h"
#include "mlx4_rxtx.h"
#include "mlx4_utils.h"
+struct mlx4_dev_list mlx4_mem_event_cb_list =
+ LIST_HEAD_INITIALIZER(mlx4_mem_event_cb_list);
+
+rte_rwlock_t mlx4_mem_event_rwlock = RTE_RWLOCK_INITIALIZER;
+
/** Configuration structure for device arguments. */
struct mlx4_conf {
struct {
@@ -92,6 +98,20 @@ mlx4_dev_configure(struct rte_eth_dev *dev)
if (ret)
ERROR("%p: interrupt handler installation failed",
(void *)dev);
+ /*
+ * Once the device is added to the list of memory event callback, its
+ * global MR cache table cannot be expanded on the fly because of
+ * deadlock. If it overflows, lookup should be done by searching MR list
+ * linearly, which is slow.
+ */
+ if (mlx4_mr_btree_init(&priv->mr.cache, MLX4_MR_BTREE_CACHE_N * 2,
+ dev->device->numa_node)) {
+ /* rte_errno is already set. */
+ return -rte_errno;
+ }
+ rte_rwlock_write_lock(&mlx4_mem_event_rwlock);
+ LIST_INSERT_HEAD(&mlx4_mem_event_cb_list, priv, mem_event_cb);
+ rte_rwlock_write_unlock(&mlx4_mem_event_rwlock);
exit:
return ret;
}
@@ -125,6 +145,9 @@ mlx4_dev_start(struct rte_eth_dev *dev)
(void *)dev, strerror(-ret));
goto err;
}
+#ifndef NDEBUG
+ mlx4_mr_dump_dev(dev);
+#endif
ret = mlx4_rxq_intr_enable(priv);
if (ret) {
ERROR("%p: interrupt handler installation failed",
@@ -200,6 +223,7 @@ mlx4_dev_close(struct rte_eth_dev *dev)
mlx4_rx_queue_release(dev->data->rx_queues[i]);
for (i = 0; i != dev->data->nb_tx_queues; ++i)
mlx4_tx_queue_release(dev->data->tx_queues[i]);
+ mlx4_mr_release(dev);
if (priv->pd != NULL) {
assert(priv->ctx != NULL);
claim_zero(mlx4_glue->dealloc_pd(priv->pd));
@@ -964,6 +988,8 @@ rte_mlx4_pmd_init(void)
}
mlx4_glue->fork_init();
rte_pci_register(&mlx4_driver);
+ rte_mem_event_callback_register("MLX4_MEM_EVENT_CB",
+ mlx4_mr_mem_event_cb);
}
RTE_PMD_EXPORT_NAME(net_mlx4, __COUNTER__);
diff --git a/drivers/net/mlx4/mlx4.h b/drivers/net/mlx4/mlx4.h
index e0e1b5d4c..300cb4d7a 100644
--- a/drivers/net/mlx4/mlx4.h
+++ b/drivers/net/mlx4/mlx4.h
@@ -23,6 +23,9 @@
#include <rte_ether.h>
#include <rte_interrupts.h>
#include <rte_mempool.h>
+#include <rte_rwlock.h>
+
+#include "mlx4_mr.h"
#ifndef IBV_RX_HASH_INNER
/** This is not necessarily defined by supported RDMA core versions. */
@@ -66,8 +69,12 @@ struct rxq;
struct txq;
struct rte_flow;
+LIST_HEAD(mlx4_dev_list, priv);
+LIST_HEAD(mlx4_mr_list, mlx4_mr);
+
/** Private data structure. */
struct priv {
+ LIST_ENTRY(priv) mem_event_cb; /* Called by memory event callback. */
struct rte_eth_dev *dev; /**< Ethernet device. */
struct ibv_context *ctx; /**< Verbs context. */
struct ibv_device_attr device_attr; /**< Device properties. */
@@ -86,6 +93,13 @@ struct priv {
uint64_t hw_rss_sup; /**< Supported RSS hash fields (Verbs format). */
struct rte_intr_handle intr_handle; /**< Port interrupt handle. */
struct mlx4_drop *drop; /**< Shared resources for drop flow rules. */
+ struct {
+ uint32_t dev_gen; /* Generation number to flush local caches. */
+ rte_rwlock_t rwlock; /* MR Lock. */
+ struct mlx4_mr_btree cache; /* Global MR cache table. */
+ struct mlx4_mr_list mr_list; /* Registered MR list. */
+ struct mlx4_mr_list mr_free_list; /* Freed MR list. */
+ } mr;
LIST_HEAD(, mlx4_rss) rss; /**< Shared targets for Rx flow rules. */
LIST_HEAD(, rte_flow) flows; /**< Configured flow rule handles. */
struct ether_addr mac[MLX4_MAX_MAC_ADDRESSES];
diff --git a/drivers/net/mlx4/mlx4_mr.c b/drivers/net/mlx4/mlx4_mr.c
index 3c87f6849..4812f7109 100644
--- a/drivers/net/mlx4/mlx4_mr.c
+++ b/drivers/net/mlx4/mlx4_mr.c
@@ -30,8 +30,1173 @@
#include <rte_malloc.h>
#include <rte_memory.h>
#include <rte_mempool.h>
+#include <rte_rwlock.h>
#include "mlx4_glue.h"
+#include "mlx4_mr.h"
#include "mlx4_rxtx.h"
#include "mlx4_utils.h"
+struct mr_find_contig_memsegs_data {
+ uintptr_t addr;
+ uintptr_t start;
+ uintptr_t end;
+ const struct rte_memseg_list *msl;
+};
+
+struct mr_update_mp_data {
+ struct rte_eth_dev *dev;
+ struct mlx4_mr_ctrl *mr_ctrl;
+ int ret;
+};
+
+/**
+ * Expand B-tree table to a given size. Can't be called with holding
+ * memory_hotplug_lock or priv->mr.rwlock due to rte_realloc().
+ *
+ * @param bt
+ * Pointer to B-tree structure.
+ * @param n
+ * Number of entries for expansion.
+ *
+ * @return
+ * 0 on success, -1 on failure.
+ */
+static int
+mr_btree_expand(struct mlx4_mr_btree *bt, int n)
+{
+ void *mem;
+ int ret = 0;
+
+ if (n <= bt->size)
+ return ret;
+ /*
+ * Downside of directly using rte_realloc() is that SOCKET_ID_ANY is
+ * used inside if there's no room to expand. Because this is a quite
+ * rare case and a part of very slow path, it is very acceptable.
+ * Initially cache_bh[] will be given practically enough space and once
+ * it is expanded, expansion wouldn't be needed again ever.
+ */
+ mem = rte_realloc(bt->table, n * sizeof(struct mlx4_mr_cache), 0);
+ if (mem == NULL) {
+ /* Not an error, B-tree search will be skipped. */
+ WARN("failed to expand MR B-tree (%p) table", (void *)bt);
+ ret = -1;
+ } else {
+ DEBUG("expanded MR B-tree table (size=%u)", n);
+ bt->table = mem;
+ bt->size = n;
+ }
+ return ret;
+}
+
+/**
+ * Look up LKey from given B-tree lookup table, store the last index and return
+ * searched LKey.
+ *
+ * @param bt
+ * Pointer to B-tree structure.
+ * @param[out] idx
+ * Pointer to index. Even on searh failure, returns index where it stops
+ * searching so that index can be used when inserting a new entry.
+ * @param addr
+ * Search key.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on no match.
+ */
+static uint32_t
+mr_btree_lookup(struct mlx4_mr_btree *bt, uint16_t *idx, uintptr_t addr)
+{
+ struct mlx4_mr_cache *lkp_tbl;
+ uint16_t n;
+ uint16_t base = 0;
+
+ assert(bt != NULL);
+ lkp_tbl = *bt->table;
+ n = bt->len;
+ /* First entry must be NULL for comparison. */
+ assert(bt->len > 0 || (lkp_tbl[0].start == 0 &&
+ lkp_tbl[0].lkey == UINT32_MAX));
+ /* Binary search. */
+ do {
+ register uint16_t delta = n >> 1;
+
+ if (addr < lkp_tbl[base + delta].start) {
+ n = delta;
+ } else {
+ base += delta;
+ n -= delta;
+ }
+ } while (n > 1);
+ assert(addr >= lkp_tbl[base].start);
+ *idx = base;
+ if (addr < lkp_tbl[base].end)
+ return lkp_tbl[base].lkey;
+ /* Not found. */
+ return UINT32_MAX;
+}
+
+/**
+ * Insert an entry to B-tree lookup table.
+ *
+ * @param bt
+ * Pointer to B-tree structure.
+ * @param entry
+ * Pointer to new entry to insert.
+ *
+ * @return
+ * 0 on success, -1 on failure.
+ */
+static int
+mr_btree_insert(struct mlx4_mr_btree *bt, struct mlx4_mr_cache *entry)
+{
+ struct mlx4_mr_cache *lkp_tbl;
+ uint16_t idx = 0;
+ size_t shift;
+
+ assert(bt != NULL);
+ assert(bt->len <= bt->size);
+ assert(bt->len > 0);
+ lkp_tbl = *bt->table;
+ /* Find out the slot for insertion. */
+ if (mr_btree_lookup(bt, &idx, entry->start) != UINT32_MAX) {
+ DEBUG("abort insertion to B-tree(%p):"
+ " already exist at idx=%u [0x%lx, 0x%lx) lkey=0x%x",
+ (void *)bt, idx, entry->start, entry->end, entry->lkey);
+ /* Already exist, return. */
+ return 0;
+ }
+ /* If table is full, return error. */
+ if (unlikely(bt->len == bt->size)) {
+ bt->overflow = 1;
+ return -1;
+ }
+ /* Insert entry. */
+ ++idx;
+ shift = (bt->len - idx) * sizeof(struct mlx4_mr_cache);
+ if (shift)
+ memmove(&lkp_tbl[idx + 1], &lkp_tbl[idx], shift);
+ lkp_tbl[idx] = *entry;
+ bt->len++;
+ DEBUG("inserted B-tree(%p)[%u], [0x%lx, 0x%lx) lkey=0x%x",
+ (void *)bt, idx, entry->start, entry->end, entry->lkey);
+ return 0;
+}
+
+/**
+ * Initialize B-tree and allocate memory for lookup table.
+ *
+ * @param bt
+ * Pointer to B-tree structure.
+ * @param n
+ * Number of entries to allocate.
+ * @param socket
+ * NUMA socket on which memory must be allocated.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx4_mr_btree_init(struct mlx4_mr_btree *bt, int n, int socket)
+{
+ if (bt == NULL) {
+ rte_errno = EINVAL;
+ return -rte_errno;
+ }
+ memset(bt, 0, sizeof(*bt));
+ bt->table = rte_calloc_socket("B-tree table",
+ n, sizeof(struct mlx4_mr_cache),
+ 0, socket);
+ if (bt->table == NULL) {
+ rte_errno = ENOMEM;
+ ERROR("failed to allocate memory for btree cache on socket %d",
+ socket);
+ return -rte_errno;
+ }
+ bt->size = n;
+ /* First entry must be NULL for binary search. */
+ (*bt->table)[bt->len++] = (struct mlx4_mr_cache) {
+ .lkey = UINT32_MAX,
+ };
+ DEBUG("initialized B-tree %p with table %p",
+ (void *)bt, (void *)bt->table);
+ return 0;
+}
+
+/**
+ * Free B-tree resources.
+ *
+ * @param bt
+ * Pointer to B-tree structure.
+ */
+void
+mlx4_mr_btree_free(struct mlx4_mr_btree *bt)
+{
+ if (bt == NULL)
+ return;
+ DEBUG("freeing B-tree %p with table %p", (void *)bt, (void *)bt->table);
+ rte_free(bt->table);
+ memset(bt, 0, sizeof(*bt));
+}
+
+#ifndef NDEBUG
+/**
+ * Dump all the entries in a B-tree
+ *
+ * @param bt
+ * Pointer to B-tree structure.
+ */
+void
+mlx4_mr_btree_dump(struct mlx4_mr_btree *bt)
+{
+ int idx;
+ struct mlx4_mr_cache *lkp_tbl;
+
+ if (bt == NULL)
+ return;
+ lkp_tbl = *bt->table;
+ for (idx = 0; idx < bt->len; ++idx) {
+ struct mlx4_mr_cache *entry = &lkp_tbl[idx];
+
+ DEBUG("B-tree(%p)[%u], [0x%lx, 0x%lx) lkey=0x%x",
+ (void *)bt, idx, entry->start, entry->end, entry->lkey);
+ }
+}
+#endif
+
+/**
+ * Find virtually contiguous memory chunk in a given MR.
+ *
+ * @param dev
+ * Pointer to MR structure.
+ * @param[out] entry
+ * Pointer to returning MR cache entry. If not found, this will not be
+ * updated.
+ * @param start_idx
+ * Start index of the memseg bitmap.
+ *
+ * @return
+ * Next index to go on lookup.
+ */
+static int
+mr_find_next_chunk(struct mlx4_mr *mr, struct mlx4_mr_cache *entry,
+ int base_idx)
+{
+ uintptr_t start = 0;
+ uintptr_t end = 0;
+ uint32_t idx = 0;
+
+ for (idx = base_idx; idx < mr->ms_bmp_n; ++idx) {
+ if (rte_bitmap_get(mr->ms_bmp, idx)) {
+ const struct rte_memseg_list *msl;
+ const struct rte_memseg *ms;
+
+ msl = mr->msl;
+ ms = rte_fbarray_get(&msl->memseg_arr,
+ mr->ms_base_idx + idx);
+ assert(msl->page_sz == ms->hugepage_sz);
+ if (!start)
+ start = ms->addr_64;
+ end = ms->addr_64 + ms->hugepage_sz;
+ } else if (start) {
+ /* Passed the end of a fragment. */
+ break;
+ }
+ }
+ if (start) {
+ /* Found one chunk. */
+ entry->start = start;
+ entry->end = end;
+ entry->lkey = rte_cpu_to_be_32(mr->ibv_mr->lkey);
+ }
+ return idx;
+}
+
+/**
+ * Insert a MR to the global B-tree cache. It may fail due to low-on-memory.
+ * Then, this entry will have to be searched by mr_lookup_dev_list() in
+ * mlx4_mr_create() on miss.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param mr
+ * Pointer to MR to insert.
+ *
+ * @return
+ * 0 on success, -1 on failure.
+ */
+static int
+mr_insert_dev_cache(struct rte_eth_dev *dev, struct mlx4_mr *mr)
+{
+ struct priv *priv = dev->data->dev_private;
+ unsigned int n;
+
+ DEBUG("port %u inserting MR(%p) to global cache",
+ dev->data->port_id, (void *)mr);
+ for (n = 0; n < mr->ms_bmp_n; ) {
+ struct mlx4_mr_cache entry = { 0, };
+
+ /* Find a contiguous chunk and advance the index. */
+ n = mr_find_next_chunk(mr, &entry, n);
+ if (!entry.end)
+ break;
+ if (mr_btree_insert(&priv->mr.cache, &entry) < 0) {
+ /*
+ * Overflowed, but the global table cannot be expanded
+ * because of deadlock.
+ */
+ return -1;
+ }
+ }
+ return 0;
+}
+
+/**
+ * Look up address in the original global MR list.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param[out] entry
+ * Pointer to returning MR cache entry. If no match, this will not be updated.
+ * @param addr
+ * Search key.
+ *
+ * @return
+ * Found MR on match, NULL otherwise.
+ */
+static struct mlx4_mr *
+mr_lookup_dev_list(struct rte_eth_dev *dev, struct mlx4_mr_cache *entry,
+ uintptr_t addr)
+{
+ struct priv *priv = dev->data->dev_private;
+ struct mlx4_mr *mr;
+
+ /* Iterate all the existing MRs. */
+ LIST_FOREACH(mr, &priv->mr.mr_list, mr) {
+ unsigned int n;
+
+ if (mr->ms_n == 0)
+ continue;
+ for (n = 0; n < mr->ms_bmp_n; ) {
+ struct mlx4_mr_cache ret = { 0, };
+
+ n = mr_find_next_chunk(mr, &ret, n);
+ if (addr >= ret.start && addr < ret.end) {
+ /* Found. */
+ *entry = ret;
+ return mr;
+ }
+ }
+ }
+ return NULL;
+}
+
+/**
+ * Look up address on device.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param[out] entry
+ * Pointer to returning MR cache entry. If no match, this will not be updated.
+ * @param addr
+ * Search key.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on failure and rte_errno is set.
+ */
+static uint32_t
+mr_lookup_dev(struct rte_eth_dev *dev, struct mlx4_mr_cache *entry,
+ uintptr_t addr)
+{
+ struct priv *priv = dev->data->dev_private;
+ uint16_t idx;
+ uint32_t lkey = UINT32_MAX;
+ struct mlx4_mr *mr;
+
+ /*
+ * If the global cache has overflowed since it failed to expand the
+ * B-tree table, it can't have all the exisitng MRs. Then, the address
+ * has to be searched by traversing the original MR list instead, which
+ * is very slow path. Otherwise, the global cache is all inclusive.
+ */
+ if (!unlikely(priv->mr.cache.overflow)) {
+ lkey = mr_btree_lookup(&priv->mr.cache, &idx, addr);
+ if (lkey != UINT32_MAX)
+ *entry = (*priv->mr.cache.table)[idx];
+ } else {
+ /* Falling back to the slowest path. */
+ mr = mr_lookup_dev_list(dev, entry, addr);
+ if (mr != NULL)
+ lkey = entry->lkey;
+ }
+ assert(lkey == UINT32_MAX || (addr >= entry->start &&
+ addr < entry->end));
+ return lkey;
+}
+
+/**
+ * Free MR resources. MR lock must not be held to avoid a deadlock. rte_free()
+ * can raise memory free event and the callback function will spin on the lock.
+ *
+ * @param mr
+ * Pointer to MR to free.
+ */
+static void
+mr_free(struct mlx4_mr *mr)
+{
+ if (mr == NULL)
+ return;
+ DEBUG("freeing MR(%p):", (void *)mr);
+ if (mr->ibv_mr != NULL)
+ claim_zero(mlx4_glue->dereg_mr(mr->ibv_mr));
+ if (mr->ms_bmp != NULL)
+ rte_bitmap_free(mr->ms_bmp);
+ rte_free(mr);
+}
+
+/**
+ * Free Memory Region (MR).
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param mr
+ * Pointer to MR to free.
+ */
+void
+mlx4_mr_free(struct rte_eth_dev *dev, struct mlx4_mr *mr)
+{
+ struct priv *priv = dev->data->dev_private;
+
+ /* Detach from the list and free resources later. */
+ rte_rwlock_write_lock(&priv->mr.rwlock);
+ LIST_REMOVE(mr, mr);
+ rte_rwlock_write_unlock(&priv->mr.rwlock);
+ /*
+ * rte_free() inside can't be called with holding the lock. This could
+ * cause deadlock when calling free callback.
+ */
+ mr_free(mr);
+ DEBUG("port %u MR(%p) freed", dev->data->port_id, (void *)mr);
+}
+
+/**
+ * Releass resources of detached MR having no online entry.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ */
+static void
+mlx4_mr_garbage_collect(struct rte_eth_dev *dev)
+{
+ struct priv *priv = dev->data->dev_private;
+ struct mlx4_mr *mr_next;
+ struct mlx4_mr_list free_list = LIST_HEAD_INITIALIZER(free_list);
+
+ /*
+ * MR can't be freed with holding the lock because rte_free() could call
+ * memory free callback function. This will be a deadlock situation.
+ */
+ rte_rwlock_write_lock(&priv->mr.rwlock);
+ /* Detach the whole free list and release it after unlocking. */
+ free_list = priv->mr.mr_free_list;
+ LIST_INIT(&priv->mr.mr_free_list);
+ rte_rwlock_write_unlock(&priv->mr.rwlock);
+ /* Release resources. */
+ mr_next = LIST_FIRST(&free_list);
+ while (mr_next != NULL) {
+ struct mlx4_mr *mr = mr_next;
+
+ mr_next = LIST_NEXT(mr, mr);
+ mr_free(mr);
+ }
+}
+
+/* Called during rte_memseg_contig_walk() by mlx4_mr_create(). */
+static int
+mr_find_contig_memsegs_cb(const struct rte_memseg_list *msl,
+ const struct rte_memseg *ms, size_t len, void *arg)
+{
+ struct mr_find_contig_memsegs_data *data = arg;
+
+ if (data->addr < ms->addr_64 || data->addr >= ms->addr_64 + len)
+ return 0;
+ /* Found, save it and stop walking. */
+ data->start = ms->addr_64;
+ data->end = ms->addr_64 + len;
+ data->msl = msl;
+ return 1;
+}
+
+/**
+ * Create a new global Memroy Region (MR) for a missing virtual address.
+ * Register entire virtually contiguous memory chunk around the address.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param[out] entry
+ * Pointer to returning MR cache entry, found in the global cache or newly
+ * created. If failed to create one, this will not be updated.
+ * @param addr
+ * Target virtual address to register.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on failure and rte_errno is set.
+ */
+static uint32_t
+mlx4_mr_create(struct rte_eth_dev *dev, struct mlx4_mr_cache *entry,
+ uintptr_t addr)
+{
+ struct priv *priv = dev->data->dev_private;
+ struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+ const struct rte_memseg_list *msl;
+ const struct rte_memseg *ms;
+ struct mlx4_mr *mr = NULL;
+ size_t len;
+ uint32_t ms_n;
+ uint32_t bmp_size;
+ void *bmp_mem;
+ int ms_idx_shift = -1;
+ unsigned int n;
+ struct mr_find_contig_memsegs_data data = {
+ .addr = addr,
+ };
+ struct mr_find_contig_memsegs_data data_re;
+
+ DEBUG("port %u creating a MR using address (%p)",
+ dev->data->port_id, (void *)addr);
+ /*
+ * Release detached MRs if any. This can't be called with holding either
+ * memory_hotplug_lock or priv->mr.rwlock. MRs on the free list have
+ * been detached by the memory free event but it couldn't be released
+ * inside the callback due to deadlock. As a result, releasing resources
+ * is quite opportunistic.
+ */
+ mlx4_mr_garbage_collect(dev);
+ /*
+ * Find out a contiguous virtual address chunk in use, to which the
+ * given address belongs, in order to register maximum range. In the
+ * best case where mempools are not dynamically recreated and
+ * '--socket-mem' is speicified as an EAL option, it is very likely to
+ * have only one MR(LKey) per a socket and per a hugepage-size even
+ * though the system memory is highly fragmented.
+ */
+ if (!rte_memseg_contig_walk(mr_find_contig_memsegs_cb, &data)) {
+ WARN("port %u unable to find virtually contigous"
+ " chunk for address (%p)."
+ " rte_memseg_contig_walk() failed.",
+ dev->data->port_id, (void *)addr);
+ rte_errno = ENXIO;
+ goto err_nolock;
+ }
+alloc_resources:
+ /* Addresses must be page-aligned. */
+ assert(rte_is_aligned((void *)data.start, data.msl->page_sz));
+ assert(rte_is_aligned((void *)data.end, data.msl->page_sz));
+ msl = data.msl;
+ ms = rte_mem_virt2memseg((void *)data.start, msl);
+ len = data.end - data.start;
+ assert(msl->page_sz == ms->hugepage_sz);
+ /* Number of memsegs in the range. */
+ ms_n = len / msl->page_sz;
+ DEBUG("port %u extending %p to [0x%lx, 0x%lx), page_sz=0x%lx, ms_n=%u",
+ dev->data->port_id, (void *)addr,
+ data.start, data.end, msl->page_sz, ms_n);
+ /* Size of memory for bitmap. */
+ bmp_size = rte_bitmap_get_memory_footprint(ms_n);
+ mr = rte_zmalloc_socket(NULL,
+ RTE_ALIGN_CEIL(sizeof(*mr),
+ RTE_CACHE_LINE_SIZE) +
+ bmp_size,
+ RTE_CACHE_LINE_SIZE, msl->socket_id);
+ if (mr == NULL) {
+ WARN("port %u unable to allocate memory for a new MR of"
+ " address (%p).",
+ dev->data->port_id, (void *)addr);
+ rte_errno = ENOMEM;
+ goto err_nolock;
+ }
+ mr->msl = msl;
+ /*
+ * Save the index of the first memseg and initialize memseg bitmap. To
+ * see if a memseg of ms_idx in the memseg-list is still valid, check:
+ * rte_bitmap_get(mr->bmp, ms_idx - mr->ms_base_idx)
+ */
+ mr->ms_base_idx = rte_fbarray_find_idx(&msl->memseg_arr, ms);
+ bmp_mem = RTE_PTR_ALIGN_CEIL(mr + 1, RTE_CACHE_LINE_SIZE);
+ mr->ms_bmp = rte_bitmap_init(ms_n, bmp_mem, bmp_size);
+ if (mr->ms_bmp == NULL) {
+ WARN("port %u unable to initialize bitamp for a new MR of"
+ " address (%p).",
+ dev->data->port_id, (void *)addr);
+ rte_errno = EINVAL;
+ goto err_nolock;
+ }
+ /*
+ * Should recheck whether the extended contiguous chunk is still valid.
+ * Because memory_hotplug_lock can't be held if there's any memory
+ * related calls in a critical path, resource allocation above can't be
+ * locked. If the memory has been changed at this point, try again with
+ * just single page. If not, go on with the big chunk atomically from
+ * here.
+ */
+ rte_rwlock_read_lock(&mcfg->memory_hotplug_lock);
+ data_re = data;
+ if (len > msl->page_sz &&
+ !rte_memseg_contig_walk(mr_find_contig_memsegs_cb, &data_re)) {
+ WARN("port %u unable to find virtually contigous"
+ " chunk for address (%p)."
+ " rte_memseg_contig_walk() failed.",
+ dev->data->port_id, (void *)addr);
+ rte_errno = ENXIO;
+ goto err_memlock;
+ }
+ if (data.start != data_re.start || data.end != data_re.end) {
+ /*
+ * The extended contiguous chunk has been changed. Try again
+ * with single memseg instead.
+ */
+ data.start = RTE_ALIGN_FLOOR(addr, msl->page_sz);
+ data.end = data.start + msl->page_sz;
+ rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
+ mr_free(mr);
+ goto alloc_resources;
+ }
+ assert(data.msl == data_re.msl);
+ rte_rwlock_write_lock(&priv->mr.rwlock);
+ /*
+ * Check the address is really missing. If other thread already created
+ * one or it is not found due to overflow, abort and return.
+ */
+ if (mr_lookup_dev(dev, entry, addr) != UINT32_MAX) {
+ /*
+ * Insert to the global cache table. It may fail due to
+ * low-on-memory. Then, this entry will have to be searched
+ * here again.
+ */
+ mr_btree_insert(&priv->mr.cache, entry);
+ DEBUG("port %u found MR for %p on final lookup, abort",
+ dev->data->port_id, (void *)addr);
+ rte_rwlock_write_unlock(&priv->mr.rwlock);
+ rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
+ /*
+ * Must be unlocked before calling rte_free() because
+ * mlx4_mr_mem_event_free_cb() can be called inside.
+ */
+ mr_free(mr);
+ return entry->lkey;
+ }
+ /*
+ * Trim start and end addresses for verbs MR. Set bits for registering
+ * memsegs but exclude already registered ones. Bitmap can be
+ * fragmented.
+ */
+ for (n = 0; n < ms_n; ++n) {
+ uintptr_t start;
+ struct mlx4_mr_cache ret = { 0, };
+
+ start = data_re.start + n * msl->page_sz;
+ /* Exclude memsegs already registered by other MRs. */
+ if (mr_lookup_dev(dev, &ret, start) == UINT32_MAX) {
+ /*
+ * Start from the first unregistered memseg in the
+ * extended range.
+ */
+ if (ms_idx_shift == -1) {
+ mr->ms_base_idx += n;
+ data.start = start;
+ ms_idx_shift = n;
+ }
+ data.end = start + msl->page_sz;
+ rte_bitmap_set(mr->ms_bmp, n - ms_idx_shift);
+ ++mr->ms_n;
+ }
+ }
+ len = data.end - data.start;
+ mr->ms_bmp_n = len / msl->page_sz;
+ assert(ms_idx_shift + mr->ms_bmp_n <= ms_n);
+ /*
+ * Finally create a verbs MR for the memory chunk. ibv_reg_mr() can be
+ * called with holding the memory lock because it doesn't use
+ * mlx4_alloc_buf_extern() which eventually calls rte_malloc_socket()
+ * through mlx4_alloc_verbs_buf().
+ */
+ mr->ibv_mr = mlx4_glue->reg_mr(priv->pd, (void *)data.start, len,
+ IBV_ACCESS_LOCAL_WRITE);
+ if (mr->ibv_mr == NULL) {
+ WARN("port %u fail to create a verbs MR for address (%p)",
+ dev->data->port_id, (void *)addr);
+ rte_errno = EINVAL;
+ goto err_mrlock;
+ }
+ assert((uintptr_t)mr->ibv_mr->addr == data.start);
+ assert(mr->ibv_mr->length == len);
+ LIST_INSERT_HEAD(&priv->mr.mr_list, mr, mr);
+ DEBUG("port %u MR CREATED (%p) for %p:\n"
+ " [0x%lx, 0x%lx), lkey=0x%x base_idx=%u ms_n=%u, ms_bmp_n=%u",
+ dev->data->port_id, (void *)mr, (void *)addr,
+ data.start, data.end, rte_cpu_to_be_32(mr->ibv_mr->lkey),
+ mr->ms_base_idx, mr->ms_n, mr->ms_bmp_n);
+ /* Insert to the global cache table. */
+ mr_insert_dev_cache(dev, mr);
+ /* Fill in output data. */
+ mr_lookup_dev(dev, entry, addr);
+ /* Lookup can't fail. */
+ assert(entry->lkey != UINT32_MAX);
+ rte_rwlock_write_unlock(&priv->mr.rwlock);
+ rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
+ return entry->lkey;
+err_mrlock:
+ rte_rwlock_write_unlock(&priv->mr.rwlock);
+err_memlock:
+ rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
+err_nolock:
+ /*
+ * In case of error, as this can be called in a datapath, a warning
+ * message per an error is preferable instead. Must be unlocked before
+ * calling rte_free() because mlx4_mr_mem_event_free_cb() can be called
+ * inside.
+ */
+ mr_free(mr);
+ return UINT32_MAX;
+}
+
+/**
+ * Rebuild the global B-tree cache of device from the original MR list.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ */
+static void
+mr_rebuild_dev_cache(struct rte_eth_dev *dev)
+{
+ struct priv *priv = dev->data->dev_private;
+ struct mlx4_mr *mr;
+
+ DEBUG("port %u rebuild dev cache[]", dev->data->port_id);
+ /* Flush cache to rebuild. */
+ priv->mr.cache.len = 1;
+ priv->mr.cache.overflow = 0;
+ /* Iterate all the existing MRs. */
+ LIST_FOREACH(mr, &priv->mr.mr_list, mr)
+ if (mr_insert_dev_cache(dev, mr) < 0)
+ return;
+}
+
+/**
+ * Callback for memory free event. Iterate freed memsegs and check whether it
+ * belongs to an existing MR. If found, clear the bit from bitmap of MR. As a
+ * result, the MR would be fragmented. If it becomes empty, the MR will be freed
+ * later by mlx4_mr_garbage_collect().
+ *
+ * The global cache must be rebuilt if there's any change and this event has to
+ * be propagated to dataplane threads to flush the local caches.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param addr
+ * Address of freed memory.
+ * @param len
+ * Size of freed memory.
+ */
+static void
+mlx4_mr_mem_event_free_cb(struct rte_eth_dev *dev, const void *addr, size_t len)
+{
+ struct priv *priv = dev->data->dev_private;
+ const struct rte_memseg_list *msl;
+ struct mlx4_mr *mr;
+ int ms_n;
+ int i;
+ int rebuild = 0;
+
+ DEBUG("port %u free callback: addr=%p, len=%lu",
+ dev->data->port_id, addr, len);
+ msl = rte_mem_virt2memseg_list(addr);
+ /* addr and len must be page-aligned. */
+ assert((uintptr_t)addr == RTE_ALIGN((uintptr_t)addr, msl->page_sz));
+ assert(len == RTE_ALIGN(len, msl->page_sz));
+ ms_n = len / msl->page_sz;
+ rte_rwlock_write_lock(&priv->mr.rwlock);
+ /* Clear bits of freed memsegs from MR. */
+ for (i = 0; i < ms_n; ++i) {
+ const struct rte_memseg *ms;
+ struct mlx4_mr_cache entry;
+ uintptr_t start;
+ int ms_idx;
+ uint32_t pos;
+
+ /* Find MR having this memseg. */
+ start = (uintptr_t)addr + i * msl->page_sz;
+ mr = mr_lookup_dev_list(dev, &entry, start);
+ if (mr == NULL)
+ continue;
+ ms = rte_mem_virt2memseg((void *)start, msl);
+ assert(ms != NULL);
+ assert(msl->page_sz == ms->hugepage_sz);
+ ms_idx = rte_fbarray_find_idx(&msl->memseg_arr, ms);
+ pos = ms_idx - mr->ms_base_idx;
+ assert(rte_bitmap_get(mr->ms_bmp, pos));
+ assert(pos < mr->ms_bmp_n);
+ DEBUG("port %u MR(%p): clear bitmap[%u] for addr %p",
+ dev->data->port_id, (void *)mr, pos, (void *)start);
+ rte_bitmap_clear(mr->ms_bmp, pos);
+ if (--mr->ms_n == 0) {
+ LIST_REMOVE(mr, mr);
+ LIST_INSERT_HEAD(&priv->mr.mr_free_list, mr, mr);
+ DEBUG("port %u remove MR(%p) from list",
+ dev->data->port_id, (void *)mr);
+ }
+ /*
+ * MR is fragmented or will be freed. the global cache must be
+ * rebuilt.
+ */
+ rebuild = 1;
+ }
+ if (rebuild) {
+ mr_rebuild_dev_cache(dev);
+ /*
+ * Flush local caches by propagating invalidation across cores.
+ * rte_smp_wmb() is enough to synchronize this event. If one of
+ * freed memsegs is seen by other core, that means the memseg
+ * has been allocated by allocator, which will come after this
+ * free call. Therefore, this store instruction (incrementing
+ * generation below) will be guaranteed to be seen by other core
+ * before the core sees the newly allocated memory.
+ */
+ ++priv->mr.dev_gen;
+ DEBUG("broadcasting local cache flush, gen=%d",
+ priv->mr.dev_gen);
+ rte_smp_wmb();
+ }
+ rte_rwlock_write_unlock(&priv->mr.rwlock);
+#ifndef NDEBUG
+ if (rebuild)
+ mlx4_mr_dump_dev(dev);
+#endif
+}
+
+/**
+ * Callback for memory event.
+ *
+ * @param event_type
+ * Memory event type.
+ * @param addr
+ * Address of memory.
+ * @param len
+ * Size of memory.
+ */
+void
+mlx4_mr_mem_event_cb(enum rte_mem_event event_type, const void *addr,
+ size_t len)
+{
+ struct priv *priv;
+
+ switch (event_type) {
+ case RTE_MEM_EVENT_FREE:
+ rte_rwlock_read_lock(&mlx4_mem_event_rwlock);
+ /* Iterate all the existing mlx4 devices. */
+ LIST_FOREACH(priv, &mlx4_mem_event_cb_list, mem_event_cb)
+ mlx4_mr_mem_event_free_cb(priv->dev, addr, len);
+ rte_rwlock_read_unlock(&mlx4_mem_event_rwlock);
+ break;
+ case RTE_MEM_EVENT_ALLOC:
+ default:
+ break;
+ }
+}
+
+/**
+ * Look up address in the global MR cache table. If not found, create a new MR.
+ * Insert the found/created entry to local bottom-half cache table.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param mr_ctrl
+ * Pointer to per-queue MR control structure.
+ * @param[out] entry
+ * Pointer to returning MR cache entry, found in the global cache or newly
+ * created. If failed to create one, this is not written.
+ * @param addr
+ * Search key.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on no match.
+ */
+static uint32_t
+mlx4_mr_lookup_dev(struct rte_eth_dev *dev, struct mlx4_mr_ctrl *mr_ctrl,
+ struct mlx4_mr_cache *entry, uintptr_t addr)
+{
+ struct priv *priv = dev->data->dev_private;
+ struct mlx4_mr_btree *bt = &mr_ctrl->cache_bh;
+ uint16_t idx;
+ uint32_t lkey;
+
+ /* If local cache table is full, try to double it. */
+ if (unlikely(bt->len == bt->size))
+ mr_btree_expand(bt, bt->size << 1);
+ /* Look up in the global cache. */
+ rte_rwlock_read_lock(&priv->mr.rwlock);
+ lkey = mr_btree_lookup(&priv->mr.cache, &idx, addr);
+ if (lkey != UINT32_MAX) {
+ /* Found. */
+ *entry = (*priv->mr.cache.table)[idx];
+ rte_rwlock_read_unlock(&priv->mr.rwlock);
+ /*
+ * Update local cache. Even if it fails, return the found entry
+ * to update top-half cache. Next time, this entry will be found
+ * in the global cache.
+ */
+ mr_btree_insert(bt, entry);
+ return lkey;
+ }
+ rte_rwlock_read_unlock(&priv->mr.rwlock);
+ /* First time to see the address? Create a new MR. */
+ lkey = mlx4_mr_create(dev, entry, addr);
+ /*
+ * Update the local cache if successfully created a new global MR. Even
+ * if failed to create one, there's no action to take in this datapath
+ * code. As returning LKey is invalid, this will eventually make HW
+ * fail.
+ */
+ if (lkey != UINT32_MAX)
+ mr_btree_insert(bt, entry);
+ return lkey;
+}
+
+/**
+ * Bottom-half of LKey search on datapath. Firstly search in cache_bh[] and if
+ * misses, search in the global MR cache table and update the new entry to
+ * per-queue local caches.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param mr_ctrl
+ * Pointer to per-queue MR control structure.
+ * @param addr
+ * Search key.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on no match.
+ */
+static uint32_t
+mlx4_mr_addr2mr_bh(struct rte_eth_dev *dev, struct mlx4_mr_ctrl *mr_ctrl,
+ uintptr_t addr)
+{
+ uint32_t lkey;
+ uint16_t bh_idx = 0;
+ /* Victim in top-half cache to replace with new entry. */
+ struct mlx4_mr_cache *repl = &mr_ctrl->cache[mr_ctrl->head];
+
+ /* Binary-search MR translation table. */
+ lkey = mr_btree_lookup(&mr_ctrl->cache_bh, &bh_idx, addr);
+ /* Update top-half cache. */
+ if (likely(lkey != UINT32_MAX)) {
+ *repl = (*mr_ctrl->cache_bh.table)[bh_idx];
+ } else {
+ /*
+ * If missed in local lookup table, search in the global cache
+ * and local cache_bh[] will be updated inside if possible.
+ * Top-half cache entry will also be updated.
+ */
+ lkey = mlx4_mr_lookup_dev(dev, mr_ctrl, repl, addr);
+ if (unlikely(lkey == UINT32_MAX))
+ return UINT32_MAX;
+ }
+ /* Update the most recently used entry. */
+ mr_ctrl->mru = mr_ctrl->head;
+ /* Point to the next victim, the oldest. */
+ mr_ctrl->head = (mr_ctrl->head + 1) % MLX4_MR_CACHE_N;
+ return lkey;
+}
+
+/**
+ * Bottom-half of LKey search on Rx.
+ *
+ * @param rxq
+ * Pointer to Rx queue structure.
+ * @param addr
+ * Search key.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on no match.
+ */
+uint32_t
+mlx4_rx_addr2mr_bh(struct rxq *rxq, uintptr_t addr)
+{
+ struct mlx4_mr_ctrl *mr_ctrl = &rxq->mr_ctrl;
+ struct priv *priv = rxq->priv;
+
+ DEBUG("Rx queue %u: miss on top-half, mru=%u, head=%u, addr=%p",
+ rxq->stats.idx, mr_ctrl->mru, mr_ctrl->head, (void *)addr);
+ return mlx4_mr_addr2mr_bh(priv->dev, mr_ctrl, addr);
+}
+
+/**
+ * Bottom-half of LKey search on Tx.
+ *
+ * @param txq
+ * Pointer to Tx queue structure.
+ * @param addr
+ * Search key.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on no match.
+ */
+uint32_t
+mlx4_tx_addr2mr_bh(struct txq *txq, uintptr_t addr)
+{
+ struct mlx4_mr_ctrl *mr_ctrl = &txq->mr_ctrl;
+ struct priv *priv = txq->priv;
+
+ DEBUG("Tx queue %u: miss on top-half, mru=%u, head=%u, addr=%p",
+ txq->stats.idx, mr_ctrl->mru, mr_ctrl->head, (void *)addr);
+ return mlx4_mr_addr2mr_bh(priv->dev, mr_ctrl, addr);
+}
+
+/**
+ * Flush all of the local cache entries.
+ *
+ * @param mr_ctrl
+ * Pointer to per-queue MR control structure.
+ */
+void
+mlx4_mr_flush_local_cache(struct mlx4_mr_ctrl *mr_ctrl)
+{
+ /* Reset the most-recently-used index. */
+ mr_ctrl->mru = 0;
+ /* Reset the linear search array. */
+ mr_ctrl->head = 0;
+ memset(mr_ctrl->cache, 0, sizeof(mr_ctrl->cache));
+ /* Reset the B-tree table. */
+ mr_ctrl->cache_bh.len = 1;
+ mr_ctrl->cache_bh.overflow = 0;
+ /* Update the generation number. */
+ mr_ctrl->cur_gen = *mr_ctrl->dev_gen_ptr;
+ DEBUG("mr_ctrl(%p): flushed, cur_gen=%d",
+ (void *)mr_ctrl, mr_ctrl->cur_gen);
+}
+
+/* Called during rte_mempool_mem_iter() by mlx4_mr_update_mp(). */
+static void
+mlx4_mr_update_mp_cb(struct rte_mempool *mp __rte_unused, void *opaque,
+ struct rte_mempool_memhdr *memhdr,
+ unsigned mem_idx __rte_unused)
+{
+ struct mr_update_mp_data *data = opaque;
+ uint32_t lkey;
+
+ /* Stop iteration if failed in the previous walk. */
+ if (data->ret < 0)
+ return;
+ /* Register address of the chunk and update local caches. */
+ lkey = mlx4_mr_addr2mr_bh(data->dev, data->mr_ctrl,
+ (uintptr_t)memhdr->addr);
+ if (lkey == UINT32_MAX)
+ data->ret = -1;
+}
+
+/**
+ * Register entire memory chunks in a Mempool.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param mr_ctrl
+ * Pointer to per-queue MR control structure.
+ * @param mp
+ * Pointer to registering Mempool.
+ *
+ * @return
+ * 0 on success, -1 on failure.
+ */
+int
+mlx4_mr_update_mp(struct rte_eth_dev *dev, struct mlx4_mr_ctrl *mr_ctrl,
+ struct rte_mempool *mp)
+{
+ struct mr_update_mp_data data = {
+ .dev = dev,
+ .mr_ctrl = mr_ctrl,
+ .ret = 0,
+ };
+
+ rte_mempool_mem_iter(mp, mlx4_mr_update_mp_cb, &data);
+ return data.ret;
+}
+
+#ifndef NDEBUG
+/**
+ * Dump all the created MRs and the global cache entries.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ */
+void
+mlx4_mr_dump_dev(struct rte_eth_dev *dev)
+{
+ struct priv *priv = dev->data->dev_private;
+ struct mlx4_mr *mr;
+ int mr_n = 0;
+ int chunk_n = 0;
+
+ rte_rwlock_read_lock(&priv->mr.rwlock);
+ /* Iterate all the existing MRs. */
+ LIST_FOREACH(mr, &priv->mr.mr_list, mr) {
+ unsigned int n;
+
+ DEBUG("port %u MR[%u], LKey = 0x%x, ms_n = %u, ms_bmp_n = %u",
+ dev->data->port_id, mr_n++,
+ rte_cpu_to_be_32(mr->ibv_mr->lkey),
+ mr->ms_n, mr->ms_bmp_n);
+ if (mr->ms_n == 0)
+ continue;
+ for (n = 0; n < mr->ms_bmp_n; ) {
+ struct mlx4_mr_cache ret = { 0, };
+
+ n = mr_find_next_chunk(mr, &ret, n);
+ if (!ret.end)
+ break;
+ DEBUG(" chunk[%u], [0x%lx, 0x%lx)",
+ chunk_n++, ret.start, ret.end);
+ }
+ }
+ DEBUG("port %u dumping global cache", dev->data->port_id);
+ mlx4_mr_btree_dump(&priv->mr.cache);
+ rte_rwlock_read_unlock(&priv->mr.rwlock);
+}
+#endif
+
+/**
+ * Release all the created MRs and resources. Remove device from memory callback
+ * list.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ */
+void
+mlx4_mr_release(struct rte_eth_dev *dev)
+{
+ struct priv *priv = dev->data->dev_private;
+ struct mlx4_mr *mr_next = LIST_FIRST(&priv->mr.mr_list);
+
+ /* Remove from memory callback device list. */
+ rte_rwlock_write_lock(&mlx4_mem_event_rwlock);
+ LIST_REMOVE(priv, mem_event_cb);
+ rte_rwlock_write_unlock(&mlx4_mem_event_rwlock);
+#ifndef NDEBUG
+ mlx4_mr_dump_dev(dev);
+#endif
+ rte_rwlock_write_lock(&priv->mr.rwlock);
+ /* Detach from MR list and move to free list. */
+ while (mr_next != NULL) {
+ struct mlx4_mr *mr = mr_next;
+
+ mr_next = LIST_NEXT(mr, mr);
+ LIST_REMOVE(mr, mr);
+ LIST_INSERT_HEAD(&priv->mr.mr_free_list, mr, mr);
+ }
+ LIST_INIT(&priv->mr.mr_list);
+ /* Free global cache. */
+ mlx4_mr_btree_free(&priv->mr.cache);
+ rte_rwlock_write_unlock(&priv->mr.rwlock);
+ /* Free all remaining MRs. */
+ mlx4_mr_garbage_collect(dev);
+}
diff --git a/drivers/net/mlx4/mlx4_mr.h b/drivers/net/mlx4/mlx4_mr.h
new file mode 100644
index 000000000..33305c0f8
--- /dev/null
+++ b/drivers/net/mlx4/mlx4_mr.h
@@ -0,0 +1,126 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2018 6WIND S.A.
+ * Copyright 2018 Mellanox Technologies, Ltd
+ */
+
+#ifndef RTE_PMD_MLX4_MR_H_
+#define RTE_PMD_MLX4_MR_H_
+
+#include <stddef.h>
+#include <stdint.h>
+#include <sys/queue.h>
+
+/* Verbs headers do not support -pedantic. */
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+#include <infiniband/verbs.h>
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
+#include <rte_eal_memconfig.h>
+#include <rte_ethdev.h>
+#include <rte_rwlock.h>
+#include <rte_bitmap.h>
+
+/* Size of per-queue MR cache array for linear search. */
+#define MLX4_MR_CACHE_N 8
+
+/* Size of MR cache table for binary search. */
+#define MLX4_MR_BTREE_CACHE_N 256
+
+/* Memory Region object. */
+struct mlx4_mr {
+ LIST_ENTRY(mlx4_mr) mr; /**< Pointer to the prev/next entry. */
+ struct ibv_mr *ibv_mr; /* Verbs Memory Region. */
+ const struct rte_memseg_list *msl;
+ int ms_base_idx; /* Start index of msl->memseg_arr[]. */
+ int ms_n; /* Number of memsegs in use. */
+ uint32_t ms_bmp_n; /* Number of bits in memsegs bit-mask. */
+ struct rte_bitmap *ms_bmp; /* Bit-mask of memsegs belonged to MR. */
+};
+
+/* Cache entry for Memory Region. */
+struct mlx4_mr_cache {
+ uintptr_t start; /* Start address of MR. */
+ uintptr_t end; /* End address of MR. */
+ uint32_t lkey; /* rte_cpu_to_be_32(ibv_mr->lkey). */
+} __rte_packed;
+
+/* MR Cache table for Binary search. */
+struct mlx4_mr_btree {
+ uint16_t len; /* Number of entries. */
+ uint16_t size; /* Total number of entries. */
+ int overflow; /* Mark failure of table expansion. */
+ struct mlx4_mr_cache (*table)[];
+} __rte_packed;
+
+/* Per-queue MR control descriptor. */
+struct mlx4_mr_ctrl {
+ uint32_t *dev_gen_ptr; /* Generation number of device to poll. */
+ uint32_t cur_gen; /* Generation number saved to flush caches. */
+ uint16_t mru; /* Index of last hit entry in top-half cache. */
+ uint16_t head; /* Index of the oldest entry in top-half cache. */
+ struct mlx4_mr_cache cache[MLX4_MR_CACHE_N]; /* Cache for top-half. */
+ struct mlx4_mr_btree cache_bh; /* Cache for bottom-half. */
+} __rte_packed;
+
+/* First entry must be NULL for comparison. */
+#define MR_N(n) ((n) - 1)
+
+/* Whether there's only one entry in MR lookup table. */
+#define IS_SINGLE_MR(n) (MR_N(n) == 1)
+
+extern struct mlx4_dev_list mlx4_mem_event_cb_list;
+extern rte_rwlock_t mlx4_mem_event_rwlock;
+
+void mlx4_mr_free(struct rte_eth_dev *dev, struct mlx4_mr *mr);
+int mlx4_mr_btree_init(struct mlx4_mr_btree *bt, int n, int socket);
+void mlx4_mr_btree_free(struct mlx4_mr_btree *bt);
+void mlx4_mr_btree_dump(struct mlx4_mr_btree *bt);
+void mlx4_mr_mem_event_cb(enum rte_mem_event event_type, const void *addr,
+ size_t len);
+int mlx4_mr_update_mp(struct rte_eth_dev *dev, struct mlx4_mr_ctrl *mr_ctrl,
+ struct rte_mempool *mp);
+void mlx4_mr_dump_dev(struct rte_eth_dev *dev);
+void mlx4_mr_release(struct rte_eth_dev *dev);
+
+/**
+ * Look up LKey from given lookup table by linear search. Firstly look up the
+ * last-hit entry. If miss, the entire array is searched. If found, update the
+ * last-hit index and return LKey.
+ *
+ * @param lkp_tbl
+ * Pointer to lookup table.
+ * @param[in,out] cached_idx
+ * Pointer to last-hit index.
+ * @param n
+ * Size of lookup table.
+ * @param addr
+ * Search key.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on no match.
+ */
+static __rte_always_inline uint32_t
+mlx4_mr_lookup_cache(struct mlx4_mr_cache *lkp_tbl, uint16_t *cached_idx,
+ uint16_t n, uintptr_t addr)
+{
+ uint16_t idx;
+
+ if (likely(addr >= lkp_tbl[*cached_idx].start &&
+ addr < lkp_tbl[*cached_idx].end))
+ return lkp_tbl[*cached_idx].lkey;
+ for (idx = 0; idx < n && lkp_tbl[idx].start != 0; ++idx) {
+ if (addr >= lkp_tbl[idx].start &&
+ addr < lkp_tbl[idx].end) {
+ /* Found. */
+ *cached_idx = idx;
+ return lkp_tbl[idx].lkey;
+ }
+ }
+ return UINT32_MAX;
+}
+
+#endif /* RTE_PMD_MLX4_MR_H_ */
diff --git a/drivers/net/mlx4/mlx4_rxq.c b/drivers/net/mlx4/mlx4_rxq.c
index 5621d5bd4..ad706be82 100644
--- a/drivers/net/mlx4/mlx4_rxq.c
+++ b/drivers/net/mlx4/mlx4_rxq.c
@@ -488,6 +488,7 @@ mlx4_rxq_attach(struct rxq *rxq)
}
struct priv *priv = rxq->priv;
+ struct rte_eth_dev *dev = priv->dev;
const uint32_t elts_n = 1 << rxq->elts_n;
const uint32_t sges_n = 1 << rxq->sges_n;
struct rte_mbuf *(*elts)[elts_n] = rxq->elts;
@@ -552,6 +553,11 @@ mlx4_rxq_attach(struct rxq *rxq)
msg = "failed to obtain device information from WQ/CQ objects";
goto error;
}
+ /* Pre-register Rx mempool. */
+ DEBUG("port %u Rx queue %u registering mp %s having %u chunks",
+ priv->dev->data->port_id, rxq->stats.idx,
+ rxq->mp->name, rxq->mp->nb_mem_chunks);
+ mlx4_mr_update_mp(dev, &rxq->mr_ctrl, rxq->mp);
wqes = (volatile struct mlx4_wqe_data_seg (*)[])
((uintptr_t)dv_rwq.buf.buf + dv_rwq.rq.offset);
for (i = 0; i != RTE_DIM(*elts); ++i) {
@@ -583,7 +589,7 @@ mlx4_rxq_attach(struct rxq *rxq)
.addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(buf,
uintptr_t)),
.byte_count = rte_cpu_to_be_32(buf->data_len),
- .lkey = UINT32_MAX,
+ .lkey = mlx4_rx_mb2mr(rxq, buf),
};
(*elts)[i] = buf;
}
@@ -883,6 +889,11 @@ mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1 << rxq->sges_n);
goto error;
}
+ if (mlx4_mr_btree_init(&rxq->mr_ctrl.cache_bh,
+ MLX4_MR_BTREE_CACHE_N, socket)) {
+ /* rte_errno is already set. */
+ goto error;
+ }
if (dev->data->dev_conf.intr_conf.rxq) {
rxq->channel = mlx4_glue->create_comp_channel(priv->ctx);
if (rxq->channel == NULL) {
@@ -940,5 +951,6 @@ mlx4_rx_queue_release(void *dpdk_rxq)
assert(!rxq->rq_db);
if (rxq->channel)
claim_zero(mlx4_glue->destroy_comp_channel(rxq->channel));
+ mlx4_mr_btree_free(&rxq->mr_ctrl.cache_bh);
rte_free(rxq);
}
diff --git a/drivers/net/mlx4/mlx4_rxtx.c b/drivers/net/mlx4/mlx4_rxtx.c
index 21ffd435a..f3899c79a 100644
--- a/drivers/net/mlx4/mlx4_rxtx.c
+++ b/drivers/net/mlx4/mlx4_rxtx.c
@@ -344,24 +344,6 @@ mlx4_txq_complete(struct txq *txq, const unsigned int elts_m,
}
/**
- * Get memory pool (MP) from mbuf. If mbuf is indirect, the pool from which
- * the cloned mbuf is allocated is returned instead.
- *
- * @param buf
- * Pointer to mbuf.
- *
- * @return
- * Memory pool where data is located for given mbuf.
- */
-static struct rte_mempool *
-mlx4_txq_mb2mp(struct rte_mbuf *buf)
-{
- if (unlikely(RTE_MBUF_INDIRECT(buf)))
- return rte_mbuf_from_indirect(buf)->pool;
- return buf->pool;
-}
-
-/**
* Write Tx data segment to the SQ.
*
* @param dseg
@@ -378,7 +360,7 @@ mlx4_fill_tx_data_seg(volatile struct mlx4_wqe_data_seg *dseg,
uint32_t lkey, uintptr_t addr, rte_be32_t byte_count)
{
dseg->addr = rte_cpu_to_be_64(addr);
- dseg->lkey = rte_cpu_to_be_32(lkey);
+ dseg->lkey = lkey;
#if RTE_CACHE_LINE_SIZE < 64
/*
* Need a barrier here before writing the byte_count
@@ -437,7 +419,7 @@ mlx4_tx_burst_segs(struct rte_mbuf *buf, struct txq *txq,
goto txbb_tail_segs;
txbb_head_seg:
/* Memory region key (big endian) for this memory pool. */
- lkey = mlx4_txq_mp2mr(txq, mlx4_txq_mb2mp(sbuf));
+ lkey = mlx4_tx_mb2mr(txq, sbuf);
if (unlikely(lkey == (uint32_t)-1)) {
DEBUG("%p: unable to get MP <-> MR association",
(void *)txq);
@@ -449,7 +431,7 @@ mlx4_tx_burst_segs(struct rte_mbuf *buf, struct txq *txq,
dseg = (volatile struct mlx4_wqe_data_seg *)
sq->buf;
dseg->addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(sbuf, uintptr_t));
- dseg->lkey = rte_cpu_to_be_32(lkey);
+ dseg->lkey = lkey;
/*
* This data segment starts at the beginning of a new
* TXBB, so we need to postpone its byte_count writing
@@ -469,7 +451,7 @@ mlx4_tx_burst_segs(struct rte_mbuf *buf, struct txq *txq,
/* Jump to default if there are more than two segments remaining. */
switch (nb_segs) {
default:
- lkey = mlx4_txq_mp2mr(txq, mlx4_txq_mb2mp(sbuf));
+ lkey = mlx4_tx_mb2mr(txq, sbuf);
if (unlikely(lkey == (uint32_t)-1)) {
DEBUG("%p: unable to get MP <-> MR association",
(void *)txq);
@@ -485,7 +467,7 @@ mlx4_tx_burst_segs(struct rte_mbuf *buf, struct txq *txq,
nb_segs--;
/* fallthrough */
case 2:
- lkey = mlx4_txq_mp2mr(txq, mlx4_txq_mb2mp(sbuf));
+ lkey = mlx4_tx_mb2mr(txq, sbuf);
if (unlikely(lkey == (uint32_t)-1)) {
DEBUG("%p: unable to get MP <-> MR association",
(void *)txq);
@@ -501,7 +483,7 @@ mlx4_tx_burst_segs(struct rte_mbuf *buf, struct txq *txq,
nb_segs--;
/* fallthrough */
case 1:
- lkey = mlx4_txq_mp2mr(txq, mlx4_txq_mb2mp(sbuf));
+ lkey = mlx4_tx_mb2mr(txq, sbuf);
if (unlikely(lkey == (uint32_t)-1)) {
DEBUG("%p: unable to get MP <-> MR association",
(void *)txq);
@@ -611,7 +593,7 @@ mlx4_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
elt->buf = NULL;
break;
}
- lkey = mlx4_txq_mp2mr(txq, mlx4_txq_mb2mp(buf));
+ lkey = mlx4_tx_mb2mr(txq, buf);
if (unlikely(lkey == (uint32_t)-1)) {
/* MR does not exist. */
DEBUG("%p: unable to get MP <-> MR association",
@@ -966,6 +948,9 @@ mlx4_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
* changes.
*/
scat->addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(rep, uintptr_t));
+ /* If there's only one MR, no need to replace LKey in WQE. */
+ if (unlikely(!IS_SINGLE_MR(rxq->mr_ctrl.cache_bh.len)))
+ scat->lkey = mlx4_rx_mb2mr(rxq, rep);
if (len > seg->data_len) {
len -= seg->data_len;
++pkt->nb_segs;
diff --git a/drivers/net/mlx4/mlx4_rxtx.h b/drivers/net/mlx4/mlx4_rxtx.h
index 2f9d3798b..4c025e3a0 100644
--- a/drivers/net/mlx4/mlx4_rxtx.h
+++ b/drivers/net/mlx4/mlx4_rxtx.h
@@ -25,6 +25,7 @@
#include "mlx4.h"
#include "mlx4_prm.h"
+#include "mlx4_mr.h"
/** Rx queue counters. */
struct mlx4_rxq_stats {
@@ -46,6 +47,7 @@ struct rxq {
uint16_t port_id; /**< Port ID for incoming packets. */
uint16_t sges_n; /**< Number of segments per packet (log2 value). */
uint16_t elts_n; /**< Mbuf queue size (log2 value). */
+ struct mlx4_mr_ctrl mr_ctrl; /* MR control descriptor. */
struct rte_mbuf *(*elts)[]; /**< Rx elements. */
volatile struct mlx4_wqe_data_seg (*wqes)[]; /**< HW queue entries. */
volatile uint32_t *rq_db; /**< RQ doorbell record. */
@@ -100,6 +102,7 @@ struct txq {
int elts_comp_cd; /**< Countdown for next completion. */
unsigned int elts_comp_cd_init; /**< Initial value for countdown. */
unsigned int elts_n; /**< (*elts)[] length. */
+ struct mlx4_mr_ctrl mr_ctrl; /* MR control descriptor. */
struct txq_elt (*elts)[]; /**< Tx elements. */
struct mlx4_txq_stats stats; /**< Tx queue counters. */
uint32_t max_inline; /**< Max inline send size. */
@@ -155,12 +158,70 @@ int mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx,
const struct rte_eth_txconf *conf);
void mlx4_tx_queue_release(void *dpdk_txq);
-static inline uint32_t
-mlx4_txq_mp2mr(struct txq *txq, struct rte_mempool *mp)
+/* mlx4_mr.c */
+
+void mlx4_mr_flush_local_cache(struct mlx4_mr_ctrl *mr_ctrl);
+uint32_t mlx4_rx_addr2mr_bh(struct rxq *rxq, uintptr_t addr);
+uint32_t mlx4_tx_addr2mr_bh(struct txq *txq, uintptr_t addr);
+
+/**
+ * Query LKey from a packet buffer for Rx. No need to flush local caches for Rx
+ * as mempool is pre-configured and static.
+ *
+ * @param rxq
+ * Pointer to Rx queue structure.
+ * @param addr
+ * Address to search.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on no match.
+ */
+static __rte_always_inline uint32_t
+mlx4_rx_addr2mr(struct rxq *rxq, uintptr_t addr)
+{
+ struct mlx4_mr_ctrl *mr_ctrl = &rxq->mr_ctrl;
+ uint32_t lkey;
+
+ /* Linear search on MR cache array. */
+ lkey = mlx4_mr_lookup_cache(mr_ctrl->cache, &mr_ctrl->mru,
+ MLX4_MR_CACHE_N, addr);
+ if (likely(lkey != UINT32_MAX))
+ return lkey;
+ /* Take slower bottom-half (Binary Search) on miss. */
+ return mlx4_rx_addr2mr_bh(rxq, addr);
+}
+
+#define mlx4_rx_mb2mr(rxq, mb) mlx4_rx_addr2mr(rxq, (uintptr_t)((mb)->buf_addr))
+
+/**
+ * Query LKey from a packet buffer for Tx. If not found, add the mempool.
+ *
+ * @param txq
+ * Pointer to Tx queue structure.
+ * @param addr
+ * Address to search.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on no match.
+ */
+static __rte_always_inline uint32_t
+mlx4_tx_addr2mr(struct txq *txq, uintptr_t addr)
{
- (void)txq;
- (void)mp;
- return UINT32_MAX;
+ struct mlx4_mr_ctrl *mr_ctrl = &txq->mr_ctrl;
+ uint32_t lkey;
+
+ /* Check generation bit to see if there's any change on existing MRs. */
+ if (unlikely(*mr_ctrl->dev_gen_ptr != mr_ctrl->cur_gen))
+ mlx4_mr_flush_local_cache(mr_ctrl);
+ /* Linear search on MR cache array. */
+ lkey = mlx4_mr_lookup_cache(mr_ctrl->cache, &mr_ctrl->mru,
+ MLX4_MR_CACHE_N, addr);
+ if (likely(lkey != UINT32_MAX))
+ return lkey;
+ /* Take slower bottom-half (binary search) on miss. */
+ return mlx4_tx_addr2mr_bh(txq, addr);
}
+#define mlx4_tx_mb2mr(rxq, mb) mlx4_tx_addr2mr(rxq, (uintptr_t)((mb)->buf_addr))
+
#endif /* MLX4_RXTX_H_ */
diff --git a/drivers/net/mlx4/mlx4_txq.c b/drivers/net/mlx4/mlx4_txq.c
index 5ea09b0b0..337ed9a1a 100644
--- a/drivers/net/mlx4/mlx4_txq.c
+++ b/drivers/net/mlx4/mlx4_txq.c
@@ -346,6 +346,13 @@ mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
/* Save first wqe pointer in the first element. */
(&(*txq->elts)[0])->wqe =
(volatile struct mlx4_wqe_ctrl_seg *)txq->msq.buf;
+ if (mlx4_mr_btree_init(&txq->mr_ctrl.cache_bh,
+ MLX4_MR_BTREE_CACHE_N, socket)) {
+ /* rte_errno is already set. */
+ goto error;
+ }
+ /* Save pointer of global generation number to check memory event. */
+ txq->mr_ctrl.dev_gen_ptr = &priv->mr.dev_gen;
DEBUG("%p: adding Tx queue %p to list", (void *)dev, (void *)txq);
dev->data->tx_queues[idx] = txq;
return 0;
@@ -386,5 +393,6 @@ mlx4_tx_queue_release(void *dpdk_txq)
claim_zero(mlx4_glue->destroy_qp(txq->qp));
if (txq->cq)
claim_zero(mlx4_glue->destroy_cq(txq->cq));
+ mlx4_mr_btree_free(&txq->mr_ctrl.cache_bh);
rte_free(txq);
}
--
2.11.0
^ permalink raw reply [flat|nested] 23+ messages in thread* [dpdk-dev] [PATCH v2 0/4] net/mlx: add new Memory Region support
2018-05-02 23:16 [dpdk-dev] [PATCH 0/5] net/mlx: add new Memory Region support Yongseok Koh
` (4 preceding siblings ...)
2018-05-02 23:16 ` [dpdk-dev] [PATCH 5/5] net/mlx4: add new " Yongseok Koh
@ 2018-05-09 11:09 ` Yongseok Koh
2018-05-09 11:09 ` [dpdk-dev] [PATCH v2 1/4] net/mlx5: remove " Yongseok Koh
` (3 more replies)
5 siblings, 4 replies; 23+ messages in thread
From: Yongseok Koh @ 2018-05-09 11:09 UTC (permalink / raw)
To: adrien.mazarguil, nelio.laranjeiro; +Cc: dev, Yongseok Koh
This is the new design of Memory Region (MR) for mlx PMD, in order to:
- Accommodate the new memory hotplug model.
- Support non-contiguous Mempool.
This patchset should be applied after:
net/mlx5: change device reference for secondary process
mem: add argument to memory event callback
v2:
* drop 'net/mlx5: trim debug messages for reference counters'.
* remove unused functions - mlx5_mr_free() and mlx4_mr_free().
* add documentation for performance tuning.
* replace IS_SINGLE_MR() with mlx5_mr_btree_len().
* fix a typo.
Yongseok Koh (4):
net/mlx5: remove Memory Region support
net/mlx5: add new Memory Region support
net/mlx4: remove Memory Region support
net/mlx4: add new Memory Region support
config/common_base | 2 -
doc/guides/nics/mlx4.rst | 14 +-
doc/guides/nics/mlx5.rst | 14 +-
drivers/net/mlx4/Makefile | 4 -
drivers/net/mlx4/mlx4.c | 26 +
drivers/net/mlx4/mlx4.h | 45 +-
drivers/net/mlx4/mlx4_mr.c | 1236 +++++++++++++++++++++++++++++++-----
drivers/net/mlx4/mlx4_mr.h | 122 ++++
drivers/net/mlx4/mlx4_rxq.c | 19 +-
drivers/net/mlx4/mlx4_rxtx.c | 35 +-
drivers/net/mlx4/mlx4_rxtx.h | 85 ++-
drivers/net/mlx4/mlx4_txq.c | 74 +--
drivers/net/mlx5/Makefile | 4 -
drivers/net/mlx5/mlx5.c | 49 +-
drivers/net/mlx5/mlx5.h | 32 +-
drivers/net/mlx5/mlx5_defs.h | 15 +-
drivers/net/mlx5/mlx5_ethdev.c | 16 +
drivers/net/mlx5/mlx5_mr.c | 1305 +++++++++++++++++++++++++++++++-------
drivers/net/mlx5/mlx5_mr.h | 117 ++++
drivers/net/mlx5/mlx5_rxq.c | 27 +-
drivers/net/mlx5/mlx5_rxtx.c | 3 +
drivers/net/mlx5/mlx5_rxtx.h | 123 ++--
drivers/net/mlx5/mlx5_rxtx_vec.h | 6 +-
drivers/net/mlx5/mlx5_trigger.c | 25 +-
drivers/net/mlx5/mlx5_txq.c | 28 +-
25 files changed, 2694 insertions(+), 732 deletions(-)
create mode 100644 drivers/net/mlx4/mlx4_mr.h
create mode 100644 drivers/net/mlx5/mlx5_mr.h
--
2.11.0
^ permalink raw reply [flat|nested] 23+ messages in thread* [dpdk-dev] [PATCH v2 1/4] net/mlx5: remove Memory Region support
2018-05-09 11:09 ` [dpdk-dev] [PATCH v2 0/4] net/mlx: " Yongseok Koh
@ 2018-05-09 11:09 ` Yongseok Koh
2018-05-09 12:03 ` Shahaf Shuler
2018-05-09 11:09 ` [dpdk-dev] [PATCH v2 2/4] net/mlx5: add new " Yongseok Koh
` (2 subsequent siblings)
3 siblings, 1 reply; 23+ messages in thread
From: Yongseok Koh @ 2018-05-09 11:09 UTC (permalink / raw)
To: adrien.mazarguil, nelio.laranjeiro; +Cc: dev, Yongseok Koh
This patch removes current support of Memory Region (MR) in order to
accommodate the dynamic memory hotplug patch. This patch can be compiled
but traffic can't flow and HW will raise faults. Subsequent patches will
add new MR support.
Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
---
config/common_base | 1 -
doc/guides/nics/mlx5.rst | 8 -
drivers/net/mlx5/Makefile | 4 -
drivers/net/mlx5/mlx5.c | 4 -
drivers/net/mlx5/mlx5.h | 10 --
drivers/net/mlx5/mlx5_defs.h | 11 --
drivers/net/mlx5/mlx5_mr.c | 346 ----------------------------------------
drivers/net/mlx5/mlx5_rxq.c | 21 +--
drivers/net/mlx5/mlx5_rxtx.h | 90 +----------
drivers/net/mlx5/mlx5_trigger.c | 14 --
drivers/net/mlx5/mlx5_txq.c | 17 --
11 files changed, 4 insertions(+), 522 deletions(-)
diff --git a/config/common_base b/config/common_base
index 0d181ace8..d525d9443 100644
--- a/config/common_base
+++ b/config/common_base
@@ -296,7 +296,6 @@ CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE=8
CONFIG_RTE_LIBRTE_MLX5_PMD=n
CONFIG_RTE_LIBRTE_MLX5_DEBUG=n
CONFIG_RTE_LIBRTE_MLX5_DLOPEN_DEPS=n
-CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE=8
#
# Compile burst-oriented Netronome NFP PMD driver
diff --git a/doc/guides/nics/mlx5.rst b/doc/guides/nics/mlx5.rst
index bc08515cf..5854106b5 100644
--- a/doc/guides/nics/mlx5.rst
+++ b/doc/guides/nics/mlx5.rst
@@ -167,14 +167,6 @@ These options can be modified in the ``.config`` file.
adds additional run-time checks and debugging messages at the cost of
lower performance.
-- ``CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE`` (default **8**)
-
- Maximum number of cached memory pools (MPs) per TX queue. Each MP from
- which buffers are to be transmitted must be associated to memory regions
- (MRs). This is a slow operation that must be cached.
-
- This value is always 1 for RX queues since they use a single MP.
-
Environment variables
~~~~~~~~~~~~~~~~~~~~~
diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile
index 3c5b4943a..13f079334 100644
--- a/drivers/net/mlx5/Makefile
+++ b/drivers/net/mlx5/Makefile
@@ -82,10 +82,6 @@ else
CFLAGS += -DNDEBUG -UPEDANTIC
endif
-ifdef CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE
-CFLAGS += -DMLX5_PMD_TX_MP_CACHE=$(CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE)
-endif
-
include $(RTE_SDK)/mk/rte.lib.mk
# Generate and clean-up mlx5_autoconf.h.
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index efc6313fe..42b019ba6 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -245,10 +245,6 @@ mlx5_dev_close(struct rte_eth_dev *dev)
if (ret)
DRV_LOG(WARNING, "port %u some flows still remain",
dev->data->port_id);
- ret = mlx5_mr_verify(dev);
- if (ret)
- DRV_LOG(WARNING, "port %u some memory region still remain",
- dev->data->port_id);
memset(priv, 0, sizeof(*priv));
}
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index f1294c54b..b34adc1ec 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -26,7 +26,6 @@
#include <rte_pci.h>
#include <rte_ether.h>
#include <rte_ethdev_driver.h>
-#include <rte_spinlock.h>
#include <rte_interrupts.h>
#include <rte_errno.h>
#include <rte_flow.h>
@@ -147,7 +146,6 @@ struct priv {
struct mlx5_hrxq_drop *flow_drop_queue; /* Flow drop queue. */
struct mlx5_flows flows; /* RTE Flow rules. */
struct mlx5_flows ctrl_flows; /* Control flow rules. */
- LIST_HEAD(mr, mlx5_mr) mr; /* Memory region. */
LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */
LIST_HEAD(rxqibv, mlx5_rxq_ibv) rxqsibv; /* Verbs Rx queues. */
LIST_HEAD(hrxq, mlx5_hrxq) hrxqs; /* Verbs Hash Rx queues. */
@@ -157,7 +155,6 @@ struct priv {
LIST_HEAD(ind_tables, mlx5_ind_table_ibv) ind_tbls;
uint32_t link_speed_capa; /* Link speed capabilities. */
struct mlx5_xstats_ctrl xstats_ctrl; /* Extended stats control. */
- rte_spinlock_t mr_lock; /* MR Lock. */
int primary_socket; /* Unix socket for primary process. */
void *uar_base; /* Reserved address space for UAR mapping */
struct rte_intr_handle intr_handle_socket; /* Interrupt handler. */
@@ -309,13 +306,6 @@ void mlx5_socket_uninit(struct rte_eth_dev *priv);
void mlx5_socket_handle(struct rte_eth_dev *priv);
int mlx5_socket_connect(struct rte_eth_dev *priv);
-/* mlx5_mr.c */
-
-struct mlx5_mr *mlx5_mr_new(struct rte_eth_dev *dev, struct rte_mempool *mp);
-struct mlx5_mr *mlx5_mr_get(struct rte_eth_dev *dev, struct rte_mempool *mp);
-int mlx5_mr_release(struct mlx5_mr *mr);
-int mlx5_mr_verify(struct rte_eth_dev *dev);
-
/* mlx5_nl.c */
int mlx5_nl_init(uint32_t nlgroups);
diff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h
index 55a86957d..f9093777d 100644
--- a/drivers/net/mlx5/mlx5_defs.h
+++ b/drivers/net/mlx5/mlx5_defs.h
@@ -38,17 +38,6 @@
#define MLX5_TX_COMP_THRESH_INLINE_DIV (1 << 3)
/*
- * Maximum number of cached Memory Pools (MPs) per TX queue. Each RTE MP
- * from which buffers are to be transmitted will have to be mapped by this
- * driver to their own Memory Region (MR). This is a slow operation.
- *
- * This value is always 1 for RX queues.
- */
-#ifndef MLX5_PMD_TX_MP_CACHE
-#define MLX5_PMD_TX_MP_CACHE 8
-#endif
-
-/*
* If defined, only use software counters. The PMD will never ask the hardware
* for these, and many of them won't be available.
*/
diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c
index 48ac84bc8..736c40ae4 100644
--- a/drivers/net/mlx5/mlx5_mr.c
+++ b/drivers/net/mlx5/mlx5_mr.c
@@ -18,349 +18,3 @@
#include "mlx5_rxtx.h"
#include "mlx5_glue.h"
-struct mlx5_check_mempool_data {
- int ret;
- char *start;
- char *end;
-};
-
-/* Called by mlx5_check_mempool() when iterating the memory chunks. */
-static void
-mlx5_check_mempool_cb(struct rte_mempool *mp __rte_unused,
- void *opaque, struct rte_mempool_memhdr *memhdr,
- unsigned int mem_idx __rte_unused)
-{
- struct mlx5_check_mempool_data *data = opaque;
-
- /* It already failed, skip the next chunks. */
- if (data->ret != 0)
- return;
- /* It is the first chunk. */
- if (data->start == NULL && data->end == NULL) {
- data->start = memhdr->addr;
- data->end = data->start + memhdr->len;
- return;
- }
- if (data->end == memhdr->addr) {
- data->end += memhdr->len;
- return;
- }
- if (data->start == (char *)memhdr->addr + memhdr->len) {
- data->start -= memhdr->len;
- return;
- }
- /* Error, mempool is not virtually contiguous. */
- data->ret = -1;
-}
-
-/**
- * Check if a mempool can be used: it must be virtually contiguous.
- *
- * @param[in] mp
- * Pointer to memory pool.
- * @param[out] start
- * Pointer to the start address of the mempool virtual memory area
- * @param[out] end
- * Pointer to the end address of the mempool virtual memory area
- *
- * @return
- * 0 on success (mempool is virtually contiguous), -1 on error.
- */
-static int
-mlx5_check_mempool(struct rte_mempool *mp, uintptr_t *start,
- uintptr_t *end)
-{
- struct mlx5_check_mempool_data data;
-
- memset(&data, 0, sizeof(data));
- rte_mempool_mem_iter(mp, mlx5_check_mempool_cb, &data);
- *start = (uintptr_t)data.start;
- *end = (uintptr_t)data.end;
- return data.ret;
-}
-
-/**
- * Register a Memory Region (MR) <-> Memory Pool (MP) association in
- * txq->mp2mr[]. If mp2mr[] is full, remove an entry first.
- *
- * @param txq
- * Pointer to TX queue structure.
- * @param[in] mp
- * Memory Pool for which a Memory Region lkey must be returned.
- * @param idx
- * Index of the next available entry.
- *
- * @return
- * mr on success, NULL on failure and rte_errno is set.
- */
-struct mlx5_mr *
-mlx5_txq_mp2mr_reg(struct mlx5_txq_data *txq, struct rte_mempool *mp,
- unsigned int idx)
-{
- struct mlx5_txq_ctrl *txq_ctrl =
- container_of(txq, struct mlx5_txq_ctrl, txq);
- struct rte_eth_dev *dev;
- struct mlx5_mr *mr;
-
- rte_spinlock_lock(&txq_ctrl->priv->mr_lock);
- /* Add a new entry, register MR first. */
- DRV_LOG(DEBUG, "port %u discovered new memory pool \"%s\" (%p)",
- PORT_ID(txq_ctrl->priv), mp->name, (void *)mp);
- dev = ETH_DEV(txq_ctrl->priv);
- mr = mlx5_mr_get(dev, mp);
- if (mr == NULL) {
- if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
- DRV_LOG(DEBUG,
- "port %u using unregistered mempool 0x%p(%s)"
- " in secondary process, please create mempool"
- " before rte_eth_dev_start()",
- PORT_ID(txq_ctrl->priv), (void *)mp, mp->name);
- rte_spinlock_unlock(&txq_ctrl->priv->mr_lock);
- rte_errno = ENOTSUP;
- return NULL;
- }
- mr = mlx5_mr_new(dev, mp);
- }
- if (unlikely(mr == NULL)) {
- DRV_LOG(DEBUG,
- "port %u unable to configure memory region,"
- " ibv_reg_mr() failed.",
- PORT_ID(txq_ctrl->priv));
- rte_spinlock_unlock(&txq_ctrl->priv->mr_lock);
- return NULL;
- }
- if (unlikely(idx == RTE_DIM(txq->mp2mr))) {
- /* Table is full, remove oldest entry. */
- DRV_LOG(DEBUG,
- "port %u memory region <-> memory pool table full, "
- " dropping oldest entry",
- PORT_ID(txq_ctrl->priv));
- --idx;
- mlx5_mr_release(txq->mp2mr[0]);
- memmove(&txq->mp2mr[0], &txq->mp2mr[1],
- (sizeof(txq->mp2mr) - sizeof(txq->mp2mr[0])));
- }
- /* Store the new entry. */
- txq_ctrl->txq.mp2mr[idx] = mr;
- DRV_LOG(DEBUG,
- "port %u new memory region lkey for MP \"%s\" (%p): 0x%08"
- PRIu32,
- PORT_ID(txq_ctrl->priv), mp->name, (void *)mp,
- txq_ctrl->txq.mp2mr[idx]->lkey);
- rte_spinlock_unlock(&txq_ctrl->priv->mr_lock);
- return mr;
-}
-
-struct mlx5_mp2mr_mbuf_check_data {
- int ret;
-};
-
-/**
- * Callback function for rte_mempool_obj_iter() to check whether a given
- * mempool object looks like a mbuf.
- *
- * @param[in] mp
- * The mempool pointer
- * @param[in] arg
- * Context data (struct txq_mp2mr_mbuf_check_data). Contains the
- * return value.
- * @param[in] obj
- * Object address.
- * @param index
- * Object index, unused.
- */
-static void
-txq_mp2mr_mbuf_check(struct rte_mempool *mp, void *arg, void *obj,
- uint32_t index __rte_unused)
-{
- struct mlx5_mp2mr_mbuf_check_data *data = arg;
- struct rte_mbuf *buf = obj;
-
- /*
- * Check whether mbuf structure fits element size and whether mempool
- * pointer is valid.
- */
- if (sizeof(*buf) > mp->elt_size || buf->pool != mp)
- data->ret = -1;
-}
-
-/**
- * Iterator function for rte_mempool_walk() to register existing mempools and
- * fill the MP to MR cache of a TX queue.
- *
- * @param[in] mp
- * Memory Pool to register.
- * @param *arg
- * Pointer to TX queue structure.
- */
-void
-mlx5_mp2mr_iter(struct rte_mempool *mp, void *arg)
-{
- struct priv *priv = (struct priv *)arg;
- struct mlx5_mp2mr_mbuf_check_data data = {
- .ret = 0,
- };
- struct mlx5_mr *mr;
-
- /* Register mempool only if the first element looks like a mbuf. */
- if (rte_mempool_obj_iter(mp, txq_mp2mr_mbuf_check, &data) == 0 ||
- data.ret == -1)
- return;
- mr = mlx5_mr_get(ETH_DEV(priv), mp);
- if (mr) {
- mlx5_mr_release(mr);
- return;
- }
- mr = mlx5_mr_new(ETH_DEV(priv), mp);
- if (!mr)
- DRV_LOG(ERR, "port %u cannot create memory region: %s",
- PORT_ID(priv), strerror(rte_errno));
-}
-
-/**
- * Register a new memory region from the mempool and store it in the memory
- * region list.
- *
- * @param dev
- * Pointer to Ethernet device.
- * @param mp
- * Pointer to the memory pool to register.
- *
- * @return
- * The memory region on success, NULL on failure and rte_errno is set.
- */
-struct mlx5_mr *
-mlx5_mr_new(struct rte_eth_dev *dev, struct rte_mempool *mp)
-{
- struct priv *priv = dev->data->dev_private;
- const struct rte_memseg *ms;
- uintptr_t start;
- uintptr_t end;
- struct mlx5_mr *mr;
-
- mr = rte_zmalloc_socket(__func__, sizeof(*mr), 0, mp->socket_id);
- if (!mr) {
- DRV_LOG(DEBUG,
- "port %u unable to configure memory region,"
- " ibv_reg_mr() failed.",
- dev->data->port_id);
- rte_errno = ENOMEM;
- return NULL;
- }
- if (mlx5_check_mempool(mp, &start, &end) != 0) {
- DRV_LOG(ERR, "port %u mempool %p: not virtually contiguous",
- dev->data->port_id, (void *)mp);
- rte_errno = ENOMEM;
- return NULL;
- }
- DRV_LOG(DEBUG, "port %u mempool %p area start=%p end=%p size=%zu",
- dev->data->port_id, (void *)mp, (void *)start, (void *)end,
- (size_t)(end - start));
- /* Save original addresses for exact MR lookup. */
- mr->start = start;
- mr->end = end;
-
- /* Round start and end to page boundary if found in memory segments. */
- ms = rte_mem_virt2memseg((void *)start, NULL);
- if (ms != NULL)
- start = RTE_ALIGN_FLOOR(start, ms->hugepage_sz);
- end = RTE_ALIGN_CEIL(end, ms->hugepage_sz);
- DRV_LOG(DEBUG,
- "port %u mempool %p using start=%p end=%p size=%zu for memory"
- " region",
- dev->data->port_id, (void *)mp, (void *)start, (void *)end,
- (size_t)(end - start));
- mr->mr = mlx5_glue->reg_mr(priv->pd, (void *)start, end - start,
- IBV_ACCESS_LOCAL_WRITE);
- if (!mr->mr) {
- rte_errno = ENOMEM;
- return NULL;
- }
- mr->mp = mp;
- mr->lkey = rte_cpu_to_be_32(mr->mr->lkey);
- rte_atomic32_inc(&mr->refcnt);
- DRV_LOG(DEBUG, "port %u new memory Region %p refcnt: %d",
- dev->data->port_id, (void *)mr, rte_atomic32_read(&mr->refcnt));
- LIST_INSERT_HEAD(&priv->mr, mr, next);
- return mr;
-}
-
-/**
- * Search the memory region object in the memory region list.
- *
- * @param dev
- * Pointer to Ethernet device.
- * @param mp
- * Pointer to the memory pool to register.
- *
- * @return
- * The memory region on success.
- */
-struct mlx5_mr *
-mlx5_mr_get(struct rte_eth_dev *dev, struct rte_mempool *mp)
-{
- struct priv *priv = dev->data->dev_private;
- struct mlx5_mr *mr;
-
- assert(mp);
- if (LIST_EMPTY(&priv->mr))
- return NULL;
- LIST_FOREACH(mr, &priv->mr, next) {
- if (mr->mp == mp) {
- rte_atomic32_inc(&mr->refcnt);
- DRV_LOG(DEBUG, "port %u memory region %p refcnt: %d",
- dev->data->port_id, (void *)mr,
- rte_atomic32_read(&mr->refcnt));
- return mr;
- }
- }
- return NULL;
-}
-
-/**
- * Release the memory region object.
- *
- * @param mr
- * Pointer to memory region to release.
- *
- * @return
- * 1 while a reference on it exists, 0 when freed.
- */
-int
-mlx5_mr_release(struct mlx5_mr *mr)
-{
- assert(mr);
- DRV_LOG(DEBUG, "memory region %p refcnt: %d", (void *)mr,
- rte_atomic32_read(&mr->refcnt));
- if (rte_atomic32_dec_and_test(&mr->refcnt)) {
- claim_zero(mlx5_glue->dereg_mr(mr->mr));
- LIST_REMOVE(mr, next);
- rte_free(mr);
- return 0;
- }
- return 1;
-}
-
-/**
- * Verify the flow list is empty
- *
- * @param dev
- * Pointer to Ethernet device.
- *
- * @return
- * The number of object not released.
- */
-int
-mlx5_mr_verify(struct rte_eth_dev *dev)
-{
- struct priv *priv = dev->data->dev_private;
- int ret = 0;
- struct mlx5_mr *mr;
-
- LIST_FOREACH(mr, &priv->mr, next) {
- DRV_LOG(DEBUG, "port %u memory region %p still referenced",
- dev->data->port_id, (void *)mr);
- ++ret;
- }
- return ret;
-}
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index aa1ddd0b6..23f909635 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -649,16 +649,6 @@ mlx5_rxq_ibv_new(struct rte_eth_dev *dev, uint16_t idx)
goto error;
}
tmpl->rxq_ctrl = rxq_ctrl;
- /* Use the entire RX mempool as the memory region. */
- tmpl->mr = mlx5_mr_get(dev, rxq_data->mp);
- if (!tmpl->mr) {
- tmpl->mr = mlx5_mr_new(dev, rxq_data->mp);
- if (!tmpl->mr) {
- DRV_LOG(ERR, "port %u: memeroy region creation failure",
- dev->data->port_id);
- goto error;
- }
- }
if (rxq_ctrl->irq) {
tmpl->channel = mlx5_glue->create_comp_channel(priv->ctx);
if (!tmpl->channel) {
@@ -799,7 +789,7 @@ mlx5_rxq_ibv_new(struct rte_eth_dev *dev, uint16_t idx)
.addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(buf,
uintptr_t)),
.byte_count = rte_cpu_to_be_32(DATA_LEN(buf)),
- .lkey = tmpl->mr->lkey,
+ .lkey = UINT32_MAX,
};
}
rxq_data->rq_db = rwq.dbrec;
@@ -835,8 +825,6 @@ mlx5_rxq_ibv_new(struct rte_eth_dev *dev, uint16_t idx)
claim_zero(mlx5_glue->destroy_cq(tmpl->cq));
if (tmpl->channel)
claim_zero(mlx5_glue->destroy_comp_channel(tmpl->channel));
- if (tmpl->mr)
- mlx5_mr_release(tmpl->mr);
priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_NONE;
rte_errno = ret; /* Restore rte_errno. */
return NULL;
@@ -866,7 +854,6 @@ mlx5_rxq_ibv_get(struct rte_eth_dev *dev, uint16_t idx)
return NULL;
rxq_ctrl = container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
if (rxq_ctrl->ibv) {
- mlx5_mr_get(dev, rxq_data->mp);
rte_atomic32_inc(&rxq_ctrl->ibv->refcnt);
DRV_LOG(DEBUG, "port %u Verbs Rx queue %u: refcnt %d",
dev->data->port_id, rxq_ctrl->idx,
@@ -887,15 +874,9 @@ mlx5_rxq_ibv_get(struct rte_eth_dev *dev, uint16_t idx)
int
mlx5_rxq_ibv_release(struct mlx5_rxq_ibv *rxq_ibv)
{
- int ret;
-
assert(rxq_ibv);
assert(rxq_ibv->wq);
assert(rxq_ibv->cq);
- assert(rxq_ibv->mr);
- ret = mlx5_mr_release(rxq_ibv->mr);
- if (!ret)
- rxq_ibv->mr = NULL;
DRV_LOG(DEBUG, "port %u Verbs Rx queue %u: refcnt %d",
PORT_ID(rxq_ibv->rxq_ctrl->priv),
rxq_ibv->rxq_ctrl->idx, rte_atomic32_read(&rxq_ibv->refcnt));
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index 2fc12a186..e8cad51aa 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -54,17 +54,6 @@ struct mlx5_txq_stats {
struct priv;
-/* Memory region queue object. */
-struct mlx5_mr {
- LIST_ENTRY(mlx5_mr) next; /**< Pointer to the next element. */
- rte_atomic32_t refcnt; /*<< Reference counter. */
- uint32_t lkey; /*<< rte_cpu_to_be_32(mr->lkey) */
- uintptr_t start; /* Start address of MR */
- uintptr_t end; /* End address of MR */
- struct ibv_mr *mr; /*<< Memory Region. */
- struct rte_mempool *mp; /*<< Memory Pool. */
-};
-
/* Compressed CQE context. */
struct rxq_zip {
uint16_t ai; /* Array index. */
@@ -114,7 +103,6 @@ struct mlx5_rxq_ibv {
struct ibv_cq *cq; /* Completion Queue. */
struct ibv_wq *wq; /* Work Queue. */
struct ibv_comp_channel *channel;
- struct mlx5_mr *mr; /* Memory Region (for mp). */
};
/* RX queue control descriptor. */
@@ -175,7 +163,6 @@ struct mlx5_txq_data {
uint16_t mpw_hdr_dseg:1; /* Enable DSEGs in the title WQEBB. */
uint16_t max_inline; /* Multiple of RTE_CACHE_LINE_SIZE to inline. */
uint16_t inline_max_packet_sz; /* Max packet size for inlining. */
- uint16_t mr_cache_idx; /* Index of last hit entry. */
uint32_t qp_num_8s; /* QP number shifted by 8. */
uint64_t offloads; /* Offloads for Tx Queue. */
volatile struct mlx5_cqe (*cqes)[]; /* Completion queue. */
@@ -183,7 +170,6 @@ struct mlx5_txq_data {
volatile uint32_t *qp_db; /* Work queue doorbell. */
volatile uint32_t *cq_db; /* Completion queue doorbell. */
volatile void *bf_reg; /* Blueflame register remapped. */
- struct mlx5_mr *mp2mr[MLX5_PMD_TX_MP_CACHE]; /* MR translation table. */
struct rte_mbuf *(*elts)[]; /* TX elements. */
struct mlx5_txq_stats stats; /* TX queue counters. */
} __rte_cache_aligned;
@@ -322,12 +308,6 @@ uint16_t mlx5_tx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts,
uint16_t mlx5_rx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts,
uint16_t pkts_n);
-/* mlx5_mr.c */
-
-void mlx5_mp2mr_iter(struct rte_mempool *mp, void *arg);
-struct mlx5_mr *mlx5_txq_mp2mr_reg(struct mlx5_txq_data *txq,
- struct rte_mempool *mp, unsigned int idx);
-
#ifndef NDEBUG
/**
* Verify or set magic value in CQE.
@@ -513,76 +493,12 @@ mlx5_tx_complete(struct mlx5_txq_data *txq)
*txq->cq_db = rte_cpu_to_be_32(cq_ci);
}
-/**
- * Get Memory Pool (MP) from mbuf. If mbuf is indirect, the pool from which
- * the cloned mbuf is allocated is returned instead.
- *
- * @param buf
- * Pointer to mbuf.
- *
- * @return
- * Memory pool where data is located for given mbuf.
- */
-static struct rte_mempool *
-mlx5_tx_mb2mp(struct rte_mbuf *buf)
-{
- if (unlikely(RTE_MBUF_INDIRECT(buf)))
- return rte_mbuf_from_indirect(buf)->pool;
- return buf->pool;
-}
-
-/**
- * Get Memory Region (MR) <-> rte_mbuf association from txq->mp2mr[].
- * Add MP to txq->mp2mr[] if it's not registered yet. If mp2mr[] is full,
- * remove an entry first.
- *
- * @param txq
- * Pointer to TX queue structure.
- * @param[in] mp
- * Memory Pool for which a Memory Region lkey must be returned.
- *
- * @return
- * mr->lkey on success, (uint32_t)-1 on failure.
- */
static __rte_always_inline uint32_t
mlx5_tx_mb2mr(struct mlx5_txq_data *txq, struct rte_mbuf *mb)
{
- uint16_t i = txq->mr_cache_idx;
- uintptr_t addr = rte_pktmbuf_mtod(mb, uintptr_t);
- struct mlx5_mr *mr;
-
- assert(i < RTE_DIM(txq->mp2mr));
- if (likely(txq->mp2mr[i]->start <= addr && txq->mp2mr[i]->end > addr))
- return txq->mp2mr[i]->lkey;
- for (i = 0; (i != RTE_DIM(txq->mp2mr)); ++i) {
- if (unlikely(txq->mp2mr[i] == NULL ||
- txq->mp2mr[i]->mr == NULL)) {
- /* Unknown MP, add a new MR for it. */
- break;
- }
- if (txq->mp2mr[i]->start <= addr &&
- txq->mp2mr[i]->end > addr) {
- assert(txq->mp2mr[i]->lkey != (uint32_t)-1);
- txq->mr_cache_idx = i;
- return txq->mp2mr[i]->lkey;
- }
- }
- mr = mlx5_txq_mp2mr_reg(txq, mlx5_tx_mb2mp(mb), i);
- /*
- * Request the reference to use in this queue, the original one is
- * kept by the control plane.
- */
- if (mr) {
- rte_atomic32_inc(&mr->refcnt);
- txq->mr_cache_idx = i >= RTE_DIM(txq->mp2mr) ? i - 1 : i;
- return mr->lkey;
- } else {
- struct rte_mempool *mp = mlx5_tx_mb2mp(mb);
-
- DRV_LOG(WARNING, "failed to register mempool 0x%p(%s)",
- (void *)mp, mp->name);
- }
- return (uint32_t)-1;
+ (void)txq;
+ (void)mb;
+ return UINT32_MAX;
}
/**
diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c
index fc56d1ee8..3db6c3f35 100644
--- a/drivers/net/mlx5/mlx5_trigger.c
+++ b/drivers/net/mlx5/mlx5_trigger.c
@@ -48,17 +48,10 @@ mlx5_txq_start(struct rte_eth_dev *dev)
/* Add memory regions to Tx queues. */
for (i = 0; i != priv->txqs_n; ++i) {
- unsigned int idx = 0;
- struct mlx5_mr *mr;
struct mlx5_txq_ctrl *txq_ctrl = mlx5_txq_get(dev, i);
if (!txq_ctrl)
continue;
- LIST_FOREACH(mr, &priv->mr, next) {
- mlx5_txq_mp2mr_reg(&txq_ctrl->txq, mr->mp, idx++);
- if (idx == MLX5_PMD_TX_MP_CACHE)
- break;
- }
txq_alloc_elts(txq_ctrl);
txq_ctrl->ibv = mlx5_txq_ibv_new(dev, i);
if (!txq_ctrl->ibv) {
@@ -144,13 +137,11 @@ int
mlx5_dev_start(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
- struct mlx5_mr *mr = NULL;
int ret;
dev->data->dev_started = 1;
DRV_LOG(DEBUG, "port %u allocating and configuring hash Rx queues",
dev->data->port_id);
- rte_mempool_walk(mlx5_mp2mr_iter, priv);
ret = mlx5_txq_start(dev);
if (ret) {
DRV_LOG(ERR, "port %u Tx queue allocation failed: %s",
@@ -190,8 +181,6 @@ mlx5_dev_start(struct rte_eth_dev *dev)
ret = rte_errno; /* Save rte_errno before cleanup. */
/* Rollback. */
dev->data->dev_started = 0;
- for (mr = LIST_FIRST(&priv->mr); mr; mr = LIST_FIRST(&priv->mr))
- mlx5_mr_release(mr);
mlx5_flow_stop(dev, &priv->flows);
mlx5_traffic_disable(dev);
mlx5_txq_stop(dev);
@@ -212,7 +201,6 @@ void
mlx5_dev_stop(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
- struct mlx5_mr *mr;
dev->data->dev_started = 0;
/* Prevent crashes when queues are still in use. */
@@ -228,8 +216,6 @@ mlx5_dev_stop(struct rte_eth_dev *dev)
mlx5_dev_interrupt_handler_uninstall(dev);
mlx5_txq_stop(dev);
mlx5_rxq_stop(dev);
- for (mr = LIST_FIRST(&priv->mr); mr; mr = LIST_FIRST(&priv->mr))
- mlx5_mr_release(mr);
}
/**
diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index af2537379..92b3ad53c 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -409,7 +409,6 @@ mlx5_txq_ibv_new(struct rte_eth_dev *dev, uint16_t idx)
return NULL;
}
memset(&tmpl, 0, sizeof(struct mlx5_txq_ibv));
- /* MRs will be registered in mp2mr[] later. */
attr.cq = (struct ibv_cq_init_attr_ex){
.comp_mask = 0,
};
@@ -804,7 +803,6 @@ mlx5_txq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
tmpl->txq.elts_n = log2above(desc);
tmpl->idx = idx;
txq_set_params(tmpl);
- /* MRs will be registered in mp2mr[] later. */
DRV_LOG(DEBUG, "port %u priv->device_attr.max_qp_wr is %d",
dev->data->port_id, priv->device_attr.orig_attr.max_qp_wr);
DRV_LOG(DEBUG, "port %u priv->device_attr.max_sge is %d",
@@ -839,15 +837,7 @@ mlx5_txq_get(struct rte_eth_dev *dev, uint16_t idx)
if ((*priv->txqs)[idx]) {
ctrl = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl,
txq);
- unsigned int i;
-
mlx5_txq_ibv_get(dev, idx);
- for (i = 0; i != MLX5_PMD_TX_MP_CACHE; ++i) {
- if (ctrl->txq.mp2mr[i])
- claim_nonzero
- (mlx5_mr_get(dev,
- ctrl->txq.mp2mr[i]->mp));
- }
rte_atomic32_inc(&ctrl->refcnt);
DRV_LOG(DEBUG, "port %u Tx queue %u refcnt %d",
dev->data->port_id,
@@ -871,7 +861,6 @@ int
mlx5_txq_release(struct rte_eth_dev *dev, uint16_t idx)
{
struct priv *priv = dev->data->dev_private;
- unsigned int i;
struct mlx5_txq_ctrl *txq;
size_t page_size = sysconf(_SC_PAGESIZE);
@@ -882,12 +871,6 @@ mlx5_txq_release(struct rte_eth_dev *dev, uint16_t idx)
txq->idx, rte_atomic32_read(&txq->refcnt));
if (txq->ibv && !mlx5_txq_ibv_release(txq->ibv))
txq->ibv = NULL;
- for (i = 0; i != MLX5_PMD_TX_MP_CACHE; ++i) {
- if (txq->txq.mp2mr[i]) {
- mlx5_mr_release(txq->txq.mp2mr[i]);
- txq->txq.mp2mr[i] = NULL;
- }
- }
if (priv->uar_base)
munmap((void *)RTE_ALIGN_FLOOR((uintptr_t)txq->txq.bf_reg,
page_size), page_size);
--
2.11.0
^ permalink raw reply [flat|nested] 23+ messages in thread* Re: [dpdk-dev] [PATCH v2 1/4] net/mlx5: remove Memory Region support
2018-05-09 11:09 ` [dpdk-dev] [PATCH v2 1/4] net/mlx5: remove " Yongseok Koh
@ 2018-05-09 12:03 ` Shahaf Shuler
0 siblings, 0 replies; 23+ messages in thread
From: Shahaf Shuler @ 2018-05-09 12:03 UTC (permalink / raw)
To: Yongseok Koh, Adrien Mazarguil, Nélio Laranjeiro; +Cc: dev, Yongseok Koh
Wednesday, May 9, 2018 2:09 PM, Yongseok Koh:
> Subject: [dpdk-dev] [PATCH v2 1/4] net/mlx5: remove Memory Region
> support
>
> This patch removes current support of Memory Region (MR) in order to
> accommodate the dynamic memory hotplug patch. This patch can be
> compiled but traffic can't flow and HW will raise faults. Subsequent patches
> will add new MR support.
>
> Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
Series applied to next-net-mlx, thanks.
^ permalink raw reply [flat|nested] 23+ messages in thread
* [dpdk-dev] [PATCH v2 2/4] net/mlx5: add new Memory Region support
2018-05-09 11:09 ` [dpdk-dev] [PATCH v2 0/4] net/mlx: " Yongseok Koh
2018-05-09 11:09 ` [dpdk-dev] [PATCH v2 1/4] net/mlx5: remove " Yongseok Koh
@ 2018-05-09 11:09 ` Yongseok Koh
2018-05-09 11:09 ` [dpdk-dev] [PATCH v2 3/4] net/mlx4: remove " Yongseok Koh
2018-05-09 11:09 ` [dpdk-dev] [PATCH v2 4/4] net/mlx4: add new " Yongseok Koh
3 siblings, 0 replies; 23+ messages in thread
From: Yongseok Koh @ 2018-05-09 11:09 UTC (permalink / raw)
To: adrien.mazarguil, nelio.laranjeiro; +Cc: dev, Yongseok Koh
This is the new design of Memory Region (MR) for mlx PMD, in order to:
- Accommodate the new memory hotplug model.
- Support non-contiguous Mempool.
There are multiple layers for MR search.
L0 is to look up the last-hit entry which is pointed by mr_ctrl->mru (Most
Recently Used). If L0 misses, L1 is to look up the address in a fixed-sized
array by linear search. L0/L1 is in an inline function -
mlx5_mr_lookup_cache().
If L1 misses, the bottom-half function is called to look up the address
from the bigger local cache of the queue. This is L2 - mlx5_mr_addr2mr_bh()
and it is not an inline function. Data structure for L2 is the Binary Tree.
If L2 misses, the search falls into the slowest path which takes locks in
order to access global device cache (priv->mr.cache) which is also a B-tree
and caches the original MR list (priv->mr.mr_list) of the device. Unless
the global cache is overflowed, it is all-inclusive of the MR list. This is
L3 - mlx5_mr_lookup_dev(). The size of the L3 cache table is limited and
can't be expanded on the fly due to deadlock. Refer to the comments in the
code for the details - mr_lookup_dev(). If L3 is overflowed, the list will
have to be searched directly bypassing the cache although it is slower.
If L3 misses, a new MR for the address should be created -
mlx5_mr_create(). When it creates a new MR, it tries to register adjacent
memsegs as much as possible which are virtually contiguous around the
address. This must take two locks - memory_hotplug_lock and
priv->mr.rwlock. Due to memory_hotplug_lock, there can't be any
allocation/free of memory inside.
In the free callback of the memory hotplug event, freed space is searched
from the MR list and corresponding bits are cleared from the bitmap of MRs.
This can fragment a MR and the MR will have multiple search entries in the
caches. Once there's a change by the event, the global cache must be
rebuilt and all the per-queue caches will be flushed as well. If memory is
frequently freed in run-time, that may cause jitter on dataplane processing
in the worst case by incurring MR cache flush and rebuild. But, it would be
the least probable scenario.
To guarantee the most optimal performance, it is highly recommended to use
an EAL option - '--socket-mem'. Then, the reserved memory will be pinned
and won't be freed dynamically. And it is also recommended to configure
per-lcore cache of Mempool. Even though there're many MRs for a device or
MRs are highly fragmented, the cache of Mempool will be much helpful to
reduce misses on per-queue caches anyway.
'--legacy-mem' is also supported.
Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
---
doc/guides/nics/mlx5.rst | 6 +
drivers/net/mlx5/mlx5.c | 45 ++
drivers/net/mlx5/mlx5.h | 22 +
drivers/net/mlx5/mlx5_defs.h | 6 +
drivers/net/mlx5/mlx5_ethdev.c | 16 +
drivers/net/mlx5/mlx5_mr.c | 1169 ++++++++++++++++++++++++++++++++++++++
drivers/net/mlx5/mlx5_mr.h | 117 ++++
drivers/net/mlx5/mlx5_rxq.c | 8 +-
drivers/net/mlx5/mlx5_rxtx.c | 3 +
drivers/net/mlx5/mlx5_rxtx.h | 73 ++-
drivers/net/mlx5/mlx5_rxtx_vec.h | 6 +-
drivers/net/mlx5/mlx5_trigger.c | 11 +
drivers/net/mlx5/mlx5_txq.c | 11 +
13 files changed, 1485 insertions(+), 8 deletions(-)
create mode 100644 drivers/net/mlx5/mlx5_mr.h
diff --git a/doc/guides/nics/mlx5.rst b/doc/guides/nics/mlx5.rst
index 5854106b5..13cc3f831 100644
--- a/doc/guides/nics/mlx5.rst
+++ b/doc/guides/nics/mlx5.rst
@@ -614,6 +614,12 @@ Performance tuning
The XXX can be different on different systems. Make sure to configure
according to the setpci output.
+7. To minimize overhead of searching Memory Regions:
+
+ - '--socket-mem' is recommended to pin memory by predictable amount.
+ - Configure per-lcore cache when creating Mempools for packet buffer.
+ - Refrain from dynamically allocating/freeing memory in run-time.
+
Notes for testpmd
-----------------
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 42b019ba6..84fc9d533 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -41,6 +41,7 @@
#include "mlx5_autoconf.h"
#include "mlx5_defs.h"
#include "mlx5_glue.h"
+#include "mlx5_mr.h"
/* Device parameter to enable RX completion queue compression. */
#define MLX5_RXQ_CQE_COMP_EN "rxq_cqe_comp_en"
@@ -84,10 +85,49 @@
#define MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP (1 << 4)
#endif
+static const char *MZ_MLX5_PMD_SHARED_DATA = "mlx5_pmd_shared_data";
+
+/* Shared memory between primary and secondary processes. */
+struct mlx5_shared_data *mlx5_shared_data;
+
+/* Spinlock for mlx5_shared_data allocation. */
+static rte_spinlock_t mlx5_shared_data_lock = RTE_SPINLOCK_INITIALIZER;
+
/** Driver-specific log messages type. */
int mlx5_logtype;
/**
+ * Prepare shared data between primary and secondary process.
+ */
+static void
+mlx5_prepare_shared_data(void)
+{
+ const struct rte_memzone *mz;
+
+ rte_spinlock_lock(&mlx5_shared_data_lock);
+ if (mlx5_shared_data == NULL) {
+ if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+ /* Allocate shared memory. */
+ mz = rte_memzone_reserve(MZ_MLX5_PMD_SHARED_DATA,
+ sizeof(*mlx5_shared_data),
+ SOCKET_ID_ANY, 0);
+ } else {
+ /* Lookup allocated shared memory. */
+ mz = rte_memzone_lookup(MZ_MLX5_PMD_SHARED_DATA);
+ }
+ if (mz == NULL)
+ rte_panic("Cannot allocate mlx5 shared data\n");
+ mlx5_shared_data = mz->addr;
+ /* Initialize shared data. */
+ if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+ LIST_INIT(&mlx5_shared_data->mem_event_cb_list);
+ rte_rwlock_init(&mlx5_shared_data->mem_event_rwlock);
+ }
+ }
+ rte_spinlock_unlock(&mlx5_shared_data_lock);
+}
+
+/**
* Retrieve integer value from environment variable.
*
* @param[in] name
@@ -201,6 +241,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
priv->txqs = NULL;
}
mlx5_flow_delete_drop_queue(dev);
+ mlx5_mr_release(dev);
if (priv->pd != NULL) {
assert(priv->ctx != NULL);
claim_zero(mlx5_glue->dealloc_pd(priv->pd));
@@ -633,6 +674,8 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
struct ibv_counter_set_description cs_desc;
#endif
+ /* Prepare shared data between primary and secondary process. */
+ mlx5_prepare_shared_data();
assert(pci_drv == &mlx5_driver);
/* Get mlx5_dev[] index. */
idx = mlx5_dev_idx(&pci_dev->addr);
@@ -1308,6 +1351,8 @@ rte_mlx5_pmd_init(void)
}
mlx5_glue->fork_init();
rte_pci_register(&mlx5_driver);
+ rte_mem_event_callback_register("MLX5_MEM_EVENT_CB",
+ mlx5_mr_mem_event_cb, NULL);
}
RTE_PMD_EXPORT_NAME(net_mlx5, __COUNTER__);
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index b34adc1ec..b2ed99087 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -26,11 +26,13 @@
#include <rte_pci.h>
#include <rte_ether.h>
#include <rte_ethdev_driver.h>
+#include <rte_rwlock.h>
#include <rte_interrupts.h>
#include <rte_errno.h>
#include <rte_flow.h>
#include "mlx5_utils.h"
+#include "mlx5_mr.h"
#include "mlx5_rxtx.h"
#include "mlx5_autoconf.h"
#include "mlx5_defs.h"
@@ -50,6 +52,16 @@ enum {
PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF = 0x101a,
};
+LIST_HEAD(mlx5_dev_list, priv);
+
+/* Shared memory between primary and secondary processes. */
+struct mlx5_shared_data {
+ struct mlx5_dev_list mem_event_cb_list;
+ rte_rwlock_t mem_event_rwlock;
+};
+
+extern struct mlx5_shared_data *mlx5_shared_data;
+
struct mlx5_xstats_ctrl {
/* Number of device stats. */
uint16_t stats_n;
@@ -119,7 +131,10 @@ struct mlx5_verbs_alloc_ctx {
const void *obj; /* Pointer to the DPDK object. */
};
+LIST_HEAD(mlx5_mr_list, mlx5_mr);
+
struct priv {
+ LIST_ENTRY(priv) mem_event_cb; /* Called by memory event callback. */
struct rte_eth_dev_data *dev_data; /* Pointer to device data. */
struct ibv_context *ctx; /* Verbs context. */
struct ibv_device_attr_ex device_attr; /* Device properties. */
@@ -146,6 +161,13 @@ struct priv {
struct mlx5_hrxq_drop *flow_drop_queue; /* Flow drop queue. */
struct mlx5_flows flows; /* RTE Flow rules. */
struct mlx5_flows ctrl_flows; /* Control flow rules. */
+ struct {
+ uint32_t dev_gen; /* Generation number to flush local caches. */
+ rte_rwlock_t rwlock; /* MR Lock. */
+ struct mlx5_mr_btree cache; /* Global MR cache table. */
+ struct mlx5_mr_list mr_list; /* Registered MR list. */
+ struct mlx5_mr_list mr_free_list; /* Freed MR list. */
+ } mr;
LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */
LIST_HEAD(rxqibv, mlx5_rxq_ibv) rxqsibv; /* Verbs Rx queues. */
LIST_HEAD(hrxq, mlx5_hrxq) hrxqs; /* Verbs Hash Rx queues. */
diff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h
index f9093777d..72e80af26 100644
--- a/drivers/net/mlx5/mlx5_defs.h
+++ b/drivers/net/mlx5/mlx5_defs.h
@@ -37,6 +37,12 @@
*/
#define MLX5_TX_COMP_THRESH_INLINE_DIV (1 << 3)
+/* Size of per-queue MR cache array for linear search. */
+#define MLX5_MR_CACHE_N 8
+
+/* Size of MR cache table for binary search. */
+#define MLX5_MR_BTREE_CACHE_N 256
+
/*
* If defined, only use software counters. The PMD will never ask the hardware
* for these, and many of them won't be available.
diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c
index 271d94a5b..610b5a006 100644
--- a/drivers/net/mlx5/mlx5_ethdev.c
+++ b/drivers/net/mlx5/mlx5_ethdev.c
@@ -34,6 +34,7 @@
#include <rte_interrupts.h>
#include <rte_malloc.h>
#include <rte_string_fns.h>
+#include <rte_rwlock.h>
#include "mlx5.h"
#include "mlx5_glue.h"
@@ -413,6 +414,21 @@ mlx5_dev_configure(struct rte_eth_dev *dev)
if (++j == rxqs_n)
j = 0;
}
+ /*
+ * Once the device is added to the list of memory event callback, its
+ * global MR cache table cannot be expanded on the fly because of
+ * deadlock. If it overflows, lookup should be done by searching MR list
+ * linearly, which is slow.
+ */
+ if (mlx5_mr_btree_init(&priv->mr.cache, MLX5_MR_BTREE_CACHE_N * 2,
+ dev->device->numa_node)) {
+ /* rte_errno is already set. */
+ return -rte_errno;
+ }
+ rte_rwlock_write_lock(&mlx5_shared_data->mem_event_rwlock);
+ LIST_INSERT_HEAD(&mlx5_shared_data->mem_event_cb_list,
+ priv, mem_event_cb);
+ rte_rwlock_write_unlock(&mlx5_shared_data->mem_event_rwlock);
return 0;
}
diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c
index 736c40ae4..026f55f4d 100644
--- a/drivers/net/mlx5/mlx5_mr.c
+++ b/drivers/net/mlx5/mlx5_mr.c
@@ -13,8 +13,1177 @@
#include <rte_mempool.h>
#include <rte_malloc.h>
+#include <rte_rwlock.h>
#include "mlx5.h"
+#include "mlx5_mr.h"
#include "mlx5_rxtx.h"
#include "mlx5_glue.h"
+struct mr_find_contig_memsegs_data {
+ uintptr_t addr;
+ uintptr_t start;
+ uintptr_t end;
+ const struct rte_memseg_list *msl;
+};
+
+struct mr_update_mp_data {
+ struct rte_eth_dev *dev;
+ struct mlx5_mr_ctrl *mr_ctrl;
+ int ret;
+};
+
+/**
+ * Expand B-tree table to a given size. Can't be called with holding
+ * memory_hotplug_lock or priv->mr.rwlock due to rte_realloc().
+ *
+ * @param bt
+ * Pointer to B-tree structure.
+ * @param n
+ * Number of entries for expansion.
+ *
+ * @return
+ * 0 on success, -1 on failure.
+ */
+static int
+mr_btree_expand(struct mlx5_mr_btree *bt, int n)
+{
+ void *mem;
+ int ret = 0;
+
+ if (n <= bt->size)
+ return ret;
+ /*
+ * Downside of directly using rte_realloc() is that SOCKET_ID_ANY is
+ * used inside if there's no room to expand. Because this is a quite
+ * rare case and a part of very slow path, it is very acceptable.
+ * Initially cache_bh[] will be given practically enough space and once
+ * it is expanded, expansion wouldn't be needed again ever.
+ */
+ mem = rte_realloc(bt->table, n * sizeof(struct mlx5_mr_cache), 0);
+ if (mem == NULL) {
+ /* Not an error, B-tree search will be skipped. */
+ DRV_LOG(WARNING, "failed to expand MR B-tree (%p) table",
+ (void *)bt);
+ ret = -1;
+ } else {
+ DRV_LOG(DEBUG, "expanded MR B-tree table (size=%u)", n);
+ bt->table = mem;
+ bt->size = n;
+ }
+ return ret;
+}
+
+/**
+ * Look up LKey from given B-tree lookup table, store the last index and return
+ * searched LKey.
+ *
+ * @param bt
+ * Pointer to B-tree structure.
+ * @param[out] idx
+ * Pointer to index. Even on search failure, returns index where it stops
+ * searching so that index can be used when inserting a new entry.
+ * @param addr
+ * Search key.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on no match.
+ */
+static uint32_t
+mr_btree_lookup(struct mlx5_mr_btree *bt, uint16_t *idx, uintptr_t addr)
+{
+ struct mlx5_mr_cache *lkp_tbl;
+ uint16_t n;
+ uint16_t base = 0;
+
+ assert(bt != NULL);
+ lkp_tbl = *bt->table;
+ n = bt->len;
+ /* First entry must be NULL for comparison. */
+ assert(bt->len > 0 || (lkp_tbl[0].start == 0 &&
+ lkp_tbl[0].lkey == UINT32_MAX));
+ /* Binary search. */
+ do {
+ register uint16_t delta = n >> 1;
+
+ if (addr < lkp_tbl[base + delta].start) {
+ n = delta;
+ } else {
+ base += delta;
+ n -= delta;
+ }
+ } while (n > 1);
+ assert(addr >= lkp_tbl[base].start);
+ *idx = base;
+ if (addr < lkp_tbl[base].end)
+ return lkp_tbl[base].lkey;
+ /* Not found. */
+ return UINT32_MAX;
+}
+
+/**
+ * Insert an entry to B-tree lookup table.
+ *
+ * @param bt
+ * Pointer to B-tree structure.
+ * @param entry
+ * Pointer to new entry to insert.
+ *
+ * @return
+ * 0 on success, -1 on failure.
+ */
+static int
+mr_btree_insert(struct mlx5_mr_btree *bt, struct mlx5_mr_cache *entry)
+{
+ struct mlx5_mr_cache *lkp_tbl;
+ uint16_t idx = 0;
+ size_t shift;
+
+ assert(bt != NULL);
+ assert(bt->len <= bt->size);
+ assert(bt->len > 0);
+ lkp_tbl = *bt->table;
+ /* Find out the slot for insertion. */
+ if (mr_btree_lookup(bt, &idx, entry->start) != UINT32_MAX) {
+ DRV_LOG(DEBUG,
+ "abort insertion to B-tree(%p):"
+ " already exist at idx=%u [0x%lx, 0x%lx) lkey=0x%x",
+ (void *)bt, idx, entry->start, entry->end, entry->lkey);
+ /* Already exist, return. */
+ return 0;
+ }
+ /* If table is full, return error. */
+ if (unlikely(bt->len == bt->size)) {
+ bt->overflow = 1;
+ return -1;
+ }
+ /* Insert entry. */
+ ++idx;
+ shift = (bt->len - idx) * sizeof(struct mlx5_mr_cache);
+ if (shift)
+ memmove(&lkp_tbl[idx + 1], &lkp_tbl[idx], shift);
+ lkp_tbl[idx] = *entry;
+ bt->len++;
+ DRV_LOG(DEBUG,
+ "inserted B-tree(%p)[%u], [0x%lx, 0x%lx) lkey=0x%x",
+ (void *)bt, idx, entry->start, entry->end, entry->lkey);
+ return 0;
+}
+
+/**
+ * Initialize B-tree and allocate memory for lookup table.
+ *
+ * @param bt
+ * Pointer to B-tree structure.
+ * @param n
+ * Number of entries to allocate.
+ * @param socket
+ * NUMA socket on which memory must be allocated.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_mr_btree_init(struct mlx5_mr_btree *bt, int n, int socket)
+{
+ if (bt == NULL) {
+ rte_errno = EINVAL;
+ return -rte_errno;
+ }
+ memset(bt, 0, sizeof(*bt));
+ bt->table = rte_calloc_socket("B-tree table",
+ n, sizeof(struct mlx5_mr_cache),
+ 0, socket);
+ if (bt->table == NULL) {
+ rte_errno = ENOMEM;
+ DRV_LOG(ERR,
+ "failed to allocate memory for btree cache on socket %d",
+ socket);
+ return -rte_errno;
+ }
+ bt->size = n;
+ /* First entry must be NULL for binary search. */
+ (*bt->table)[bt->len++] = (struct mlx5_mr_cache) {
+ .lkey = UINT32_MAX,
+ };
+ DRV_LOG(DEBUG, "initialized B-tree %p with table %p",
+ (void *)bt, (void *)bt->table);
+ return 0;
+}
+
+/**
+ * Free B-tree resources.
+ *
+ * @param bt
+ * Pointer to B-tree structure.
+ */
+void
+mlx5_mr_btree_free(struct mlx5_mr_btree *bt)
+{
+ if (bt == NULL)
+ return;
+ DRV_LOG(DEBUG, "freeing B-tree %p with table %p",
+ (void *)bt, (void *)bt->table);
+ rte_free(bt->table);
+ memset(bt, 0, sizeof(*bt));
+}
+
+/**
+ * Dump all the entries in a B-tree
+ *
+ * @param bt
+ * Pointer to B-tree structure.
+ */
+static void
+mlx5_mr_btree_dump(struct mlx5_mr_btree *bt)
+{
+ int idx;
+ struct mlx5_mr_cache *lkp_tbl;
+
+ if (bt == NULL)
+ return;
+ lkp_tbl = *bt->table;
+ for (idx = 0; idx < bt->len; ++idx) {
+ struct mlx5_mr_cache *entry = &lkp_tbl[idx];
+
+ DRV_LOG(DEBUG,
+ "B-tree(%p)[%u], [0x%lx, 0x%lx) lkey=0x%x",
+ (void *)bt, idx, entry->start, entry->end, entry->lkey);
+ }
+}
+
+/**
+ * Find virtually contiguous memory chunk in a given MR.
+ *
+ * @param dev
+ * Pointer to MR structure.
+ * @param[out] entry
+ * Pointer to returning MR cache entry. If not found, this will not be
+ * updated.
+ * @param start_idx
+ * Start index of the memseg bitmap.
+ *
+ * @return
+ * Next index to go on lookup.
+ */
+static int
+mr_find_next_chunk(struct mlx5_mr *mr, struct mlx5_mr_cache *entry,
+ int base_idx)
+{
+ uintptr_t start = 0;
+ uintptr_t end = 0;
+ uint32_t idx = 0;
+
+ for (idx = base_idx; idx < mr->ms_bmp_n; ++idx) {
+ if (rte_bitmap_get(mr->ms_bmp, idx)) {
+ const struct rte_memseg_list *msl;
+ const struct rte_memseg *ms;
+
+ msl = mr->msl;
+ ms = rte_fbarray_get(&msl->memseg_arr,
+ mr->ms_base_idx + idx);
+ assert(msl->page_sz == ms->hugepage_sz);
+ if (!start)
+ start = ms->addr_64;
+ end = ms->addr_64 + ms->hugepage_sz;
+ } else if (start) {
+ /* Passed the end of a fragment. */
+ break;
+ }
+ }
+ if (start) {
+ /* Found one chunk. */
+ entry->start = start;
+ entry->end = end;
+ entry->lkey = rte_cpu_to_be_32(mr->ibv_mr->lkey);
+ }
+ return idx;
+}
+
+/**
+ * Insert a MR to the global B-tree cache. It may fail due to low-on-memory.
+ * Then, this entry will have to be searched by mr_lookup_dev_list() in
+ * mlx5_mr_create() on miss.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param mr
+ * Pointer to MR to insert.
+ *
+ * @return
+ * 0 on success, -1 on failure.
+ */
+static int
+mr_insert_dev_cache(struct rte_eth_dev *dev, struct mlx5_mr *mr)
+{
+ struct priv *priv = dev->data->dev_private;
+ unsigned int n;
+
+ DRV_LOG(DEBUG, "port %u inserting MR(%p) to global cache",
+ dev->data->port_id, (void *)mr);
+ for (n = 0; n < mr->ms_bmp_n; ) {
+ struct mlx5_mr_cache entry = { 0, };
+
+ /* Find a contiguous chunk and advance the index. */
+ n = mr_find_next_chunk(mr, &entry, n);
+ if (!entry.end)
+ break;
+ if (mr_btree_insert(&priv->mr.cache, &entry) < 0) {
+ /*
+ * Overflowed, but the global table cannot be expanded
+ * because of deadlock.
+ */
+ return -1;
+ }
+ }
+ return 0;
+}
+
+/**
+ * Look up address in the original global MR list.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param[out] entry
+ * Pointer to returning MR cache entry. If no match, this will not be updated.
+ * @param addr
+ * Search key.
+ *
+ * @return
+ * Found MR on match, NULL otherwise.
+ */
+static struct mlx5_mr *
+mr_lookup_dev_list(struct rte_eth_dev *dev, struct mlx5_mr_cache *entry,
+ uintptr_t addr)
+{
+ struct priv *priv = dev->data->dev_private;
+ struct mlx5_mr *mr;
+
+ /* Iterate all the existing MRs. */
+ LIST_FOREACH(mr, &priv->mr.mr_list, mr) {
+ unsigned int n;
+
+ if (mr->ms_n == 0)
+ continue;
+ for (n = 0; n < mr->ms_bmp_n; ) {
+ struct mlx5_mr_cache ret = { 0, };
+
+ n = mr_find_next_chunk(mr, &ret, n);
+ if (addr >= ret.start && addr < ret.end) {
+ /* Found. */
+ *entry = ret;
+ return mr;
+ }
+ }
+ }
+ return NULL;
+}
+
+/**
+ * Look up address on device.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param[out] entry
+ * Pointer to returning MR cache entry. If no match, this will not be updated.
+ * @param addr
+ * Search key.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on failure and rte_errno is set.
+ */
+static uint32_t
+mr_lookup_dev(struct rte_eth_dev *dev, struct mlx5_mr_cache *entry,
+ uintptr_t addr)
+{
+ struct priv *priv = dev->data->dev_private;
+ uint16_t idx;
+ uint32_t lkey = UINT32_MAX;
+ struct mlx5_mr *mr;
+
+ /*
+ * If the global cache has overflowed since it failed to expand the
+ * B-tree table, it can't have all the exisitng MRs. Then, the address
+ * has to be searched by traversing the original MR list instead, which
+ * is very slow path. Otherwise, the global cache is all inclusive.
+ */
+ if (!unlikely(priv->mr.cache.overflow)) {
+ lkey = mr_btree_lookup(&priv->mr.cache, &idx, addr);
+ if (lkey != UINT32_MAX)
+ *entry = (*priv->mr.cache.table)[idx];
+ } else {
+ /* Falling back to the slowest path. */
+ mr = mr_lookup_dev_list(dev, entry, addr);
+ if (mr != NULL)
+ lkey = entry->lkey;
+ }
+ assert(lkey == UINT32_MAX || (addr >= entry->start &&
+ addr < entry->end));
+ return lkey;
+}
+
+/**
+ * Free MR resources. MR lock must not be held to avoid a deadlock. rte_free()
+ * can raise memory free event and the callback function will spin on the lock.
+ *
+ * @param mr
+ * Pointer to MR to free.
+ */
+static void
+mr_free(struct mlx5_mr *mr)
+{
+ if (mr == NULL)
+ return;
+ DRV_LOG(DEBUG, "freeing MR(%p):", (void *)mr);
+ if (mr->ibv_mr != NULL)
+ claim_zero(mlx5_glue->dereg_mr(mr->ibv_mr));
+ if (mr->ms_bmp != NULL)
+ rte_bitmap_free(mr->ms_bmp);
+ rte_free(mr);
+}
+
+/**
+ * Releass resources of detached MR having no online entry.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ */
+static void
+mlx5_mr_garbage_collect(struct rte_eth_dev *dev)
+{
+ struct priv *priv = dev->data->dev_private;
+ struct mlx5_mr *mr_next;
+ struct mlx5_mr_list free_list = LIST_HEAD_INITIALIZER(free_list);
+
+ /* Must be called from the primary process. */
+ assert(rte_eal_process_type() == RTE_PROC_PRIMARY);
+ /*
+ * MR can't be freed with holding the lock because rte_free() could call
+ * memory free callback function. This will be a deadlock situation.
+ */
+ rte_rwlock_write_lock(&priv->mr.rwlock);
+ /* Detach the whole free list and release it after unlocking. */
+ free_list = priv->mr.mr_free_list;
+ LIST_INIT(&priv->mr.mr_free_list);
+ rte_rwlock_write_unlock(&priv->mr.rwlock);
+ /* Release resources. */
+ mr_next = LIST_FIRST(&free_list);
+ while (mr_next != NULL) {
+ struct mlx5_mr *mr = mr_next;
+
+ mr_next = LIST_NEXT(mr, mr);
+ mr_free(mr);
+ }
+}
+
+/* Called during rte_memseg_contig_walk() by mlx5_mr_create(). */
+static int
+mr_find_contig_memsegs_cb(const struct rte_memseg_list *msl,
+ const struct rte_memseg *ms, size_t len, void *arg)
+{
+ struct mr_find_contig_memsegs_data *data = arg;
+
+ if (data->addr < ms->addr_64 || data->addr >= ms->addr_64 + len)
+ return 0;
+ /* Found, save it and stop walking. */
+ data->start = ms->addr_64;
+ data->end = ms->addr_64 + len;
+ data->msl = msl;
+ return 1;
+}
+
+/**
+ * Create a new global Memroy Region (MR) for a missing virtual address.
+ * Register entire virtually contiguous memory chunk around the address.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param[out] entry
+ * Pointer to returning MR cache entry, found in the global cache or newly
+ * created. If failed to create one, this will not be updated.
+ * @param addr
+ * Target virtual address to register.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on failure and rte_errno is set.
+ */
+static uint32_t
+mlx5_mr_create(struct rte_eth_dev *dev, struct mlx5_mr_cache *entry,
+ uintptr_t addr)
+{
+ struct priv *priv = dev->data->dev_private;
+ struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+ const struct rte_memseg_list *msl;
+ const struct rte_memseg *ms;
+ struct mlx5_mr *mr = NULL;
+ size_t len;
+ uint32_t ms_n;
+ uint32_t bmp_size;
+ void *bmp_mem;
+ int ms_idx_shift = -1;
+ unsigned int n;
+ struct mr_find_contig_memsegs_data data = {
+ .addr = addr,
+ };
+ struct mr_find_contig_memsegs_data data_re;
+
+ DRV_LOG(DEBUG, "port %u creating a MR using address (%p)",
+ dev->data->port_id, (void *)addr);
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+ DRV_LOG(WARNING,
+ "port %u using address (%p) of unregistered mempool"
+ " in secondary process, please create mempool"
+ " before rte_eth_dev_start()",
+ dev->data->port_id, (void *)addr);
+ rte_errno = EPERM;
+ goto err_nolock;
+ }
+ /*
+ * Release detached MRs if any. This can't be called with holding either
+ * memory_hotplug_lock or priv->mr.rwlock. MRs on the free list have
+ * been detached by the memory free event but it couldn't be released
+ * inside the callback due to deadlock. As a result, releasing resources
+ * is quite opportunistic.
+ */
+ mlx5_mr_garbage_collect(dev);
+ /*
+ * Find out a contiguous virtual address chunk in use, to which the
+ * given address belongs, in order to register maximum range. In the
+ * best case where mempools are not dynamically recreated and
+ * '--socket-mem' is speicified as an EAL option, it is very likely to
+ * have only one MR(LKey) per a socket and per a hugepage-size even
+ * though the system memory is highly fragmented.
+ */
+ if (!rte_memseg_contig_walk(mr_find_contig_memsegs_cb, &data)) {
+ DRV_LOG(WARNING,
+ "port %u unable to find virtually contigous"
+ " chunk for address (%p)."
+ " rte_memseg_contig_walk() failed.",
+ dev->data->port_id, (void *)addr);
+ rte_errno = ENXIO;
+ goto err_nolock;
+ }
+alloc_resources:
+ /* Addresses must be page-aligned. */
+ assert(rte_is_aligned((void *)data.start, data.msl->page_sz));
+ assert(rte_is_aligned((void *)data.end, data.msl->page_sz));
+ msl = data.msl;
+ ms = rte_mem_virt2memseg((void *)data.start, msl);
+ len = data.end - data.start;
+ assert(msl->page_sz == ms->hugepage_sz);
+ /* Number of memsegs in the range. */
+ ms_n = len / msl->page_sz;
+ DRV_LOG(DEBUG,
+ "port %u extending %p to [0x%lx, 0x%lx), page_sz=0x%lx, ms_n=%u",
+ dev->data->port_id, (void *)addr,
+ data.start, data.end, msl->page_sz, ms_n);
+ /* Size of memory for bitmap. */
+ bmp_size = rte_bitmap_get_memory_footprint(ms_n);
+ mr = rte_zmalloc_socket(NULL,
+ RTE_ALIGN_CEIL(sizeof(*mr),
+ RTE_CACHE_LINE_SIZE) +
+ bmp_size,
+ RTE_CACHE_LINE_SIZE, msl->socket_id);
+ if (mr == NULL) {
+ DRV_LOG(WARNING,
+ "port %u unable to allocate memory for a new MR of"
+ " address (%p).",
+ dev->data->port_id, (void *)addr);
+ rte_errno = ENOMEM;
+ goto err_nolock;
+ }
+ mr->msl = msl;
+ /*
+ * Save the index of the first memseg and initialize memseg bitmap. To
+ * see if a memseg of ms_idx in the memseg-list is still valid, check:
+ * rte_bitmap_get(mr->bmp, ms_idx - mr->ms_base_idx)
+ */
+ mr->ms_base_idx = rte_fbarray_find_idx(&msl->memseg_arr, ms);
+ bmp_mem = RTE_PTR_ALIGN_CEIL(mr + 1, RTE_CACHE_LINE_SIZE);
+ mr->ms_bmp = rte_bitmap_init(ms_n, bmp_mem, bmp_size);
+ if (mr->ms_bmp == NULL) {
+ DRV_LOG(WARNING,
+ "port %u unable to initialize bitamp for a new MR of"
+ " address (%p).",
+ dev->data->port_id, (void *)addr);
+ rte_errno = EINVAL;
+ goto err_nolock;
+ }
+ /*
+ * Should recheck whether the extended contiguous chunk is still valid.
+ * Because memory_hotplug_lock can't be held if there's any memory
+ * related calls in a critical path, resource allocation above can't be
+ * locked. If the memory has been changed at this point, try again with
+ * just single page. If not, go on with the big chunk atomically from
+ * here.
+ */
+ rte_rwlock_read_lock(&mcfg->memory_hotplug_lock);
+ data_re = data;
+ if (len > msl->page_sz &&
+ !rte_memseg_contig_walk(mr_find_contig_memsegs_cb, &data_re)) {
+ DRV_LOG(WARNING,
+ "port %u unable to find virtually contigous"
+ " chunk for address (%p)."
+ " rte_memseg_contig_walk() failed.",
+ dev->data->port_id, (void *)addr);
+ rte_errno = ENXIO;
+ goto err_memlock;
+ }
+ if (data.start != data_re.start || data.end != data_re.end) {
+ /*
+ * The extended contiguous chunk has been changed. Try again
+ * with single memseg instead.
+ */
+ data.start = RTE_ALIGN_FLOOR(addr, msl->page_sz);
+ data.end = data.start + msl->page_sz;
+ rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
+ mr_free(mr);
+ goto alloc_resources;
+ }
+ assert(data.msl == data_re.msl);
+ rte_rwlock_write_lock(&priv->mr.rwlock);
+ /*
+ * Check the address is really missing. If other thread already created
+ * one or it is not found due to overflow, abort and return.
+ */
+ if (mr_lookup_dev(dev, entry, addr) != UINT32_MAX) {
+ /*
+ * Insert to the global cache table. It may fail due to
+ * low-on-memory. Then, this entry will have to be searched
+ * here again.
+ */
+ mr_btree_insert(&priv->mr.cache, entry);
+ DRV_LOG(DEBUG,
+ "port %u found MR for %p on final lookup, abort",
+ dev->data->port_id, (void *)addr);
+ rte_rwlock_write_unlock(&priv->mr.rwlock);
+ rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
+ /*
+ * Must be unlocked before calling rte_free() because
+ * mlx5_mr_mem_event_free_cb() can be called inside.
+ */
+ mr_free(mr);
+ return entry->lkey;
+ }
+ /*
+ * Trim start and end addresses for verbs MR. Set bits for registering
+ * memsegs but exclude already registered ones. Bitmap can be
+ * fragmented.
+ */
+ for (n = 0; n < ms_n; ++n) {
+ uintptr_t start;
+ struct mlx5_mr_cache ret = { 0, };
+
+ start = data_re.start + n * msl->page_sz;
+ /* Exclude memsegs already registered by other MRs. */
+ if (mr_lookup_dev(dev, &ret, start) == UINT32_MAX) {
+ /*
+ * Start from the first unregistered memseg in the
+ * extended range.
+ */
+ if (ms_idx_shift == -1) {
+ mr->ms_base_idx += n;
+ data.start = start;
+ ms_idx_shift = n;
+ }
+ data.end = start + msl->page_sz;
+ rte_bitmap_set(mr->ms_bmp, n - ms_idx_shift);
+ ++mr->ms_n;
+ }
+ }
+ len = data.end - data.start;
+ mr->ms_bmp_n = len / msl->page_sz;
+ assert(ms_idx_shift + mr->ms_bmp_n <= ms_n);
+ /*
+ * Finally create a verbs MR for the memory chunk. ibv_reg_mr() can be
+ * called with holding the memory lock because it doesn't use
+ * mlx5_alloc_buf_extern() which eventually calls rte_malloc_socket()
+ * through mlx5_alloc_verbs_buf().
+ */
+ mr->ibv_mr = mlx5_glue->reg_mr(priv->pd, (void *)data.start, len,
+ IBV_ACCESS_LOCAL_WRITE);
+ if (mr->ibv_mr == NULL) {
+ DRV_LOG(WARNING,
+ "port %u fail to create a verbs MR for address (%p)",
+ dev->data->port_id, (void *)addr);
+ rte_errno = EINVAL;
+ goto err_mrlock;
+ }
+ assert((uintptr_t)mr->ibv_mr->addr == data.start);
+ assert(mr->ibv_mr->length == len);
+ LIST_INSERT_HEAD(&priv->mr.mr_list, mr, mr);
+ DRV_LOG(DEBUG,
+ "port %u MR CREATED (%p) for %p:\n"
+ " [0x%lx, 0x%lx), lkey=0x%x base_idx=%u ms_n=%u, ms_bmp_n=%u",
+ dev->data->port_id, (void *)mr, (void *)addr,
+ data.start, data.end, rte_cpu_to_be_32(mr->ibv_mr->lkey),
+ mr->ms_base_idx, mr->ms_n, mr->ms_bmp_n);
+ /* Insert to the global cache table. */
+ mr_insert_dev_cache(dev, mr);
+ /* Fill in output data. */
+ mr_lookup_dev(dev, entry, addr);
+ /* Lookup can't fail. */
+ assert(entry->lkey != UINT32_MAX);
+ rte_rwlock_write_unlock(&priv->mr.rwlock);
+ rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
+ return entry->lkey;
+err_mrlock:
+ rte_rwlock_write_unlock(&priv->mr.rwlock);
+err_memlock:
+ rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
+err_nolock:
+ /*
+ * In case of error, as this can be called in a datapath, a warning
+ * message per an error is preferable instead. Must be unlocked before
+ * calling rte_free() because mlx5_mr_mem_event_free_cb() can be called
+ * inside.
+ */
+ mr_free(mr);
+ return UINT32_MAX;
+}
+
+/**
+ * Rebuild the global B-tree cache of device from the original MR list.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ */
+static void
+mr_rebuild_dev_cache(struct rte_eth_dev *dev)
+{
+ struct priv *priv = dev->data->dev_private;
+ struct mlx5_mr *mr;
+
+ DRV_LOG(DEBUG, "port %u rebuild dev cache[]", dev->data->port_id);
+ /* Flush cache to rebuild. */
+ priv->mr.cache.len = 1;
+ priv->mr.cache.overflow = 0;
+ /* Iterate all the existing MRs. */
+ LIST_FOREACH(mr, &priv->mr.mr_list, mr)
+ if (mr_insert_dev_cache(dev, mr) < 0)
+ return;
+}
+
+/**
+ * Callback for memory free event. Iterate freed memsegs and check whether it
+ * belongs to an existing MR. If found, clear the bit from bitmap of MR. As a
+ * result, the MR would be fragmented. If it becomes empty, the MR will be freed
+ * later by mlx5_mr_garbage_collect(). Even if this callback is called from a
+ * secondary process, the garbage collector will be called in primary process
+ * as the secondary process can't call mlx5_mr_create().
+ *
+ * The global cache must be rebuilt if there's any change and this event has to
+ * be propagated to dataplane threads to flush the local caches.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param addr
+ * Address of freed memory.
+ * @param len
+ * Size of freed memory.
+ */
+static void
+mlx5_mr_mem_event_free_cb(struct rte_eth_dev *dev, const void *addr, size_t len)
+{
+ struct priv *priv = dev->data->dev_private;
+ const struct rte_memseg_list *msl;
+ struct mlx5_mr *mr;
+ int ms_n;
+ int i;
+ int rebuild = 0;
+
+ DRV_LOG(DEBUG, "port %u free callback: addr=%p, len=%lu",
+ dev->data->port_id, addr, len);
+ msl = rte_mem_virt2memseg_list(addr);
+ /* addr and len must be page-aligned. */
+ assert((uintptr_t)addr == RTE_ALIGN((uintptr_t)addr, msl->page_sz));
+ assert(len == RTE_ALIGN(len, msl->page_sz));
+ ms_n = len / msl->page_sz;
+ rte_rwlock_write_lock(&priv->mr.rwlock);
+ /* Clear bits of freed memsegs from MR. */
+ for (i = 0; i < ms_n; ++i) {
+ const struct rte_memseg *ms;
+ struct mlx5_mr_cache entry;
+ uintptr_t start;
+ int ms_idx;
+ uint32_t pos;
+
+ /* Find MR having this memseg. */
+ start = (uintptr_t)addr + i * msl->page_sz;
+ mr = mr_lookup_dev_list(dev, &entry, start);
+ if (mr == NULL)
+ continue;
+ ms = rte_mem_virt2memseg((void *)start, msl);
+ assert(ms != NULL);
+ assert(msl->page_sz == ms->hugepage_sz);
+ ms_idx = rte_fbarray_find_idx(&msl->memseg_arr, ms);
+ pos = ms_idx - mr->ms_base_idx;
+ assert(rte_bitmap_get(mr->ms_bmp, pos));
+ assert(pos < mr->ms_bmp_n);
+ DRV_LOG(DEBUG, "port %u MR(%p): clear bitmap[%u] for addr %p",
+ dev->data->port_id, (void *)mr, pos, (void *)start);
+ rte_bitmap_clear(mr->ms_bmp, pos);
+ if (--mr->ms_n == 0) {
+ LIST_REMOVE(mr, mr);
+ LIST_INSERT_HEAD(&priv->mr.mr_free_list, mr, mr);
+ DRV_LOG(DEBUG, "port %u remove MR(%p) from list",
+ dev->data->port_id, (void *)mr);
+ }
+ /*
+ * MR is fragmented or will be freed. the global cache must be
+ * rebuilt.
+ */
+ rebuild = 1;
+ }
+ if (rebuild) {
+ mr_rebuild_dev_cache(dev);
+ /*
+ * Flush local caches by propagating invalidation across cores.
+ * rte_smp_wmb() is enough to synchronize this event. If one of
+ * freed memsegs is seen by other core, that means the memseg
+ * has been allocated by allocator, which will come after this
+ * free call. Therefore, this store instruction (incrementing
+ * generation below) will be guaranteed to be seen by other core
+ * before the core sees the newly allocated memory.
+ */
+ ++priv->mr.dev_gen;
+ DRV_LOG(DEBUG, "broadcasting local cache flush, gen=%d",
+ priv->mr.dev_gen);
+ rte_smp_wmb();
+ }
+ rte_rwlock_write_unlock(&priv->mr.rwlock);
+ if (rebuild && rte_log_get_level(mlx5_logtype) == RTE_LOG_DEBUG)
+ mlx5_mr_dump_dev(dev);
+}
+
+/**
+ * Callback for memory event. This can be called from both primary and secondary
+ * process.
+ *
+ * @param event_type
+ * Memory event type.
+ * @param addr
+ * Address of memory.
+ * @param len
+ * Size of memory.
+ */
+void
+mlx5_mr_mem_event_cb(enum rte_mem_event event_type, const void *addr,
+ size_t len, void *arg __rte_unused)
+{
+ struct priv *priv;
+ struct mlx5_dev_list *dev_list = &mlx5_shared_data->mem_event_cb_list;
+
+ switch (event_type) {
+ case RTE_MEM_EVENT_FREE:
+ rte_rwlock_write_lock(&mlx5_shared_data->mem_event_rwlock);
+ /* Iterate all the existing mlx5 devices. */
+ LIST_FOREACH(priv, dev_list, mem_event_cb)
+ mlx5_mr_mem_event_free_cb(ETH_DEV(priv), addr, len);
+ rte_rwlock_write_unlock(&mlx5_shared_data->mem_event_rwlock);
+ break;
+ case RTE_MEM_EVENT_ALLOC:
+ default:
+ break;
+ }
+}
+
+/**
+ * Look up address in the global MR cache table. If not found, create a new MR.
+ * Insert the found/created entry to local bottom-half cache table.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param mr_ctrl
+ * Pointer to per-queue MR control structure.
+ * @param[out] entry
+ * Pointer to returning MR cache entry, found in the global cache or newly
+ * created. If failed to create one, this is not written.
+ * @param addr
+ * Search key.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on no match.
+ */
+static uint32_t
+mlx5_mr_lookup_dev(struct rte_eth_dev *dev, struct mlx5_mr_ctrl *mr_ctrl,
+ struct mlx5_mr_cache *entry, uintptr_t addr)
+{
+ struct priv *priv = dev->data->dev_private;
+ struct mlx5_mr_btree *bt = &mr_ctrl->cache_bh;
+ uint16_t idx;
+ uint32_t lkey;
+
+ /* If local cache table is full, try to double it. */
+ if (unlikely(bt->len == bt->size))
+ mr_btree_expand(bt, bt->size << 1);
+ /* Look up in the global cache. */
+ rte_rwlock_read_lock(&priv->mr.rwlock);
+ lkey = mr_btree_lookup(&priv->mr.cache, &idx, addr);
+ if (lkey != UINT32_MAX) {
+ /* Found. */
+ *entry = (*priv->mr.cache.table)[idx];
+ rte_rwlock_read_unlock(&priv->mr.rwlock);
+ /*
+ * Update local cache. Even if it fails, return the found entry
+ * to update top-half cache. Next time, this entry will be found
+ * in the global cache.
+ */
+ mr_btree_insert(bt, entry);
+ return lkey;
+ }
+ rte_rwlock_read_unlock(&priv->mr.rwlock);
+ /* First time to see the address? Create a new MR. */
+ lkey = mlx5_mr_create(dev, entry, addr);
+ /*
+ * Update the local cache if successfully created a new global MR. Even
+ * if failed to create one, there's no action to take in this datapath
+ * code. As returning LKey is invalid, this will eventually make HW
+ * fail.
+ */
+ if (lkey != UINT32_MAX)
+ mr_btree_insert(bt, entry);
+ return lkey;
+}
+
+/**
+ * Bottom-half of LKey search on datapath. Firstly search in cache_bh[] and if
+ * misses, search in the global MR cache table and update the new entry to
+ * per-queue local caches.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param mr_ctrl
+ * Pointer to per-queue MR control structure.
+ * @param addr
+ * Search key.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on no match.
+ */
+static uint32_t
+mlx5_mr_addr2mr_bh(struct rte_eth_dev *dev, struct mlx5_mr_ctrl *mr_ctrl,
+ uintptr_t addr)
+{
+ uint32_t lkey;
+ uint16_t bh_idx = 0;
+ /* Victim in top-half cache to replace with new entry. */
+ struct mlx5_mr_cache *repl = &mr_ctrl->cache[mr_ctrl->head];
+
+ /* Binary-search MR translation table. */
+ lkey = mr_btree_lookup(&mr_ctrl->cache_bh, &bh_idx, addr);
+ /* Update top-half cache. */
+ if (likely(lkey != UINT32_MAX)) {
+ *repl = (*mr_ctrl->cache_bh.table)[bh_idx];
+ } else {
+ /*
+ * If missed in local lookup table, search in the global cache
+ * and local cache_bh[] will be updated inside if possible.
+ * Top-half cache entry will also be updated.
+ */
+ lkey = mlx5_mr_lookup_dev(dev, mr_ctrl, repl, addr);
+ if (unlikely(lkey == UINT32_MAX))
+ return UINT32_MAX;
+ }
+ /* Update the most recently used entry. */
+ mr_ctrl->mru = mr_ctrl->head;
+ /* Point to the next victim, the oldest. */
+ mr_ctrl->head = (mr_ctrl->head + 1) % MLX5_MR_CACHE_N;
+ return lkey;
+}
+
+/**
+ * Bottom-half of LKey search on Rx.
+ *
+ * @param rxq
+ * Pointer to Rx queue structure.
+ * @param addr
+ * Search key.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on no match.
+ */
+uint32_t
+mlx5_rx_addr2mr_bh(struct mlx5_rxq_data *rxq, uintptr_t addr)
+{
+ struct mlx5_rxq_ctrl *rxq_ctrl =
+ container_of(rxq, struct mlx5_rxq_ctrl, rxq);
+ struct mlx5_mr_ctrl *mr_ctrl = &rxq->mr_ctrl;
+ struct priv *priv = rxq_ctrl->priv;
+
+ DRV_LOG(DEBUG,
+ "Rx queue %u: miss on top-half, mru=%u, head=%u, addr=%p",
+ rxq_ctrl->idx, mr_ctrl->mru, mr_ctrl->head, (void *)addr);
+ return mlx5_mr_addr2mr_bh(ETH_DEV(priv), mr_ctrl, addr);
+}
+
+/**
+ * Bottom-half of LKey search on Tx.
+ *
+ * @param txq
+ * Pointer to Tx queue structure.
+ * @param addr
+ * Search key.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on no match.
+ */
+uint32_t
+mlx5_tx_addr2mr_bh(struct mlx5_txq_data *txq, uintptr_t addr)
+{
+ struct mlx5_txq_ctrl *txq_ctrl =
+ container_of(txq, struct mlx5_txq_ctrl, txq);
+ struct mlx5_mr_ctrl *mr_ctrl = &txq->mr_ctrl;
+ struct priv *priv = txq_ctrl->priv;
+
+ DRV_LOG(DEBUG,
+ "Tx queue %u: miss on top-half, mru=%u, head=%u, addr=%p",
+ txq_ctrl->idx, mr_ctrl->mru, mr_ctrl->head, (void *)addr);
+ return mlx5_mr_addr2mr_bh(ETH_DEV(priv), mr_ctrl, addr);
+}
+
+/**
+ * Flush all of the local cache entries.
+ *
+ * @param mr_ctrl
+ * Pointer to per-queue MR control structure.
+ */
+void
+mlx5_mr_flush_local_cache(struct mlx5_mr_ctrl *mr_ctrl)
+{
+ /* Reset the most-recently-used index. */
+ mr_ctrl->mru = 0;
+ /* Reset the linear search array. */
+ mr_ctrl->head = 0;
+ memset(mr_ctrl->cache, 0, sizeof(mr_ctrl->cache));
+ /* Reset the B-tree table. */
+ mr_ctrl->cache_bh.len = 1;
+ mr_ctrl->cache_bh.overflow = 0;
+ /* Update the generation number. */
+ mr_ctrl->cur_gen = *mr_ctrl->dev_gen_ptr;
+ DRV_LOG(DEBUG, "mr_ctrl(%p): flushed, cur_gen=%d",
+ (void *)mr_ctrl, mr_ctrl->cur_gen);
+}
+
+/* Called during rte_mempool_mem_iter() by mlx5_mr_update_mp(). */
+static void
+mlx5_mr_update_mp_cb(struct rte_mempool *mp __rte_unused, void *opaque,
+ struct rte_mempool_memhdr *memhdr,
+ unsigned mem_idx __rte_unused)
+{
+ struct mr_update_mp_data *data = opaque;
+ uint32_t lkey;
+
+ /* Stop iteration if failed in the previous walk. */
+ if (data->ret < 0)
+ return;
+ /* Register address of the chunk and update local caches. */
+ lkey = mlx5_mr_addr2mr_bh(data->dev, data->mr_ctrl,
+ (uintptr_t)memhdr->addr);
+ if (lkey == UINT32_MAX)
+ data->ret = -1;
+}
+
+/**
+ * Register entire memory chunks in a Mempool.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param mr_ctrl
+ * Pointer to per-queue MR control structure.
+ * @param mp
+ * Pointer to registering Mempool.
+ *
+ * @return
+ * 0 on success, -1 on failure.
+ */
+int
+mlx5_mr_update_mp(struct rte_eth_dev *dev, struct mlx5_mr_ctrl *mr_ctrl,
+ struct rte_mempool *mp)
+{
+ struct mr_update_mp_data data = {
+ .dev = dev,
+ .mr_ctrl = mr_ctrl,
+ .ret = 0,
+ };
+
+ rte_mempool_mem_iter(mp, mlx5_mr_update_mp_cb, &data);
+ return data.ret;
+}
+
+/**
+ * Dump all the created MRs and the global cache entries.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ */
+void
+mlx5_mr_dump_dev(struct rte_eth_dev *dev)
+{
+ struct priv *priv = dev->data->dev_private;
+ struct mlx5_mr *mr;
+ int mr_n = 0;
+ int chunk_n = 0;
+
+ rte_rwlock_read_lock(&priv->mr.rwlock);
+ /* Iterate all the existing MRs. */
+ LIST_FOREACH(mr, &priv->mr.mr_list, mr) {
+ unsigned int n;
+
+ DRV_LOG(DEBUG,
+ "port %u MR[%u], LKey = 0x%x, ms_n = %u, ms_bmp_n = %u",
+ dev->data->port_id, mr_n++,
+ rte_cpu_to_be_32(mr->ibv_mr->lkey),
+ mr->ms_n, mr->ms_bmp_n);
+ if (mr->ms_n == 0)
+ continue;
+ for (n = 0; n < mr->ms_bmp_n; ) {
+ struct mlx5_mr_cache ret = { 0, };
+
+ n = mr_find_next_chunk(mr, &ret, n);
+ if (!ret.end)
+ break;
+ DRV_LOG(DEBUG, " chunk[%u], [0x%lx, 0x%lx)",
+ chunk_n++, ret.start, ret.end);
+ }
+ }
+ DRV_LOG(DEBUG, "port %u dumping global cache", dev->data->port_id);
+ mlx5_mr_btree_dump(&priv->mr.cache);
+ rte_rwlock_read_unlock(&priv->mr.rwlock);
+}
+
+/**
+ * Release all the created MRs and resources. Remove device from memory callback
+ * list.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ */
+void
+mlx5_mr_release(struct rte_eth_dev *dev)
+{
+ struct priv *priv = dev->data->dev_private;
+ struct mlx5_mr *mr_next = LIST_FIRST(&priv->mr.mr_list);
+
+ /* Remove from memory callback device list. */
+ rte_rwlock_write_lock(&mlx5_shared_data->mem_event_rwlock);
+ LIST_REMOVE(priv, mem_event_cb);
+ rte_rwlock_write_unlock(&mlx5_shared_data->mem_event_rwlock);
+ if (rte_log_get_level(mlx5_logtype) == RTE_LOG_DEBUG)
+ mlx5_mr_dump_dev(dev);
+ rte_rwlock_write_lock(&priv->mr.rwlock);
+ /* Detach from MR list and move to free list. */
+ while (mr_next != NULL) {
+ struct mlx5_mr *mr = mr_next;
+
+ mr_next = LIST_NEXT(mr, mr);
+ LIST_REMOVE(mr, mr);
+ LIST_INSERT_HEAD(&priv->mr.mr_free_list, mr, mr);
+ }
+ LIST_INIT(&priv->mr.mr_list);
+ /* Free global cache. */
+ mlx5_mr_btree_free(&priv->mr.cache);
+ rte_rwlock_write_unlock(&priv->mr.rwlock);
+ /* Free all remaining MRs. */
+ mlx5_mr_garbage_collect(dev);
+}
diff --git a/drivers/net/mlx5/mlx5_mr.h b/drivers/net/mlx5/mlx5_mr.h
new file mode 100644
index 000000000..e0b28215c
--- /dev/null
+++ b/drivers/net/mlx5/mlx5_mr.h
@@ -0,0 +1,117 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2018 6WIND S.A.
+ * Copyright 2018 Mellanox Technologies, Ltd
+ */
+
+#ifndef RTE_PMD_MLX5_MR_H_
+#define RTE_PMD_MLX5_MR_H_
+
+#include <stddef.h>
+#include <stdint.h>
+#include <sys/queue.h>
+
+/* Verbs header. */
+/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+#include <infiniband/verbs.h>
+#include <infiniband/mlx5dv.h>
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
+#include <rte_eal_memconfig.h>
+#include <rte_ethdev.h>
+#include <rte_rwlock.h>
+#include <rte_bitmap.h>
+
+/* Memory Region object. */
+struct mlx5_mr {
+ LIST_ENTRY(mlx5_mr) mr; /**< Pointer to the prev/next entry. */
+ struct ibv_mr *ibv_mr; /* Verbs Memory Region. */
+ const struct rte_memseg_list *msl;
+ int ms_base_idx; /* Start index of msl->memseg_arr[]. */
+ int ms_n; /* Number of memsegs in use. */
+ uint32_t ms_bmp_n; /* Number of bits in memsegs bit-mask. */
+ struct rte_bitmap *ms_bmp; /* Bit-mask of memsegs belonged to MR. */
+};
+
+/* Cache entry for Memory Region. */
+struct mlx5_mr_cache {
+ uintptr_t start; /* Start address of MR. */
+ uintptr_t end; /* End address of MR. */
+ uint32_t lkey; /* rte_cpu_to_be_32(ibv_mr->lkey). */
+} __rte_packed;
+
+/* MR Cache table for Binary search. */
+struct mlx5_mr_btree {
+ uint16_t len; /* Number of entries. */
+ uint16_t size; /* Total number of entries. */
+ int overflow; /* Mark failure of table expansion. */
+ struct mlx5_mr_cache (*table)[];
+} __rte_packed;
+
+/* Per-queue MR control descriptor. */
+struct mlx5_mr_ctrl {
+ uint32_t *dev_gen_ptr; /* Generation number of device to poll. */
+ uint32_t cur_gen; /* Generation number saved to flush caches. */
+ uint16_t mru; /* Index of last hit entry in top-half cache. */
+ uint16_t head; /* Index of the oldest entry in top-half cache. */
+ struct mlx5_mr_cache cache[MLX5_MR_CACHE_N]; /* Cache for top-half. */
+ struct mlx5_mr_btree cache_bh; /* Cache for bottom-half. */
+} __rte_packed;
+
+extern struct mlx5_dev_list mlx5_mem_event_cb_list;
+extern rte_rwlock_t mlx5_mem_event_rwlock;
+
+/* First entry must be NULL for comparison. */
+#define mlx5_mr_btree_len(bt) ((bt)->len - 1)
+
+int mlx5_mr_btree_init(struct mlx5_mr_btree *bt, int n, int socket);
+void mlx5_mr_btree_free(struct mlx5_mr_btree *bt);
+void mlx5_mr_mem_event_cb(enum rte_mem_event event_type, const void *addr,
+ size_t len, void *arg);
+int mlx5_mr_update_mp(struct rte_eth_dev *dev, struct mlx5_mr_ctrl *mr_ctrl,
+ struct rte_mempool *mp);
+void mlx5_mr_dump_dev(struct rte_eth_dev *dev);
+void mlx5_mr_release(struct rte_eth_dev *dev);
+
+/**
+ * Look up LKey from given lookup table by linear search. Firstly look up the
+ * last-hit entry. If miss, the entire array is searched. If found, update the
+ * last-hit index and return LKey.
+ *
+ * @param lkp_tbl
+ * Pointer to lookup table.
+ * @param[in,out] cached_idx
+ * Pointer to last-hit index.
+ * @param n
+ * Size of lookup table.
+ * @param addr
+ * Search key.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on no match.
+ */
+static __rte_always_inline uint32_t
+mlx5_mr_lookup_cache(struct mlx5_mr_cache *lkp_tbl, uint16_t *cached_idx,
+ uint16_t n, uintptr_t addr)
+{
+ uint16_t idx;
+
+ if (likely(addr >= lkp_tbl[*cached_idx].start &&
+ addr < lkp_tbl[*cached_idx].end))
+ return lkp_tbl[*cached_idx].lkey;
+ for (idx = 0; idx < n && lkp_tbl[idx].start != 0; ++idx) {
+ if (addr >= lkp_tbl[idx].start &&
+ addr < lkp_tbl[idx].end) {
+ /* Found. */
+ *cached_idx = idx;
+ return lkp_tbl[idx].lkey;
+ }
+ }
+ return UINT32_MAX;
+}
+
+#endif /* RTE_PMD_MLX5_MR_H_ */
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index 23f909635..dbfc837d6 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -789,7 +789,7 @@ mlx5_rxq_ibv_new(struct rte_eth_dev *dev, uint16_t idx)
.addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(buf,
uintptr_t)),
.byte_count = rte_cpu_to_be_32(DATA_LEN(buf)),
- .lkey = UINT32_MAX,
+ .lkey = mlx5_rx_mb2mr(rxq_data, buf),
};
}
rxq_data->rq_db = rwq.dbrec;
@@ -970,6 +970,11 @@ mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
rte_errno = ENOMEM;
return NULL;
}
+ if (mlx5_mr_btree_init(&tmpl->rxq.mr_ctrl.cache_bh,
+ MLX5_MR_BTREE_CACHE_N, socket)) {
+ /* rte_errno is already set. */
+ goto error;
+ }
tmpl->socket = socket;
if (dev->data->dev_conf.intr_conf.rxq)
tmpl->irq = 1;
@@ -1125,6 +1130,7 @@ mlx5_rxq_release(struct rte_eth_dev *dev, uint16_t idx)
DRV_LOG(DEBUG, "port %u Rx queue %u: refcnt %d", dev->data->port_id,
rxq_ctrl->idx, rte_atomic32_read(&rxq_ctrl->refcnt));
if (rte_atomic32_dec_and_test(&rxq_ctrl->refcnt)) {
+ mlx5_mr_btree_free(&rxq_ctrl->rxq.mr_ctrl.cache_bh);
LIST_REMOVE(rxq_ctrl, next);
rte_free(rxq_ctrl);
(*priv->rxqs)[idx] = NULL;
diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 56c243495..4a4113fbe 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -1965,6 +1965,9 @@ mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
* changes.
*/
wqe->addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(rep, uintptr_t));
+ /* If there's only one MR, no need to replace LKey in WQE. */
+ if (unlikely(mlx5_mr_btree_len(&rxq->mr_ctrl.cache_bh) > 1))
+ wqe->lkey = mlx5_rx_mb2mr(rxq, rep);
if (len > DATA_LEN(seg)) {
len -= DATA_LEN(seg);
++NB_SEGS(pkt);
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index e8cad51aa..74581cf9b 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -29,6 +29,7 @@
#include "mlx5_utils.h"
#include "mlx5.h"
+#include "mlx5_mr.h"
#include "mlx5_autoconf.h"
#include "mlx5_defs.h"
#include "mlx5_prm.h"
@@ -81,6 +82,7 @@ struct mlx5_rxq_data {
uint16_t rq_ci;
uint16_t rq_pi;
uint16_t cq_ci;
+ struct mlx5_mr_ctrl mr_ctrl; /* MR control descriptor. */
volatile struct mlx5_wqe_data_seg(*wqes)[];
volatile struct mlx5_cqe(*cqes)[];
struct rxq_zip zip; /* Compressed context. */
@@ -109,8 +111,8 @@ struct mlx5_rxq_ibv {
struct mlx5_rxq_ctrl {
LIST_ENTRY(mlx5_rxq_ctrl) next; /* Pointer to the next element. */
rte_atomic32_t refcnt; /* Reference counter. */
- struct priv *priv; /* Back pointer to private data. */
struct mlx5_rxq_ibv *ibv; /* Verbs elements. */
+ struct priv *priv; /* Back pointer to private data. */
struct mlx5_rxq_data rxq; /* Data path structure. */
unsigned int socket; /* CPU socket ID for allocations. */
uint32_t tunnel_types[16]; /* Tunnel type counter. */
@@ -165,6 +167,7 @@ struct mlx5_txq_data {
uint16_t inline_max_packet_sz; /* Max packet size for inlining. */
uint32_t qp_num_8s; /* QP number shifted by 8. */
uint64_t offloads; /* Offloads for Tx Queue. */
+ struct mlx5_mr_ctrl mr_ctrl; /* MR control descriptor. */
volatile struct mlx5_cqe (*cqes)[]; /* Completion queue. */
volatile void *wqes; /* Work queue (use volatile to write into). */
volatile uint32_t *qp_db; /* Work queue doorbell. */
@@ -187,11 +190,11 @@ struct mlx5_txq_ibv {
struct mlx5_txq_ctrl {
LIST_ENTRY(mlx5_txq_ctrl) next; /* Pointer to the next element. */
rte_atomic32_t refcnt; /* Reference counter. */
- struct priv *priv; /* Back pointer to private data. */
unsigned int socket; /* CPU socket ID for allocations. */
unsigned int max_inline_data; /* Max inline data. */
unsigned int max_tso_header; /* Max TSO header size. */
struct mlx5_txq_ibv *ibv; /* Verbs queue object. */
+ struct priv *priv; /* Back pointer to private data. */
struct mlx5_txq_data txq; /* Data path structure. */
off_t uar_mmap_offset; /* UAR mmap offset for non-primary process. */
volatile void *bf_reg_orig; /* Blueflame register from verbs. */
@@ -308,6 +311,12 @@ uint16_t mlx5_tx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts,
uint16_t mlx5_rx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts,
uint16_t pkts_n);
+/* mlx5_mr.c */
+
+void mlx5_mr_flush_local_cache(struct mlx5_mr_ctrl *mr_ctrl);
+uint32_t mlx5_rx_addr2mr_bh(struct mlx5_rxq_data *rxq, uintptr_t addr);
+uint32_t mlx5_tx_addr2mr_bh(struct mlx5_txq_data *txq, uintptr_t addr);
+
#ifndef NDEBUG
/**
* Verify or set magic value in CQE.
@@ -493,14 +502,66 @@ mlx5_tx_complete(struct mlx5_txq_data *txq)
*txq->cq_db = rte_cpu_to_be_32(cq_ci);
}
+/**
+ * Query LKey from a packet buffer for Rx. No need to flush local caches for Rx
+ * as mempool is pre-configured and static.
+ *
+ * @param rxq
+ * Pointer to Rx queue structure.
+ * @param addr
+ * Address to search.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on no match.
+ */
static __rte_always_inline uint32_t
-mlx5_tx_mb2mr(struct mlx5_txq_data *txq, struct rte_mbuf *mb)
+mlx5_rx_addr2mr(struct mlx5_rxq_data *rxq, uintptr_t addr)
{
- (void)txq;
- (void)mb;
- return UINT32_MAX;
+ struct mlx5_mr_ctrl *mr_ctrl = &rxq->mr_ctrl;
+ uint32_t lkey;
+
+ /* Linear search on MR cache array. */
+ lkey = mlx5_mr_lookup_cache(mr_ctrl->cache, &mr_ctrl->mru,
+ MLX5_MR_CACHE_N, addr);
+ if (likely(lkey != UINT32_MAX))
+ return lkey;
+ /* Take slower bottom-half (Binary Search) on miss. */
+ return mlx5_rx_addr2mr_bh(rxq, addr);
}
+#define mlx5_rx_mb2mr(rxq, mb) mlx5_rx_addr2mr(rxq, (uintptr_t)((mb)->buf_addr))
+
+/**
+ * Query LKey from a packet buffer for Tx. If not found, add the mempool.
+ *
+ * @param txq
+ * Pointer to Tx queue structure.
+ * @param addr
+ * Address to search.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on no match.
+ */
+static __rte_always_inline uint32_t
+mlx5_tx_addr2mr(struct mlx5_txq_data *txq, uintptr_t addr)
+{
+ struct mlx5_mr_ctrl *mr_ctrl = &txq->mr_ctrl;
+ uint32_t lkey;
+
+ /* Check generation bit to see if there's any change on existing MRs. */
+ if (unlikely(*mr_ctrl->dev_gen_ptr != mr_ctrl->cur_gen))
+ mlx5_mr_flush_local_cache(mr_ctrl);
+ /* Linear search on MR cache array. */
+ lkey = mlx5_mr_lookup_cache(mr_ctrl->cache, &mr_ctrl->mru,
+ MLX5_MR_CACHE_N, addr);
+ if (likely(lkey != UINT32_MAX))
+ return lkey;
+ /* Take slower bottom-half (binary search) on miss. */
+ return mlx5_tx_addr2mr_bh(txq, addr);
+}
+
+#define mlx5_tx_mb2mr(rxq, mb) mlx5_tx_addr2mr(rxq, (uintptr_t)((mb)->buf_addr))
+
/**
* Ring TX queue doorbell and flush the update if requested.
*
diff --git a/drivers/net/mlx5/mlx5_rxtx_vec.h b/drivers/net/mlx5/mlx5_rxtx_vec.h
index 56c5a1b0c..64442836b 100644
--- a/drivers/net/mlx5/mlx5_rxtx_vec.h
+++ b/drivers/net/mlx5/mlx5_rxtx_vec.h
@@ -99,9 +99,13 @@ mlx5_rx_replenish_bulk_mbuf(struct mlx5_rxq_data *rxq, uint16_t n)
rxq->stats.rx_nombuf += n;
return;
}
- for (i = 0; i < n; ++i)
+ for (i = 0; i < n; ++i) {
wq[i].addr = rte_cpu_to_be_64((uintptr_t)elts[i]->buf_addr +
RTE_PKTMBUF_HEADROOM);
+ /* If there's only one MR, no need to replace LKey in WQE. */
+ if (unlikely(mlx5_mr_btree_len(&rxq->mr_ctrl.cache_bh) > 1))
+ wq[i].lkey = mlx5_rx_mb2mr(rxq, elts[i]);
+ }
rxq->rq_ci += n;
/* Prevent overflowing into consumed mbufs. */
elts_idx = rxq->rq_ci & q_mask;
diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c
index 3db6c3f35..36b7c9e2f 100644
--- a/drivers/net/mlx5/mlx5_trigger.c
+++ b/drivers/net/mlx5/mlx5_trigger.c
@@ -104,9 +104,18 @@ mlx5_rxq_start(struct rte_eth_dev *dev)
for (i = 0; i != priv->rxqs_n; ++i) {
struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_get(dev, i);
+ struct rte_mempool *mp;
if (!rxq_ctrl)
continue;
+ /* Pre-register Rx mempool. */
+ mp = rxq_ctrl->rxq.mp;
+ DRV_LOG(DEBUG,
+ "port %u Rx queue %u registering"
+ " mp %s having %u chunks",
+ dev->data->port_id, rxq_ctrl->idx,
+ mp->name, mp->nb_mem_chunks);
+ mlx5_mr_update_mp(dev, &rxq_ctrl->rxq.mr_ctrl, mp);
ret = rxq_alloc_elts(rxq_ctrl);
if (ret)
goto error;
@@ -154,6 +163,8 @@ mlx5_dev_start(struct rte_eth_dev *dev)
dev->data->port_id, strerror(rte_errno));
goto error;
}
+ if (rte_log_get_level(mlx5_logtype) == RTE_LOG_DEBUG)
+ mlx5_mr_dump_dev(dev);
ret = mlx5_rx_intr_vec_enable(dev);
if (ret) {
DRV_LOG(ERR, "port %u Rx interrupt vector creation failed",
diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index 92b3ad53c..4a37ff420 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -796,6 +796,13 @@ mlx5_txq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
rte_errno = ENOMEM;
return NULL;
}
+ if (mlx5_mr_btree_init(&tmpl->txq.mr_ctrl.cache_bh,
+ MLX5_MR_BTREE_CACHE_N, socket)) {
+ /* rte_errno is already set. */
+ goto error;
+ }
+ /* Save pointer of global generation number to check memory event. */
+ tmpl->txq.mr_ctrl.dev_gen_ptr = &priv->mr.dev_gen;
assert(desc > MLX5_TX_COMP_THRESH);
tmpl->txq.offloads = conf->offloads;
tmpl->priv = priv;
@@ -815,6 +822,9 @@ mlx5_txq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
idx, rte_atomic32_read(&tmpl->refcnt));
LIST_INSERT_HEAD(&priv->txqsctrl, tmpl, next);
return tmpl;
+error:
+ rte_free(tmpl);
+ return NULL;
}
/**
@@ -876,6 +886,7 @@ mlx5_txq_release(struct rte_eth_dev *dev, uint16_t idx)
page_size), page_size);
if (rte_atomic32_dec_and_test(&txq->refcnt)) {
txq_free_elts(txq);
+ mlx5_mr_btree_free(&txq->txq.mr_ctrl.cache_bh);
LIST_REMOVE(txq, next);
rte_free(txq);
(*priv->txqs)[idx] = NULL;
--
2.11.0
^ permalink raw reply [flat|nested] 23+ messages in thread* [dpdk-dev] [PATCH v2 3/4] net/mlx4: remove Memory Region support
2018-05-09 11:09 ` [dpdk-dev] [PATCH v2 0/4] net/mlx: " Yongseok Koh
2018-05-09 11:09 ` [dpdk-dev] [PATCH v2 1/4] net/mlx5: remove " Yongseok Koh
2018-05-09 11:09 ` [dpdk-dev] [PATCH v2 2/4] net/mlx5: add new " Yongseok Koh
@ 2018-05-09 11:09 ` Yongseok Koh
2018-05-09 11:09 ` [dpdk-dev] [PATCH v2 4/4] net/mlx4: add new " Yongseok Koh
3 siblings, 0 replies; 23+ messages in thread
From: Yongseok Koh @ 2018-05-09 11:09 UTC (permalink / raw)
To: adrien.mazarguil, nelio.laranjeiro; +Cc: dev, Yongseok Koh
This patch removes current support of Memory Region (MR) in order to
accommodate the dynamic memory hotplug patch. This patch can be compiled
but traffic can't flow and HW will raise faults. Subsequent patches will
add new MR support.
Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
---
config/common_base | 1 -
doc/guides/nics/mlx4.rst | 8 --
drivers/net/mlx4/Makefile | 4 -
drivers/net/mlx4/mlx4.h | 33 -------
drivers/net/mlx4/mlx4_mr.c | 222 -------------------------------------------
drivers/net/mlx4/mlx4_rxq.c | 11 +--
drivers/net/mlx4/mlx4_rxtx.h | 34 +------
drivers/net/mlx4/mlx4_txq.c | 66 -------------
8 files changed, 4 insertions(+), 375 deletions(-)
diff --git a/config/common_base b/config/common_base
index d525d9443..d55816e23 100644
--- a/config/common_base
+++ b/config/common_base
@@ -288,7 +288,6 @@ CONFIG_RTE_LIBRTE_AVF_16BYTE_RX_DESC=n
CONFIG_RTE_LIBRTE_MLX4_PMD=n
CONFIG_RTE_LIBRTE_MLX4_DEBUG=n
CONFIG_RTE_LIBRTE_MLX4_DLOPEN_DEPS=n
-CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE=8
#
# Compile burst-oriented Mellanox ConnectX-4 & ConnectX-5 (MLX5) PMD
diff --git a/doc/guides/nics/mlx4.rst b/doc/guides/nics/mlx4.rst
index 9564f890a..25d339d94 100644
--- a/doc/guides/nics/mlx4.rst
+++ b/doc/guides/nics/mlx4.rst
@@ -85,14 +85,6 @@ These options can be modified in the ``.config`` file.
adds additional run-time checks and debugging messages at the cost of
lower performance.
-- ``CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE`` (default **8**)
-
- Maximum number of cached memory pools (MPs) per TX queue. Each MP from
- which buffers are to be transmitted must be associated to memory regions
- (MRs). This is a slow operation that must be cached.
-
- This value is always 1 for RX queues since they use a single MP.
-
Environment variables
~~~~~~~~~~~~~~~~~~~~~
diff --git a/drivers/net/mlx4/Makefile b/drivers/net/mlx4/Makefile
index ac5b67f10..73f9d4056 100644
--- a/drivers/net/mlx4/Makefile
+++ b/drivers/net/mlx4/Makefile
@@ -69,10 +69,6 @@ else
CFLAGS += -DNDEBUG -UPEDANTIC
endif
-ifdef CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE
-CFLAGS += -DMLX4_PMD_TX_MP_CACHE=$(CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE)
-endif
-
include $(RTE_SDK)/mk/rte.lib.mk
# Generate and clean-up mlx4_autoconf.h.
diff --git a/drivers/net/mlx4/mlx4.h b/drivers/net/mlx4/mlx4.h
index 415b7d40f..e0e1b5d4c 100644
--- a/drivers/net/mlx4/mlx4.h
+++ b/drivers/net/mlx4/mlx4.h
@@ -23,7 +23,6 @@
#include <rte_ether.h>
#include <rte_interrupts.h>
#include <rte_mempool.h>
-#include <rte_spinlock.h>
#ifndef IBV_RX_HASH_INNER
/** This is not necessarily defined by supported RDMA core versions. */
@@ -42,17 +41,6 @@
/** Fixed RSS hash key size in bytes. Cannot be modified. */
#define MLX4_RSS_HASH_KEY_SIZE 40
-/**
- * Maximum number of cached Memory Pools (MPs) per TX queue. Each RTE MP
- * from which buffers are to be transmitted will have to be mapped by this
- * driver to their own Memory Region (MR). This is a slow operation.
- *
- * This value is always 1 for RX queues.
- */
-#ifndef MLX4_PMD_TX_MP_CACHE
-#define MLX4_PMD_TX_MP_CACHE 8
-#endif
-
/** Interrupt alarm timeout value in microseconds. */
#define MLX4_INTR_ALARM_TIMEOUT 100000
@@ -78,18 +66,6 @@ struct rxq;
struct txq;
struct rte_flow;
-/** Memory region descriptor. */
-struct mlx4_mr {
- LIST_ENTRY(mlx4_mr) next; /**< Next entry in list. */
- uintptr_t start; /**< Base address for memory region. */
- uintptr_t end; /**< End address for memory region. */
- uint32_t lkey; /**< L_Key extracted from @p mr. */
- uint32_t refcnt; /**< Reference count for this object. */
- struct priv *priv; /**< Back pointer to private data. */
- struct ibv_mr *mr; /**< Memory region associated with @p mp. */
- struct rte_mempool *mp; /**< Target memory pool (mempool). */
-};
-
/** Private data structure. */
struct priv {
struct rte_eth_dev *dev; /**< Ethernet device. */
@@ -112,8 +88,6 @@ struct priv {
struct mlx4_drop *drop; /**< Shared resources for drop flow rules. */
LIST_HEAD(, mlx4_rss) rss; /**< Shared targets for Rx flow rules. */
LIST_HEAD(, rte_flow) flows; /**< Configured flow rule handles. */
- LIST_HEAD(, mlx4_mr) mr; /**< Registered memory regions. */
- rte_spinlock_t mr_lock; /**< Lock for @p mr access. */
struct ether_addr mac[MLX4_MAX_MAC_ADDRESSES];
/**< Configured MAC addresses. Unused entries are zeroed. */
};
@@ -156,11 +130,4 @@ void mlx4_rxq_intr_disable(struct priv *priv);
int mlx4_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx);
int mlx4_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx);
-/* mlx4_mr.c */
-
-struct mlx4_mr *mlx4_mr_get(struct priv *priv, struct rte_mempool *mp);
-void mlx4_mr_put(struct mlx4_mr *mr);
-uint32_t mlx4_txq_add_mr(struct txq *txq, struct rte_mempool *mp,
- uint32_t i);
-
#endif /* RTE_PMD_MLX4_H_ */
diff --git a/drivers/net/mlx4/mlx4_mr.c b/drivers/net/mlx4/mlx4_mr.c
index 8d5a6741d..3c87f6849 100644
--- a/drivers/net/mlx4/mlx4_mr.c
+++ b/drivers/net/mlx4/mlx4_mr.c
@@ -30,230 +30,8 @@
#include <rte_malloc.h>
#include <rte_memory.h>
#include <rte_mempool.h>
-#include <rte_spinlock.h>
#include "mlx4_glue.h"
#include "mlx4_rxtx.h"
#include "mlx4_utils.h"
-struct mlx4_check_mempool_data {
- int ret;
- char *start;
- char *end;
-};
-
-/**
- * Called by mlx4_check_mempool() when iterating the memory chunks.
- *
- * @param[in] mp
- * Pointer to memory pool (unused).
- * @param[in, out] data
- * Pointer to shared buffer with mlx4_check_mempool().
- * @param[in] memhdr
- * Pointer to mempool chunk header.
- * @param mem_idx
- * Mempool element index (unused).
- */
-static void
-mlx4_check_mempool_cb(struct rte_mempool *mp, void *opaque,
- struct rte_mempool_memhdr *memhdr,
- unsigned int mem_idx)
-{
- struct mlx4_check_mempool_data *data = opaque;
-
- (void)mp;
- (void)mem_idx;
- /* It already failed, skip the next chunks. */
- if (data->ret != 0)
- return;
- /* It is the first chunk. */
- if (data->start == NULL && data->end == NULL) {
- data->start = memhdr->addr;
- data->end = data->start + memhdr->len;
- return;
- }
- if (data->end == memhdr->addr) {
- data->end += memhdr->len;
- return;
- }
- if (data->start == (char *)memhdr->addr + memhdr->len) {
- data->start -= memhdr->len;
- return;
- }
- /* Error, mempool is not virtually contiguous. */
- data->ret = -1;
-}
-
-/**
- * Check if a mempool can be used: it must be virtually contiguous.
- *
- * @param[in] mp
- * Pointer to memory pool.
- * @param[out] start
- * Pointer to the start address of the mempool virtual memory area.
- * @param[out] end
- * Pointer to the end address of the mempool virtual memory area.
- *
- * @return
- * 0 on success (mempool is virtually contiguous), -1 on error.
- */
-static int
-mlx4_check_mempool(struct rte_mempool *mp, uintptr_t *start, uintptr_t *end)
-{
- struct mlx4_check_mempool_data data;
-
- memset(&data, 0, sizeof(data));
- rte_mempool_mem_iter(mp, mlx4_check_mempool_cb, &data);
- *start = (uintptr_t)data.start;
- *end = (uintptr_t)data.end;
- return data.ret;
-}
-
-/**
- * Obtain a memory region from a memory pool.
- *
- * If a matching memory region already exists, it is returned with its
- * reference count incremented, otherwise a new one is registered.
- *
- * @param priv
- * Pointer to private structure.
- * @param mp
- * Pointer to memory pool.
- *
- * @return
- * Memory region pointer, NULL in case of error and rte_errno is set.
- */
-struct mlx4_mr *
-mlx4_mr_get(struct priv *priv, struct rte_mempool *mp)
-{
- const struct rte_memseg *ms;
- uintptr_t start;
- uintptr_t end;
- struct mlx4_mr *mr;
-
- if (mlx4_check_mempool(mp, &start, &end) != 0) {
- rte_errno = EINVAL;
- ERROR("mempool %p: not virtually contiguous",
- (void *)mp);
- return NULL;
- }
- DEBUG("mempool %p area start=%p end=%p size=%zu",
- (void *)mp, (void *)start, (void *)end,
- (size_t)(end - start));
- /* Round start and end to page boundary if found in memory segments. */
- ms = rte_mem_virt2memseg((void *)start, NULL);
- if (ms != NULL)
- start = RTE_ALIGN_FLOOR(start, ms->hugepage_sz);
- end = RTE_ALIGN_CEIL(end, ms->hugepage_sz);
- DEBUG("mempool %p using start=%p end=%p size=%zu for MR",
- (void *)mp, (void *)start, (void *)end,
- (size_t)(end - start));
- rte_spinlock_lock(&priv->mr_lock);
- LIST_FOREACH(mr, &priv->mr, next)
- if (mp == mr->mp && start >= mr->start && end <= mr->end)
- break;
- if (mr) {
- ++mr->refcnt;
- goto release;
- }
- mr = rte_malloc(__func__, sizeof(*mr), 0);
- if (!mr) {
- rte_errno = ENOMEM;
- goto release;
- }
- *mr = (struct mlx4_mr){
- .start = start,
- .end = end,
- .refcnt = 1,
- .priv = priv,
- .mr = mlx4_glue->reg_mr(priv->pd, (void *)start, end - start,
- IBV_ACCESS_LOCAL_WRITE),
- .mp = mp,
- };
- if (mr->mr) {
- mr->lkey = mr->mr->lkey;
- LIST_INSERT_HEAD(&priv->mr, mr, next);
- } else {
- rte_free(mr);
- mr = NULL;
- rte_errno = errno ? errno : EINVAL;
- }
-release:
- rte_spinlock_unlock(&priv->mr_lock);
- return mr;
-}
-
-/**
- * Release a memory region.
- *
- * This function decrements its reference count and destroys it after
- * reaching 0.
- *
- * Note to avoid race conditions given this function may be used from the
- * data plane, it's extremely important that each user holds its own
- * reference.
- *
- * @param mr
- * Memory region to release.
- */
-void
-mlx4_mr_put(struct mlx4_mr *mr)
-{
- struct priv *priv = mr->priv;
-
- rte_spinlock_lock(&priv->mr_lock);
- assert(mr->refcnt);
- if (--mr->refcnt)
- goto release;
- LIST_REMOVE(mr, next);
- claim_zero(mlx4_glue->dereg_mr(mr->mr));
- rte_free(mr);
-release:
- rte_spinlock_unlock(&priv->mr_lock);
-}
-
-/**
- * Add memory region (MR) <-> memory pool (MP) association to txq->mp2mr[].
- * If mp2mr[] is full, remove an entry first.
- *
- * @param txq
- * Pointer to Tx queue structure.
- * @param[in] mp
- * Memory pool for which a memory region lkey must be added.
- * @param[in] i
- * Index in memory pool (MP) where to add memory region (MR).
- *
- * @return
- * Added mr->lkey on success, (uint32_t)-1 on failure.
- */
-uint32_t
-mlx4_txq_add_mr(struct txq *txq, struct rte_mempool *mp, uint32_t i)
-{
- struct mlx4_mr *mr;
-
- /* Add a new entry, register MR first. */
- DEBUG("%p: discovered new memory pool \"%s\" (%p)",
- (void *)txq, mp->name, (void *)mp);
- mr = mlx4_mr_get(txq->priv, mp);
- if (unlikely(mr == NULL)) {
- DEBUG("%p: unable to configure MR, mlx4_mr_get() failed",
- (void *)txq);
- return (uint32_t)-1;
- }
- if (unlikely(i == RTE_DIM(txq->mp2mr))) {
- /* Table is full, remove oldest entry. */
- DEBUG("%p: MR <-> MP table full, dropping oldest entry.",
- (void *)txq);
- --i;
- mlx4_mr_put(txq->mp2mr[0].mr);
- memmove(&txq->mp2mr[0], &txq->mp2mr[1],
- (sizeof(txq->mp2mr) - sizeof(txq->mp2mr[0])));
- }
- /* Store the new entry. */
- txq->mp2mr[i].mp = mp;
- txq->mp2mr[i].mr = mr;
- txq->mp2mr[i].lkey = mr->lkey;
- DEBUG("%p: new MR lkey for MP \"%s\" (%p): 0x%08" PRIu32,
- (void *)txq, mp->name, (void *)mp, txq->mp2mr[i].lkey);
- return txq->mp2mr[i].lkey;
-}
diff --git a/drivers/net/mlx4/mlx4_rxq.c b/drivers/net/mlx4/mlx4_rxq.c
index 65f099423..5621d5bd4 100644
--- a/drivers/net/mlx4/mlx4_rxq.c
+++ b/drivers/net/mlx4/mlx4_rxq.c
@@ -583,7 +583,7 @@ mlx4_rxq_attach(struct rxq *rxq)
.addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(buf,
uintptr_t)),
.byte_count = rte_cpu_to_be_32(buf->data_len),
- .lkey = rte_cpu_to_be_32(rxq->mr->lkey),
+ .lkey = UINT32_MAX,
};
(*elts)[i] = buf;
}
@@ -883,13 +883,6 @@ mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1 << rxq->sges_n);
goto error;
}
- /* Use the entire Rx mempool as the memory region. */
- rxq->mr = mlx4_mr_get(priv, mp);
- if (!rxq->mr) {
- ERROR("%p: MR creation failure: %s",
- (void *)dev, strerror(rte_errno));
- goto error;
- }
if (dev->data->dev_conf.intr_conf.rxq) {
rxq->channel = mlx4_glue->create_comp_channel(priv->ctx);
if (rxq->channel == NULL) {
@@ -947,7 +940,5 @@ mlx4_rx_queue_release(void *dpdk_rxq)
assert(!rxq->rq_db);
if (rxq->channel)
claim_zero(mlx4_glue->destroy_comp_channel(rxq->channel));
- if (rxq->mr)
- mlx4_mr_put(rxq->mr);
rte_free(rxq);
}
diff --git a/drivers/net/mlx4/mlx4_rxtx.h b/drivers/net/mlx4/mlx4_rxtx.h
index 2dfee957f..2f9d3798b 100644
--- a/drivers/net/mlx4/mlx4_rxtx.h
+++ b/drivers/net/mlx4/mlx4_rxtx.h
@@ -39,7 +39,6 @@ struct mlx4_rxq_stats {
struct rxq {
struct priv *priv; /**< Back pointer to private data. */
struct rte_mempool *mp; /**< Memory pool for allocations. */
- struct mlx4_mr *mr; /**< Memory region. */
struct ibv_cq *cq; /**< Completion queue. */
struct ibv_wq *wq; /**< Work queue. */
struct ibv_comp_channel *channel; /**< Rx completion channel. */
@@ -109,11 +108,6 @@ struct txq {
uint32_t lb:1; /**< Whether packets should be looped back by eSwitch. */
uint8_t *bounce_buf;
/**< Memory used for storing the first DWORD of data TXBBs. */
- struct {
- const struct rte_mempool *mp; /**< Cached memory pool. */
- struct mlx4_mr *mr; /**< Memory region (for mp). */
- uint32_t lkey; /**< mr->lkey copy. */
- } mp2mr[MLX4_PMD_TX_MP_CACHE]; /**< MP to MR translation table. */
struct priv *priv; /**< Back pointer to private data. */
unsigned int socket; /**< CPU socket ID for allocations. */
struct ibv_cq *cq; /**< Completion queue. */
@@ -161,34 +155,12 @@ int mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx,
const struct rte_eth_txconf *conf);
void mlx4_tx_queue_release(void *dpdk_txq);
-/**
- * Get memory region (MR) <-> memory pool (MP) association from txq->mp2mr[].
- * Call mlx4_txq_add_mr() if MP is not registered yet.
- *
- * @param txq
- * Pointer to Tx queue structure.
- * @param[in] mp
- * Memory pool for which a memory region lkey must be returned.
- *
- * @return
- * mr->lkey on success, (uint32_t)-1 on failure.
- */
static inline uint32_t
mlx4_txq_mp2mr(struct txq *txq, struct rte_mempool *mp)
{
- unsigned int i;
-
- for (i = 0; (i != RTE_DIM(txq->mp2mr)); ++i) {
- if (unlikely(txq->mp2mr[i].mp == NULL)) {
- /* Unknown MP, add a new MR for it. */
- break;
- }
- if (txq->mp2mr[i].mp == mp) {
- /* MP found MP. */
- return txq->mp2mr[i].lkey;
- }
- }
- return mlx4_txq_add_mr(txq, mp, i);
+ (void)txq;
+ (void)mp;
+ return UINT32_MAX;
}
#endif /* MLX4_RXTX_H_ */
diff --git a/drivers/net/mlx4/mlx4_txq.c b/drivers/net/mlx4/mlx4_txq.c
index fe6a8e07e..5ea09b0b0 100644
--- a/drivers/net/mlx4/mlx4_txq.c
+++ b/drivers/net/mlx4/mlx4_txq.c
@@ -63,64 +63,6 @@ mlx4_txq_free_elts(struct txq *txq)
txq->elts_tail = txq->elts_head;
}
-struct txq_mp2mr_mbuf_check_data {
- int ret;
-};
-
-/**
- * Callback function for rte_mempool_obj_iter() to check whether a given
- * mempool object looks like a mbuf.
- *
- * @param[in] mp
- * The mempool pointer
- * @param[in] arg
- * Context data (struct mlx4_txq_mp2mr_mbuf_check_data). Contains the
- * return value.
- * @param[in] obj
- * Object address.
- * @param index
- * Object index, unused.
- */
-static void
-mlx4_txq_mp2mr_mbuf_check(struct rte_mempool *mp, void *arg, void *obj,
- uint32_t index)
-{
- struct txq_mp2mr_mbuf_check_data *data = arg;
- struct rte_mbuf *buf = obj;
-
- (void)index;
- /*
- * Check whether mbuf structure fits element size and whether mempool
- * pointer is valid.
- */
- if (sizeof(*buf) > mp->elt_size || buf->pool != mp)
- data->ret = -1;
-}
-
-/**
- * Iterator function for rte_mempool_walk() to register existing mempools and
- * fill the MP to MR cache of a Tx queue.
- *
- * @param[in] mp
- * Memory Pool to register.
- * @param *arg
- * Pointer to Tx queue structure.
- */
-static void
-mlx4_txq_mp2mr_iter(struct rte_mempool *mp, void *arg)
-{
- struct txq *txq = arg;
- struct txq_mp2mr_mbuf_check_data data = {
- .ret = 0,
- };
-
- /* Register mempool only if the first element looks like a mbuf. */
- if (rte_mempool_obj_iter(mp, mlx4_txq_mp2mr_mbuf_check, &data) == 0 ||
- data.ret == -1)
- return;
- mlx4_txq_mp2mr(txq, mp);
-}
-
/**
* Retrieves information needed in order to directly access the Tx queue.
*
@@ -404,8 +346,6 @@ mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
/* Save first wqe pointer in the first element. */
(&(*txq->elts)[0])->wqe =
(volatile struct mlx4_wqe_ctrl_seg *)txq->msq.buf;
- /* Pre-register known mempools. */
- rte_mempool_walk(mlx4_txq_mp2mr_iter, txq);
DEBUG("%p: adding Tx queue %p to list", (void *)dev, (void *)txq);
dev->data->tx_queues[idx] = txq;
return 0;
@@ -446,11 +386,5 @@ mlx4_tx_queue_release(void *dpdk_txq)
claim_zero(mlx4_glue->destroy_qp(txq->qp));
if (txq->cq)
claim_zero(mlx4_glue->destroy_cq(txq->cq));
- for (i = 0; i != RTE_DIM(txq->mp2mr); ++i) {
- if (!txq->mp2mr[i].mp)
- break;
- assert(txq->mp2mr[i].mr);
- mlx4_mr_put(txq->mp2mr[i].mr);
- }
rte_free(txq);
}
--
2.11.0
^ permalink raw reply [flat|nested] 23+ messages in thread* [dpdk-dev] [PATCH v2 4/4] net/mlx4: add new Memory Region support
2018-05-09 11:09 ` [dpdk-dev] [PATCH v2 0/4] net/mlx: " Yongseok Koh
` (2 preceding siblings ...)
2018-05-09 11:09 ` [dpdk-dev] [PATCH v2 3/4] net/mlx4: remove " Yongseok Koh
@ 2018-05-09 11:09 ` Yongseok Koh
2018-05-09 23:12 ` Ferruh Yigit
3 siblings, 1 reply; 23+ messages in thread
From: Yongseok Koh @ 2018-05-09 11:09 UTC (permalink / raw)
To: adrien.mazarguil, nelio.laranjeiro; +Cc: dev, Yongseok Koh
This is the new design of Memory Region (MR) for mlx PMD, in order to:
- Accommodate the new memory hotplug model.
- Support non-contiguous Mempool.
There are multiple layers for MR search.
L0 is to look up the last-hit entry which is pointed by mr_ctrl->mru (Most
Recently Used). If L0 misses, L1 is to look up the address in a fixed-sized
array by linear search. L0/L1 is in an inline function -
mlx4_mr_lookup_cache().
If L1 misses, the bottom-half function is called to look up the address
from the bigger local cache of the queue. This is L2 - mlx4_mr_addr2mr_bh()
and it is not an inline function. Data structure for L2 is the Binary Tree.
If L2 misses, the search falls into the slowest path which takes locks in
order to access global device cache (priv->mr.cache) which is also a B-tree
and caches the original MR list (priv->mr.mr_list) of the device. Unless
the global cache is overflowed, it is all-inclusive of the MR list. This is
L3 - mlx4_mr_lookup_dev(). The size of the L3 cache table is limited and
can't be expanded on the fly due to deadlock. Refer to the comments in the
code for the details - mr_lookup_dev(). If L3 is overflowed, the list will
have to be searched directly bypassing the cache although it is slower.
If L3 misses, a new MR for the address should be created -
mlx4_mr_create(). When it creates a new MR, it tries to register adjacent
memsegs as much as possible which are virtually contiguous around the
address. This must take two locks - memory_hotplug_lock and
priv->mr.rwlock. Due to memory_hotplug_lock, there can't be any
allocation/free of memory inside.
In the free callback of the memory hotplug event, freed space is searched
from the MR list and corresponding bits are cleared from the bitmap of MRs.
This can fragment a MR and the MR will have multiple search entries in the
caches. Once there's a change by the event, the global cache must be
rebuilt and all the per-queue caches will be flushed as well. If memory is
frequently freed in run-time, that may cause jitter on dataplane processing
in the worst case by incurring MR cache flush and rebuild. But, it would be
the least probable scenario.
To guarantee the most optimal performance, it is highly recommended to use
an EAL option - '--socket-mem'. Then, the reserved memory will be pinned
and won't be freed dynamically. And it is also recommended to configure
per-lcore cache of Mempool. Even though there're many MRs for a device or
MRs are highly fragmented, the cache of Mempool will be much helpful to
reduce misses on per-queue caches anyway.
'--legacy-mem' is also supported.
Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
---
doc/guides/nics/mlx4.rst | 6 +
drivers/net/mlx4/mlx4.c | 26 +
drivers/net/mlx4/mlx4.h | 14 +
drivers/net/mlx4/mlx4_mr.c | 1140 ++++++++++++++++++++++++++++++++++++++++++
drivers/net/mlx4/mlx4_mr.h | 122 +++++
drivers/net/mlx4/mlx4_rxq.c | 14 +-
drivers/net/mlx4/mlx4_rxtx.c | 35 +-
drivers/net/mlx4/mlx4_rxtx.h | 71 ++-
drivers/net/mlx4/mlx4_txq.c | 8 +
9 files changed, 1405 insertions(+), 31 deletions(-)
create mode 100644 drivers/net/mlx4/mlx4_mr.h
diff --git a/doc/guides/nics/mlx4.rst b/doc/guides/nics/mlx4.rst
index 25d339d94..0c4c03780 100644
--- a/doc/guides/nics/mlx4.rst
+++ b/doc/guides/nics/mlx4.rst
@@ -363,6 +363,12 @@ Performance tuning
The XXX can be different on different systems. Make sure to configure
according to the setpci output.
+6. To minimize overhead of searching Memory Regions:
+
+ - '--socket-mem' is recommended to pin memory by predictable amount.
+ - Configure per-lcore cache when creating Mempools for packet buffer.
+ - Refrain from dynamically allocating/freeing memory in run-time.
+
Usage example
-------------
diff --git a/drivers/net/mlx4/mlx4.c b/drivers/net/mlx4/mlx4.c
index 47451b651..abed2f5dc 100644
--- a/drivers/net/mlx4/mlx4.c
+++ b/drivers/net/mlx4/mlx4.c
@@ -44,9 +44,15 @@
#include "mlx4.h"
#include "mlx4_glue.h"
#include "mlx4_flow.h"
+#include "mlx4_mr.h"
#include "mlx4_rxtx.h"
#include "mlx4_utils.h"
+struct mlx4_dev_list mlx4_mem_event_cb_list =
+ LIST_HEAD_INITIALIZER(mlx4_mem_event_cb_list);
+
+rte_rwlock_t mlx4_mem_event_rwlock = RTE_RWLOCK_INITIALIZER;
+
/** Configuration structure for device arguments. */
struct mlx4_conf {
struct {
@@ -92,6 +98,20 @@ mlx4_dev_configure(struct rte_eth_dev *dev)
if (ret)
ERROR("%p: interrupt handler installation failed",
(void *)dev);
+ /*
+ * Once the device is added to the list of memory event callback, its
+ * global MR cache table cannot be expanded on the fly because of
+ * deadlock. If it overflows, lookup should be done by searching MR list
+ * linearly, which is slow.
+ */
+ if (mlx4_mr_btree_init(&priv->mr.cache, MLX4_MR_BTREE_CACHE_N * 2,
+ dev->device->numa_node)) {
+ /* rte_errno is already set. */
+ return -rte_errno;
+ }
+ rte_rwlock_write_lock(&mlx4_mem_event_rwlock);
+ LIST_INSERT_HEAD(&mlx4_mem_event_cb_list, priv, mem_event_cb);
+ rte_rwlock_write_unlock(&mlx4_mem_event_rwlock);
exit:
return ret;
}
@@ -125,6 +145,9 @@ mlx4_dev_start(struct rte_eth_dev *dev)
(void *)dev, strerror(-ret));
goto err;
}
+#ifndef NDEBUG
+ mlx4_mr_dump_dev(dev);
+#endif
ret = mlx4_rxq_intr_enable(priv);
if (ret) {
ERROR("%p: interrupt handler installation failed",
@@ -200,6 +223,7 @@ mlx4_dev_close(struct rte_eth_dev *dev)
mlx4_rx_queue_release(dev->data->rx_queues[i]);
for (i = 0; i != dev->data->nb_tx_queues; ++i)
mlx4_tx_queue_release(dev->data->tx_queues[i]);
+ mlx4_mr_release(dev);
if (priv->pd != NULL) {
assert(priv->ctx != NULL);
claim_zero(mlx4_glue->dealloc_pd(priv->pd));
@@ -964,6 +988,8 @@ rte_mlx4_pmd_init(void)
}
mlx4_glue->fork_init();
rte_pci_register(&mlx4_driver);
+ rte_mem_event_callback_register("MLX4_MEM_EVENT_CB",
+ mlx4_mr_mem_event_cb, NULL);
}
RTE_PMD_EXPORT_NAME(net_mlx4, __COUNTER__);
diff --git a/drivers/net/mlx4/mlx4.h b/drivers/net/mlx4/mlx4.h
index e0e1b5d4c..300cb4d7a 100644
--- a/drivers/net/mlx4/mlx4.h
+++ b/drivers/net/mlx4/mlx4.h
@@ -23,6 +23,9 @@
#include <rte_ether.h>
#include <rte_interrupts.h>
#include <rte_mempool.h>
+#include <rte_rwlock.h>
+
+#include "mlx4_mr.h"
#ifndef IBV_RX_HASH_INNER
/** This is not necessarily defined by supported RDMA core versions. */
@@ -66,8 +69,12 @@ struct rxq;
struct txq;
struct rte_flow;
+LIST_HEAD(mlx4_dev_list, priv);
+LIST_HEAD(mlx4_mr_list, mlx4_mr);
+
/** Private data structure. */
struct priv {
+ LIST_ENTRY(priv) mem_event_cb; /* Called by memory event callback. */
struct rte_eth_dev *dev; /**< Ethernet device. */
struct ibv_context *ctx; /**< Verbs context. */
struct ibv_device_attr device_attr; /**< Device properties. */
@@ -86,6 +93,13 @@ struct priv {
uint64_t hw_rss_sup; /**< Supported RSS hash fields (Verbs format). */
struct rte_intr_handle intr_handle; /**< Port interrupt handle. */
struct mlx4_drop *drop; /**< Shared resources for drop flow rules. */
+ struct {
+ uint32_t dev_gen; /* Generation number to flush local caches. */
+ rte_rwlock_t rwlock; /* MR Lock. */
+ struct mlx4_mr_btree cache; /* Global MR cache table. */
+ struct mlx4_mr_list mr_list; /* Registered MR list. */
+ struct mlx4_mr_list mr_free_list; /* Freed MR list. */
+ } mr;
LIST_HEAD(, mlx4_rss) rss; /**< Shared targets for Rx flow rules. */
LIST_HEAD(, rte_flow) flows; /**< Configured flow rule handles. */
struct ether_addr mac[MLX4_MAX_MAC_ADDRESSES];
diff --git a/drivers/net/mlx4/mlx4_mr.c b/drivers/net/mlx4/mlx4_mr.c
index 3c87f6849..4e729efda 100644
--- a/drivers/net/mlx4/mlx4_mr.c
+++ b/drivers/net/mlx4/mlx4_mr.c
@@ -30,8 +30,1148 @@
#include <rte_malloc.h>
#include <rte_memory.h>
#include <rte_mempool.h>
+#include <rte_rwlock.h>
#include "mlx4_glue.h"
+#include "mlx4_mr.h"
#include "mlx4_rxtx.h"
#include "mlx4_utils.h"
+struct mr_find_contig_memsegs_data {
+ uintptr_t addr;
+ uintptr_t start;
+ uintptr_t end;
+ const struct rte_memseg_list *msl;
+};
+
+struct mr_update_mp_data {
+ struct rte_eth_dev *dev;
+ struct mlx4_mr_ctrl *mr_ctrl;
+ int ret;
+};
+
+/**
+ * Expand B-tree table to a given size. Can't be called with holding
+ * memory_hotplug_lock or priv->mr.rwlock due to rte_realloc().
+ *
+ * @param bt
+ * Pointer to B-tree structure.
+ * @param n
+ * Number of entries for expansion.
+ *
+ * @return
+ * 0 on success, -1 on failure.
+ */
+static int
+mr_btree_expand(struct mlx4_mr_btree *bt, int n)
+{
+ void *mem;
+ int ret = 0;
+
+ if (n <= bt->size)
+ return ret;
+ /*
+ * Downside of directly using rte_realloc() is that SOCKET_ID_ANY is
+ * used inside if there's no room to expand. Because this is a quite
+ * rare case and a part of very slow path, it is very acceptable.
+ * Initially cache_bh[] will be given practically enough space and once
+ * it is expanded, expansion wouldn't be needed again ever.
+ */
+ mem = rte_realloc(bt->table, n * sizeof(struct mlx4_mr_cache), 0);
+ if (mem == NULL) {
+ /* Not an error, B-tree search will be skipped. */
+ WARN("failed to expand MR B-tree (%p) table", (void *)bt);
+ ret = -1;
+ } else {
+ DEBUG("expanded MR B-tree table (size=%u)", n);
+ bt->table = mem;
+ bt->size = n;
+ }
+ return ret;
+}
+
+/**
+ * Look up LKey from given B-tree lookup table, store the last index and return
+ * searched LKey.
+ *
+ * @param bt
+ * Pointer to B-tree structure.
+ * @param[out] idx
+ * Pointer to index. Even on search failure, returns index where it stops
+ * searching so that index can be used when inserting a new entry.
+ * @param addr
+ * Search key.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on no match.
+ */
+static uint32_t
+mr_btree_lookup(struct mlx4_mr_btree *bt, uint16_t *idx, uintptr_t addr)
+{
+ struct mlx4_mr_cache *lkp_tbl;
+ uint16_t n;
+ uint16_t base = 0;
+
+ assert(bt != NULL);
+ lkp_tbl = *bt->table;
+ n = bt->len;
+ /* First entry must be NULL for comparison. */
+ assert(bt->len > 0 || (lkp_tbl[0].start == 0 &&
+ lkp_tbl[0].lkey == UINT32_MAX));
+ /* Binary search. */
+ do {
+ register uint16_t delta = n >> 1;
+
+ if (addr < lkp_tbl[base + delta].start) {
+ n = delta;
+ } else {
+ base += delta;
+ n -= delta;
+ }
+ } while (n > 1);
+ assert(addr >= lkp_tbl[base].start);
+ *idx = base;
+ if (addr < lkp_tbl[base].end)
+ return lkp_tbl[base].lkey;
+ /* Not found. */
+ return UINT32_MAX;
+}
+
+/**
+ * Insert an entry to B-tree lookup table.
+ *
+ * @param bt
+ * Pointer to B-tree structure.
+ * @param entry
+ * Pointer to new entry to insert.
+ *
+ * @return
+ * 0 on success, -1 on failure.
+ */
+static int
+mr_btree_insert(struct mlx4_mr_btree *bt, struct mlx4_mr_cache *entry)
+{
+ struct mlx4_mr_cache *lkp_tbl;
+ uint16_t idx = 0;
+ size_t shift;
+
+ assert(bt != NULL);
+ assert(bt->len <= bt->size);
+ assert(bt->len > 0);
+ lkp_tbl = *bt->table;
+ /* Find out the slot for insertion. */
+ if (mr_btree_lookup(bt, &idx, entry->start) != UINT32_MAX) {
+ DEBUG("abort insertion to B-tree(%p):"
+ " already exist at idx=%u [0x%lx, 0x%lx) lkey=0x%x",
+ (void *)bt, idx, entry->start, entry->end, entry->lkey);
+ /* Already exist, return. */
+ return 0;
+ }
+ /* If table is full, return error. */
+ if (unlikely(bt->len == bt->size)) {
+ bt->overflow = 1;
+ return -1;
+ }
+ /* Insert entry. */
+ ++idx;
+ shift = (bt->len - idx) * sizeof(struct mlx4_mr_cache);
+ if (shift)
+ memmove(&lkp_tbl[idx + 1], &lkp_tbl[idx], shift);
+ lkp_tbl[idx] = *entry;
+ bt->len++;
+ DEBUG("inserted B-tree(%p)[%u], [0x%lx, 0x%lx) lkey=0x%x",
+ (void *)bt, idx, entry->start, entry->end, entry->lkey);
+ return 0;
+}
+
+/**
+ * Initialize B-tree and allocate memory for lookup table.
+ *
+ * @param bt
+ * Pointer to B-tree structure.
+ * @param n
+ * Number of entries to allocate.
+ * @param socket
+ * NUMA socket on which memory must be allocated.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx4_mr_btree_init(struct mlx4_mr_btree *bt, int n, int socket)
+{
+ if (bt == NULL) {
+ rte_errno = EINVAL;
+ return -rte_errno;
+ }
+ memset(bt, 0, sizeof(*bt));
+ bt->table = rte_calloc_socket("B-tree table",
+ n, sizeof(struct mlx4_mr_cache),
+ 0, socket);
+ if (bt->table == NULL) {
+ rte_errno = ENOMEM;
+ ERROR("failed to allocate memory for btree cache on socket %d",
+ socket);
+ return -rte_errno;
+ }
+ bt->size = n;
+ /* First entry must be NULL for binary search. */
+ (*bt->table)[bt->len++] = (struct mlx4_mr_cache) {
+ .lkey = UINT32_MAX,
+ };
+ DEBUG("initialized B-tree %p with table %p",
+ (void *)bt, (void *)bt->table);
+ return 0;
+}
+
+/**
+ * Free B-tree resources.
+ *
+ * @param bt
+ * Pointer to B-tree structure.
+ */
+void
+mlx4_mr_btree_free(struct mlx4_mr_btree *bt)
+{
+ if (bt == NULL)
+ return;
+ DEBUG("freeing B-tree %p with table %p", (void *)bt, (void *)bt->table);
+ rte_free(bt->table);
+ memset(bt, 0, sizeof(*bt));
+}
+
+#ifndef NDEBUG
+/**
+ * Dump all the entries in a B-tree
+ *
+ * @param bt
+ * Pointer to B-tree structure.
+ */
+void
+mlx4_mr_btree_dump(struct mlx4_mr_btree *bt)
+{
+ int idx;
+ struct mlx4_mr_cache *lkp_tbl;
+
+ if (bt == NULL)
+ return;
+ lkp_tbl = *bt->table;
+ for (idx = 0; idx < bt->len; ++idx) {
+ struct mlx4_mr_cache *entry = &lkp_tbl[idx];
+
+ DEBUG("B-tree(%p)[%u], [0x%lx, 0x%lx) lkey=0x%x",
+ (void *)bt, idx, entry->start, entry->end, entry->lkey);
+ }
+}
+#endif
+
+/**
+ * Find virtually contiguous memory chunk in a given MR.
+ *
+ * @param dev
+ * Pointer to MR structure.
+ * @param[out] entry
+ * Pointer to returning MR cache entry. If not found, this will not be
+ * updated.
+ * @param start_idx
+ * Start index of the memseg bitmap.
+ *
+ * @return
+ * Next index to go on lookup.
+ */
+static int
+mr_find_next_chunk(struct mlx4_mr *mr, struct mlx4_mr_cache *entry,
+ int base_idx)
+{
+ uintptr_t start = 0;
+ uintptr_t end = 0;
+ uint32_t idx = 0;
+
+ for (idx = base_idx; idx < mr->ms_bmp_n; ++idx) {
+ if (rte_bitmap_get(mr->ms_bmp, idx)) {
+ const struct rte_memseg_list *msl;
+ const struct rte_memseg *ms;
+
+ msl = mr->msl;
+ ms = rte_fbarray_get(&msl->memseg_arr,
+ mr->ms_base_idx + idx);
+ assert(msl->page_sz == ms->hugepage_sz);
+ if (!start)
+ start = ms->addr_64;
+ end = ms->addr_64 + ms->hugepage_sz;
+ } else if (start) {
+ /* Passed the end of a fragment. */
+ break;
+ }
+ }
+ if (start) {
+ /* Found one chunk. */
+ entry->start = start;
+ entry->end = end;
+ entry->lkey = rte_cpu_to_be_32(mr->ibv_mr->lkey);
+ }
+ return idx;
+}
+
+/**
+ * Insert a MR to the global B-tree cache. It may fail due to low-on-memory.
+ * Then, this entry will have to be searched by mr_lookup_dev_list() in
+ * mlx4_mr_create() on miss.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param mr
+ * Pointer to MR to insert.
+ *
+ * @return
+ * 0 on success, -1 on failure.
+ */
+static int
+mr_insert_dev_cache(struct rte_eth_dev *dev, struct mlx4_mr *mr)
+{
+ struct priv *priv = dev->data->dev_private;
+ unsigned int n;
+
+ DEBUG("port %u inserting MR(%p) to global cache",
+ dev->data->port_id, (void *)mr);
+ for (n = 0; n < mr->ms_bmp_n; ) {
+ struct mlx4_mr_cache entry = { 0, };
+
+ /* Find a contiguous chunk and advance the index. */
+ n = mr_find_next_chunk(mr, &entry, n);
+ if (!entry.end)
+ break;
+ if (mr_btree_insert(&priv->mr.cache, &entry) < 0) {
+ /*
+ * Overflowed, but the global table cannot be expanded
+ * because of deadlock.
+ */
+ return -1;
+ }
+ }
+ return 0;
+}
+
+/**
+ * Look up address in the original global MR list.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param[out] entry
+ * Pointer to returning MR cache entry. If no match, this will not be updated.
+ * @param addr
+ * Search key.
+ *
+ * @return
+ * Found MR on match, NULL otherwise.
+ */
+static struct mlx4_mr *
+mr_lookup_dev_list(struct rte_eth_dev *dev, struct mlx4_mr_cache *entry,
+ uintptr_t addr)
+{
+ struct priv *priv = dev->data->dev_private;
+ struct mlx4_mr *mr;
+
+ /* Iterate all the existing MRs. */
+ LIST_FOREACH(mr, &priv->mr.mr_list, mr) {
+ unsigned int n;
+
+ if (mr->ms_n == 0)
+ continue;
+ for (n = 0; n < mr->ms_bmp_n; ) {
+ struct mlx4_mr_cache ret = { 0, };
+
+ n = mr_find_next_chunk(mr, &ret, n);
+ if (addr >= ret.start && addr < ret.end) {
+ /* Found. */
+ *entry = ret;
+ return mr;
+ }
+ }
+ }
+ return NULL;
+}
+
+/**
+ * Look up address on device.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param[out] entry
+ * Pointer to returning MR cache entry. If no match, this will not be updated.
+ * @param addr
+ * Search key.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on failure and rte_errno is set.
+ */
+static uint32_t
+mr_lookup_dev(struct rte_eth_dev *dev, struct mlx4_mr_cache *entry,
+ uintptr_t addr)
+{
+ struct priv *priv = dev->data->dev_private;
+ uint16_t idx;
+ uint32_t lkey = UINT32_MAX;
+ struct mlx4_mr *mr;
+
+ /*
+ * If the global cache has overflowed since it failed to expand the
+ * B-tree table, it can't have all the exisitng MRs. Then, the address
+ * has to be searched by traversing the original MR list instead, which
+ * is very slow path. Otherwise, the global cache is all inclusive.
+ */
+ if (!unlikely(priv->mr.cache.overflow)) {
+ lkey = mr_btree_lookup(&priv->mr.cache, &idx, addr);
+ if (lkey != UINT32_MAX)
+ *entry = (*priv->mr.cache.table)[idx];
+ } else {
+ /* Falling back to the slowest path. */
+ mr = mr_lookup_dev_list(dev, entry, addr);
+ if (mr != NULL)
+ lkey = entry->lkey;
+ }
+ assert(lkey == UINT32_MAX || (addr >= entry->start &&
+ addr < entry->end));
+ return lkey;
+}
+
+/**
+ * Free MR resources. MR lock must not be held to avoid a deadlock. rte_free()
+ * can raise memory free event and the callback function will spin on the lock.
+ *
+ * @param mr
+ * Pointer to MR to free.
+ */
+static void
+mr_free(struct mlx4_mr *mr)
+{
+ if (mr == NULL)
+ return;
+ DEBUG("freeing MR(%p):", (void *)mr);
+ if (mr->ibv_mr != NULL)
+ claim_zero(mlx4_glue->dereg_mr(mr->ibv_mr));
+ if (mr->ms_bmp != NULL)
+ rte_bitmap_free(mr->ms_bmp);
+ rte_free(mr);
+}
+
+/**
+ * Releass resources of detached MR having no online entry.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ */
+static void
+mlx4_mr_garbage_collect(struct rte_eth_dev *dev)
+{
+ struct priv *priv = dev->data->dev_private;
+ struct mlx4_mr *mr_next;
+ struct mlx4_mr_list free_list = LIST_HEAD_INITIALIZER(free_list);
+
+ /*
+ * MR can't be freed with holding the lock because rte_free() could call
+ * memory free callback function. This will be a deadlock situation.
+ */
+ rte_rwlock_write_lock(&priv->mr.rwlock);
+ /* Detach the whole free list and release it after unlocking. */
+ free_list = priv->mr.mr_free_list;
+ LIST_INIT(&priv->mr.mr_free_list);
+ rte_rwlock_write_unlock(&priv->mr.rwlock);
+ /* Release resources. */
+ mr_next = LIST_FIRST(&free_list);
+ while (mr_next != NULL) {
+ struct mlx4_mr *mr = mr_next;
+
+ mr_next = LIST_NEXT(mr, mr);
+ mr_free(mr);
+ }
+}
+
+/* Called during rte_memseg_contig_walk() by mlx4_mr_create(). */
+static int
+mr_find_contig_memsegs_cb(const struct rte_memseg_list *msl,
+ const struct rte_memseg *ms, size_t len, void *arg)
+{
+ struct mr_find_contig_memsegs_data *data = arg;
+
+ if (data->addr < ms->addr_64 || data->addr >= ms->addr_64 + len)
+ return 0;
+ /* Found, save it and stop walking. */
+ data->start = ms->addr_64;
+ data->end = ms->addr_64 + len;
+ data->msl = msl;
+ return 1;
+}
+
+/**
+ * Create a new global Memroy Region (MR) for a missing virtual address.
+ * Register entire virtually contiguous memory chunk around the address.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param[out] entry
+ * Pointer to returning MR cache entry, found in the global cache or newly
+ * created. If failed to create one, this will not be updated.
+ * @param addr
+ * Target virtual address to register.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on failure and rte_errno is set.
+ */
+static uint32_t
+mlx4_mr_create(struct rte_eth_dev *dev, struct mlx4_mr_cache *entry,
+ uintptr_t addr)
+{
+ struct priv *priv = dev->data->dev_private;
+ struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+ const struct rte_memseg_list *msl;
+ const struct rte_memseg *ms;
+ struct mlx4_mr *mr = NULL;
+ size_t len;
+ uint32_t ms_n;
+ uint32_t bmp_size;
+ void *bmp_mem;
+ int ms_idx_shift = -1;
+ unsigned int n;
+ struct mr_find_contig_memsegs_data data = {
+ .addr = addr,
+ };
+ struct mr_find_contig_memsegs_data data_re;
+
+ DEBUG("port %u creating a MR using address (%p)",
+ dev->data->port_id, (void *)addr);
+ /*
+ * Release detached MRs if any. This can't be called with holding either
+ * memory_hotplug_lock or priv->mr.rwlock. MRs on the free list have
+ * been detached by the memory free event but it couldn't be released
+ * inside the callback due to deadlock. As a result, releasing resources
+ * is quite opportunistic.
+ */
+ mlx4_mr_garbage_collect(dev);
+ /*
+ * Find out a contiguous virtual address chunk in use, to which the
+ * given address belongs, in order to register maximum range. In the
+ * best case where mempools are not dynamically recreated and
+ * '--socket-mem' is speicified as an EAL option, it is very likely to
+ * have only one MR(LKey) per a socket and per a hugepage-size even
+ * though the system memory is highly fragmented.
+ */
+ if (!rte_memseg_contig_walk(mr_find_contig_memsegs_cb, &data)) {
+ WARN("port %u unable to find virtually contigous"
+ " chunk for address (%p)."
+ " rte_memseg_contig_walk() failed.",
+ dev->data->port_id, (void *)addr);
+ rte_errno = ENXIO;
+ goto err_nolock;
+ }
+alloc_resources:
+ /* Addresses must be page-aligned. */
+ assert(rte_is_aligned((void *)data.start, data.msl->page_sz));
+ assert(rte_is_aligned((void *)data.end, data.msl->page_sz));
+ msl = data.msl;
+ ms = rte_mem_virt2memseg((void *)data.start, msl);
+ len = data.end - data.start;
+ assert(msl->page_sz == ms->hugepage_sz);
+ /* Number of memsegs in the range. */
+ ms_n = len / msl->page_sz;
+ DEBUG("port %u extending %p to [0x%lx, 0x%lx), page_sz=0x%lx, ms_n=%u",
+ dev->data->port_id, (void *)addr,
+ data.start, data.end, msl->page_sz, ms_n);
+ /* Size of memory for bitmap. */
+ bmp_size = rte_bitmap_get_memory_footprint(ms_n);
+ mr = rte_zmalloc_socket(NULL,
+ RTE_ALIGN_CEIL(sizeof(*mr),
+ RTE_CACHE_LINE_SIZE) +
+ bmp_size,
+ RTE_CACHE_LINE_SIZE, msl->socket_id);
+ if (mr == NULL) {
+ WARN("port %u unable to allocate memory for a new MR of"
+ " address (%p).",
+ dev->data->port_id, (void *)addr);
+ rte_errno = ENOMEM;
+ goto err_nolock;
+ }
+ mr->msl = msl;
+ /*
+ * Save the index of the first memseg and initialize memseg bitmap. To
+ * see if a memseg of ms_idx in the memseg-list is still valid, check:
+ * rte_bitmap_get(mr->bmp, ms_idx - mr->ms_base_idx)
+ */
+ mr->ms_base_idx = rte_fbarray_find_idx(&msl->memseg_arr, ms);
+ bmp_mem = RTE_PTR_ALIGN_CEIL(mr + 1, RTE_CACHE_LINE_SIZE);
+ mr->ms_bmp = rte_bitmap_init(ms_n, bmp_mem, bmp_size);
+ if (mr->ms_bmp == NULL) {
+ WARN("port %u unable to initialize bitamp for a new MR of"
+ " address (%p).",
+ dev->data->port_id, (void *)addr);
+ rte_errno = EINVAL;
+ goto err_nolock;
+ }
+ /*
+ * Should recheck whether the extended contiguous chunk is still valid.
+ * Because memory_hotplug_lock can't be held if there's any memory
+ * related calls in a critical path, resource allocation above can't be
+ * locked. If the memory has been changed at this point, try again with
+ * just single page. If not, go on with the big chunk atomically from
+ * here.
+ */
+ rte_rwlock_read_lock(&mcfg->memory_hotplug_lock);
+ data_re = data;
+ if (len > msl->page_sz &&
+ !rte_memseg_contig_walk(mr_find_contig_memsegs_cb, &data_re)) {
+ WARN("port %u unable to find virtually contigous"
+ " chunk for address (%p)."
+ " rte_memseg_contig_walk() failed.",
+ dev->data->port_id, (void *)addr);
+ rte_errno = ENXIO;
+ goto err_memlock;
+ }
+ if (data.start != data_re.start || data.end != data_re.end) {
+ /*
+ * The extended contiguous chunk has been changed. Try again
+ * with single memseg instead.
+ */
+ data.start = RTE_ALIGN_FLOOR(addr, msl->page_sz);
+ data.end = data.start + msl->page_sz;
+ rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
+ mr_free(mr);
+ goto alloc_resources;
+ }
+ assert(data.msl == data_re.msl);
+ rte_rwlock_write_lock(&priv->mr.rwlock);
+ /*
+ * Check the address is really missing. If other thread already created
+ * one or it is not found due to overflow, abort and return.
+ */
+ if (mr_lookup_dev(dev, entry, addr) != UINT32_MAX) {
+ /*
+ * Insert to the global cache table. It may fail due to
+ * low-on-memory. Then, this entry will have to be searched
+ * here again.
+ */
+ mr_btree_insert(&priv->mr.cache, entry);
+ DEBUG("port %u found MR for %p on final lookup, abort",
+ dev->data->port_id, (void *)addr);
+ rte_rwlock_write_unlock(&priv->mr.rwlock);
+ rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
+ /*
+ * Must be unlocked before calling rte_free() because
+ * mlx4_mr_mem_event_free_cb() can be called inside.
+ */
+ mr_free(mr);
+ return entry->lkey;
+ }
+ /*
+ * Trim start and end addresses for verbs MR. Set bits for registering
+ * memsegs but exclude already registered ones. Bitmap can be
+ * fragmented.
+ */
+ for (n = 0; n < ms_n; ++n) {
+ uintptr_t start;
+ struct mlx4_mr_cache ret = { 0, };
+
+ start = data_re.start + n * msl->page_sz;
+ /* Exclude memsegs already registered by other MRs. */
+ if (mr_lookup_dev(dev, &ret, start) == UINT32_MAX) {
+ /*
+ * Start from the first unregistered memseg in the
+ * extended range.
+ */
+ if (ms_idx_shift == -1) {
+ mr->ms_base_idx += n;
+ data.start = start;
+ ms_idx_shift = n;
+ }
+ data.end = start + msl->page_sz;
+ rte_bitmap_set(mr->ms_bmp, n - ms_idx_shift);
+ ++mr->ms_n;
+ }
+ }
+ len = data.end - data.start;
+ mr->ms_bmp_n = len / msl->page_sz;
+ assert(ms_idx_shift + mr->ms_bmp_n <= ms_n);
+ /*
+ * Finally create a verbs MR for the memory chunk. ibv_reg_mr() can be
+ * called with holding the memory lock because it doesn't use
+ * mlx4_alloc_buf_extern() which eventually calls rte_malloc_socket()
+ * through mlx4_alloc_verbs_buf().
+ */
+ mr->ibv_mr = mlx4_glue->reg_mr(priv->pd, (void *)data.start, len,
+ IBV_ACCESS_LOCAL_WRITE);
+ if (mr->ibv_mr == NULL) {
+ WARN("port %u fail to create a verbs MR for address (%p)",
+ dev->data->port_id, (void *)addr);
+ rte_errno = EINVAL;
+ goto err_mrlock;
+ }
+ assert((uintptr_t)mr->ibv_mr->addr == data.start);
+ assert(mr->ibv_mr->length == len);
+ LIST_INSERT_HEAD(&priv->mr.mr_list, mr, mr);
+ DEBUG("port %u MR CREATED (%p) for %p:\n"
+ " [0x%lx, 0x%lx), lkey=0x%x base_idx=%u ms_n=%u, ms_bmp_n=%u",
+ dev->data->port_id, (void *)mr, (void *)addr,
+ data.start, data.end, rte_cpu_to_be_32(mr->ibv_mr->lkey),
+ mr->ms_base_idx, mr->ms_n, mr->ms_bmp_n);
+ /* Insert to the global cache table. */
+ mr_insert_dev_cache(dev, mr);
+ /* Fill in output data. */
+ mr_lookup_dev(dev, entry, addr);
+ /* Lookup can't fail. */
+ assert(entry->lkey != UINT32_MAX);
+ rte_rwlock_write_unlock(&priv->mr.rwlock);
+ rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
+ return entry->lkey;
+err_mrlock:
+ rte_rwlock_write_unlock(&priv->mr.rwlock);
+err_memlock:
+ rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
+err_nolock:
+ /*
+ * In case of error, as this can be called in a datapath, a warning
+ * message per an error is preferable instead. Must be unlocked before
+ * calling rte_free() because mlx4_mr_mem_event_free_cb() can be called
+ * inside.
+ */
+ mr_free(mr);
+ return UINT32_MAX;
+}
+
+/**
+ * Rebuild the global B-tree cache of device from the original MR list.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ */
+static void
+mr_rebuild_dev_cache(struct rte_eth_dev *dev)
+{
+ struct priv *priv = dev->data->dev_private;
+ struct mlx4_mr *mr;
+
+ DEBUG("port %u rebuild dev cache[]", dev->data->port_id);
+ /* Flush cache to rebuild. */
+ priv->mr.cache.len = 1;
+ priv->mr.cache.overflow = 0;
+ /* Iterate all the existing MRs. */
+ LIST_FOREACH(mr, &priv->mr.mr_list, mr)
+ if (mr_insert_dev_cache(dev, mr) < 0)
+ return;
+}
+
+/**
+ * Callback for memory free event. Iterate freed memsegs and check whether it
+ * belongs to an existing MR. If found, clear the bit from bitmap of MR. As a
+ * result, the MR would be fragmented. If it becomes empty, the MR will be freed
+ * later by mlx4_mr_garbage_collect().
+ *
+ * The global cache must be rebuilt if there's any change and this event has to
+ * be propagated to dataplane threads to flush the local caches.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param addr
+ * Address of freed memory.
+ * @param len
+ * Size of freed memory.
+ */
+static void
+mlx4_mr_mem_event_free_cb(struct rte_eth_dev *dev, const void *addr, size_t len)
+{
+ struct priv *priv = dev->data->dev_private;
+ const struct rte_memseg_list *msl;
+ struct mlx4_mr *mr;
+ int ms_n;
+ int i;
+ int rebuild = 0;
+
+ DEBUG("port %u free callback: addr=%p, len=%lu",
+ dev->data->port_id, addr, len);
+ msl = rte_mem_virt2memseg_list(addr);
+ /* addr and len must be page-aligned. */
+ assert((uintptr_t)addr == RTE_ALIGN((uintptr_t)addr, msl->page_sz));
+ assert(len == RTE_ALIGN(len, msl->page_sz));
+ ms_n = len / msl->page_sz;
+ rte_rwlock_write_lock(&priv->mr.rwlock);
+ /* Clear bits of freed memsegs from MR. */
+ for (i = 0; i < ms_n; ++i) {
+ const struct rte_memseg *ms;
+ struct mlx4_mr_cache entry;
+ uintptr_t start;
+ int ms_idx;
+ uint32_t pos;
+
+ /* Find MR having this memseg. */
+ start = (uintptr_t)addr + i * msl->page_sz;
+ mr = mr_lookup_dev_list(dev, &entry, start);
+ if (mr == NULL)
+ continue;
+ ms = rte_mem_virt2memseg((void *)start, msl);
+ assert(ms != NULL);
+ assert(msl->page_sz == ms->hugepage_sz);
+ ms_idx = rte_fbarray_find_idx(&msl->memseg_arr, ms);
+ pos = ms_idx - mr->ms_base_idx;
+ assert(rte_bitmap_get(mr->ms_bmp, pos));
+ assert(pos < mr->ms_bmp_n);
+ DEBUG("port %u MR(%p): clear bitmap[%u] for addr %p",
+ dev->data->port_id, (void *)mr, pos, (void *)start);
+ rte_bitmap_clear(mr->ms_bmp, pos);
+ if (--mr->ms_n == 0) {
+ LIST_REMOVE(mr, mr);
+ LIST_INSERT_HEAD(&priv->mr.mr_free_list, mr, mr);
+ DEBUG("port %u remove MR(%p) from list",
+ dev->data->port_id, (void *)mr);
+ }
+ /*
+ * MR is fragmented or will be freed. the global cache must be
+ * rebuilt.
+ */
+ rebuild = 1;
+ }
+ if (rebuild) {
+ mr_rebuild_dev_cache(dev);
+ /*
+ * Flush local caches by propagating invalidation across cores.
+ * rte_smp_wmb() is enough to synchronize this event. If one of
+ * freed memsegs is seen by other core, that means the memseg
+ * has been allocated by allocator, which will come after this
+ * free call. Therefore, this store instruction (incrementing
+ * generation below) will be guaranteed to be seen by other core
+ * before the core sees the newly allocated memory.
+ */
+ ++priv->mr.dev_gen;
+ DEBUG("broadcasting local cache flush, gen=%d",
+ priv->mr.dev_gen);
+ rte_smp_wmb();
+ }
+ rte_rwlock_write_unlock(&priv->mr.rwlock);
+#ifndef NDEBUG
+ if (rebuild)
+ mlx4_mr_dump_dev(dev);
+#endif
+}
+
+/**
+ * Callback for memory event.
+ *
+ * @param event_type
+ * Memory event type.
+ * @param addr
+ * Address of memory.
+ * @param len
+ * Size of memory.
+ */
+void
+mlx4_mr_mem_event_cb(enum rte_mem_event event_type, const void *addr,
+ size_t len, void *arg __rte_unused)
+{
+ struct priv *priv;
+
+ switch (event_type) {
+ case RTE_MEM_EVENT_FREE:
+ rte_rwlock_read_lock(&mlx4_mem_event_rwlock);
+ /* Iterate all the existing mlx4 devices. */
+ LIST_FOREACH(priv, &mlx4_mem_event_cb_list, mem_event_cb)
+ mlx4_mr_mem_event_free_cb(priv->dev, addr, len);
+ rte_rwlock_read_unlock(&mlx4_mem_event_rwlock);
+ break;
+ case RTE_MEM_EVENT_ALLOC:
+ default:
+ break;
+ }
+}
+
+/**
+ * Look up address in the global MR cache table. If not found, create a new MR.
+ * Insert the found/created entry to local bottom-half cache table.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param mr_ctrl
+ * Pointer to per-queue MR control structure.
+ * @param[out] entry
+ * Pointer to returning MR cache entry, found in the global cache or newly
+ * created. If failed to create one, this is not written.
+ * @param addr
+ * Search key.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on no match.
+ */
+static uint32_t
+mlx4_mr_lookup_dev(struct rte_eth_dev *dev, struct mlx4_mr_ctrl *mr_ctrl,
+ struct mlx4_mr_cache *entry, uintptr_t addr)
+{
+ struct priv *priv = dev->data->dev_private;
+ struct mlx4_mr_btree *bt = &mr_ctrl->cache_bh;
+ uint16_t idx;
+ uint32_t lkey;
+
+ /* If local cache table is full, try to double it. */
+ if (unlikely(bt->len == bt->size))
+ mr_btree_expand(bt, bt->size << 1);
+ /* Look up in the global cache. */
+ rte_rwlock_read_lock(&priv->mr.rwlock);
+ lkey = mr_btree_lookup(&priv->mr.cache, &idx, addr);
+ if (lkey != UINT32_MAX) {
+ /* Found. */
+ *entry = (*priv->mr.cache.table)[idx];
+ rte_rwlock_read_unlock(&priv->mr.rwlock);
+ /*
+ * Update local cache. Even if it fails, return the found entry
+ * to update top-half cache. Next time, this entry will be found
+ * in the global cache.
+ */
+ mr_btree_insert(bt, entry);
+ return lkey;
+ }
+ rte_rwlock_read_unlock(&priv->mr.rwlock);
+ /* First time to see the address? Create a new MR. */
+ lkey = mlx4_mr_create(dev, entry, addr);
+ /*
+ * Update the local cache if successfully created a new global MR. Even
+ * if failed to create one, there's no action to take in this datapath
+ * code. As returning LKey is invalid, this will eventually make HW
+ * fail.
+ */
+ if (lkey != UINT32_MAX)
+ mr_btree_insert(bt, entry);
+ return lkey;
+}
+
+/**
+ * Bottom-half of LKey search on datapath. Firstly search in cache_bh[] and if
+ * misses, search in the global MR cache table and update the new entry to
+ * per-queue local caches.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param mr_ctrl
+ * Pointer to per-queue MR control structure.
+ * @param addr
+ * Search key.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on no match.
+ */
+static uint32_t
+mlx4_mr_addr2mr_bh(struct rte_eth_dev *dev, struct mlx4_mr_ctrl *mr_ctrl,
+ uintptr_t addr)
+{
+ uint32_t lkey;
+ uint16_t bh_idx = 0;
+ /* Victim in top-half cache to replace with new entry. */
+ struct mlx4_mr_cache *repl = &mr_ctrl->cache[mr_ctrl->head];
+
+ /* Binary-search MR translation table. */
+ lkey = mr_btree_lookup(&mr_ctrl->cache_bh, &bh_idx, addr);
+ /* Update top-half cache. */
+ if (likely(lkey != UINT32_MAX)) {
+ *repl = (*mr_ctrl->cache_bh.table)[bh_idx];
+ } else {
+ /*
+ * If missed in local lookup table, search in the global cache
+ * and local cache_bh[] will be updated inside if possible.
+ * Top-half cache entry will also be updated.
+ */
+ lkey = mlx4_mr_lookup_dev(dev, mr_ctrl, repl, addr);
+ if (unlikely(lkey == UINT32_MAX))
+ return UINT32_MAX;
+ }
+ /* Update the most recently used entry. */
+ mr_ctrl->mru = mr_ctrl->head;
+ /* Point to the next victim, the oldest. */
+ mr_ctrl->head = (mr_ctrl->head + 1) % MLX4_MR_CACHE_N;
+ return lkey;
+}
+
+/**
+ * Bottom-half of LKey search on Rx.
+ *
+ * @param rxq
+ * Pointer to Rx queue structure.
+ * @param addr
+ * Search key.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on no match.
+ */
+uint32_t
+mlx4_rx_addr2mr_bh(struct rxq *rxq, uintptr_t addr)
+{
+ struct mlx4_mr_ctrl *mr_ctrl = &rxq->mr_ctrl;
+ struct priv *priv = rxq->priv;
+
+ DEBUG("Rx queue %u: miss on top-half, mru=%u, head=%u, addr=%p",
+ rxq->stats.idx, mr_ctrl->mru, mr_ctrl->head, (void *)addr);
+ return mlx4_mr_addr2mr_bh(priv->dev, mr_ctrl, addr);
+}
+
+/**
+ * Bottom-half of LKey search on Tx.
+ *
+ * @param txq
+ * Pointer to Tx queue structure.
+ * @param addr
+ * Search key.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on no match.
+ */
+uint32_t
+mlx4_tx_addr2mr_bh(struct txq *txq, uintptr_t addr)
+{
+ struct mlx4_mr_ctrl *mr_ctrl = &txq->mr_ctrl;
+ struct priv *priv = txq->priv;
+
+ DEBUG("Tx queue %u: miss on top-half, mru=%u, head=%u, addr=%p",
+ txq->stats.idx, mr_ctrl->mru, mr_ctrl->head, (void *)addr);
+ return mlx4_mr_addr2mr_bh(priv->dev, mr_ctrl, addr);
+}
+
+/**
+ * Flush all of the local cache entries.
+ *
+ * @param mr_ctrl
+ * Pointer to per-queue MR control structure.
+ */
+void
+mlx4_mr_flush_local_cache(struct mlx4_mr_ctrl *mr_ctrl)
+{
+ /* Reset the most-recently-used index. */
+ mr_ctrl->mru = 0;
+ /* Reset the linear search array. */
+ mr_ctrl->head = 0;
+ memset(mr_ctrl->cache, 0, sizeof(mr_ctrl->cache));
+ /* Reset the B-tree table. */
+ mr_ctrl->cache_bh.len = 1;
+ mr_ctrl->cache_bh.overflow = 0;
+ /* Update the generation number. */
+ mr_ctrl->cur_gen = *mr_ctrl->dev_gen_ptr;
+ DEBUG("mr_ctrl(%p): flushed, cur_gen=%d",
+ (void *)mr_ctrl, mr_ctrl->cur_gen);
+}
+
+/* Called during rte_mempool_mem_iter() by mlx4_mr_update_mp(). */
+static void
+mlx4_mr_update_mp_cb(struct rte_mempool *mp __rte_unused, void *opaque,
+ struct rte_mempool_memhdr *memhdr,
+ unsigned mem_idx __rte_unused)
+{
+ struct mr_update_mp_data *data = opaque;
+ uint32_t lkey;
+
+ /* Stop iteration if failed in the previous walk. */
+ if (data->ret < 0)
+ return;
+ /* Register address of the chunk and update local caches. */
+ lkey = mlx4_mr_addr2mr_bh(data->dev, data->mr_ctrl,
+ (uintptr_t)memhdr->addr);
+ if (lkey == UINT32_MAX)
+ data->ret = -1;
+}
+
+/**
+ * Register entire memory chunks in a Mempool.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ * @param mr_ctrl
+ * Pointer to per-queue MR control structure.
+ * @param mp
+ * Pointer to registering Mempool.
+ *
+ * @return
+ * 0 on success, -1 on failure.
+ */
+int
+mlx4_mr_update_mp(struct rte_eth_dev *dev, struct mlx4_mr_ctrl *mr_ctrl,
+ struct rte_mempool *mp)
+{
+ struct mr_update_mp_data data = {
+ .dev = dev,
+ .mr_ctrl = mr_ctrl,
+ .ret = 0,
+ };
+
+ rte_mempool_mem_iter(mp, mlx4_mr_update_mp_cb, &data);
+ return data.ret;
+}
+
+#ifndef NDEBUG
+/**
+ * Dump all the created MRs and the global cache entries.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ */
+void
+mlx4_mr_dump_dev(struct rte_eth_dev *dev)
+{
+ struct priv *priv = dev->data->dev_private;
+ struct mlx4_mr *mr;
+ int mr_n = 0;
+ int chunk_n = 0;
+
+ rte_rwlock_read_lock(&priv->mr.rwlock);
+ /* Iterate all the existing MRs. */
+ LIST_FOREACH(mr, &priv->mr.mr_list, mr) {
+ unsigned int n;
+
+ DEBUG("port %u MR[%u], LKey = 0x%x, ms_n = %u, ms_bmp_n = %u",
+ dev->data->port_id, mr_n++,
+ rte_cpu_to_be_32(mr->ibv_mr->lkey),
+ mr->ms_n, mr->ms_bmp_n);
+ if (mr->ms_n == 0)
+ continue;
+ for (n = 0; n < mr->ms_bmp_n; ) {
+ struct mlx4_mr_cache ret = { 0, };
+
+ n = mr_find_next_chunk(mr, &ret, n);
+ if (!ret.end)
+ break;
+ DEBUG(" chunk[%u], [0x%lx, 0x%lx)",
+ chunk_n++, ret.start, ret.end);
+ }
+ }
+ DEBUG("port %u dumping global cache", dev->data->port_id);
+ mlx4_mr_btree_dump(&priv->mr.cache);
+ rte_rwlock_read_unlock(&priv->mr.rwlock);
+}
+#endif
+
+/**
+ * Release all the created MRs and resources. Remove device from memory callback
+ * list.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ */
+void
+mlx4_mr_release(struct rte_eth_dev *dev)
+{
+ struct priv *priv = dev->data->dev_private;
+ struct mlx4_mr *mr_next = LIST_FIRST(&priv->mr.mr_list);
+
+ /* Remove from memory callback device list. */
+ rte_rwlock_write_lock(&mlx4_mem_event_rwlock);
+ LIST_REMOVE(priv, mem_event_cb);
+ rte_rwlock_write_unlock(&mlx4_mem_event_rwlock);
+#ifndef NDEBUG
+ mlx4_mr_dump_dev(dev);
+#endif
+ rte_rwlock_write_lock(&priv->mr.rwlock);
+ /* Detach from MR list and move to free list. */
+ while (mr_next != NULL) {
+ struct mlx4_mr *mr = mr_next;
+
+ mr_next = LIST_NEXT(mr, mr);
+ LIST_REMOVE(mr, mr);
+ LIST_INSERT_HEAD(&priv->mr.mr_free_list, mr, mr);
+ }
+ LIST_INIT(&priv->mr.mr_list);
+ /* Free global cache. */
+ mlx4_mr_btree_free(&priv->mr.cache);
+ rte_rwlock_write_unlock(&priv->mr.rwlock);
+ /* Free all remaining MRs. */
+ mlx4_mr_garbage_collect(dev);
+}
diff --git a/drivers/net/mlx4/mlx4_mr.h b/drivers/net/mlx4/mlx4_mr.h
new file mode 100644
index 000000000..37a365a8b
--- /dev/null
+++ b/drivers/net/mlx4/mlx4_mr.h
@@ -0,0 +1,122 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2018 6WIND S.A.
+ * Copyright 2018 Mellanox Technologies, Ltd
+ */
+
+#ifndef RTE_PMD_MLX4_MR_H_
+#define RTE_PMD_MLX4_MR_H_
+
+#include <stddef.h>
+#include <stdint.h>
+#include <sys/queue.h>
+
+/* Verbs headers do not support -pedantic. */
+#ifdef PEDANTIC
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+#include <infiniband/verbs.h>
+#ifdef PEDANTIC
+#pragma GCC diagnostic error "-Wpedantic"
+#endif
+
+#include <rte_eal_memconfig.h>
+#include <rte_ethdev.h>
+#include <rte_rwlock.h>
+#include <rte_bitmap.h>
+
+/* Size of per-queue MR cache array for linear search. */
+#define MLX4_MR_CACHE_N 8
+
+/* Size of MR cache table for binary search. */
+#define MLX4_MR_BTREE_CACHE_N 256
+
+/* Memory Region object. */
+struct mlx4_mr {
+ LIST_ENTRY(mlx4_mr) mr; /**< Pointer to the prev/next entry. */
+ struct ibv_mr *ibv_mr; /* Verbs Memory Region. */
+ const struct rte_memseg_list *msl;
+ int ms_base_idx; /* Start index of msl->memseg_arr[]. */
+ int ms_n; /* Number of memsegs in use. */
+ uint32_t ms_bmp_n; /* Number of bits in memsegs bit-mask. */
+ struct rte_bitmap *ms_bmp; /* Bit-mask of memsegs belonged to MR. */
+};
+
+/* Cache entry for Memory Region. */
+struct mlx4_mr_cache {
+ uintptr_t start; /* Start address of MR. */
+ uintptr_t end; /* End address of MR. */
+ uint32_t lkey; /* rte_cpu_to_be_32(ibv_mr->lkey). */
+} __rte_packed;
+
+/* MR Cache table for Binary search. */
+struct mlx4_mr_btree {
+ uint16_t len; /* Number of entries. */
+ uint16_t size; /* Total number of entries. */
+ int overflow; /* Mark failure of table expansion. */
+ struct mlx4_mr_cache (*table)[];
+} __rte_packed;
+
+/* Per-queue MR control descriptor. */
+struct mlx4_mr_ctrl {
+ uint32_t *dev_gen_ptr; /* Generation number of device to poll. */
+ uint32_t cur_gen; /* Generation number saved to flush caches. */
+ uint16_t mru; /* Index of last hit entry in top-half cache. */
+ uint16_t head; /* Index of the oldest entry in top-half cache. */
+ struct mlx4_mr_cache cache[MLX4_MR_CACHE_N]; /* Cache for top-half. */
+ struct mlx4_mr_btree cache_bh; /* Cache for bottom-half. */
+} __rte_packed;
+
+extern struct mlx4_dev_list mlx4_mem_event_cb_list;
+extern rte_rwlock_t mlx4_mem_event_rwlock;
+
+/* First entry must be NULL for comparison. */
+#define mlx4_mr_btree_len(bt) ((bt)->len - 1)
+
+int mlx4_mr_btree_init(struct mlx4_mr_btree *bt, int n, int socket);
+void mlx4_mr_btree_free(struct mlx4_mr_btree *bt);
+void mlx4_mr_btree_dump(struct mlx4_mr_btree *bt);
+void mlx4_mr_mem_event_cb(enum rte_mem_event event_type, const void *addr,
+ size_t len, void *arg);
+int mlx4_mr_update_mp(struct rte_eth_dev *dev, struct mlx4_mr_ctrl *mr_ctrl,
+ struct rte_mempool *mp);
+void mlx4_mr_dump_dev(struct rte_eth_dev *dev);
+void mlx4_mr_release(struct rte_eth_dev *dev);
+
+/**
+ * Look up LKey from given lookup table by linear search. Firstly look up the
+ * last-hit entry. If miss, the entire array is searched. If found, update the
+ * last-hit index and return LKey.
+ *
+ * @param lkp_tbl
+ * Pointer to lookup table.
+ * @param[in,out] cached_idx
+ * Pointer to last-hit index.
+ * @param n
+ * Size of lookup table.
+ * @param addr
+ * Search key.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on no match.
+ */
+static __rte_always_inline uint32_t
+mlx4_mr_lookup_cache(struct mlx4_mr_cache *lkp_tbl, uint16_t *cached_idx,
+ uint16_t n, uintptr_t addr)
+{
+ uint16_t idx;
+
+ if (likely(addr >= lkp_tbl[*cached_idx].start &&
+ addr < lkp_tbl[*cached_idx].end))
+ return lkp_tbl[*cached_idx].lkey;
+ for (idx = 0; idx < n && lkp_tbl[idx].start != 0; ++idx) {
+ if (addr >= lkp_tbl[idx].start &&
+ addr < lkp_tbl[idx].end) {
+ /* Found. */
+ *cached_idx = idx;
+ return lkp_tbl[idx].lkey;
+ }
+ }
+ return UINT32_MAX;
+}
+
+#endif /* RTE_PMD_MLX4_MR_H_ */
diff --git a/drivers/net/mlx4/mlx4_rxq.c b/drivers/net/mlx4/mlx4_rxq.c
index 5621d5bd4..ad706be82 100644
--- a/drivers/net/mlx4/mlx4_rxq.c
+++ b/drivers/net/mlx4/mlx4_rxq.c
@@ -488,6 +488,7 @@ mlx4_rxq_attach(struct rxq *rxq)
}
struct priv *priv = rxq->priv;
+ struct rte_eth_dev *dev = priv->dev;
const uint32_t elts_n = 1 << rxq->elts_n;
const uint32_t sges_n = 1 << rxq->sges_n;
struct rte_mbuf *(*elts)[elts_n] = rxq->elts;
@@ -552,6 +553,11 @@ mlx4_rxq_attach(struct rxq *rxq)
msg = "failed to obtain device information from WQ/CQ objects";
goto error;
}
+ /* Pre-register Rx mempool. */
+ DEBUG("port %u Rx queue %u registering mp %s having %u chunks",
+ priv->dev->data->port_id, rxq->stats.idx,
+ rxq->mp->name, rxq->mp->nb_mem_chunks);
+ mlx4_mr_update_mp(dev, &rxq->mr_ctrl, rxq->mp);
wqes = (volatile struct mlx4_wqe_data_seg (*)[])
((uintptr_t)dv_rwq.buf.buf + dv_rwq.rq.offset);
for (i = 0; i != RTE_DIM(*elts); ++i) {
@@ -583,7 +589,7 @@ mlx4_rxq_attach(struct rxq *rxq)
.addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(buf,
uintptr_t)),
.byte_count = rte_cpu_to_be_32(buf->data_len),
- .lkey = UINT32_MAX,
+ .lkey = mlx4_rx_mb2mr(rxq, buf),
};
(*elts)[i] = buf;
}
@@ -883,6 +889,11 @@ mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1 << rxq->sges_n);
goto error;
}
+ if (mlx4_mr_btree_init(&rxq->mr_ctrl.cache_bh,
+ MLX4_MR_BTREE_CACHE_N, socket)) {
+ /* rte_errno is already set. */
+ goto error;
+ }
if (dev->data->dev_conf.intr_conf.rxq) {
rxq->channel = mlx4_glue->create_comp_channel(priv->ctx);
if (rxq->channel == NULL) {
@@ -940,5 +951,6 @@ mlx4_rx_queue_release(void *dpdk_rxq)
assert(!rxq->rq_db);
if (rxq->channel)
claim_zero(mlx4_glue->destroy_comp_channel(rxq->channel));
+ mlx4_mr_btree_free(&rxq->mr_ctrl.cache_bh);
rte_free(rxq);
}
diff --git a/drivers/net/mlx4/mlx4_rxtx.c b/drivers/net/mlx4/mlx4_rxtx.c
index 21ffd435a..44efeb447 100644
--- a/drivers/net/mlx4/mlx4_rxtx.c
+++ b/drivers/net/mlx4/mlx4_rxtx.c
@@ -344,24 +344,6 @@ mlx4_txq_complete(struct txq *txq, const unsigned int elts_m,
}
/**
- * Get memory pool (MP) from mbuf. If mbuf is indirect, the pool from which
- * the cloned mbuf is allocated is returned instead.
- *
- * @param buf
- * Pointer to mbuf.
- *
- * @return
- * Memory pool where data is located for given mbuf.
- */
-static struct rte_mempool *
-mlx4_txq_mb2mp(struct rte_mbuf *buf)
-{
- if (unlikely(RTE_MBUF_INDIRECT(buf)))
- return rte_mbuf_from_indirect(buf)->pool;
- return buf->pool;
-}
-
-/**
* Write Tx data segment to the SQ.
*
* @param dseg
@@ -378,7 +360,7 @@ mlx4_fill_tx_data_seg(volatile struct mlx4_wqe_data_seg *dseg,
uint32_t lkey, uintptr_t addr, rte_be32_t byte_count)
{
dseg->addr = rte_cpu_to_be_64(addr);
- dseg->lkey = rte_cpu_to_be_32(lkey);
+ dseg->lkey = lkey;
#if RTE_CACHE_LINE_SIZE < 64
/*
* Need a barrier here before writing the byte_count
@@ -437,7 +419,7 @@ mlx4_tx_burst_segs(struct rte_mbuf *buf, struct txq *txq,
goto txbb_tail_segs;
txbb_head_seg:
/* Memory region key (big endian) for this memory pool. */
- lkey = mlx4_txq_mp2mr(txq, mlx4_txq_mb2mp(sbuf));
+ lkey = mlx4_tx_mb2mr(txq, sbuf);
if (unlikely(lkey == (uint32_t)-1)) {
DEBUG("%p: unable to get MP <-> MR association",
(void *)txq);
@@ -449,7 +431,7 @@ mlx4_tx_burst_segs(struct rte_mbuf *buf, struct txq *txq,
dseg = (volatile struct mlx4_wqe_data_seg *)
sq->buf;
dseg->addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(sbuf, uintptr_t));
- dseg->lkey = rte_cpu_to_be_32(lkey);
+ dseg->lkey = lkey;
/*
* This data segment starts at the beginning of a new
* TXBB, so we need to postpone its byte_count writing
@@ -469,7 +451,7 @@ mlx4_tx_burst_segs(struct rte_mbuf *buf, struct txq *txq,
/* Jump to default if there are more than two segments remaining. */
switch (nb_segs) {
default:
- lkey = mlx4_txq_mp2mr(txq, mlx4_txq_mb2mp(sbuf));
+ lkey = mlx4_tx_mb2mr(txq, sbuf);
if (unlikely(lkey == (uint32_t)-1)) {
DEBUG("%p: unable to get MP <-> MR association",
(void *)txq);
@@ -485,7 +467,7 @@ mlx4_tx_burst_segs(struct rte_mbuf *buf, struct txq *txq,
nb_segs--;
/* fallthrough */
case 2:
- lkey = mlx4_txq_mp2mr(txq, mlx4_txq_mb2mp(sbuf));
+ lkey = mlx4_tx_mb2mr(txq, sbuf);
if (unlikely(lkey == (uint32_t)-1)) {
DEBUG("%p: unable to get MP <-> MR association",
(void *)txq);
@@ -501,7 +483,7 @@ mlx4_tx_burst_segs(struct rte_mbuf *buf, struct txq *txq,
nb_segs--;
/* fallthrough */
case 1:
- lkey = mlx4_txq_mp2mr(txq, mlx4_txq_mb2mp(sbuf));
+ lkey = mlx4_tx_mb2mr(txq, sbuf);
if (unlikely(lkey == (uint32_t)-1)) {
DEBUG("%p: unable to get MP <-> MR association",
(void *)txq);
@@ -611,7 +593,7 @@ mlx4_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
elt->buf = NULL;
break;
}
- lkey = mlx4_txq_mp2mr(txq, mlx4_txq_mb2mp(buf));
+ lkey = mlx4_tx_mb2mr(txq, buf);
if (unlikely(lkey == (uint32_t)-1)) {
/* MR does not exist. */
DEBUG("%p: unable to get MP <-> MR association",
@@ -966,6 +948,9 @@ mlx4_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
* changes.
*/
scat->addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(rep, uintptr_t));
+ /* If there's only one MR, no need to replace LKey in WQE. */
+ if (unlikely(mlx4_mr_btree_len(&rxq->mr_ctrl.cache_bh) > 1))
+ scat->lkey = mlx4_rx_mb2mr(rxq, rep);
if (len > seg->data_len) {
len -= seg->data_len;
++pkt->nb_segs;
diff --git a/drivers/net/mlx4/mlx4_rxtx.h b/drivers/net/mlx4/mlx4_rxtx.h
index 2f9d3798b..4c025e3a0 100644
--- a/drivers/net/mlx4/mlx4_rxtx.h
+++ b/drivers/net/mlx4/mlx4_rxtx.h
@@ -25,6 +25,7 @@
#include "mlx4.h"
#include "mlx4_prm.h"
+#include "mlx4_mr.h"
/** Rx queue counters. */
struct mlx4_rxq_stats {
@@ -46,6 +47,7 @@ struct rxq {
uint16_t port_id; /**< Port ID for incoming packets. */
uint16_t sges_n; /**< Number of segments per packet (log2 value). */
uint16_t elts_n; /**< Mbuf queue size (log2 value). */
+ struct mlx4_mr_ctrl mr_ctrl; /* MR control descriptor. */
struct rte_mbuf *(*elts)[]; /**< Rx elements. */
volatile struct mlx4_wqe_data_seg (*wqes)[]; /**< HW queue entries. */
volatile uint32_t *rq_db; /**< RQ doorbell record. */
@@ -100,6 +102,7 @@ struct txq {
int elts_comp_cd; /**< Countdown for next completion. */
unsigned int elts_comp_cd_init; /**< Initial value for countdown. */
unsigned int elts_n; /**< (*elts)[] length. */
+ struct mlx4_mr_ctrl mr_ctrl; /* MR control descriptor. */
struct txq_elt (*elts)[]; /**< Tx elements. */
struct mlx4_txq_stats stats; /**< Tx queue counters. */
uint32_t max_inline; /**< Max inline send size. */
@@ -155,12 +158,70 @@ int mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx,
const struct rte_eth_txconf *conf);
void mlx4_tx_queue_release(void *dpdk_txq);
-static inline uint32_t
-mlx4_txq_mp2mr(struct txq *txq, struct rte_mempool *mp)
+/* mlx4_mr.c */
+
+void mlx4_mr_flush_local_cache(struct mlx4_mr_ctrl *mr_ctrl);
+uint32_t mlx4_rx_addr2mr_bh(struct rxq *rxq, uintptr_t addr);
+uint32_t mlx4_tx_addr2mr_bh(struct txq *txq, uintptr_t addr);
+
+/**
+ * Query LKey from a packet buffer for Rx. No need to flush local caches for Rx
+ * as mempool is pre-configured and static.
+ *
+ * @param rxq
+ * Pointer to Rx queue structure.
+ * @param addr
+ * Address to search.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on no match.
+ */
+static __rte_always_inline uint32_t
+mlx4_rx_addr2mr(struct rxq *rxq, uintptr_t addr)
+{
+ struct mlx4_mr_ctrl *mr_ctrl = &rxq->mr_ctrl;
+ uint32_t lkey;
+
+ /* Linear search on MR cache array. */
+ lkey = mlx4_mr_lookup_cache(mr_ctrl->cache, &mr_ctrl->mru,
+ MLX4_MR_CACHE_N, addr);
+ if (likely(lkey != UINT32_MAX))
+ return lkey;
+ /* Take slower bottom-half (Binary Search) on miss. */
+ return mlx4_rx_addr2mr_bh(rxq, addr);
+}
+
+#define mlx4_rx_mb2mr(rxq, mb) mlx4_rx_addr2mr(rxq, (uintptr_t)((mb)->buf_addr))
+
+/**
+ * Query LKey from a packet buffer for Tx. If not found, add the mempool.
+ *
+ * @param txq
+ * Pointer to Tx queue structure.
+ * @param addr
+ * Address to search.
+ *
+ * @return
+ * Searched LKey on success, UINT32_MAX on no match.
+ */
+static __rte_always_inline uint32_t
+mlx4_tx_addr2mr(struct txq *txq, uintptr_t addr)
{
- (void)txq;
- (void)mp;
- return UINT32_MAX;
+ struct mlx4_mr_ctrl *mr_ctrl = &txq->mr_ctrl;
+ uint32_t lkey;
+
+ /* Check generation bit to see if there's any change on existing MRs. */
+ if (unlikely(*mr_ctrl->dev_gen_ptr != mr_ctrl->cur_gen))
+ mlx4_mr_flush_local_cache(mr_ctrl);
+ /* Linear search on MR cache array. */
+ lkey = mlx4_mr_lookup_cache(mr_ctrl->cache, &mr_ctrl->mru,
+ MLX4_MR_CACHE_N, addr);
+ if (likely(lkey != UINT32_MAX))
+ return lkey;
+ /* Take slower bottom-half (binary search) on miss. */
+ return mlx4_tx_addr2mr_bh(txq, addr);
}
+#define mlx4_tx_mb2mr(rxq, mb) mlx4_tx_addr2mr(rxq, (uintptr_t)((mb)->buf_addr))
+
#endif /* MLX4_RXTX_H_ */
diff --git a/drivers/net/mlx4/mlx4_txq.c b/drivers/net/mlx4/mlx4_txq.c
index 5ea09b0b0..337ed9a1a 100644
--- a/drivers/net/mlx4/mlx4_txq.c
+++ b/drivers/net/mlx4/mlx4_txq.c
@@ -346,6 +346,13 @@ mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
/* Save first wqe pointer in the first element. */
(&(*txq->elts)[0])->wqe =
(volatile struct mlx4_wqe_ctrl_seg *)txq->msq.buf;
+ if (mlx4_mr_btree_init(&txq->mr_ctrl.cache_bh,
+ MLX4_MR_BTREE_CACHE_N, socket)) {
+ /* rte_errno is already set. */
+ goto error;
+ }
+ /* Save pointer of global generation number to check memory event. */
+ txq->mr_ctrl.dev_gen_ptr = &priv->mr.dev_gen;
DEBUG("%p: adding Tx queue %p to list", (void *)dev, (void *)txq);
dev->data->tx_queues[idx] = txq;
return 0;
@@ -386,5 +393,6 @@ mlx4_tx_queue_release(void *dpdk_txq)
claim_zero(mlx4_glue->destroy_qp(txq->qp));
if (txq->cq)
claim_zero(mlx4_glue->destroy_cq(txq->cq));
+ mlx4_mr_btree_free(&txq->mr_ctrl.cache_bh);
rte_free(txq);
}
--
2.11.0
^ permalink raw reply [flat|nested] 23+ messages in thread* Re: [dpdk-dev] [PATCH v2 4/4] net/mlx4: add new Memory Region support
2018-05-09 11:09 ` [dpdk-dev] [PATCH v2 4/4] net/mlx4: add new " Yongseok Koh
@ 2018-05-09 23:12 ` Ferruh Yigit
2018-05-10 3:00 ` Yongseok Koh
0 siblings, 1 reply; 23+ messages in thread
From: Ferruh Yigit @ 2018-05-09 23:12 UTC (permalink / raw)
To: Yongseok Koh, adrien.mazarguil, nelio.laranjeiro; +Cc: dev
On 5/9/2018 12:09 PM, Yongseok Koh wrote:
> This is the new design of Memory Region (MR) for mlx PMD, in order to:
> - Accommodate the new memory hotplug model.
> - Support non-contiguous Mempool.
>
> There are multiple layers for MR search.
>
> L0 is to look up the last-hit entry which is pointed by mr_ctrl->mru (Most
> Recently Used). If L0 misses, L1 is to look up the address in a fixed-sized
> array by linear search. L0/L1 is in an inline function -
> mlx4_mr_lookup_cache().
>
> If L1 misses, the bottom-half function is called to look up the address
> from the bigger local cache of the queue. This is L2 - mlx4_mr_addr2mr_bh()
> and it is not an inline function. Data structure for L2 is the Binary Tree.
>
> If L2 misses, the search falls into the slowest path which takes locks in
> order to access global device cache (priv->mr.cache) which is also a B-tree
> and caches the original MR list (priv->mr.mr_list) of the device. Unless
> the global cache is overflowed, it is all-inclusive of the MR list. This is
> L3 - mlx4_mr_lookup_dev(). The size of the L3 cache table is limited and
> can't be expanded on the fly due to deadlock. Refer to the comments in the
> code for the details - mr_lookup_dev(). If L3 is overflowed, the list will
> have to be searched directly bypassing the cache although it is slower.
>
> If L3 misses, a new MR for the address should be created -
> mlx4_mr_create(). When it creates a new MR, it tries to register adjacent
> memsegs as much as possible which are virtually contiguous around the
> address. This must take two locks - memory_hotplug_lock and
> priv->mr.rwlock. Due to memory_hotplug_lock, there can't be any
> allocation/free of memory inside.
>
> In the free callback of the memory hotplug event, freed space is searched
> from the MR list and corresponding bits are cleared from the bitmap of MRs.
> This can fragment a MR and the MR will have multiple search entries in the
> caches. Once there's a change by the event, the global cache must be
> rebuilt and all the per-queue caches will be flushed as well. If memory is
> frequently freed in run-time, that may cause jitter on dataplane processing
> in the worst case by incurring MR cache flush and rebuild. But, it would be
> the least probable scenario.
>
> To guarantee the most optimal performance, it is highly recommended to use
> an EAL option - '--socket-mem'. Then, the reserved memory will be pinned
> and won't be freed dynamically. And it is also recommended to configure
> per-lcore cache of Mempool. Even though there're many MRs for a device or
> MRs are highly fragmented, the cache of Mempool will be much helpful to
> reduce misses on per-queue caches anyway.
>
> '--legacy-mem' is also supported.
>
> Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
<...>
> +/**
> + * Insert an entry to B-tree lookup table.
> + *
> + * @param bt
> + * Pointer to B-tree structure.
> + * @param entry
> + * Pointer to new entry to insert.
> + *
> + * @return
> + * 0 on success, -1 on failure.
> + */
> +static int
> +mr_btree_insert(struct mlx4_mr_btree *bt, struct mlx4_mr_cache *entry)
> +{
> + struct mlx4_mr_cache *lkp_tbl;
> + uint16_t idx = 0;
> + size_t shift;
> +
> + assert(bt != NULL);
> + assert(bt->len <= bt->size);
> + assert(bt->len > 0);
> + lkp_tbl = *bt->table;
> + /* Find out the slot for insertion. */
> + if (mr_btree_lookup(bt, &idx, entry->start) != UINT32_MAX) {
> + DEBUG("abort insertion to B-tree(%p):"
> + " already exist at idx=%u [0x%lx, 0x%lx) lkey=0x%x",
> + (void *)bt, idx, entry->start, entry->end, entry->lkey);
This and various other logs causing 32bits build error because of %lx usage. Can
you please check them?
I am feeling sad to complain a patch like this just because of log format issue,
we should find a solution to this issue as community, either checkpatch checks
or automated 32bit builds, I don't know.
^ permalink raw reply [flat|nested] 23+ messages in thread* Re: [dpdk-dev] [PATCH v2 4/4] net/mlx4: add new Memory Region support
2018-05-09 23:12 ` Ferruh Yigit
@ 2018-05-10 3:00 ` Yongseok Koh
2018-05-10 6:01 ` Yongseok Koh
0 siblings, 1 reply; 23+ messages in thread
From: Yongseok Koh @ 2018-05-10 3:00 UTC (permalink / raw)
To: Ferruh Yigit; +Cc: Adrien Mazarguil, Nélio Laranjeiro, dev
> On May 9, 2018, at 4:12 PM, Ferruh Yigit <ferruh.yigit@intel.com> wrote:
>
> On 5/9/2018 12:09 PM, Yongseok Koh wrote:
>> This is the new design of Memory Region (MR) for mlx PMD, in order to:
>> - Accommodate the new memory hotplug model.
>> - Support non-contiguous Mempool.
>>
>> There are multiple layers for MR search.
>>
>> L0 is to look up the last-hit entry which is pointed by mr_ctrl->mru (Most
>> Recently Used). If L0 misses, L1 is to look up the address in a fixed-sized
>> array by linear search. L0/L1 is in an inline function -
>> mlx4_mr_lookup_cache().
>>
>> If L1 misses, the bottom-half function is called to look up the address
>> from the bigger local cache of the queue. This is L2 - mlx4_mr_addr2mr_bh()
>> and it is not an inline function. Data structure for L2 is the Binary Tree.
>>
>> If L2 misses, the search falls into the slowest path which takes locks in
>> order to access global device cache (priv->mr.cache) which is also a B-tree
>> and caches the original MR list (priv->mr.mr_list) of the device. Unless
>> the global cache is overflowed, it is all-inclusive of the MR list. This is
>> L3 - mlx4_mr_lookup_dev(). The size of the L3 cache table is limited and
>> can't be expanded on the fly due to deadlock. Refer to the comments in the
>> code for the details - mr_lookup_dev(). If L3 is overflowed, the list will
>> have to be searched directly bypassing the cache although it is slower.
>>
>> If L3 misses, a new MR for the address should be created -
>> mlx4_mr_create(). When it creates a new MR, it tries to register adjacent
>> memsegs as much as possible which are virtually contiguous around the
>> address. This must take two locks - memory_hotplug_lock and
>> priv->mr.rwlock. Due to memory_hotplug_lock, there can't be any
>> allocation/free of memory inside.
>>
>> In the free callback of the memory hotplug event, freed space is searched
>> from the MR list and corresponding bits are cleared from the bitmap of MRs.
>> This can fragment a MR and the MR will have multiple search entries in the
>> caches. Once there's a change by the event, the global cache must be
>> rebuilt and all the per-queue caches will be flushed as well. If memory is
>> frequently freed in run-time, that may cause jitter on dataplane processing
>> in the worst case by incurring MR cache flush and rebuild. But, it would be
>> the least probable scenario.
>>
>> To guarantee the most optimal performance, it is highly recommended to use
>> an EAL option - '--socket-mem'. Then, the reserved memory will be pinned
>> and won't be freed dynamically. And it is also recommended to configure
>> per-lcore cache of Mempool. Even though there're many MRs for a device or
>> MRs are highly fragmented, the cache of Mempool will be much helpful to
>> reduce misses on per-queue caches anyway.
>>
>> '--legacy-mem' is also supported.
>>
>> Signed-off-by: Yongseok Koh <yskoh@mellanox.com>
>
> <...>
>
>> +/**
>> + * Insert an entry to B-tree lookup table.
>> + *
>> + * @param bt
>> + * Pointer to B-tree structure.
>> + * @param entry
>> + * Pointer to new entry to insert.
>> + *
>> + * @return
>> + * 0 on success, -1 on failure.
>> + */
>> +static int
>> +mr_btree_insert(struct mlx4_mr_btree *bt, struct mlx4_mr_cache *entry)
>> +{
>> + struct mlx4_mr_cache *lkp_tbl;
>> + uint16_t idx = 0;
>> + size_t shift;
>> +
>> + assert(bt != NULL);
>> + assert(bt->len <= bt->size);
>> + assert(bt->len > 0);
>> + lkp_tbl = *bt->table;
>> + /* Find out the slot for insertion. */
>> + if (mr_btree_lookup(bt, &idx, entry->start) != UINT32_MAX) {
>> + DEBUG("abort insertion to B-tree(%p):"
>> + " already exist at idx=%u [0x%lx, 0x%lx) lkey=0x%x",
>> + (void *)bt, idx, entry->start, entry->end, entry->lkey);
>
> This and various other logs causing 32bits build error because of %lx usage. Can
> you please check them?
>
> I am feeling sad to complain a patch like this just because of log format issue,
> we should find a solution to this issue as community, either checkpatch checks
> or automated 32bit builds, I don't know.
Bummer. I have to change my bad habit of using %lx. And we will add 32-bit build
check to our internal system to filter this kind of mistakes beforehand.
Will work with Shahaf to fix it and rebase next-net-mlx.
Thanks,
Yongseok
^ permalink raw reply [flat|nested] 23+ messages in thread* Re: [dpdk-dev] [PATCH v2 4/4] net/mlx4: add new Memory Region support
2018-05-10 3:00 ` Yongseok Koh
@ 2018-05-10 6:01 ` Yongseok Koh
2018-05-10 19:29 ` Ferruh Yigit
0 siblings, 1 reply; 23+ messages in thread
From: Yongseok Koh @ 2018-05-10 6:01 UTC (permalink / raw)
To: Ferruh Yigit; +Cc: Adrien Mazarguil, Nélio Laranjeiro, dev
> On May 9, 2018, at 8:00 PM, Yongseok Koh <yskoh@mellanox.com> wrote:
>
>
>> On May 9, 2018, at 4:12 PM, Ferruh Yigit <ferruh.yigit@intel.com> wrote:
>>
>> On 5/9/2018 12:09 PM, Yongseok Koh wrote:
>> <...>
>>
>>> +/**
>>> + * Insert an entry to B-tree lookup table.
>>> + *
>>> + * @param bt
>>> + * Pointer to B-tree structure.
>>> + * @param entry
>>> + * Pointer to new entry to insert.
>>> + *
>>> + * @return
>>> + * 0 on success, -1 on failure.
>>> + */
>>> +static int
>>> +mr_btree_insert(struct mlx4_mr_btree *bt, struct mlx4_mr_cache *entry)
>>> +{
>>> + struct mlx4_mr_cache *lkp_tbl;
>>> + uint16_t idx = 0;
>>> + size_t shift;
>>> +
>>> + assert(bt != NULL);
>>> + assert(bt->len <= bt->size);
>>> + assert(bt->len > 0);
>>> + lkp_tbl = *bt->table;
>>> + /* Find out the slot for insertion. */
>>> + if (mr_btree_lookup(bt, &idx, entry->start) != UINT32_MAX) {
>>> + DEBUG("abort insertion to B-tree(%p):"
>>> + " already exist at idx=%u [0x%lx, 0x%lx) lkey=0x%x",
>>> + (void *)bt, idx, entry->start, entry->end, entry->lkey);
>>
>> This and various other logs causing 32bits build error because of %lx usage. Can
>> you please check them?
>>
>> I am feeling sad to complain a patch like this just because of log format issue,
>> we should find a solution to this issue as community, either checkpatch checks
>> or automated 32bit builds, I don't know.
>
> Bummer. I have to change my bad habit of using %lx. And we will add 32-bit build
> check to our internal system to filter this kind of mistakes beforehand.
>
> Will work with Shahaf to fix it and rebase next-net-mlx.
Ferruh, I've sent out a patch to Shahaf to change printing format specifiers and
Shahaf will squash it into the previous patches.
However, it seems we had stopped supporting 32-bit compilation since Nelio's
commit [1]
Not sure I'm doing right but I'm compiling it for T=i686-native-linuxapp-gcc and
still having a few more errors even except for my code. And even if I fix all of
the errors, linkage fails as explained in the commit message of [1].
Are you sure you encountered this 32b compilation issue for the first time?
[1] http://dpdk.org/browse/dpdk/commit/?id=ebbb81eb27daca0a89ee8f228fcf141d9eb6ef1c
Thanks,
Yongseok
^ permalink raw reply [flat|nested] 23+ messages in thread* Re: [dpdk-dev] [PATCH v2 4/4] net/mlx4: add new Memory Region support
2018-05-10 6:01 ` Yongseok Koh
@ 2018-05-10 19:29 ` Ferruh Yigit
2018-05-15 9:00 ` Nélio Laranjeiro
0 siblings, 1 reply; 23+ messages in thread
From: Ferruh Yigit @ 2018-05-10 19:29 UTC (permalink / raw)
To: Yongseok Koh; +Cc: Adrien Mazarguil, Nélio Laranjeiro, dev
On 5/10/2018 7:01 AM, Yongseok Koh wrote:
>
>> On May 9, 2018, at 8:00 PM, Yongseok Koh <yskoh@mellanox.com> wrote:
>>
>>
>>> On May 9, 2018, at 4:12 PM, Ferruh Yigit <ferruh.yigit@intel.com> wrote:
>>>
>>> On 5/9/2018 12:09 PM, Yongseok Koh wrote:
>>> <...>
>>>
>>>> +/**
>>>> + * Insert an entry to B-tree lookup table.
>>>> + *
>>>> + * @param bt
>>>> + * Pointer to B-tree structure.
>>>> + * @param entry
>>>> + * Pointer to new entry to insert.
>>>> + *
>>>> + * @return
>>>> + * 0 on success, -1 on failure.
>>>> + */
>>>> +static int
>>>> +mr_btree_insert(struct mlx4_mr_btree *bt, struct mlx4_mr_cache *entry)
>>>> +{
>>>> + struct mlx4_mr_cache *lkp_tbl;
>>>> + uint16_t idx = 0;
>>>> + size_t shift;
>>>> +
>>>> + assert(bt != NULL);
>>>> + assert(bt->len <= bt->size);
>>>> + assert(bt->len > 0);
>>>> + lkp_tbl = *bt->table;
>>>> + /* Find out the slot for insertion. */
>>>> + if (mr_btree_lookup(bt, &idx, entry->start) != UINT32_MAX) {
>>>> + DEBUG("abort insertion to B-tree(%p):"
>>>> + " already exist at idx=%u [0x%lx, 0x%lx) lkey=0x%x",
>>>> + (void *)bt, idx, entry->start, entry->end, entry->lkey);
>>>
>>> This and various other logs causing 32bits build error because of %lx usage. Can
>>> you please check them?
>>>
>>> I am feeling sad to complain a patch like this just because of log format issue,
>>> we should find a solution to this issue as community, either checkpatch checks
>>> or automated 32bit builds, I don't know.
>>
>> Bummer. I have to change my bad habit of using %lx. And we will add 32-bit build
>> check to our internal system to filter this kind of mistakes beforehand.
>>
>> Will work with Shahaf to fix it and rebase next-net-mlx.
>
> Ferruh, I've sent out a patch to Shahaf to change printing format specifiers and
> Shahaf will squash it into the previous patches.
>
> However, it seems we had stopped supporting 32-bit compilation since Nelio's
> commit [1]
>
> Not sure I'm doing right but I'm compiling it for T=i686-native-linuxapp-gcc and
> still having a few more errors even except for my code. And even if I fix all of
> the errors, linkage fails as explained in the commit message of [1].
>
> Are you sure you encountered this 32b compilation issue for the first time?
I do just compilation on mlx drivers.
And building only mlx4 for 32bits, mlx5 doesn't support 32bits as you point out
below patch. mlx4 32bit compiles fine with me as same config you have used.
Also features documentation [2] verifies this, mlx4 supports 32bits but mlx5 not.
[2]
https://dpdk.org/browse/next/dpdk-next-net/tree/doc/guides/nics/features/mlx4.ini?h=v18.02#n32
https://dpdk.org/browse/next/dpdk-next-net/tree/doc/guides/nics/features/mlx5.ini?h=v18.02#n42
>
>
> [1] http://dpdk.org/browse/dpdk/commit/?id=ebbb81eb27daca0a89ee8f228fcf141d9eb6ef1c
>
>
> Thanks,
> Yongseok
>
>
^ permalink raw reply [flat|nested] 23+ messages in thread* Re: [dpdk-dev] [PATCH v2 4/4] net/mlx4: add new Memory Region support
2018-05-10 19:29 ` Ferruh Yigit
@ 2018-05-15 9:00 ` Nélio Laranjeiro
0 siblings, 0 replies; 23+ messages in thread
From: Nélio Laranjeiro @ 2018-05-15 9:00 UTC (permalink / raw)
To: Ferruh Yigit; +Cc: Yongseok Koh, Adrien Mazarguil, dev
On Thu, May 10, 2018 at 08:29:03PM +0100, Ferruh Yigit wrote:
> On 5/10/2018 7:01 AM, Yongseok Koh wrote:
> >
> >> On May 9, 2018, at 8:00 PM, Yongseok Koh <yskoh@mellanox.com> wrote:
> >>
> >>
> >>> On May 9, 2018, at 4:12 PM, Ferruh Yigit <ferruh.yigit@intel.com> wrote:
> >>>
> >>> On 5/9/2018 12:09 PM, Yongseok Koh wrote:
> >>> <...>
> >>>
> >>>> +/**
> >>>> + * Insert an entry to B-tree lookup table.
> >>>> + *
> >>>> + * @param bt
> >>>> + * Pointer to B-tree structure.
> >>>> + * @param entry
> >>>> + * Pointer to new entry to insert.
> >>>> + *
> >>>> + * @return
> >>>> + * 0 on success, -1 on failure.
> >>>> + */
> >>>> +static int
> >>>> +mr_btree_insert(struct mlx4_mr_btree *bt, struct mlx4_mr_cache *entry)
> >>>> +{
> >>>> + struct mlx4_mr_cache *lkp_tbl;
> >>>> + uint16_t idx = 0;
> >>>> + size_t shift;
> >>>> +
> >>>> + assert(bt != NULL);
> >>>> + assert(bt->len <= bt->size);
> >>>> + assert(bt->len > 0);
> >>>> + lkp_tbl = *bt->table;
> >>>> + /* Find out the slot for insertion. */
> >>>> + if (mr_btree_lookup(bt, &idx, entry->start) != UINT32_MAX) {
> >>>> + DEBUG("abort insertion to B-tree(%p):"
> >>>> + " already exist at idx=%u [0x%lx, 0x%lx) lkey=0x%x",
> >>>> + (void *)bt, idx, entry->start, entry->end, entry->lkey);
> >>>
> >>> This and various other logs causing 32bits build error because of %lx usage. Can
> >>> you please check them?
> >>>
> >>> I am feeling sad to complain a patch like this just because of log format issue,
> >>> we should find a solution to this issue as community, either checkpatch checks
> >>> or automated 32bit builds, I don't know.
> >>
> >> Bummer. I have to change my bad habit of using %lx. And we will add 32-bit build
> >> check to our internal system to filter this kind of mistakes beforehand.
> >>
> >> Will work with Shahaf to fix it and rebase next-net-mlx.
> >
> > Ferruh, I've sent out a patch to Shahaf to change printing format specifiers and
> > Shahaf will squash it into the previous patches.
> >
> > However, it seems we had stopped supporting 32-bit compilation since Nelio's
> > commit [1]
> >
> > Not sure I'm doing right but I'm compiling it for T=i686-native-linuxapp-gcc and
> > still having a few more errors even except for my code. And even if I fix all of
> > the errors, linkage fails as explained in the commit message of [1].
> >
> > Are you sure you encountered this 32b compilation issue for the first time?
On mlx5 32 bits has been disabled as Mellanox OFED does not support
32bits compilation whereas RDMA-Core supports it.
I've just taken a look on Mellanox Website, Mellanox OFED 4.3-3.0.2.1 is
still not available for 32bits. We cannot assume the support exists.
> I do just compilation on mlx drivers.
> And building only mlx4 for 32bits, mlx5 doesn't support 32bits as you point out
> below patch. mlx4 32bit compiles fine with me as same config you have used.
>
> Also features documentation [2] verifies this, mlx4 supports 32bits but mlx5 not.
>
> [2]
> https://dpdk.org/browse/next/dpdk-next-net/tree/doc/guides/nics/features/mlx4.ini?h=v18.02#n32
> https://dpdk.org/browse/next/dpdk-next-net/tree/doc/guides/nics/features/mlx5.ini?h=v18.02#n42
>
> >
> >
> > [1] http://dpdk.org/browse/dpdk/commit/?id=ebbb81eb27daca0a89ee8f228fcf141d9eb6ef1c
> >
> >
> > Thanks,
> > Yongseok
> >
> >
>
--
Nélio Laranjeiro
6WIND
^ permalink raw reply [flat|nested] 23+ messages in thread