[dpdk-dev] [PATCH 1/2] common/mlx5: add virtio queue protection domain

[dpdk-dev] [PATCH 1/2] common/mlx5: add virtio queue protection domain

[Matan Azrad]

Starting from FW version 22.27.4002, it is required to configure
protection domain (PD) for each virtq created by DevX.

Add PD requirement in virtq DevX APIs.

Cc: stable@dpdk.org

Signed-off-by: Matan Azrad <matan@mellanox.com>
Signed-off-by: Xueming Li <xuemingl@mellanox.com>
---
 drivers/common/mlx5/mlx5_devx_cmds.c | 1 +
 drivers/common/mlx5/mlx5_devx_cmds.h | 1 +
 drivers/common/mlx5/mlx5_prm.h       | 4 +++-
 3 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c
index 4bf22ce..3b6a857 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.c
+++ b/drivers/common/mlx5/mlx5_devx_cmds.c
@@ -1262,6 +1262,7 @@ struct mlx5_devx_obj *
 	MLX5_SET(virtio_q, virtctx, umem_3_size, attr->umems[2].size);
 	MLX5_SET64(virtio_q, virtctx, umem_3_offset, attr->umems[2].offset);
 	MLX5_SET(virtio_q, virtctx, counter_set_id, attr->counters_obj_id);
+	MLX5_SET(virtio_q, virtctx, pd, attr->pd);
 	MLX5_SET(virtio_net_q, virtq, tisn_or_qpn, attr->tis_id);
 	virtq_obj->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in), out,
 						    sizeof(out));
diff --git a/drivers/common/mlx5/mlx5_devx_cmds.h b/drivers/common/mlx5/mlx5_devx_cmds.h
index 59a70a0..3ce44c3 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.h
+++ b/drivers/common/mlx5/mlx5_devx_cmds.h
@@ -259,6 +259,7 @@ struct mlx5_devx_virtq_attr {
 	uint16_t hw_available_index;
 	uint16_t hw_used_index;
 	uint16_t q_size;
+	uint32_t pd:24;
 	uint32_t virtio_version_1_0:1;
 	uint32_t tso_ipv4:1;
 	uint32_t tso_ipv6:1;
diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h
index 5fc10d6..c63795f 100644
--- a/drivers/common/mlx5/mlx5_prm.h
+++ b/drivers/common/mlx5/mlx5_prm.h
@@ -2088,7 +2088,9 @@ struct mlx5_ifc_virtio_q_bits {
 	u8 umem_3_size[0x20];
 	u8 umem_3_offset[0x40];
 	u8 counter_set_id[0x20];
-	u8 reserved_at_320[0xe0];
+	u8 reserved_at_320[0x8];
+	u8 pd[0x18];
+	u8 reserved_at_340[0xc0];
 };
 
 struct mlx5_ifc_virtio_net_q_bits {
-- 
1.8.3.1

[dpdk-dev] [PATCH 2/2] vdpa/mlx5: adjust virtio queue protection domain

[Matan Azrad]

In other to fill the new requirement for virtq configuration,
set the single PD managed by the driver for all the virtqs.

Cc: stable@dpdk.org

Signed-off-by: Matan Azrad <matan@mellanox.com>
Signed-off-by: Xueming Li <xuemingl@mellanox.com>
---
 drivers/vdpa/mlx5/mlx5_vdpa.c       | 38 ++++++++++++++++++++++++++++++++++-
 drivers/vdpa/mlx5/mlx5_vdpa_mem.c   | 40 -------------------------------------
 drivers/vdpa/mlx5/mlx5_vdpa_virtq.c |  1 +
 3 files changed, 38 insertions(+), 41 deletions(-)

diff --git a/drivers/vdpa/mlx5/mlx5_vdpa.c b/drivers/vdpa/mlx5/mlx5_vdpa.c
index a80e3f4..94cac66 100644
--- a/drivers/vdpa/mlx5/mlx5_vdpa.c
+++ b/drivers/vdpa/mlx5/mlx5_vdpa.c
@@ -192,6 +192,37 @@
 }
 
 static int
+mlx5_vdpa_pd_create(struct mlx5_vdpa_priv *priv)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+	priv->pd = mlx5_glue->alloc_pd(priv->ctx);
+	if (priv->pd == NULL) {
+		DRV_LOG(ERR, "Failed to allocate PD.");
+		return errno ? -errno : -ENOMEM;
+	}
+	struct mlx5dv_obj obj;
+	struct mlx5dv_pd pd_info;
+	int ret = 0;
+
+	obj.pd.in = priv->pd;
+	obj.pd.out = &pd_info;
+	ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_PD);
+	if (ret) {
+		DRV_LOG(ERR, "Fail to get PD object info.");
+		mlx5_glue->dealloc_pd(priv->pd);
+		priv->pd = NULL;
+		return -errno;
+	}
+	priv->pdn = pd_info.pdn;
+	return 0;
+#else
+	(void)priv;
+	DRV_LOG(ERR, "Cannot get pdn - no DV support.");
+	return -ENOTSUP;
+#endif /* HAVE_IBV_FLOW_DV_SUPPORT */
+}
+
+static int
 mlx5_vdpa_dev_close(int vid)
 {
 	int did = rte_vhost_get_vdpa_device_id(vid);
@@ -209,6 +240,10 @@
 	mlx5_vdpa_virtqs_release(priv);
 	mlx5_vdpa_event_qp_global_release(priv);
 	mlx5_vdpa_mem_dereg(priv);
+	if (priv->pd) {
+		claim_zero(mlx5_glue->dealloc_pd(priv->pd));
+		priv->pd = NULL;
+	}
 	priv->configured = 0;
 	priv->vid = 0;
 	DRV_LOG(INFO, "vDPA device %d was closed.", vid);
@@ -230,7 +265,8 @@
 		return -1;
 	}
 	priv->vid = vid;
-	if (mlx5_vdpa_mem_register(priv) || mlx5_vdpa_direct_db_prepare(priv) ||
+	if (mlx5_vdpa_pd_create(priv) || mlx5_vdpa_mem_register(priv) ||
+	    mlx5_vdpa_direct_db_prepare(priv) ||
 	    mlx5_vdpa_virtqs_prepare(priv) || mlx5_vdpa_steer_setup(priv) ||
 	    mlx5_vdpa_cqe_event_setup(priv)) {
 		mlx5_vdpa_dev_close(vid);
diff --git a/drivers/vdpa/mlx5/mlx5_vdpa_mem.c b/drivers/vdpa/mlx5/mlx5_vdpa_mem.c
index da31b47..b6c7cb8 100644
--- a/drivers/vdpa/mlx5/mlx5_vdpa_mem.c
+++ b/drivers/vdpa/mlx5/mlx5_vdpa_mem.c
@@ -14,39 +14,6 @@
 #include "mlx5_vdpa_utils.h"
 #include "mlx5_vdpa.h"
 
-static int
-mlx5_vdpa_pd_prepare(struct mlx5_vdpa_priv *priv)
-{
-#ifdef HAVE_IBV_FLOW_DV_SUPPORT
-	if (priv->pd)
-		return 0;
-	priv->pd = mlx5_glue->alloc_pd(priv->ctx);
-	if (priv->pd == NULL) {
-		DRV_LOG(ERR, "Failed to allocate PD.");
-		return errno ? -errno : -ENOMEM;
-	}
-	struct mlx5dv_obj obj;
-	struct mlx5dv_pd pd_info;
-	int ret = 0;
-
-	obj.pd.in = priv->pd;
-	obj.pd.out = &pd_info;
-	ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_PD);
-	if (ret) {
-		DRV_LOG(ERR, "Fail to get PD object info.");
-		mlx5_glue->dealloc_pd(priv->pd);
-		priv->pd = NULL;
-		return -errno;
-	}
-	priv->pdn = pd_info.pdn;
-	return 0;
-#else
-	(void)priv;
-	DRV_LOG(ERR, "Cannot get pdn - no DV support.");
-	return -ENOTSUP;
-#endif /* HAVE_IBV_FLOW_DV_SUPPORT */
-}
-
 void
 mlx5_vdpa_mem_dereg(struct mlx5_vdpa_priv *priv)
 {
@@ -68,10 +35,6 @@
 		claim_zero(mlx5_glue->dereg_mr(priv->null_mr));
 		priv->null_mr = NULL;
 	}
-	if (priv->pd) {
-		claim_zero(mlx5_glue->dealloc_pd(priv->pd));
-		priv->pd = NULL;
-	}
 	if (priv->vmem) {
 		free(priv->vmem);
 		priv->vmem = NULL;
@@ -230,9 +193,6 @@
 	if (!mem)
 		return -rte_errno;
 	priv->vmem = mem;
-	ret = mlx5_vdpa_pd_prepare(priv);
-	if (ret)
-		goto error;
 	priv->null_mr = mlx5_glue->alloc_null_mr(priv->pd);
 	if (!priv->null_mr) {
 		DRV_LOG(ERR, "Failed to allocate null MR.");
diff --git a/drivers/vdpa/mlx5/mlx5_vdpa_virtq.c b/drivers/vdpa/mlx5/mlx5_vdpa_virtq.c
index d57ed59..86aded2 100644
--- a/drivers/vdpa/mlx5/mlx5_vdpa_virtq.c
+++ b/drivers/vdpa/mlx5/mlx5_vdpa_virtq.c
@@ -284,6 +284,7 @@
 	attr.mkey = priv->gpa_mkey_index;
 	attr.tis_id = priv->tis->id;
 	attr.queue_index = index;
+	attr.pd = priv->pdn;
 	virtq->virtq = mlx5_devx_cmd_create_virtq(priv->ctx, &attr);
 	virtq->priv = priv;
 	if (!virtq->virtq)
-- 
1.8.3.1

Re: [dpdk-dev] [PATCH 2/2] vdpa/mlx5: adjust virtio queue protection domain

[Maxime Coquelin]

On 6/2/20 5:51 PM, Matan Azrad wrote:
> In other to fill the new requirement for virtq configuration,
> set the single PD managed by the driver for all the virtqs.
> 
> Cc: stable@dpdk.org
> 
> Signed-off-by: Matan Azrad <matan@mellanox.com>
> Signed-off-by: Xueming Li <xuemingl@mellanox.com>
> ---
>  drivers/vdpa/mlx5/mlx5_vdpa.c       | 38 ++++++++++++++++++++++++++++++++++-
>  drivers/vdpa/mlx5/mlx5_vdpa_mem.c   | 40 -------------------------------------
>  drivers/vdpa/mlx5/mlx5_vdpa_virtq.c |  1 +
>  3 files changed, 38 insertions(+), 41 deletions(-)
> 

Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>

Re: [dpdk-dev] [PATCH 2/2] vdpa/mlx5: adjust virtio queue protection domain

[Maxime Coquelin]

On 6/2/20 5:51 PM, Matan Azrad wrote:
> In other to fill the new requirement for virtq configuration,
> set the single PD managed by the driver for all the virtqs.
> 
> Cc: stable@dpdk.org
> 
> Signed-off-by: Matan Azrad <matan@mellanox.com>
> Signed-off-by: Xueming Li <xuemingl@mellanox.com>
> ---
>  drivers/vdpa/mlx5/mlx5_vdpa.c       | 38 ++++++++++++++++++++++++++++++++++-
>  drivers/vdpa/mlx5/mlx5_vdpa_mem.c   | 40 -------------------------------------
>  drivers/vdpa/mlx5/mlx5_vdpa_virtq.c |  1 +
>  3 files changed, 38 insertions(+), 41 deletions(-)
> 


Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>

Re: [dpdk-dev] [PATCH 2/2] vdpa/mlx5: adjust virtio queue protection domain

[Maxime Coquelin]

On 6/2/20 5:51 PM, Matan Azrad wrote:
> In other to fill the new requirement for virtq configuration,
> set the single PD managed by the driver for all the virtqs.
> 
> Cc: stable@dpdk.org
> 
> Signed-off-by: Matan Azrad <matan@mellanox.com>
> Signed-off-by: Xueming Li <xuemingl@mellanox.com>
> ---
>  drivers/vdpa/mlx5/mlx5_vdpa.c       | 38 ++++++++++++++++++++++++++++++++++-
>  drivers/vdpa/mlx5/mlx5_vdpa_mem.c   | 40 -------------------------------------
>  drivers/vdpa/mlx5/mlx5_vdpa_virtq.c |  1 +
>  3 files changed, 38 insertions(+), 41 deletions(-)
> 

Applied to dpdk-next-virtio/master

Thanks,
Maxime

Re: [dpdk-dev] [PATCH 1/2] common/mlx5: add virtio queue protection domain

[Maxime Coquelin]

On 6/2/20 5:51 PM, Matan Azrad wrote:
> Starting from FW version 22.27.4002, it is required to configure
> protection domain (PD) for each virtq created by DevX.
> 
> Add PD requirement in virtq DevX APIs.
> 
> Cc: stable@dpdk.org
> 
> Signed-off-by: Matan Azrad <matan@mellanox.com>
> Signed-off-by: Xueming Li <xuemingl@mellanox.com>
> ---
>  drivers/common/mlx5/mlx5_devx_cmds.c | 1 +
>  drivers/common/mlx5/mlx5_devx_cmds.h | 1 +
>  drivers/common/mlx5/mlx5_prm.h       | 4 +++-
>  3 files changed, 5 insertions(+), 1 deletion(-)
> 

Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>

Re: [dpdk-dev] [PATCH 1/2] common/mlx5: add virtio queue protection domain

[Maxime Coquelin]

On 6/2/20 5:51 PM, Matan Azrad wrote:
> Starting from FW version 22.27.4002, it is required to configure
> protection domain (PD) for each virtq created by DevX.
> 
> Add PD requirement in virtq DevX APIs.
> 
> Cc: stable@dpdk.org
> 
> Signed-off-by: Matan Azrad <matan@mellanox.com>
> Signed-off-by: Xueming Li <xuemingl@mellanox.com>
> ---
>  drivers/common/mlx5/mlx5_devx_cmds.c | 1 +
>  drivers/common/mlx5/mlx5_devx_cmds.h | 1 +
>  drivers/common/mlx5/mlx5_prm.h       | 4 +++-
>  3 files changed, 5 insertions(+), 1 deletion(-)
> 


Applied to dpdk-next-virtio/master

Thanks,
Maxime

Re: [dpdk-dev] [PATCH 1/2] common/mlx5: add virtio queue protection domain

[Maxime Coquelin]

Hi Ferruh,

On 6/18/20 6:30 PM, Maxime Coquelin wrote:
> 
> 
> On 6/2/20 5:51 PM, Matan Azrad wrote:
>> Starting from FW version 22.27.4002, it is required to configure
>> protection domain (PD) for each virtq created by DevX.
>>
>> Add PD requirement in virtq DevX APIs.
>>
>> Cc: stable@dpdk.org
>>
>> Signed-off-by: Matan Azrad <matan@mellanox.com>
>> Signed-off-by: Xueming Li <xuemingl@mellanox.com>
>> ---
>>  drivers/common/mlx5/mlx5_devx_cmds.c | 1 +
>>  drivers/common/mlx5/mlx5_devx_cmds.h | 1 +
>>  drivers/common/mlx5/mlx5_prm.h       | 4 +++-
>>  3 files changed, 5 insertions(+), 1 deletion(-)
>>
> 
> 
> Applied to dpdk-next-virtio/master


Finally not, this version has also been superseded.
I removed it from my tree.

Maxime

> Thanks,
> Maxime
> 

Re: [dpdk-dev] [PATCH 1/2] common/mlx5: add virtio queue protection domain

[Maxime Coquelin]

On 6/19/20 8:01 AM, Maxime Coquelin wrote:
> Hi Ferruh,
> 
> On 6/18/20 6:30 PM, Maxime Coquelin wrote:
>>
>>
>> On 6/2/20 5:51 PM, Matan Azrad wrote:
>>> Starting from FW version 22.27.4002, it is required to configure
>>> protection domain (PD) for each virtq created by DevX.
>>>
>>> Add PD requirement in virtq DevX APIs.
>>>
>>> Cc: stable@dpdk.org
>>>
>>> Signed-off-by: Matan Azrad <matan@mellanox.com>
>>> Signed-off-by: Xueming Li <xuemingl@mellanox.com>
>>> ---
>>>  drivers/common/mlx5/mlx5_devx_cmds.c | 1 +
>>>  drivers/common/mlx5/mlx5_devx_cmds.h | 1 +
>>>  drivers/common/mlx5/mlx5_prm.h       | 4 +++-
>>>  3 files changed, 5 insertions(+), 1 deletion(-)
>>>
>>
>>
>> Applied to dpdk-next-virtio/master
> 
> 
> Finally not, this version has also been superseded.
> I removed it from my tree.

Nevermind for this series, Matan did not send a new revision for it.
Sorry,
Maxime

> Maxime
> 
>> Thanks,
>> Maxime
>>
>