Re: [dpdk-dev] [PATCH] net/mlx5: creating relaxed ordering memory regions

[Raslan Darawsheh <rasland@mellanox.com>]

Hi,

> -----Original Message-----
> From: Shiri Kuzin <shirik@mellanox.com>
> Sent: Tuesday, March 24, 2020 1:40 PM
> To: dev@dpdk.org
> Cc: Matan Azrad <matan@mellanox.com>; Raslan Darawsheh
> <rasland@mellanox.com>; Slava Ovsiienko <viacheslavo@mellanox.com>
> Subject: [PATCH] net/mlx5: creating relaxed ordering memory regions
> 
> In the current state, when preforming read/write
> transactions we must wait for a completion in order
> to run the next transaction, and all transactions are
> performed by order.
> 
> Relaxed Ordering is a PCI optimization which by enabling it
> we allow the system to perform read/writes in a different
> order without having to wait for completion and improve
> the performance in that matter.
> 
> This commit introduces the creation of relaxed ordering
> memory regions in mlx5.
> As relaxed ordering is an optimization, drivers that
> do not support it can simply ignore it and therefore
> it is enabled by default.
> 
> Signed-off-by: Shiri Kuzin <shirik@mellanox.com>
> Acked-by: Matan Azrad <matan@mellanox.com>
> ---
>  doc/guides/rel_notes/release_20_05.rst | 2 +-
>  drivers/common/mlx5/mlx5_devx_cmds.c   | 4 ++++
>  drivers/common/mlx5/mlx5_devx_cmds.h   | 1 +
>  drivers/common/mlx5/mlx5_glue.h        | 4 ++++
>  drivers/common/mlx5/mlx5_prm.h         | 4 +++-
>  drivers/net/mlx5/mlx5_flow_dv.c        | 1 +
>  drivers/net/mlx5/mlx5_mr.c             | 6 ++++--
>  drivers/vdpa/mlx5/mlx5_vdpa_lm.c       | 1 +
>  drivers/vdpa/mlx5/mlx5_vdpa_mem.c      | 1 +
>  9 files changed, 20 insertions(+), 4 deletions(-)
> 
> diff --git a/doc/guides/rel_notes/release_20_05.rst
> b/doc/guides/rel_notes/release_20_05.rst
> index 000bbf5..c960fd2 100644
> --- a/doc/guides/rel_notes/release_20_05.rst
> +++ b/doc/guides/rel_notes/release_20_05.rst
> @@ -61,7 +61,7 @@ New Features
>    Updated Mellanox mlx5 driver with new features and improvements,
> including:
> 
>    * Added support for matching on IPv4 Time To Live and IPv6 Hop Limit.
> -
> +  * Added support for creating Relaxed Ordering Memory Regions.
> 
>  Removed Items
>  -------------
> diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c
> b/drivers/common/mlx5/mlx5_devx_cmds.c
> index d960bc9..1157a44 100644
> --- a/drivers/common/mlx5/mlx5_devx_cmds.c
> +++ b/drivers/common/mlx5/mlx5_devx_cmds.c
> @@ -196,6 +196,10 @@ struct mlx5_devx_obj *
>  	MLX5_SET(mkc, mkc, pd, attr->pd);
>  	MLX5_SET(mkc, mkc, mkey_7_0, attr->umem_id & 0xFF);
>  	MLX5_SET(mkc, mkc, translations_octword_size, translation_size);
> +	if (attr->relaxed_ordering == 1) {
> +		MLX5_SET(mkc, mkc, relaxed_ordering_write, 0x1);
> +		MLX5_SET(mkc, mkc, relaxed_ordering_read, 0x1);
> +	}
>  	MLX5_SET64(mkc, mkc, start_addr, attr->addr);
>  	MLX5_SET64(mkc, mkc, len, attr->size);
>  	mkey->obj = mlx5_glue->devx_obj_create(ctx, in, in_size_dw * 4,
> out,
> diff --git a/drivers/common/mlx5/mlx5_devx_cmds.h
> b/drivers/common/mlx5/mlx5_devx_cmds.h
> index 6912dc6..20bb294 100644
> --- a/drivers/common/mlx5/mlx5_devx_cmds.h
> +++ b/drivers/common/mlx5/mlx5_devx_cmds.h
> @@ -33,6 +33,7 @@ struct mlx5_devx_mkey_attr {
>  	uint32_t pd;
>  	uint32_t log_entity_size;
>  	uint32_t pg_access:1;
> +	uint32_t relaxed_ordering:1;
>  	struct mlx5_klm *klm_array;
>  	int klm_num;
>  };
> diff --git a/drivers/common/mlx5/mlx5_glue.h
> b/drivers/common/mlx5/mlx5_glue.h
> index 6238b43..cd1136f 100644
> --- a/drivers/common/mlx5/mlx5_glue.h
> +++ b/drivers/common/mlx5/mlx5_glue.h
> @@ -98,6 +98,10 @@
>  			uint64_t comp_mask; };
>  #endif
> 
> +#ifndef IBV_ACCESS_RELAXED_ORDERING
> +#define IBV_ACCESS_RELAXED_ORDERING 0
> +#endif
> +
>  /* LIB_GLUE_VERSION must be updated every time this structure is
> modified. */
>  struct mlx5_glue {
>  	const char *version;
> diff --git a/drivers/common/mlx5/mlx5_prm.h
> b/drivers/common/mlx5/mlx5_prm.h
> index eee3a4b..00fd7c1 100644
> --- a/drivers/common/mlx5/mlx5_prm.h
> +++ b/drivers/common/mlx5/mlx5_prm.h
> @@ -882,7 +882,9 @@ struct mlx5_ifc_mkc_bits {
> 
>  	u8         translations_octword_size[0x20];
> 
> -	u8         reserved_at_1c0[0x1b];
> +	u8         reserved_at_1c0[0x19];
> +	u8		   relaxed_ordering_read[0x1];
> +	u8		   reserved_at_1da[0x1];
>  	u8         log_page_size[0x5];
> 
>  	u8         reserved_at_1e0[0x20];
> diff --git a/drivers/net/mlx5/mlx5_flow_dv.c
> b/drivers/net/mlx5/mlx5_flow_dv.c
> index 2090631..809833b 100644
> --- a/drivers/net/mlx5/mlx5_flow_dv.c
> +++ b/drivers/net/mlx5/mlx5_flow_dv.c
> @@ -3962,6 +3962,7 @@ struct field_modify_info modify_tcp[] = {
>  	mkey_attr.pg_access = 0;
>  	mkey_attr.klm_array = NULL;
>  	mkey_attr.klm_num = 0;
> +	mkey_attr.relaxed_ordering = 1;
>  	mem_mng->dm = mlx5_devx_cmd_mkey_create(sh->ctx,
> &mkey_attr);
>  	if (!mem_mng->dm) {
>  		mlx5_glue->devx_umem_dereg(mem_mng->umem);
> diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c
> index cb97c87..fd1245d 100644
> --- a/drivers/net/mlx5/mlx5_mr.c
> +++ b/drivers/net/mlx5/mlx5_mr.c
> @@ -768,7 +768,8 @@ struct mr_update_mp_data {
>  	 * through mlx5_alloc_verbs_buf().
>  	 */
>  	mr->ibv_mr = mlx5_glue->reg_mr(sh->pd, (void *)data.start, len,
> -				       IBV_ACCESS_LOCAL_WRITE);
> +				       IBV_ACCESS_LOCAL_WRITE |
> +					   IBV_ACCESS_RELAXED_ORDERING);
>  	if (mr->ibv_mr == NULL) {
>  		DEBUG("port %u fail to create a verbs MR for address (%p)",
>  		      dev->data->port_id, (void *)addr);
> @@ -1217,7 +1218,8 @@ struct mr_update_mp_data {
>  	if (mr == NULL)
>  		return NULL;
>  	mr->ibv_mr = mlx5_glue->reg_mr(priv->sh->pd, (void *)addr, len,
> -				       IBV_ACCESS_LOCAL_WRITE);
> +				       IBV_ACCESS_LOCAL_WRITE |
> +					   IBV_ACCESS_RELAXED_ORDERING);
>  	if (mr->ibv_mr == NULL) {
>  		DRV_LOG(WARNING,
>  			"port %u fail to create a verbs MR for address (%p)",
> diff --git a/drivers/vdpa/mlx5/mlx5_vdpa_lm.c
> b/drivers/vdpa/mlx5/mlx5_vdpa_lm.c
> index 3358704..4457760 100644
> --- a/drivers/vdpa/mlx5/mlx5_vdpa_lm.c
> +++ b/drivers/vdpa/mlx5/mlx5_vdpa_lm.c
> @@ -39,6 +39,7 @@
>  			.pg_access = 1,
>  			.klm_array = NULL,
>  			.klm_num = 0,
> +			.relaxed_ordering = 0,
>  	};
>  	struct mlx5_devx_virtq_attr attr = {
>  		.type =
> MLX5_VIRTQ_MODIFY_TYPE_DIRTY_BITMAP_PARAMS,
> diff --git a/drivers/vdpa/mlx5/mlx5_vdpa_mem.c
> b/drivers/vdpa/mlx5/mlx5_vdpa_mem.c
> index 398ca35..da31b47 100644
> --- a/drivers/vdpa/mlx5/mlx5_vdpa_mem.c
> +++ b/drivers/vdpa/mlx5/mlx5_vdpa_mem.c
> @@ -263,6 +263,7 @@
>  		mkey_attr.pg_access = 1;
>  		mkey_attr.klm_array = NULL;
>  		mkey_attr.klm_num = 0;
> +		mkey_attr.relaxed_ordering = 0;
>  		entry->mkey = mlx5_devx_cmd_mkey_create(priv->ctx,
> &mkey_attr);
>  		if (!entry->mkey) {
>  			DRV_LOG(ERR, "Failed to create direct Mkey.");
> --
> 1.8.3.1


Patch applied to next-net-mlx,

Kindest regards,
Raslan Darawsheh