Re: [dpdk-dev] [PATCH 1/5] common/mlx5: add timestamp format support to DevX

[Matan Azrad <matan@nvidia.com>]

From: Viacheslav Ovsiienko
> This patch handles the NIC supported timestamp formats via DevX.
> There are two different timestamp format can be provided potentially.
> The free-running is some opaque values captured from internal clock counter
> clocked by some independent oscillator. The free-running frequency is not pre-
> defined and should be queried from the NIC.
> The real-time timestamps are expressed in nanosecond, captured from the
> dedicated UTC counter, that can be adjusted on the fly and synchronized with
> some external master clock.
> 
> Depending on the version and configuration the hardware might support either
> FR or RT timestamps, or both in the same time on per queue basis. The commit
> provides the querying information about the supported timestamp formats and
> provides the means to configure ones at queue creation.
> 
> Fixes: e2b4925ef7c1 ("net/mlx5: support Direct Rules E-Switch")
> Cc: stable@dpdk.org
> 
> Signed-off-by: Viacheslav Ovsiienko <viacheslavo@nvidia.com>
> ---
>  drivers/common/mlx5/mlx5_devx_cmds.c | 35 ++++++++++++++++++++++--
> drivers/common/mlx5/mlx5_devx_cmds.h |  7 +++++
>  drivers/common/mlx5/mlx5_prm.h       | 41 +++++++++++++++++++++++++---
>  3 files changed, 76 insertions(+), 7 deletions(-)
> 
> diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c
> b/drivers/common/mlx5/mlx5_devx_cmds.c
> index 0060c37fc0..bf9921b78a 100644
> --- a/drivers/common/mlx5/mlx5_devx_cmds.c
> +++ b/drivers/common/mlx5/mlx5_devx_cmds.c
> @@ -715,6 +715,9 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,
>  				      device_frequency_khz);
>  	attr->scatter_fcs_w_decap_disable =
>  		MLX5_GET(cmd_hca_cap, hcattr,
> scatter_fcs_w_decap_disable);
> +	attr->roce = MLX5_GET(cmd_hca_cap, hcattr, roce);
> +	attr->rq_ts_format = MLX5_GET(cmd_hca_cap, hcattr, rq_ts_format);
> +	attr->sq_ts_format = MLX5_GET(cmd_hca_cap, hcattr, sq_ts_format);
>  	attr->regex = MLX5_GET(cmd_hca_cap, hcattr, regexp);
>  	attr->regexp_num_of_engines = MLX5_GET(cmd_hca_cap, hcattr,
>  					       regexp_num_of_engines);
> @@ -859,9 +862,32 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,
>  	attr->rss_ind_tbl_cap = MLX5_GET
> 
> 	(per_protocol_networking_offload_caps,
>  					 hcattr, rss_ind_tbl_cap);
> -	if (attr->wqe_inline_mode !=
> MLX5_CAP_INLINE_MODE_VPORT_CONTEXT)
> -		return 0;
> -	if (attr->eth_virt) {
> +	/* Query HCA attribute for ROCE. */
> +	if (attr->roce) {
> +		memset(in, 0, sizeof(in));
> +		memset(out, 0, sizeof(out));
> +		MLX5_SET(query_hca_cap_in, in, opcode,
> +			 MLX5_CMD_OP_QUERY_HCA_CAP);
> +		MLX5_SET(query_hca_cap_in, in, op_mod,
> +			 MLX5_GET_HCA_CAP_OP_MOD_ROCE |
> +			 MLX5_HCA_CAP_OPMOD_GET_CUR);
> +		rc = mlx5_glue->devx_general_cmd(ctx, in, sizeof(in),
> +						 out, sizeof(out));
> +		if (rc)
> +			goto error;
> +		status = MLX5_GET(query_hca_cap_out, out, status);

I think, like other DevX commands, status and syndrome are not filled in error case...

> +		syndrome = MLX5_GET(query_hca_cap_out, out, syndrome);
> +		if (status) {
> +			DRV_LOG(DEBUG,
> +				"Failed to query devx HCA ROCE capabilities, "
> +				"status %x, syndrome = %x", status, syndrome);
> +			return -1;
> +		}
> +		hcattr = MLX5_ADDR_OF(query_hca_cap_out, out, capability);
> +		attr->qp_ts_format = MLX5_GET(roce_caps, hcattr,
> qp_ts_format);
> +	}
> +	if (attr->eth_virt &&
> +	    attr->wqe_inline_mode ==
> MLX5_CAP_INLINE_MODE_VPORT_CONTEXT) {
>  		rc = mlx5_devx_cmd_query_nic_vport_context(ctx, 0, attr);
>  		if (rc) {
>  			attr->eth_virt = 0;
> @@ -1004,6 +1030,7 @@ mlx5_devx_cmd_create_rq(void *ctx,
>  	MLX5_SET(rqc, rq_ctx, cqn, rq_attr->cqn);
>  	MLX5_SET(rqc, rq_ctx, counter_set_id, rq_attr->counter_set_id);
>  	MLX5_SET(rqc, rq_ctx, rmpn, rq_attr->rmpn);
> +	MLX5_SET(sqc, rq_ctx, ts_format, rq_attr->ts_format);
>  	wq_ctx = MLX5_ADDR_OF(rqc, rq_ctx, wq);
>  	wq_attr = &rq_attr->wq_attr;
>  	devx_cmd_fill_wq_data(wq_ctx, wq_attr); @@ -1376,6 +1403,7 @@
> mlx5_devx_cmd_create_sq(void *ctx,
>  		 sq_attr->packet_pacing_rate_limit_index);
>  	MLX5_SET(sqc, sq_ctx, tis_lst_sz, sq_attr->tis_lst_sz);
>  	MLX5_SET(sqc, sq_ctx, tis_num_0, sq_attr->tis_num);
> +	MLX5_SET(sqc, sq_ctx, ts_format, sq_attr->ts_format);
>  	wq_ctx = MLX5_ADDR_OF(sqc, sq_ctx, wq);
>  	wq_attr = &sq_attr->wq_attr;
>  	devx_cmd_fill_wq_data(wq_ctx, wq_attr); @@ -1825,6 +1853,7 @@
> mlx5_devx_cmd_create_qp(void *ctx,
>  	MLX5_SET(create_qp_in, in, opcode, MLX5_CMD_OP_CREATE_QP);
>  	MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC);
>  	MLX5_SET(qpc, qpc, pd, attr->pd);
> +	MLX5_SET(qpc, qpc, ts_format, attr->ts_format);
>  	if (attr->uar_index) {
>  		MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
>  		MLX5_SET(qpc, qpc, uar_page, attr->uar_index); diff --git
> a/drivers/common/mlx5/mlx5_devx_cmds.h
> b/drivers/common/mlx5/mlx5_devx_cmds.h
> index bc66d28e83..2826c0b2c6 100644
> --- a/drivers/common/mlx5/mlx5_devx_cmds.h
> +++ b/drivers/common/mlx5/mlx5_devx_cmds.h
> @@ -121,6 +121,10 @@ struct mlx5_hca_attr {
>  	uint32_t dev_freq_khz; /* Timestamp counter frequency, kHz. */
>  	uint32_t scatter_fcs_w_decap_disable:1;
>  	uint32_t flow_hit_aso:1; /* General obj type FLOW_HIT_ASO
> supported. */
> +	uint32_t roce:1;
> +	uint32_t rq_ts_format:2;
> +	uint32_t sq_ts_format:2;
> +	uint32_t qp_ts_format:2;
>  	uint32_t regex:1;
>  	uint32_t reg_c_preserve:1;
>  	uint32_t regexp_num_of_engines;
> @@ -188,6 +192,7 @@ struct mlx5_devx_create_rq_attr {
>  	uint32_t state:4;
>  	uint32_t flush_in_error_en:1;
>  	uint32_t hairpin:1;
> +	uint32_t ts_format:2;
>  	uint32_t user_index:24;
>  	uint32_t cqn:24;
>  	uint32_t counter_set_id:8;
> @@ -271,6 +276,7 @@ struct mlx5_devx_create_sq_attr {
>  	uint32_t hairpin:1;
>  	uint32_t non_wire:1;
>  	uint32_t static_sq_wq:1;
> +	uint32_t ts_format:2;
>  	uint32_t user_index:24;
>  	uint32_t cqn:24;
>  	uint32_t packet_pacing_rate_limit_index:16;
> @@ -354,6 +360,7 @@ struct mlx5_devx_qp_attr {
>  	uint32_t rq_size:17; /* Must be power of 2. */
>  	uint32_t log_rq_stride:3;
>  	uint32_t sq_size:17; /* Must be power of 2. */
> +	uint32_t ts_format:2;
>  	uint32_t dbr_umem_valid:1;
>  	uint32_t dbr_umem_id;
>  	uint64_t dbr_address;
> diff --git a/drivers/common/mlx5/mlx5_prm.h
> b/drivers/common/mlx5/mlx5_prm.h index 01a039f1f7..b0120e05d7 100644
> --- a/drivers/common/mlx5/mlx5_prm.h
> +++ b/drivers/common/mlx5/mlx5_prm.h
> @@ -1116,6 +1116,7 @@ enum {
>  	MLX5_GET_HCA_CAP_OP_MOD_GENERAL_DEVICE = 0x0 << 1,
>  	MLX5_GET_HCA_CAP_OP_MOD_ETHERNET_OFFLOAD_CAPS = 0x1 <<
> 1,
>  	MLX5_GET_HCA_CAP_OP_MOD_QOS_CAP = 0xc << 1,
> +	MLX5_GET_HCA_CAP_OP_MOD_ROCE = 0x4 << 1,
>  	MLX5_GET_HCA_CAP_OP_MOD_NIC_FLOW_TABLE = 0x7 << 1,
>  	MLX5_GET_HCA_CAP_OP_MOD_VDPA_EMULATION = 0x13 << 1,  };
> @@ -1153,6 +1154,20 @@ enum {
>  	MLX5_INLINE_MODE_INNER_TCP_UDP,
>  };
> 
> +/* The supported timestamp formats reported in HCA attributes. */ enum
> +{
> +	MLX5_HCA_CAP_TIMESTAMP_FORMAT_FR = 0x0,
> +	MLX5_HCA_CAP_TIMESTAMP_FORMAT_RT = 0x1,
> +	MLX5_HCA_CAP_TIMESTAMP_FORMAT_FR_RT = 0x2, };
> +
> +/* The timestamp format attributes to configure queues (RQ/SQ/QP). */
> +enum {
> +	MLX5_QPC_TIMESTAMP_FORMAT_FREE_RUNNING = 0x0,
> +	MLX5_QPC_TIMESTAMP_FORMAT_DEFAULT      = 0x1,
> +	MLX5_QPC_TIMESTAMP_FORMAT_REAL_TIME    = 0x2,
> +};
> +
>  /* HCA bit masks indicating which Flex parser protocols are already enabled.
> */  #define MLX5_HCA_FLEX_IPV4_OVER_VXLAN_ENABLED (1UL << 0)  #define
> MLX5_HCA_FLEX_IPV6_OVER_VXLAN_ENABLED (1UL << 1) @@ -1431,7
> +1446,9 @@ struct mlx5_ifc_cmd_hca_cap_bits {
>  	u8 reserved_at_3f8[0x3];
>  	u8 log_max_current_uc_list[0x5];
>  	u8 general_obj_types[0x40];
> -	u8 reserved_at_440[0x20];
> +	u8 sq_ts_format[0x2];
> +	u8 rq_ts_format[0x2];
> +	u8 reserved_at_444[0x1C];
>  	u8 reserved_at_460[0x10];
>  	u8 max_num_eqs[0x10];
>  	u8 reserved_at_480[0x3];
> @@ -1623,6 +1640,15 @@ struct mlx5_ifc_flow_table_prop_layout_bits {
>  	u8 reserved_at_c0[0x140];
>  };
> 
> +struct mlx5_ifc_roce_caps_bits {
> +	u8 reserved_0[0x5];
> +	u8 fl_rc_qp_when_roce_disabled[0x1];
> +	u8 fl_rc_qp_when_roce_enabled[0x1];
> +	u8 reserved_at_7[0x17];
> +	u8 qp_ts_format[0x2];
> +	u8 reserved_at_20[0x7e0];
> +};
> +
>  struct mlx5_ifc_flow_table_nic_cap_bits {
>  	u8	   reserved_at_0[0x200];
>  	struct mlx5_ifc_flow_table_prop_layout_bits flow_table_properties;
> @@ -1635,6 +1661,7 @@ union mlx5_ifc_hca_cap_union_bits {
>  	struct mlx5_ifc_qos_cap_bits qos_cap;
>  	struct mlx5_ifc_virtio_emulation_cap_bits vdpa_caps;
>  	struct mlx5_ifc_flow_table_nic_cap_bits flow_table_nic_cap;
> +	struct mlx5_ifc_roce_caps_bits roce_caps;
>  	u8 reserved_at_0[0x8000];
>  };
> 
> @@ -1851,7 +1878,9 @@ struct mlx5_ifc_rqc_bits {
>  	u8 reserved_at_c[0x1];
>  	u8 flush_in_error_en[0x1];
>  	u8 hairpin[0x1];
> -	u8 reserved_at_f[0x11];
> +	u8 reserved_at_f[0xB];
> +	u8 ts_format[0x02];
> +	u8 reserved_at_1c[0x4];
>  	u8 reserved_at_20[0x8];
>  	u8 user_index[0x18];
>  	u8 reserved_at_40[0x8];
> @@ -2157,7 +2186,9 @@ struct mlx5_ifc_sqc_bits {
>  	u8 hairpin[0x1];
>  	u8 non_wire[0x1];
>  	u8 static_sq_wq[0x1];
> -	u8 reserved_at_11[0xf];
> +	u8 reserved_at_11[0x9];
> +	u8 ts_format[0x02];
> +	u8 reserved_at_1c[0x4];
>  	u8 reserved_at_20[0x8];
>  	u8 user_index[0x18];
>  	u8 reserved_at_40[0x8];
> @@ -2684,7 +2715,9 @@ struct mlx5_ifc_qpc_bits {
>  	u8 log_rq_stride[0x3];
>  	u8 no_sq[0x1];
>  	u8 log_sq_size[0x4];
> -	u8 reserved_at_55[0x6];
> +	u8 reserved_at_55[0x3];
> +	u8 ts_format[0x2];
> +	u8 reserved_at_5a[0x1];
>  	u8 rlky[0x1];
>  	u8 ulp_stateless_offload_mode[0x4];
>  	u8 counter_set_id[0x8];
> --
> 2.28.0

Beside above,
Acked-by: Matan Azrad <matan@nvidia.com>