Re: [dpdk-dev] [PATCH] net/mlx5: introduce dev parameter for hairpin

[Slava Ovsiienko <viacheslavo@mellanox.com>]

> -----Original Message-----
> From: Bing Zhao <bingz@mellanox.com>
> Sent: Tuesday, March 24, 2020 14:59
> To: Ori Kam <orika@mellanox.com>; Matan Azrad <matan@mellanox.com>
> Cc: Raslan Darawsheh <rasland@mellanox.com>; Slava Ovsiienko
> <viacheslavo@mellanox.com>; dev@dpdk.org
> Subject: [PATCH] net/mlx5: introduce dev parameter for hairpin
> 
> When creating a hairpin queue, the total data size and the maximal number
> of packets are interrelated. The differ is the stride size.
> Larger buffer size means big packet like jumbo could be supported, but in the
> meanwhile, it will introduce more cache misses and have a side effect on the
> performance.
> Now a new device parameter "hp_buf_log_sz" is introduced for applications
> to set the total data buffer size (the logarithm value).
> Then the maximal number of packets will also be calculated automaticlly by
> this value.
> Applications could also change this value to a larger one in order to support
> larger packets in hairpin case. A smaller value will be beneficial for memory
> consumption.
> If it is not set, the default value will be used.
> 
> Signed-off-by: Bing Zhao <bingz@mellanox.com>
Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>

> ---
>  drivers/net/mlx5/mlx5.c      | 10 ++++++++++
>  drivers/net/mlx5/mlx5.h      |  1 +
>  drivers/net/mlx5/mlx5_defs.h |  2 +-
>  drivers/net/mlx5/mlx5_rxq.c  | 17 ++++++++++++++---
> drivers/net/mlx5/mlx5_txq.c  | 17 ++++++++++++++---
>  5 files changed, 40 insertions(+), 7 deletions(-)
> 
> diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index
> 94aaa60..3e6eaa0 100644
> --- a/drivers/net/mlx5/mlx5.c
> +++ b/drivers/net/mlx5/mlx5.c
> @@ -150,6 +150,12 @@
>  /* Configure timeout of LRO session (in microseconds). */  #define
> MLX5_LRO_TIMEOUT_USEC "lro_timeout_usec"
> 
> +/*
> + * Device parameter to configure the total data buffer size for a
> +single
> + * hairpin queue (logarithm value).
> + */
> +#define MLX5_HP_BUF_SIZE "hp_buf_log_sz"
> +
>  #ifndef HAVE_IBV_MLX5_MOD_MPW
>  #define MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED (1 << 2)  #define
> MLX5DV_CONTEXT_FLAGS_ENHANCED_MPW (1 << 3) @@ -1580,6 +1586,8
> @@ struct mlx5_flow_id_pool *
>  		config->lro.timeout = tmp;
>  	} else if (strcmp(MLX5_CLASS_ARG_NAME, key) == 0) {
>  		DRV_LOG(DEBUG, "class argument is %s.", val);
> +	} else if (strcmp(MLX5_HP_BUF_SIZE, key) == 0) {
> +		config->log_hp_size = tmp;
>  	} else {
>  		DRV_LOG(WARNING, "%s: unknown parameter", key);
>  		rte_errno = EINVAL;
> @@ -1632,6 +1640,7 @@ struct mlx5_flow_id_pool *
>  		MLX5_MAX_DUMP_FILES_NUM,
>  		MLX5_LRO_TIMEOUT_USEC,
>  		MLX5_CLASS_ARG_NAME,
> +		MLX5_HP_BUF_SIZE,
>  		NULL,
>  	};
>  	struct rte_kvargs *kvlist;
> @@ -3342,6 +3351,7 @@ struct mlx5_flow_id_pool *
>  		},
>  		.dv_esw_en = 1,
>  		.dv_flow_en = 1,
> +		.log_hp_size = MLX5_ARG_UNSET,
>  	};
>  	/* Device specific configuration. */
>  	switch (pci_dev->id.device_id) {
> diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index
> d7c519b..53f35fd 100644
> --- a/drivers/net/mlx5/mlx5.h
> +++ b/drivers/net/mlx5/mlx5.h
> @@ -191,6 +191,7 @@ struct mlx5_dev_config {
>  	unsigned int tso_max_payload_sz; /* Maximum TCP payload for TSO.
> */
>  	unsigned int ind_table_max_size; /* Maximum indirection table size.
> */
>  	unsigned int max_dump_files_num; /* Maximum dump files per
> queue. */
> +	unsigned int log_hp_size; /* Single hairpin queue data size in total.
> +*/
>  	int txqs_inline; /* Queue number threshold for inlining. */
>  	int txq_inline_min; /* Minimal amount of data bytes to inline. */
>  	int txq_inline_max; /* Max packet size for inlining with SEND. */ diff --
> git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h index
> 83ca367..19e8253 100644
> --- a/drivers/net/mlx5/mlx5_defs.h
> +++ b/drivers/net/mlx5/mlx5_defs.h
> @@ -175,7 +175,7 @@
> 
>  /* Hairpin TX/RX queue configuration parameters. */  #define
> MLX5_HAIRPIN_QUEUE_STRIDE 6 -#define MLX5_HAIRPIN_JUMBO_LOG_SIZE
> (15 + 2)
> +#define MLX5_HAIRPIN_JUMBO_LOG_SIZE (14 + 2)
> 
>  /* Definition of static_assert found in /usr/include/assert.h */  #ifndef
> HAVE_STATIC_ASSERT diff --git a/drivers/net/mlx5/mlx5_rxq.c
> b/drivers/net/mlx5/mlx5_rxq.c index 8a6b410..3fdb632 100644
> --- a/drivers/net/mlx5/mlx5_rxq.c
> +++ b/drivers/net/mlx5/mlx5_rxq.c
> @@ -1286,9 +1286,20 @@
>  	attr.hairpin = 1;
>  	max_wq_data = priv->config.hca_attr.log_max_hairpin_wq_data_sz;
>  	/* Jumbo frames > 9KB should be supported, and more packets. */
> -	attr.wq_attr.log_hairpin_data_sz =
> -			(max_wq_data < MLX5_HAIRPIN_JUMBO_LOG_SIZE) ?
> -			max_wq_data : MLX5_HAIRPIN_JUMBO_LOG_SIZE;
> +	if (priv->config.log_hp_size != (uint32_t)MLX5_ARG_UNSET) {
> +		if (priv->config.log_hp_size > max_wq_data) {
> +			DRV_LOG(ERR, "total data size %u power of 2 is "
> +				"too large for hairpin",
> +				priv->config.log_hp_size);
> +			rte_errno = ERANGE;
> +			return NULL;
> +		}
> +		attr.wq_attr.log_hairpin_data_sz = priv->config.log_hp_size;
> +	} else {
> +		attr.wq_attr.log_hairpin_data_sz =
> +				(max_wq_data <
> MLX5_HAIRPIN_JUMBO_LOG_SIZE) ?
> +				 max_wq_data :
> MLX5_HAIRPIN_JUMBO_LOG_SIZE;
> +	}
>  	/* Set the packets number to the maximum value for performance. */
>  	attr.wq_attr.log_hairpin_num_packets =
>  			attr.wq_attr.log_hairpin_data_sz -
> diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c index
> 57bc116..0653f4c 100644
> --- a/drivers/net/mlx5/mlx5_txq.c
> +++ b/drivers/net/mlx5/mlx5_txq.c
> @@ -512,9 +512,20 @@
>  	attr.tis_lst_sz = 1;
>  	max_wq_data = priv->config.hca_attr.log_max_hairpin_wq_data_sz;
>  	/* Jumbo frames > 9KB should be supported, and more packets. */
> -	attr.wq_attr.log_hairpin_data_sz =
> -			(max_wq_data < MLX5_HAIRPIN_JUMBO_LOG_SIZE) ?
> -			max_wq_data : MLX5_HAIRPIN_JUMBO_LOG_SIZE;
> +	if (priv->config.log_hp_size != (uint32_t)MLX5_ARG_UNSET) {
> +		if (priv->config.log_hp_size > max_wq_data) {
> +			DRV_LOG(ERR, "total data size %u power of 2 is "
> +				"too large for hairpin",
> +				priv->config.log_hp_size);
> +			rte_errno = ERANGE;
> +			return NULL;
> +		}
> +		attr.wq_attr.log_hairpin_data_sz = priv->config.log_hp_size;
> +	} else {
> +		attr.wq_attr.log_hairpin_data_sz =
> +				(max_wq_data <
> MLX5_HAIRPIN_JUMBO_LOG_SIZE) ?
> +				 max_wq_data :
> MLX5_HAIRPIN_JUMBO_LOG_SIZE;
> +	}
>  	/* Set the packets number to the maximum value for performance. */
>  	attr.wq_attr.log_hairpin_num_packets =
>  			attr.wq_attr.log_hairpin_data_sz -
> --
> 1.8.3.1