DPDK patches and discussions
 help / color / mirror / Atom feed
* [dpdk-dev] [RFC] mlx5: flow counters support on the linux-rdma v19 base
@ 2018-08-20 13:34 viacheslavo
  2018-09-27 23:40 ` Yongseok Koh
  2018-10-03 15:29 ` [dpdk-dev] [PATCH] net/mlx5: flow counters support on the Linux-rdma " Slava Ovsiienko
  0 siblings, 2 replies; 39+ messages in thread
From: viacheslavo @ 2018-08-20 13:34 UTC (permalink / raw)
  To: dev, adrien.mazarguil, thomas, ferruh.yigit, shahafs; +Cc: orika, viacheslavo

Mellanox mlx5 PMD supports Flow Counters via Verbs library.
The current implementation is based on the Mellanox proprietary
Verbs library included in MLNX OFED packages. The Flow Counter
support is recently added into linux-rdma release (v19),
so the mlx5 PMD update is needed to provide Counter feature
on the base of linux-rdma.

mlx5 PMD can be compiled with MLNX OFED or linux-rdma v19
and provide flow counters for both.

Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
---
 drivers/net/mlx5/Makefile    | 10 +++++++
 drivers/net/mlx5/mlx5.c      |  6 ++++
 drivers/net/mlx5/mlx5_flow.c | 67 ++++++++++++++++++++++++++++++++++++--------
 drivers/net/mlx5/mlx5_glue.c | 41 +++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_glue.h | 16 +++++++++++
 5 files changed, 128 insertions(+), 12 deletions(-)

diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile
index 2e70dec..ca4c143 100644
--- a/drivers/net/mlx5/Makefile
+++ b/drivers/net/mlx5/Makefile
@@ -155,6 +155,16 @@ mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
 		type 'struct ibv_counter_set_init_attr' \
 		$(AUTOCONF_OUTPUT)
 	$Q sh -- '$<' '$@' \
+		HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT \
+		infiniband/verbs.h \
+		type 'struct ibv_counters_init_attr' \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45 \
+		infiniband/verbs.h \
+		type 'struct ibv_counters_init_attr' \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
 		HAVE_RDMA_NL_NLDEV \
 		rdma/rdma_netlink.h \
 		enum RDMA_NL_NLDEV \
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index ec63bc6..ad289f0 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -730,8 +730,10 @@
 	unsigned int mprq_min_stride_num_n = 0;
 	unsigned int mprq_max_stride_num_n = 0;
 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
 	struct ibv_counter_set_description cs_desc = { .counter_type = 0 };
 #endif
+#endif
 	struct ether_addr mac;
 	char name[RTE_ETH_NAME_MAX_LEN];
 	int own_domain_id = 0;
@@ -1001,11 +1003,15 @@
 	DRV_LOG(DEBUG, "checksum offloading is %ssupported",
 		(config.hw_csum ? "" : "not "));
 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
 	config.flow_counter_en = !!attr.max_counter_sets;
 	mlx5_glue->describe_counter_set(ctx, 0, &cs_desc);
 	DRV_LOG(DEBUG, "counter type = %d, num of cs = %ld, attributes = %d",
 		cs_desc.counter_type, cs_desc.num_of_cs,
 		cs_desc.attributes);
+#else
+	config.flow_counter_en = 1;
+#endif
 #endif
 	config.ind_table_max_size =
 		attr.rss_caps.max_rwq_indirection_table_size;
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index ca4625b..4d762dd 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -290,7 +290,11 @@ struct mlx5_flow_counter {
 	uint32_t shared:1; /**< Share counter ID with other flow rules. */
 	uint32_t ref_cnt:31; /**< Reference counter. */
 	uint32_t id; /**< Counter ID. */
-	struct ibv_counter_set *cs; /**< Holds the counters for the rule. */
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
+struct ibv_counters *cs; /**< Holds the counters for the rule. */
+#else
+struct ibv_counter_set *cs; /**< Holds the counters for the rule. */
+#endif
 	uint64_t hits; /**< Number of packets matched by the rule. */
 	uint64_t bytes; /**< Number of bytes matched by the rule. */
 };
@@ -519,27 +523,32 @@ struct mlx5_flow_tunnel_info {
 static struct mlx5_flow_counter *
 mlx5_flow_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id)
 {
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
 	struct priv *priv = dev->data->dev_private;
 	struct mlx5_flow_counter *cnt;
 
-	LIST_FOREACH(cnt, &priv->flow_counters, next) {
-		if (!cnt->shared || cnt->shared != shared)
-			continue;
-		if (cnt->id != id)
-			continue;
-		cnt->ref_cnt++;
-		return cnt;
+	if (shared) {
+		LIST_FOREACH(cnt, &priv->flow_counters, next)
+		if (cnt->shared && cnt->id == id) {
+			cnt->ref_cnt++;
+			return cnt;
+		}
 	}
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
 
 	struct mlx5_flow_counter tmpl = {
 		.shared = shared,
 		.id = id,
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
+		.cs = mlx5_glue->create_counter_set
+			(priv->ctx,
+			 &(struct ibv_counters_init_attr){0}),
+#else
 		.cs = mlx5_glue->create_counter_set
 			(priv->ctx,
 			 &(struct ibv_counter_set_init_attr){
 				 .counter_set_id = id,
 			 }),
+#endif
 		.hits = 0,
 		.bytes = 0,
 	};
@@ -548,17 +557,40 @@ struct mlx5_flow_tunnel_info {
 		rte_errno = errno;
 		return NULL;
 	}
+
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
+	struct ibv_counter_attach_attr attach_attr = {0};
+	int ret;
+
+	attach_attr.counter_desc = IBV_COUNTER_PACKETS;
+	attach_attr.index = 0;
+	ret = ibv_attach_counters_point_flow(tmpl.cs, &attach_attr, NULL);
+	if (!ret) {
+		attach_attr.counter_desc = IBV_COUNTER_BYTES;
+		attach_attr.index = 1;
+		ret = ibv_attach_counters_point_flow(tmpl.cs,
+						     &attach_attr,
+						     NULL);
+	}
+	if (ret) {
+		claim_zero(mlx5_glue->destroy_counter_set(tmpl.cs));
+		rte_errno = ret;
+		return NULL;
+	}
+#endif
 	cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
 	if (!cnt) {
+		claim_zero(mlx5_glue->destroy_counter_set(tmpl.cs));
 		rte_errno = ENOMEM;
 		return NULL;
 	}
 	*cnt = tmpl;
 	LIST_INSERT_HEAD(&priv->flow_counters, cnt, next);
 	return cnt;
-#endif
+#else
 	rte_errno = ENOTSUP;
 	return NULL;
+#endif
 }
 
 /**
@@ -2307,7 +2339,7 @@ struct mlx5_flow_tunnel_info {
 		flow->counter = mlx5_flow_counter_new(dev, count->shared,
 						      count->id);
 		if (!flow->counter)
-			return rte_flow_error_set(error, ENOTSUP,
+			return rte_flow_error_set(error, rte_errno,
 						  RTE_FLOW_ERROR_TYPE_ACTION,
 						  action,
 						  "cannot get counter"
@@ -2320,7 +2352,11 @@ struct mlx5_flow_tunnel_info {
 					  "flow counters are not supported.");
 	flow->modifier |= MLX5_FLOW_MOD_COUNT;
 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
+	counter.counters = flow->counter->cs;
+#else
 	counter.counter_set_handle = flow->counter->cs->handle;
+#endif
 	if (size <= flow_size)
 		mlx5_flow_spec_verbs_add(flow, &counter, size);
 	return size;
@@ -3373,6 +3409,13 @@ struct rte_flow *
 	if (flow->modifier & MLX5_FLOW_MOD_COUNT) {
 		struct rte_flow_query_count *qc = data;
 		uint64_t counters[2] = {0, 0};
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
+		int err = mlx5_glue->query_counter_set(
+				flow->counter->cs,
+				counters,
+				RTE_DIM(counters),
+				IBV_READ_COUNTERS_ATTR_PREFER_CACHED);
+#else
 		struct ibv_query_counter_set_attr query_cs_attr = {
 			.cs = flow->counter->cs,
 			.query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
@@ -3383,7 +3426,7 @@ struct rte_flow *
 		};
 		int err = mlx5_glue->query_counter_set(&query_cs_attr,
 						       &query_out);
-
+#endif
 		if (err)
 			return rte_flow_error_set
 				(error, err,
diff --git a/drivers/net/mlx5/mlx5_glue.c b/drivers/net/mlx5/mlx5_glue.c
index 84f9492..b2b39ad 100644
--- a/drivers/net/mlx5/mlx5_glue.c
+++ b/drivers/net/mlx5/mlx5_glue.c
@@ -211,6 +211,39 @@
 	return ibv_dereg_mr(mr);
 }
 
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
+static struct ibv_counters *
+mlx5_glue_create_counters(struct ibv_context *context,
+			  struct ibv_counters_init_attr *init_attr)
+{
+	return ibv_create_counters(context, init_attr);
+}
+
+static int
+mlx5_glue_destroy_counters(struct ibv_counters *counters)
+{
+	return ibv_destroy_counters(counters);
+}
+
+static int
+mlx5_glue_attach_counters(struct ibv_counters *counters,
+		     struct ibv_counter_attach_attr *attr,
+		     struct ibv_flow *flow)
+{
+	return ibv_attach_counters_point_flow(counters, attr, flow);
+}
+
+static int
+mlx5_glue_query_counters(struct ibv_counters *counters,
+			 uint64_t *counters_value,
+			 uint32_t ncounters,
+			 uint32_t flags)
+{
+	return ibv_read_counters(counters, counters_value, ncounters, flags);
+}
+#endif
+
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
 static struct ibv_counter_set *
 mlx5_glue_create_counter_set(struct ibv_context *context,
 			     struct ibv_counter_set_init_attr *init_attr)
@@ -262,6 +295,7 @@
 	return ibv_query_counter_set(query_attr, cs_data);
 #endif
 }
+#endif
 
 static void
 mlx5_glue_ack_async_event(struct ibv_async_event *event)
@@ -378,10 +412,17 @@
 	.modify_qp = mlx5_glue_modify_qp,
 	.reg_mr = mlx5_glue_reg_mr,
 	.dereg_mr = mlx5_glue_dereg_mr,
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
+	.create_counter_set = mlx5_glue_create_counters,
+	.destroy_counter_set = mlx5_glue_destroy_counters,
+	.attach_counter_set = mlx5_glue_attach_counters,
+	.query_counter_set = mlx5_glue_query_counters,
+#else
 	.create_counter_set = mlx5_glue_create_counter_set,
 	.destroy_counter_set = mlx5_glue_destroy_counter_set,
 	.describe_counter_set = mlx5_glue_describe_counter_set,
 	.query_counter_set = mlx5_glue_query_counter_set,
+#endif
 	.ack_async_event = mlx5_glue_ack_async_event,
 	.get_async_event = mlx5_glue_get_async_event,
 	.port_state_str = mlx5_glue_port_state_str,
diff --git a/drivers/net/mlx5/mlx5_glue.h b/drivers/net/mlx5/mlx5_glue.h
index e584d36..c5ff3cb 100644
--- a/drivers/net/mlx5/mlx5_glue.h
+++ b/drivers/net/mlx5/mlx5_glue.h
@@ -89,6 +89,21 @@ struct mlx5_glue {
 	struct ibv_mr *(*reg_mr)(struct ibv_pd *pd, void *addr,
 				 size_t length, int access);
 	int (*dereg_mr)(struct ibv_mr *mr);
+#ifdef	HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
+	struct ibv_counters *(*create_counter_set)
+		(struct ibv_context *context,
+		 struct ibv_counters_init_attr *init_attr);
+	int (*destroy_counter_set)(struct ibv_counters *cs);
+	int (*attach_counter_set)
+		(struct ibv_counters *cs,
+		 struct ibv_counter_attach_attr *attr,
+		 struct ibv_flow *flow);
+	int (*query_counter_set)
+		(struct ibv_counters *cs,
+		 uint64_t *counters_value,
+		 uint32_t ncounters,
+		 uint32_t flags);
+#else
 	struct ibv_counter_set *(*create_counter_set)
 		(struct ibv_context *context,
 		 struct ibv_counter_set_init_attr *init_attr);
@@ -99,6 +114,7 @@ struct mlx5_glue {
 		 struct ibv_counter_set_description *cs_desc);
 	int (*query_counter_set)(struct ibv_query_counter_set_attr *query_attr,
 				 struct ibv_counter_set_data *cs_data);
+#endif
 	void (*ack_async_event)(struct ibv_async_event *event);
 	int (*get_async_event)(struct ibv_context *context,
 			       struct ibv_async_event *event);
-- 
1.8.3.1

^ permalink raw reply	[flat|nested] 39+ messages in thread

* Re: [dpdk-dev] [RFC] mlx5: flow counters support on the linux-rdma v19 base
  2018-08-20 13:34 [dpdk-dev] [RFC] mlx5: flow counters support on the linux-rdma v19 base viacheslavo
@ 2018-09-27 23:40 ` Yongseok Koh
  2018-10-03 15:29 ` [dpdk-dev] [PATCH] net/mlx5: flow counters support on the Linux-rdma " Slava Ovsiienko
  1 sibling, 0 replies; 39+ messages in thread
From: Yongseok Koh @ 2018-09-27 23:40 UTC (permalink / raw)
  To: Slava Ovsiienko; +Cc: dev, Shahaf Shuler

> On Aug 20, 2018, at 6:34 AM, viacheslavo <viacheslavo@mellanox.com> wrote:
> 
> Mellanox mlx5 PMD supports Flow Counters via Verbs library.
> The current implementation is based on the Mellanox proprietary
> Verbs library included in MLNX OFED packages. The Flow Counter
> support is recently added into linux-rdma release (v19),
> so the mlx5 PMD update is needed to provide Counter feature
> on the base of linux-rdma.
> 
> mlx5 PMD can be compiled with MLNX OFED or linux-rdma v19
> and provide flow counters for both.
> 
> Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
> ---
> drivers/net/mlx5/Makefile    | 10 +++++++
> drivers/net/mlx5/mlx5.c      |  6 ++++
> drivers/net/mlx5/mlx5_flow.c | 67 ++++++++++++++++++++++++++++++++++++--------
> drivers/net/mlx5/mlx5_glue.c | 41 +++++++++++++++++++++++++++
> drivers/net/mlx5/mlx5_glue.h | 16 +++++++++++
> 5 files changed, 128 insertions(+), 12 deletions(-)
> 
> diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile
> index 2e70dec..ca4c143 100644
> --- a/drivers/net/mlx5/Makefile
> +++ b/drivers/net/mlx5/Makefile
> @@ -155,6 +155,16 @@ mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
> 		type 'struct ibv_counter_set_init_attr' \
> 		$(AUTOCONF_OUTPUT)
> 	$Q sh -- '$<' '$@' \
> +		HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT \
> +		infiniband/verbs.h \
> +		type 'struct ibv_counters_init_attr' \
> +		$(AUTOCONF_OUTPUT)
> +	$Q sh -- '$<' '$@' \
> +		HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45 \
> +		infiniband/verbs.h \
> +		type 'struct ibv_counters_init_attr' \
> +		$(AUTOCONF_OUTPUT)
> +	$Q sh -- '$<' '$@' \

I don't see any difference between the two -
HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT and HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45??

Thanks
Yongseok

> 		HAVE_RDMA_NL_NLDEV \
> 		rdma/rdma_netlink.h \
> 		enum RDMA_NL_NLDEV \
> diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
> index ec63bc6..ad289f0 100644
> --- a/drivers/net/mlx5/mlx5.c
> +++ b/drivers/net/mlx5/mlx5.c
> @@ -730,8 +730,10 @@
> 	unsigned int mprq_min_stride_num_n = 0;
> 	unsigned int mprq_max_stride_num_n = 0;
> #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
> +#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
> 	struct ibv_counter_set_description cs_desc = { .counter_type = 0 };
> #endif
> +#endif
> 	struct ether_addr mac;
> 	char name[RTE_ETH_NAME_MAX_LEN];
> 	int own_domain_id = 0;
> @@ -1001,11 +1003,15 @@
> 	DRV_LOG(DEBUG, "checksum offloading is %ssupported",
> 		(config.hw_csum ? "" : "not "));
> #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
> +#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
> 	config.flow_counter_en = !!attr.max_counter_sets;
> 	mlx5_glue->describe_counter_set(ctx, 0, &cs_desc);
> 	DRV_LOG(DEBUG, "counter type = %d, num of cs = %ld, attributes = %d",
> 		cs_desc.counter_type, cs_desc.num_of_cs,
> 		cs_desc.attributes);
> +#else
> +	config.flow_counter_en = 1;
> +#endif
> #endif
> 	config.ind_table_max_size =
> 		attr.rss_caps.max_rwq_indirection_table_size;
> diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
> index ca4625b..4d762dd 100644
> --- a/drivers/net/mlx5/mlx5_flow.c
> +++ b/drivers/net/mlx5/mlx5_flow.c
> @@ -290,7 +290,11 @@ struct mlx5_flow_counter {
> 	uint32_t shared:1; /**< Share counter ID with other flow rules. */
> 	uint32_t ref_cnt:31; /**< Reference counter. */
> 	uint32_t id; /**< Counter ID. */
> -	struct ibv_counter_set *cs; /**< Holds the counters for the rule. */
> +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
> +struct ibv_counters *cs; /**< Holds the counters for the rule. */
> +#else
> +struct ibv_counter_set *cs; /**< Holds the counters for the rule. */
> +#endif
> 	uint64_t hits; /**< Number of packets matched by the rule. */
> 	uint64_t bytes; /**< Number of bytes matched by the rule. */
> };
> @@ -519,27 +523,32 @@ struct mlx5_flow_tunnel_info {
> static struct mlx5_flow_counter *
> mlx5_flow_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id)
> {
> +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
> 	struct priv *priv = dev->data->dev_private;
> 	struct mlx5_flow_counter *cnt;
> 
> -	LIST_FOREACH(cnt, &priv->flow_counters, next) {
> -		if (!cnt->shared || cnt->shared != shared)
> -			continue;
> -		if (cnt->id != id)
> -			continue;
> -		cnt->ref_cnt++;
> -		return cnt;
> +	if (shared) {
> +		LIST_FOREACH(cnt, &priv->flow_counters, next)
> +		if (cnt->shared && cnt->id == id) {
> +			cnt->ref_cnt++;
> +			return cnt;
> +		}
> 	}
> -#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
> 
> 	struct mlx5_flow_counter tmpl = {
> 		.shared = shared,
> 		.id = id,
> +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
> +		.cs = mlx5_glue->create_counter_set
> +			(priv->ctx,
> +			 &(struct ibv_counters_init_attr){0}),
> +#else
> 		.cs = mlx5_glue->create_counter_set
> 			(priv->ctx,
> 			 &(struct ibv_counter_set_init_attr){
> 				 .counter_set_id = id,
> 			 }),
> +#endif
> 		.hits = 0,
> 		.bytes = 0,
> 	};
> @@ -548,17 +557,40 @@ struct mlx5_flow_tunnel_info {
> 		rte_errno = errno;
> 		return NULL;
> 	}
> +
> +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
> +	struct ibv_counter_attach_attr attach_attr = {0};
> +	int ret;
> +
> +	attach_attr.counter_desc = IBV_COUNTER_PACKETS;
> +	attach_attr.index = 0;
> +	ret = ibv_attach_counters_point_flow(tmpl.cs, &attach_attr, NULL);
> +	if (!ret) {
> +		attach_attr.counter_desc = IBV_COUNTER_BYTES;
> +		attach_attr.index = 1;
> +		ret = ibv_attach_counters_point_flow(tmpl.cs,
> +						     &attach_attr,
> +						     NULL);
> +	}
> +	if (ret) {
> +		claim_zero(mlx5_glue->destroy_counter_set(tmpl.cs));
> +		rte_errno = ret;
> +		return NULL;
> +	}
> +#endif
> 	cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
> 	if (!cnt) {
> +		claim_zero(mlx5_glue->destroy_counter_set(tmpl.cs));
> 		rte_errno = ENOMEM;
> 		return NULL;
> 	}
> 	*cnt = tmpl;
> 	LIST_INSERT_HEAD(&priv->flow_counters, cnt, next);
> 	return cnt;
> -#endif
> +#else
> 	rte_errno = ENOTSUP;
> 	return NULL;
> +#endif
> }
> 
> /**
> @@ -2307,7 +2339,7 @@ struct mlx5_flow_tunnel_info {
> 		flow->counter = mlx5_flow_counter_new(dev, count->shared,
> 						      count->id);
> 		if (!flow->counter)
> -			return rte_flow_error_set(error, ENOTSUP,
> +			return rte_flow_error_set(error, rte_errno,
> 						  RTE_FLOW_ERROR_TYPE_ACTION,
> 						  action,
> 						  "cannot get counter"
> @@ -2320,7 +2352,11 @@ struct mlx5_flow_tunnel_info {
> 					  "flow counters are not supported.");
> 	flow->modifier |= MLX5_FLOW_MOD_COUNT;
> #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
> +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
> +	counter.counters = flow->counter->cs;
> +#else
> 	counter.counter_set_handle = flow->counter->cs->handle;
> +#endif
> 	if (size <= flow_size)
> 		mlx5_flow_spec_verbs_add(flow, &counter, size);
> 	return size;
> @@ -3373,6 +3409,13 @@ struct rte_flow *
> 	if (flow->modifier & MLX5_FLOW_MOD_COUNT) {
> 		struct rte_flow_query_count *qc = data;
> 		uint64_t counters[2] = {0, 0};
> +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
> +		int err = mlx5_glue->query_counter_set(
> +				flow->counter->cs,
> +				counters,
> +				RTE_DIM(counters),
> +				IBV_READ_COUNTERS_ATTR_PREFER_CACHED);
> +#else
> 		struct ibv_query_counter_set_attr query_cs_attr = {
> 			.cs = flow->counter->cs,
> 			.query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
> @@ -3383,7 +3426,7 @@ struct rte_flow *
> 		};
> 		int err = mlx5_glue->query_counter_set(&query_cs_attr,
> 						       &query_out);
> -
> +#endif
> 		if (err)
> 			return rte_flow_error_set
> 				(error, err,
> diff --git a/drivers/net/mlx5/mlx5_glue.c b/drivers/net/mlx5/mlx5_glue.c
> index 84f9492..b2b39ad 100644
> --- a/drivers/net/mlx5/mlx5_glue.c
> +++ b/drivers/net/mlx5/mlx5_glue.c
> @@ -211,6 +211,39 @@
> 	return ibv_dereg_mr(mr);
> }
> 
> +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
> +static struct ibv_counters *
> +mlx5_glue_create_counters(struct ibv_context *context,
> +			  struct ibv_counters_init_attr *init_attr)
> +{
> +	return ibv_create_counters(context, init_attr);
> +}
> +
> +static int
> +mlx5_glue_destroy_counters(struct ibv_counters *counters)
> +{
> +	return ibv_destroy_counters(counters);
> +}
> +
> +static int
> +mlx5_glue_attach_counters(struct ibv_counters *counters,
> +		     struct ibv_counter_attach_attr *attr,
> +		     struct ibv_flow *flow)
> +{
> +	return ibv_attach_counters_point_flow(counters, attr, flow);
> +}
> +
> +static int
> +mlx5_glue_query_counters(struct ibv_counters *counters,
> +			 uint64_t *counters_value,
> +			 uint32_t ncounters,
> +			 uint32_t flags)
> +{
> +	return ibv_read_counters(counters, counters_value, ncounters, flags);
> +}
> +#endif
> +
> +#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
> static struct ibv_counter_set *
> mlx5_glue_create_counter_set(struct ibv_context *context,
> 			     struct ibv_counter_set_init_attr *init_attr)
> @@ -262,6 +295,7 @@
> 	return ibv_query_counter_set(query_attr, cs_data);
> #endif
> }
> +#endif
> 
> static void
> mlx5_glue_ack_async_event(struct ibv_async_event *event)
> @@ -378,10 +412,17 @@
> 	.modify_qp = mlx5_glue_modify_qp,
> 	.reg_mr = mlx5_glue_reg_mr,
> 	.dereg_mr = mlx5_glue_dereg_mr,
> +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
> +	.create_counter_set = mlx5_glue_create_counters,
> +	.destroy_counter_set = mlx5_glue_destroy_counters,
> +	.attach_counter_set = mlx5_glue_attach_counters,
> +	.query_counter_set = mlx5_glue_query_counters,
> +#else
> 	.create_counter_set = mlx5_glue_create_counter_set,
> 	.destroy_counter_set = mlx5_glue_destroy_counter_set,
> 	.describe_counter_set = mlx5_glue_describe_counter_set,
> 	.query_counter_set = mlx5_glue_query_counter_set,
> +#endif
> 	.ack_async_event = mlx5_glue_ack_async_event,
> 	.get_async_event = mlx5_glue_get_async_event,
> 	.port_state_str = mlx5_glue_port_state_str,
> diff --git a/drivers/net/mlx5/mlx5_glue.h b/drivers/net/mlx5/mlx5_glue.h
> index e584d36..c5ff3cb 100644
> --- a/drivers/net/mlx5/mlx5_glue.h
> +++ b/drivers/net/mlx5/mlx5_glue.h
> @@ -89,6 +89,21 @@ struct mlx5_glue {
> 	struct ibv_mr *(*reg_mr)(struct ibv_pd *pd, void *addr,
> 				 size_t length, int access);
> 	int (*dereg_mr)(struct ibv_mr *mr);
> +#ifdef	HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
> +	struct ibv_counters *(*create_counter_set)
> +		(struct ibv_context *context,
> +		 struct ibv_counters_init_attr *init_attr);
> +	int (*destroy_counter_set)(struct ibv_counters *cs);
> +	int (*attach_counter_set)
> +		(struct ibv_counters *cs,
> +		 struct ibv_counter_attach_attr *attr,
> +		 struct ibv_flow *flow);
> +	int (*query_counter_set)
> +		(struct ibv_counters *cs,
> +		 uint64_t *counters_value,
> +		 uint32_t ncounters,
> +		 uint32_t flags);
> +#else
> 	struct ibv_counter_set *(*create_counter_set)
> 		(struct ibv_context *context,
> 		 struct ibv_counter_set_init_attr *init_attr);
> @@ -99,6 +114,7 @@ struct mlx5_glue {
> 		 struct ibv_counter_set_description *cs_desc);
> 	int (*query_counter_set)(struct ibv_query_counter_set_attr *query_attr,
> 				 struct ibv_counter_set_data *cs_data);
> +#endif
> 	void (*ack_async_event)(struct ibv_async_event *event);
> 	int (*get_async_event)(struct ibv_context *context,
> 			       struct ibv_async_event *event);
> -- 
> 1.8.3.1
> 

^ permalink raw reply	[flat|nested] 39+ messages in thread

* [dpdk-dev] [PATCH] net/mlx5: flow counters support on the Linux-rdma v19 base
  2018-08-20 13:34 [dpdk-dev] [RFC] mlx5: flow counters support on the linux-rdma v19 base viacheslavo
  2018-09-27 23:40 ` Yongseok Koh
@ 2018-10-03 15:29 ` Slava Ovsiienko
  2018-10-03 23:48   ` Yongseok Koh
  2018-10-17 13:53   ` [dpdk-dev] [PATCH v2] " Viacheslav Ovsiienko
  1 sibling, 2 replies; 39+ messages in thread
From: Slava Ovsiienko @ 2018-10-03 15:29 UTC (permalink / raw)
  To: dev; +Cc: Shahaf Shuler, Slava Ovsiienko

Mellanox mlx5 PMD supports Flow Counters via Verbs library.
The current implementation is based on the Mellanox proprietary
Verbs library included in MLNX OFED packages. The Flow Counter
support is recently added into linux-rdma release (v19),
so the mlx5 PMD update is needed to provide Counter feature
on the base of linux-rdma.

mlx5 PMD can be compiled with MLNX OFED or linux-rdma v19+
and provide flow counters for both.

Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
---
 drivers/net/mlx5/Makefile          | 10 ++++++++
 drivers/net/mlx5/mlx5.c            |  6 +++++
 drivers/net/mlx5/mlx5_flow.c       |  9 ++++++-
 drivers/net/mlx5/mlx5_flow.h       |  4 +++
 drivers/net/mlx5/mlx5_flow_verbs.c | 52 ++++++++++++++++++++++++++++++--------
 drivers/net/mlx5/mlx5_glue.c       | 41 ++++++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_glue.h       | 16 ++++++++++++
 7 files changed, 127 insertions(+), 11 deletions(-)

diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile
index ca1de9f..e3d2156 100644
--- a/drivers/net/mlx5/Makefile
+++ b/drivers/net/mlx5/Makefile
@@ -162,6 +162,16 @@ mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
 		type 'struct ibv_counter_set_init_attr' \
 		$(AUTOCONF_OUTPUT)
 	$Q sh -- '$<' '$@' \
+		HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT \
+		infiniband/verbs.h \
+		type 'struct ibv_counters_init_attr' \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45 \
+		infiniband/verbs.h \
+		type 'struct ibv_counters_init_attr' \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
 		HAVE_RDMA_NL_NLDEV \
 		rdma/rdma_netlink.h \
 		enum RDMA_NL_NLDEV \
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 4be6a1c..81f6ba1 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -739,8 +739,10 @@
 	unsigned int mprq_min_stride_num_n = 0;
 	unsigned int mprq_max_stride_num_n = 0;
 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
 	struct ibv_counter_set_description cs_desc = { .counter_type = 0 };
 #endif
+#endif
 	struct ether_addr mac;
 	char name[RTE_ETH_NAME_MAX_LEN];
 	int own_domain_id = 0;
@@ -1009,11 +1011,15 @@
 	DRV_LOG(DEBUG, "checksum offloading is %ssupported",
 		(config.hw_csum ? "" : "not "));
 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
 	config.flow_counter_en = !!attr.max_counter_sets;
 	mlx5_glue->describe_counter_set(ctx, 0, &cs_desc);
 	DRV_LOG(DEBUG, "counter type = %d, num of cs = %ld, attributes = %d",
 		cs_desc.counter_type, cs_desc.num_of_cs,
 		cs_desc.attributes);
+#else
+	config.flow_counter_en = 1;
+#endif
 #endif
 	config.ind_table_max_size =
 		attr.rss_caps.max_rwq_indirection_table_size;
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 8007bf1..652580c 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -2306,6 +2306,13 @@ struct rte_flow *
 	if (flow->actions & MLX5_ACTION_COUNT) {
 		struct rte_flow_query_count *qc = data;
 		uint64_t counters[2] = {0, 0};
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
+		int err = mlx5_glue->query_counter_set(
+				flow->counter->cs,
+				counters,
+				RTE_DIM(counters),
+				IBV_READ_COUNTERS_ATTR_PREFER_CACHED);
+#else
 		struct ibv_query_counter_set_attr query_cs_attr = {
 			.cs = flow->counter->cs,
 			.query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
@@ -2316,7 +2323,7 @@ struct rte_flow *
 		};
 		int err = mlx5_glue->query_counter_set(&query_cs_attr,
 						       &query_out);
-
+#endif
 		if (err)
 			return rte_flow_error_set
 				(error, err,
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 10d700a..a3b82dd 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -222,7 +222,11 @@ struct mlx5_flow_counter {
 	uint32_t shared:1; /**< Share counter ID with other flow rules. */
 	uint32_t ref_cnt:31; /**< Reference counter. */
 	uint32_t id; /**< Counter ID. */
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
+	struct ibv_counters *cs; /**< Holds the counters for the rule. */
+#else
 	struct ibv_counter_set *cs; /**< Holds the counters for the rule. */
+#endif
 	uint64_t hits; /**< Number of packets matched by the rule. */
 	uint64_t bytes; /**< Number of bytes matched by the rule. */
 };
diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c b/drivers/net/mlx5/mlx5_flow_verbs.c
index 05ab5fd..1c8bdba 100644
--- a/drivers/net/mlx5/mlx5_flow_verbs.c
+++ b/drivers/net/mlx5/mlx5_flow_verbs.c
@@ -48,27 +48,32 @@
 static struct mlx5_flow_counter *
 flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id)
 {
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
 	struct priv *priv = dev->data->dev_private;
 	struct mlx5_flow_counter *cnt;
 
-	LIST_FOREACH(cnt, &priv->flow_counters, next) {
-		if (!cnt->shared || cnt->shared != shared)
-			continue;
-		if (cnt->id != id)
-			continue;
-		cnt->ref_cnt++;
-		return cnt;
+	if (shared) {
+		LIST_FOREACH(cnt, &priv->flow_counters, next)
+		if (cnt->shared && cnt->id == id) {
+			cnt->ref_cnt++;
+			return cnt;
+		}
 	}
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
 
 	struct mlx5_flow_counter tmpl = {
 		.shared = shared,
 		.id = id,
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
+		.cs = mlx5_glue->create_counter_set
+			(priv->ctx,
+			 &(struct ibv_counters_init_attr){0}),
+#else
 		.cs = mlx5_glue->create_counter_set
 			(priv->ctx,
 			 &(struct ibv_counter_set_init_attr){
 				 .counter_set_id = id,
 			 }),
+#endif
 		.hits = 0,
 		.bytes = 0,
 	};
@@ -77,17 +82,40 @@
 		rte_errno = errno;
 		return NULL;
 	}
+
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
+	struct ibv_counter_attach_attr attach_attr = {0};
+	int ret;
+
+	attach_attr.counter_desc = IBV_COUNTER_PACKETS;
+	attach_attr.index = 0;
+	ret = ibv_attach_counters_point_flow(tmpl.cs, &attach_attr, NULL);
+	if (!ret) {
+		attach_attr.counter_desc = IBV_COUNTER_BYTES;
+		attach_attr.index = 1;
+		ret = ibv_attach_counters_point_flow(tmpl.cs,
+						     &attach_attr,
+						     NULL);
+	}
+	if (ret) {
+		claim_zero(mlx5_glue->destroy_counter_set(tmpl.cs));
+		rte_errno = ret;
+		return NULL;
+	}
+#endif
 	cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
 	if (!cnt) {
+		claim_zero(mlx5_glue->destroy_counter_set(tmpl.cs));
 		rte_errno = ENOMEM;
 		return NULL;
 	}
 	*cnt = tmpl;
 	LIST_INSERT_HEAD(&priv->flow_counters, cnt, next);
 	return cnt;
-#endif
+#else
 	rte_errno = ENOTSUP;
 	return NULL;
+#endif
 }
 
 /**
@@ -947,7 +975,7 @@
 		flow->counter = flow_verbs_counter_new(dev, count->shared,
 						       count->id);
 		if (!flow->counter)
-			return rte_flow_error_set(error, ENOTSUP,
+			return rte_flow_error_set(error, rte_errno,
 						  RTE_FLOW_ERROR_TYPE_ACTION,
 						  action,
 						  "cannot get counter"
@@ -955,7 +983,11 @@
 	}
 	*action_flags |= MLX5_ACTION_COUNT;
 #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
+	counter.counters = flow->counter->cs;
+#else
 	counter.counter_set_handle = flow->counter->cs->handle;
+#endif
 	flow_verbs_spec_add(dev_flow, &counter, size);
 #endif
 	return 0;
diff --git a/drivers/net/mlx5/mlx5_glue.c b/drivers/net/mlx5/mlx5_glue.c
index 48590df..785234c 100644
--- a/drivers/net/mlx5/mlx5_glue.c
+++ b/drivers/net/mlx5/mlx5_glue.c
@@ -211,6 +211,39 @@
 	return ibv_dereg_mr(mr);
 }
 
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
+static struct ibv_counters *
+mlx5_glue_create_counters(struct ibv_context *context,
+			  struct ibv_counters_init_attr *init_attr)
+{
+	return ibv_create_counters(context, init_attr);
+}
+
+static int
+mlx5_glue_destroy_counters(struct ibv_counters *counters)
+{
+	return ibv_destroy_counters(counters);
+}
+
+static int
+mlx5_glue_attach_counters(struct ibv_counters *counters,
+		     struct ibv_counter_attach_attr *attr,
+		     struct ibv_flow *flow)
+{
+	return ibv_attach_counters_point_flow(counters, attr, flow);
+}
+
+static int
+mlx5_glue_query_counters(struct ibv_counters *counters,
+			 uint64_t *counters_value,
+			 uint32_t ncounters,
+			 uint32_t flags)
+{
+	return ibv_read_counters(counters, counters_value, ncounters, flags);
+}
+#endif
+
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
 static struct ibv_counter_set *
 mlx5_glue_create_counter_set(struct ibv_context *context,
 			     struct ibv_counter_set_init_attr *init_attr)
@@ -262,6 +295,7 @@
 	return ibv_query_counter_set(query_attr, cs_data);
 #endif
 }
+#endif
 
 static void
 mlx5_glue_ack_async_event(struct ibv_async_event *event)
@@ -420,10 +454,17 @@
 	.modify_qp = mlx5_glue_modify_qp,
 	.reg_mr = mlx5_glue_reg_mr,
 	.dereg_mr = mlx5_glue_dereg_mr,
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
+	.create_counter_set = mlx5_glue_create_counters,
+	.destroy_counter_set = mlx5_glue_destroy_counters,
+	.attach_counter_set = mlx5_glue_attach_counters,
+	.query_counter_set = mlx5_glue_query_counters,
+#else
 	.create_counter_set = mlx5_glue_create_counter_set,
 	.destroy_counter_set = mlx5_glue_destroy_counter_set,
 	.describe_counter_set = mlx5_glue_describe_counter_set,
 	.query_counter_set = mlx5_glue_query_counter_set,
+#endif
 	.ack_async_event = mlx5_glue_ack_async_event,
 	.get_async_event = mlx5_glue_get_async_event,
 	.port_state_str = mlx5_glue_port_state_str,
diff --git a/drivers/net/mlx5/mlx5_glue.h b/drivers/net/mlx5/mlx5_glue.h
index f6e4e38..504d487 100644
--- a/drivers/net/mlx5/mlx5_glue.h
+++ b/drivers/net/mlx5/mlx5_glue.h
@@ -96,6 +96,21 @@ struct mlx5_glue {
 	struct ibv_mr *(*reg_mr)(struct ibv_pd *pd, void *addr,
 				 size_t length, int access);
 	int (*dereg_mr)(struct ibv_mr *mr);
+#ifdef	HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
+	struct ibv_counters *(*create_counter_set)
+		(struct ibv_context *context,
+		 struct ibv_counters_init_attr *init_attr);
+	int (*destroy_counter_set)(struct ibv_counters *cs);
+	int (*attach_counter_set)
+		(struct ibv_counters *cs,
+		 struct ibv_counter_attach_attr *attr,
+		 struct ibv_flow *flow);
+	int (*query_counter_set)
+		(struct ibv_counters *cs,
+		 uint64_t *counters_value,
+		 uint32_t ncounters,
+		 uint32_t flags);
+#else
 	struct ibv_counter_set *(*create_counter_set)
 		(struct ibv_context *context,
 		 struct ibv_counter_set_init_attr *init_attr);
@@ -106,6 +121,7 @@ struct mlx5_glue {
 		 struct ibv_counter_set_description *cs_desc);
 	int (*query_counter_set)(struct ibv_query_counter_set_attr *query_attr,
 				 struct ibv_counter_set_data *cs_data);
+#endif
 	void (*ack_async_event)(struct ibv_async_event *event);
 	int (*get_async_event)(struct ibv_context *context,
 			       struct ibv_async_event *event);
-- 
1.8.3.1

^ permalink raw reply	[flat|nested] 39+ messages in thread

* Re: [dpdk-dev] [PATCH] net/mlx5: flow counters support on the Linux-rdma v19 base
  2018-10-03 15:29 ` [dpdk-dev] [PATCH] net/mlx5: flow counters support on the Linux-rdma " Slava Ovsiienko
@ 2018-10-03 23:48   ` Yongseok Koh
  2018-10-09 13:45     ` Shahaf Shuler
  2018-10-11 14:51     ` Slava Ovsiienko
  2018-10-17 13:53   ` [dpdk-dev] [PATCH v2] " Viacheslav Ovsiienko
  1 sibling, 2 replies; 39+ messages in thread
From: Yongseok Koh @ 2018-10-03 23:48 UTC (permalink / raw)
  To: Slava Ovsiienko; +Cc: dev, Shahaf Shuler

On Wed, Oct 03, 2018 at 03:29:12PM +0000, Slava Ovsiienko wrote:
> Mellanox mlx5 PMD supports Flow Counters via Verbs library.
> The current implementation is based on the Mellanox proprietary
> Verbs library included in MLNX OFED packages. The Flow Counter
> support is recently added into linux-rdma release (v19),
> so the mlx5 PMD update is needed to provide Counter feature
> on the base of linux-rdma.
> 
> mlx5 PMD can be compiled with MLNX OFED or linux-rdma v19+
> and provide flow counters for both.
> 
> Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
> ---
>  drivers/net/mlx5/Makefile          | 10 ++++++++
>  drivers/net/mlx5/mlx5.c            |  6 +++++
>  drivers/net/mlx5/mlx5_flow.c       |  9 ++++++-
>  drivers/net/mlx5/mlx5_flow.h       |  4 +++
>  drivers/net/mlx5/mlx5_flow_verbs.c | 52 ++++++++++++++++++++++++++++++--------
>  drivers/net/mlx5/mlx5_glue.c       | 41 ++++++++++++++++++++++++++++++
>  drivers/net/mlx5/mlx5_glue.h       | 16 ++++++++++++
>  7 files changed, 127 insertions(+), 11 deletions(-)
> 
> diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile
> index ca1de9f..e3d2156 100644
> --- a/drivers/net/mlx5/Makefile
> +++ b/drivers/net/mlx5/Makefile
> @@ -162,6 +162,16 @@ mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
>  		type 'struct ibv_counter_set_init_attr' \
>  		$(AUTOCONF_OUTPUT)
>  	$Q sh -- '$<' '$@' \
> +		HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT \
> +		infiniband/verbs.h \
> +		type 'struct ibv_counters_init_attr' \
> +		$(AUTOCONF_OUTPUT)
> +	$Q sh -- '$<' '$@' \
> +		HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45 \
> +		infiniband/verbs.h \
> +		type 'struct ibv_counters_init_attr' \
> +		$(AUTOCONF_OUTPUT)
> +	$Q sh -- '$<' '$@' \

I still don't understand what is different between the two. These are exactly
same checking, then why do you need to have two different macros? From this
script, HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT is same as
HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45, isn't it?

And if you make some changes in Makefile, you should also make corresponding
changes for the meson build.

>  		HAVE_RDMA_NL_NLDEV \
>  		rdma/rdma_netlink.h \
>  		enum RDMA_NL_NLDEV \
> diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
> index 4be6a1c..81f6ba1 100644
> --- a/drivers/net/mlx5/mlx5.c
> +++ b/drivers/net/mlx5/mlx5.c
> @@ -739,8 +739,10 @@
>  	unsigned int mprq_min_stride_num_n = 0;
>  	unsigned int mprq_max_stride_num_n = 0;
>  #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
> +#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
>  	struct ibv_counter_set_description cs_desc = { .counter_type = 0 };
>  #endif
> +#endif
>  	struct ether_addr mac;
>  	char name[RTE_ETH_NAME_MAX_LEN];
>  	int own_domain_id = 0;
> @@ -1009,11 +1011,15 @@
>  	DRV_LOG(DEBUG, "checksum offloading is %ssupported",
>  		(config.hw_csum ? "" : "not "));
>  #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
> +#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
>  	config.flow_counter_en = !!attr.max_counter_sets;
>  	mlx5_glue->describe_counter_set(ctx, 0, &cs_desc);
>  	DRV_LOG(DEBUG, "counter type = %d, num of cs = %ld, attributes = %d",
>  		cs_desc.counter_type, cs_desc.num_of_cs,
>  		cs_desc.attributes);
> +#else
> +	config.flow_counter_en = 1;
> +#endif
>  #endif
>  	config.ind_table_max_size =
>  		attr.rss_caps.max_rwq_indirection_table_size;
> diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
> index 8007bf1..652580c 100644
> --- a/drivers/net/mlx5/mlx5_flow.c
> +++ b/drivers/net/mlx5/mlx5_flow.c
> @@ -2306,6 +2306,13 @@ struct rte_flow *
>  	if (flow->actions & MLX5_ACTION_COUNT) {
>  		struct rte_flow_query_count *qc = data;
>  		uint64_t counters[2] = {0, 0};
> +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
> +		int err = mlx5_glue->query_counter_set(
> +				flow->counter->cs,
> +				counters,
> +				RTE_DIM(counters),
> +				IBV_READ_COUNTERS_ATTR_PREFER_CACHED);
> +#else
>  		struct ibv_query_counter_set_attr query_cs_attr = {
>  			.cs = flow->counter->cs,
>  			.query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
> @@ -2316,7 +2323,7 @@ struct rte_flow *
>  		};
>  		int err = mlx5_glue->query_counter_set(&query_cs_attr,
>  						       &query_out);
> -
> +#endif
>  		if (err)
>  			return rte_flow_error_set
>  				(error, err,
> diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
> index 10d700a..a3b82dd 100644
> --- a/drivers/net/mlx5/mlx5_flow.h
> +++ b/drivers/net/mlx5/mlx5_flow.h
> @@ -222,7 +222,11 @@ struct mlx5_flow_counter {
>  	uint32_t shared:1; /**< Share counter ID with other flow rules. */
>  	uint32_t ref_cnt:31; /**< Reference counter. */
>  	uint32_t id; /**< Counter ID. */
> +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
> +	struct ibv_counters *cs; /**< Holds the counters for the rule. */
> +#else
>  	struct ibv_counter_set *cs; /**< Holds the counters for the rule. */
> +#endif
>  	uint64_t hits; /**< Number of packets matched by the rule. */
>  	uint64_t bytes; /**< Number of bytes matched by the rule. */
>  };
> diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c b/drivers/net/mlx5/mlx5_flow_verbs.c
> index 05ab5fd..1c8bdba 100644
> --- a/drivers/net/mlx5/mlx5_flow_verbs.c
> +++ b/drivers/net/mlx5/mlx5_flow_verbs.c
> @@ -48,27 +48,32 @@
>  static struct mlx5_flow_counter *
>  flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id)
>  {
> +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
>  	struct priv *priv = dev->data->dev_private;
>  	struct mlx5_flow_counter *cnt;
>  
> -	LIST_FOREACH(cnt, &priv->flow_counters, next) {
> -		if (!cnt->shared || cnt->shared != shared)
> -			continue;
> -		if (cnt->id != id)
> -			continue;
> -		cnt->ref_cnt++;
> -		return cnt;
> +	if (shared) {
> +		LIST_FOREACH(cnt, &priv->flow_counters, next)
> +		if (cnt->shared && cnt->id == id) {
> +			cnt->ref_cnt++;
> +			return cnt;
> +		}
>  	}
> -#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
>  
>  	struct mlx5_flow_counter tmpl = {
>  		.shared = shared,
>  		.id = id,
> +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
> +		.cs = mlx5_glue->create_counter_set
> +			(priv->ctx,
> +			 &(struct ibv_counters_init_attr){0}),
> +#else
>  		.cs = mlx5_glue->create_counter_set
>  			(priv->ctx,
>  			 &(struct ibv_counter_set_init_attr){
>  				 .counter_set_id = id,
>  			 }),
> +#endif
>  		.hits = 0,
>  		.bytes = 0,
>  	};
> @@ -77,17 +82,40 @@
>  		rte_errno = errno;
>  		return NULL;
>  	}
> +
> +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
> +	struct ibv_counter_attach_attr attach_attr = {0};
> +	int ret;
> +
> +	attach_attr.counter_desc = IBV_COUNTER_PACKETS;
> +	attach_attr.index = 0;
> +	ret = ibv_attach_counters_point_flow(tmpl.cs, &attach_attr, NULL);
> +	if (!ret) {
> +		attach_attr.counter_desc = IBV_COUNTER_BYTES;
> +		attach_attr.index = 1;
> +		ret = ibv_attach_counters_point_flow(tmpl.cs,
> +						     &attach_attr,
> +						     NULL);
> +	}
> +	if (ret) {
> +		claim_zero(mlx5_glue->destroy_counter_set(tmpl.cs));
> +		rte_errno = ret;
> +		return NULL;
> +	}
> +#endif
>  	cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
>  	if (!cnt) {
> +		claim_zero(mlx5_glue->destroy_counter_set(tmpl.cs));
>  		rte_errno = ENOMEM;
>  		return NULL;
>  	}
>  	*cnt = tmpl;
>  	LIST_INSERT_HEAD(&priv->flow_counters, cnt, next);
>  	return cnt;
> -#endif
> +#else
>  	rte_errno = ENOTSUP;
>  	return NULL;
> +#endif
>  }
>  
>  /**
> @@ -947,7 +975,7 @@
>  		flow->counter = flow_verbs_counter_new(dev, count->shared,
>  						       count->id);
>  		if (!flow->counter)
> -			return rte_flow_error_set(error, ENOTSUP,
> +			return rte_flow_error_set(error, rte_errno,
>  						  RTE_FLOW_ERROR_TYPE_ACTION,
>  						  action,
>  						  "cannot get counter"
> @@ -955,7 +983,11 @@
>  	}
>  	*action_flags |= MLX5_ACTION_COUNT;
>  #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
> +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
> +	counter.counters = flow->counter->cs;
> +#else
>  	counter.counter_set_handle = flow->counter->cs->handle;
> +#endif
>  	flow_verbs_spec_add(dev_flow, &counter, size);
>  #endif
>  	return 0;
> diff --git a/drivers/net/mlx5/mlx5_glue.c b/drivers/net/mlx5/mlx5_glue.c
> index 48590df..785234c 100644
> --- a/drivers/net/mlx5/mlx5_glue.c
> +++ b/drivers/net/mlx5/mlx5_glue.c
> @@ -211,6 +211,39 @@
>  	return ibv_dereg_mr(mr);
>  }
>  
> +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
> +static struct ibv_counters *
> +mlx5_glue_create_counters(struct ibv_context *context,
> +			  struct ibv_counters_init_attr *init_attr)
> +{
> +	return ibv_create_counters(context, init_attr);
> +}
> +
> +static int
> +mlx5_glue_destroy_counters(struct ibv_counters *counters)
> +{
> +	return ibv_destroy_counters(counters);
> +}
> +
> +static int
> +mlx5_glue_attach_counters(struct ibv_counters *counters,
> +		     struct ibv_counter_attach_attr *attr,
> +		     struct ibv_flow *flow)
> +{
> +	return ibv_attach_counters_point_flow(counters, attr, flow);
> +}
> +
> +static int
> +mlx5_glue_query_counters(struct ibv_counters *counters,
> +			 uint64_t *counters_value,
> +			 uint32_t ncounters,
> +			 uint32_t flags)
> +{
> +	return ibv_read_counters(counters, counters_value, ncounters, flags);
> +}
> +#endif
> +
> +#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
>  static struct ibv_counter_set *
>  mlx5_glue_create_counter_set(struct ibv_context *context,
>  			     struct ibv_counter_set_init_attr *init_attr)
> @@ -262,6 +295,7 @@
>  	return ibv_query_counter_set(query_attr, cs_data);
>  #endif
>  }
> +#endif
>  
>  static void
>  mlx5_glue_ack_async_event(struct ibv_async_event *event)
> @@ -420,10 +454,17 @@
>  	.modify_qp = mlx5_glue_modify_qp,
>  	.reg_mr = mlx5_glue_reg_mr,
>  	.dereg_mr = mlx5_glue_dereg_mr,
> +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
> +	.create_counter_set = mlx5_glue_create_counters,
> +	.destroy_counter_set = mlx5_glue_destroy_counters,
> +	.attach_counter_set = mlx5_glue_attach_counters,
> +	.query_counter_set = mlx5_glue_query_counters,
> +#else
>  	.create_counter_set = mlx5_glue_create_counter_set,
>  	.destroy_counter_set = mlx5_glue_destroy_counter_set,
>  	.describe_counter_set = mlx5_glue_describe_counter_set,
>  	.query_counter_set = mlx5_glue_query_counter_set,
> +#endif
>  	.ack_async_event = mlx5_glue_ack_async_event,
>  	.get_async_event = mlx5_glue_get_async_event,
>  	.port_state_str = mlx5_glue_port_state_str,
> diff --git a/drivers/net/mlx5/mlx5_glue.h b/drivers/net/mlx5/mlx5_glue.h
> index f6e4e38..504d487 100644
> --- a/drivers/net/mlx5/mlx5_glue.h
> +++ b/drivers/net/mlx5/mlx5_glue.h
> @@ -96,6 +96,21 @@ struct mlx5_glue {
>  	struct ibv_mr *(*reg_mr)(struct ibv_pd *pd, void *addr,
>  				 size_t length, int access);
>  	int (*dereg_mr)(struct ibv_mr *mr);
> +#ifdef	HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
> +	struct ibv_counters *(*create_counter_set)
> +		(struct ibv_context *context,
> +		 struct ibv_counters_init_attr *init_attr);
> +	int (*destroy_counter_set)(struct ibv_counters *cs);
> +	int (*attach_counter_set)
> +		(struct ibv_counters *cs,
> +		 struct ibv_counter_attach_attr *attr,
> +		 struct ibv_flow *flow);
> +	int (*query_counter_set)
> +		(struct ibv_counters *cs,
> +		 uint64_t *counters_value,
> +		 uint32_t ncounters,
> +		 uint32_t flags);
> +#else
>  	struct ibv_counter_set *(*create_counter_set)
>  		(struct ibv_context *context,
>  		 struct ibv_counter_set_init_attr *init_attr);
> @@ -106,6 +121,7 @@ struct mlx5_glue {
>  		 struct ibv_counter_set_description *cs_desc);
>  	int (*query_counter_set)(struct ibv_query_counter_set_attr *query_attr,
>  				 struct ibv_counter_set_data *cs_data);
> +#endif
>  	void (*ack_async_event)(struct ibv_async_event *event);
>  	int (*get_async_event)(struct ibv_context *context,
>  			       struct ibv_async_event *event);
> -- 
> 1.8.3.1
> 

^ permalink raw reply	[flat|nested] 39+ messages in thread

* Re: [dpdk-dev] [PATCH] net/mlx5: flow counters support on the Linux-rdma v19 base
  2018-10-03 23:48   ` Yongseok Koh
@ 2018-10-09 13:45     ` Shahaf Shuler
  2018-10-11 14:51     ` Slava Ovsiienko
  1 sibling, 0 replies; 39+ messages in thread
From: Shahaf Shuler @ 2018-10-09 13:45 UTC (permalink / raw)
  To: Yongseok Koh, Slava Ovsiienko; +Cc: dev

Hi Slava,

Adding some more comments,

Thursday, October 4, 2018 2:48 AM, Yongseok Koh:
> Subject: Re: [dpdk-dev] [PATCH] net/mlx5: flow counters support on the
> Linux-rdma v19 base
> 
> On Wed, Oct 03, 2018 at 03:29:12PM +0000, Slava Ovsiienko wrote:
> > Mellanox mlx5 PMD supports Flow Counters via Verbs library.
> > The current implementation is based on the Mellanox proprietary Verbs
> > library included in MLNX OFED packages. The Flow Counter support is
> > recently added into linux-rdma release (v19), so the mlx5 PMD update
> > is needed to provide Counter feature on the base of linux-rdma.
> >
> > mlx5 PMD can be compiled with MLNX OFED or linux-rdma v19+ and
> provide
> > flow counters for both.
> >
> > Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
> > ---
> >  drivers/net/mlx5/Makefile          | 10 ++++++++
> >  drivers/net/mlx5/mlx5.c            |  6 +++++
> >  drivers/net/mlx5/mlx5_flow.c       |  9 ++++++-
> >  drivers/net/mlx5/mlx5_flow.h       |  4 +++
> >  drivers/net/mlx5/mlx5_flow_verbs.c | 52
> ++++++++++++++++++++++++++++++--------
> >  drivers/net/mlx5/mlx5_glue.c       | 41
> ++++++++++++++++++++++++++++++
> >  drivers/net/mlx5/mlx5_glue.h       | 16 ++++++++++++
> >  7 files changed, 127 insertions(+), 11 deletions(-)
> >
> > diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile
> > index ca1de9f..e3d2156 100644
> > --- a/drivers/net/mlx5/Makefile
> > +++ b/drivers/net/mlx5/Makefile
> > @@ -162,6 +162,16 @@ mlx5_autoconf.h.new:
> $(RTE_SDK)/buildtools/auto-config-h.sh
> >  		type 'struct ibv_counter_set_init_attr' \
> >  		$(AUTOCONF_OUTPUT)
> >  	$Q sh -- '$<' '$@' \
> > +		HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT \
> > +		infiniband/verbs.h \
> > +		type 'struct ibv_counters_init_attr' \
> > +		$(AUTOCONF_OUTPUT)
> > +	$Q sh -- '$<' '$@' \
> > +		HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45 \
> > +		infiniband/verbs.h \
> > +		type 'struct ibv_counters_init_attr' \
> > +		$(AUTOCONF_OUTPUT)
> > +	$Q sh -- '$<' '$@' \
> 
> I still don't understand what is different between the two. These are exactly
> same checking, then why do you need to have two different macros? From
> this script, HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT is same as
> HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45, isn't it?
> 
> And if you make some changes in Makefile, you should also make
> corresponding changes for the meson build.
> 
> >  		HAVE_RDMA_NL_NLDEV \
> >  		rdma/rdma_netlink.h \
> >  		enum RDMA_NL_NLDEV \
> > diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index
> > 4be6a1c..81f6ba1 100644
> > --- a/drivers/net/mlx5/mlx5.c
> > +++ b/drivers/net/mlx5/mlx5.c
> > @@ -739,8 +739,10 @@
> >  	unsigned int mprq_min_stride_num_n = 0;
> >  	unsigned int mprq_max_stride_num_n = 0;  #ifdef
> > HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
> > +#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45

Is there really a rdma-core version on which both types of counters exists together? 

> >  	struct ibv_counter_set_description cs_desc = { .counter_type = 0 };
> > #endif
> > +#endif
> >  	struct ether_addr mac;
> >  	char name[RTE_ETH_NAME_MAX_LEN];
> >  	int own_domain_id = 0;
> > @@ -1009,11 +1011,15 @@
> >  	DRV_LOG(DEBUG, "checksum offloading is %ssupported",
> >  		(config.hw_csum ? "" : "not "));
> >  #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
> > +#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
> >  	config.flow_counter_en = !!attr.max_counter_sets;
> >  	mlx5_glue->describe_counter_set(ctx, 0, &cs_desc);
> >  	DRV_LOG(DEBUG, "counter type = %d, num of cs = %ld, attributes =
> %d",
> >  		cs_desc.counter_type, cs_desc.num_of_cs,
> >  		cs_desc.attributes);
> > +#else
> > +	config.flow_counter_en = 1;
> > +#endif
> >  #endif
> >  	config.ind_table_max_size =
> >  		attr.rss_caps.max_rwq_indirection_table_size;
> > diff --git a/drivers/net/mlx5/mlx5_flow.c
> > b/drivers/net/mlx5/mlx5_flow.c index 8007bf1..652580c 100644
> > --- a/drivers/net/mlx5/mlx5_flow.c
> > +++ b/drivers/net/mlx5/mlx5_flow.c
> > @@ -2306,6 +2306,13 @@ struct rte_flow *
> >  	if (flow->actions & MLX5_ACTION_COUNT) {
> >  		struct rte_flow_query_count *qc = data;
> >  		uint64_t counters[2] = {0, 0};
> > +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
> > +		int err = mlx5_glue->query_counter_set(
> > +				flow->counter->cs,
> > +				counters,
> > +				RTE_DIM(counters),
> > +
> 	IBV_READ_COUNTERS_ATTR_PREFER_CACHED);
> > +#else

This part is under #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT. So on rdma-core versions were the "old" counters are not supported anymore this code wil never be reached. 

> >  		struct ibv_query_counter_set_attr query_cs_attr = {
> >  			.cs = flow->counter->cs,
> >  			.query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
> @@ -2316,7 +2323,7 @@
> > struct rte_flow *
> >  		};
> >  		int err = mlx5_glue->query_counter_set(&query_cs_attr,
> >  						       &query_out);
> > -
> > +#endif
> >  		if (err)
> >  			return rte_flow_error_set
> >  				(error, err,
> > diff --git a/drivers/net/mlx5/mlx5_flow.h
> > b/drivers/net/mlx5/mlx5_flow.h index 10d700a..a3b82dd 100644
> > --- a/drivers/net/mlx5/mlx5_flow.h
> > +++ b/drivers/net/mlx5/mlx5_flow.h
> > @@ -222,7 +222,11 @@ struct mlx5_flow_counter {
> >  	uint32_t shared:1; /**< Share counter ID with other flow rules. */
> >  	uint32_t ref_cnt:31; /**< Reference counter. */
> >  	uint32_t id; /**< Counter ID. */
> > +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
> > +	struct ibv_counters *cs; /**< Holds the counters for the rule. */
> > +#else
> >  	struct ibv_counter_set *cs; /**< Holds the counters for the rule. */
> > +#endif
> >  	uint64_t hits; /**< Number of packets matched by the rule. */
> >  	uint64_t bytes; /**< Number of bytes matched by the rule. */  };
> > diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c
> > b/drivers/net/mlx5/mlx5_flow_verbs.c
> > index 05ab5fd..1c8bdba 100644
> > --- a/drivers/net/mlx5/mlx5_flow_verbs.c
> > +++ b/drivers/net/mlx5/mlx5_flow_verbs.c
> > @@ -48,27 +48,32 @@
> >  static struct mlx5_flow_counter *
> >  flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t shared,
> > uint32_t id)  {
> > +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT

I don't understand why you choose to move the ifdef. 

> >  	struct priv *priv = dev->data->dev_private;
> >  	struct mlx5_flow_counter *cnt;
> >
> > -	LIST_FOREACH(cnt, &priv->flow_counters, next) {
> > -		if (!cnt->shared || cnt->shared != shared)
> > -			continue;
> > -		if (cnt->id != id)
> > -			continue;
> > -		cnt->ref_cnt++;
> > -		return cnt;
> > +	if (shared) {
> > +		LIST_FOREACH(cnt, &priv->flow_counters, next)
> > +		if (cnt->shared && cnt->id == id) {
> > +			cnt->ref_cnt++;
> > +			return cnt;
> > +		}

Nor this code. Is it related to the support patch or you are fixing bug on the way. If bug fix it deserve a separate commit. 

> >  	}
> > -#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
> >
> >  	struct mlx5_flow_counter tmpl = {
> >  		.shared = shared,
> >  		.id = id,
> > +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45

Same - you will never reach this code if the old flow count is not supported on the underlying rdma-core version. 

> > +		.cs = mlx5_glue->create_counter_set
> > +			(priv->ctx,
> > +			 &(struct ibv_counters_init_attr){0}), #else
 > >  		.cs = mlx5_glue->create_counter_set
> >  			(priv->ctx,
> >  			 &(struct ibv_counter_set_init_attr){
> >  				 .counter_set_id = id,
> >  			 }),

Here from the old counters perspective the counter is created...

> > +#endif
> >  		.hits = 0,
> >  		.bytes = 0,
> >  	};
> > @@ -77,17 +82,40 @@
> >  		rte_errno = errno;
> >  		return NULL;
> >  	}
> > +
> > +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
> > +	struct ibv_counter_attach_attr attach_attr = {0};
> > +	int ret;
> > +
> > +	attach_attr.counter_desc = IBV_COUNTER_PACKETS;
> > +	attach_attr.index = 0;
> > +	ret = ibv_attach_counters_point_flow(tmpl.cs, &attach_attr, NULL);
> > +	if (!ret) {
> > +		attach_attr.counter_desc = IBV_COUNTER_BYTES;
> > +		attach_attr.index = 1;
> > +		ret = ibv_attach_counters_point_flow(tmpl.cs,
> > +						     &attach_attr,
> > +						     NULL);


Here it is the end of the counter creation on the new API. My suggestion is to have a single ifdef which create the new/old counter (and not two). 

> > +	}
> > +	if (ret) {
> > +		claim_zero(mlx5_glue->destroy_counter_set(tmpl.cs));
> > +		rte_errno = ret;
> > +		return NULL;
> > +	}
> > +#endif
> >  	cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
> >  	if (!cnt) {
> > +		claim_zero(mlx5_glue->destroy_counter_set(tmpl.cs));

Another bug fix? 

> >  		rte_errno = ENOMEM;
> >  		return NULL;
> >  	}
> >  	*cnt = tmpl;
> >  	LIST_INSERT_HEAD(&priv->flow_counters, cnt, next);
> >  	return cnt;
> > -#endif
> > +#else
> >  	rte_errno = ENOTSUP;
> >  	return NULL;
> > +#endif
> >  }
> >
> >  /**
> > @@ -947,7 +975,7 @@
> >  		flow->counter = flow_verbs_counter_new(dev, count-
> >shared,
> >  						       count->id);
> >  		if (!flow->counter)
> > -			return rte_flow_error_set(error, ENOTSUP,
> > +			return rte_flow_error_set(error, rte_errno,
> >
> RTE_FLOW_ERROR_TYPE_ACTION,
> >  						  action,
> >  						  "cannot get counter"
> > @@ -955,7 +983,11 @@
> >  	}
> >  	*action_flags |= MLX5_ACTION_COUNT;
> >  #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
> > +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45

Same comment about the code never reached. 

> > +	counter.counters = flow->counter->cs; #else
> >  	counter.counter_set_handle = flow->counter->cs->handle;
> > +#endif
> >  	flow_verbs_spec_add(dev_flow, &counter, size);  #endif
> >  	return 0;
> > diff --git a/drivers/net/mlx5/mlx5_glue.c
> > b/drivers/net/mlx5/mlx5_glue.c index 48590df..785234c 100644
> > --- a/drivers/net/mlx5/mlx5_glue.c
> > +++ b/drivers/net/mlx5/mlx5_glue.c
> > @@ -211,6 +211,39 @@
> >  	return ibv_dereg_mr(mr);
> >  }
> >
> > +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
> > +static struct ibv_counters *
> > +mlx5_glue_create_counters(struct ibv_context *context,
> > +			  struct ibv_counters_init_attr *init_attr) {
> > +	return ibv_create_counters(context, init_attr); }
> > +
> > +static int
> > +mlx5_glue_destroy_counters(struct ibv_counters *counters) {
> > +	return ibv_destroy_counters(counters); }
> > +
> > +static int
> > +mlx5_glue_attach_counters(struct ibv_counters *counters,
> > +		     struct ibv_counter_attach_attr *attr,
> > +		     struct ibv_flow *flow)
> > +{
> > +	return ibv_attach_counters_point_flow(counters, attr, flow); }
> > +
> > +static int
> > +mlx5_glue_query_counters(struct ibv_counters *counters,
> > +			 uint64_t *counters_value,
> > +			 uint32_t ncounters,
> > +			 uint32_t flags)
> > +{
> > +	return ibv_read_counters(counters, counters_value, ncounters,
> > +flags); } #endif
> > +
> > +#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
> >  static struct ibv_counter_set *
> >  mlx5_glue_create_counter_set(struct ibv_context *context,
> >  			     struct ibv_counter_set_init_attr *init_attr) @@ -
> 262,6 +295,7
> > @@
> >  	return ibv_query_counter_set(query_attr, cs_data);  #endif  }
> > +#endif
> >
> >  static void
> >  mlx5_glue_ack_async_event(struct ibv_async_event *event) @@ -420,10
> > +454,17 @@
> >  	.modify_qp = mlx5_glue_modify_qp,
> >  	.reg_mr = mlx5_glue_reg_mr,
> >  	.dereg_mr = mlx5_glue_dereg_mr,
> > +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
> > +	.create_counter_set = mlx5_glue_create_counters,
> > +	.destroy_counter_set = mlx5_glue_destroy_counters,
> > +	.attach_counter_set = mlx5_glue_attach_counters,
> > +	.query_counter_set = mlx5_glue_query_counters, #else
> >  	.create_counter_set = mlx5_glue_create_counter_set,
> >  	.destroy_counter_set = mlx5_glue_destroy_counter_set,
> >  	.describe_counter_set = mlx5_glue_describe_counter_set,
> >  	.query_counter_set = mlx5_glue_query_counter_set,
> > +#endif
> >  	.ack_async_event = mlx5_glue_ack_async_event,
> >  	.get_async_event = mlx5_glue_get_async_event,
> >  	.port_state_str = mlx5_glue_port_state_str, diff --git
> > a/drivers/net/mlx5/mlx5_glue.h b/drivers/net/mlx5/mlx5_glue.h index
> > f6e4e38..504d487 100644
> > --- a/drivers/net/mlx5/mlx5_glue.h
> > +++ b/drivers/net/mlx5/mlx5_glue.h
> > @@ -96,6 +96,21 @@ struct mlx5_glue {
> >  	struct ibv_mr *(*reg_mr)(struct ibv_pd *pd, void *addr,
> >  				 size_t length, int access);
> >  	int (*dereg_mr)(struct ibv_mr *mr);
> > +#ifdef	HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
> > +	struct ibv_counters *(*create_counter_set)
> > +		(struct ibv_context *context,
> > +		 struct ibv_counters_init_attr *init_attr);
> > +	int (*destroy_counter_set)(struct ibv_counters *cs);
> > +	int (*attach_counter_set)
> > +		(struct ibv_counters *cs,
> > +		 struct ibv_counter_attach_attr *attr,
> > +		 struct ibv_flow *flow);
> > +	int (*query_counter_set)
> > +		(struct ibv_counters *cs,
> > +		 uint64_t *counters_value,
> > +		 uint32_t ncounters,
> > +		 uint32_t flags);
> > +#else
> >  	struct ibv_counter_set *(*create_counter_set)
> >  		(struct ibv_context *context,
> >  		 struct ibv_counter_set_init_attr *init_attr); @@ -106,6
> +121,7 @@
> > struct mlx5_glue {
> >  		 struct ibv_counter_set_description *cs_desc);
> >  	int (*query_counter_set)(struct ibv_query_counter_set_attr
> *query_attr,
> >  				 struct ibv_counter_set_data *cs_data);
> > +#endif

Pay attention to how the old counter supported was added to the mlx5_glue lib.
There are no ifdefs of the function pointer declaration nor on the function declaration. This was because it was causing link issues between this lib to verbs. 

Same should be for the new counters. Need to declare all needed structs if not declared on the mlx5_glue.h file, and always the function pointer and the declaration. 
The ifdef is only inside the function implementation. 

> >  	void (*ack_async_event)(struct ibv_async_event *event);
> >  	int (*get_async_event)(struct ibv_context *context,
> >  			       struct ibv_async_event *event);
> > --
> > 1.8.3.1
> >

^ permalink raw reply	[flat|nested] 39+ messages in thread

* Re: [dpdk-dev] [PATCH] net/mlx5: flow counters support on the Linux-rdma v19 base
  2018-10-03 23:48   ` Yongseok Koh
  2018-10-09 13:45     ` Shahaf Shuler
@ 2018-10-11 14:51     ` Slava Ovsiienko
  2018-10-14  5:32       ` Shahaf Shuler
  1 sibling, 1 reply; 39+ messages in thread
From: Slava Ovsiienko @ 2018-10-11 14:51 UTC (permalink / raw)
  To: Yongseok Koh; +Cc: dev, Shahaf Shuler

> -----Original Message-----
> From: Yongseok Koh
> Sent: Thursday, October 4, 2018 2:48
> To: Slava Ovsiienko <viacheslavo@mellanox.com>
> Cc: dev@dpdk.org; Shahaf Shuler <shahafs@mellanox.com>
> Subject: Re: [dpdk-dev] [PATCH] net/mlx5: flow counters support on the
> Linux-rdma v19 base
> 
> On Wed, Oct 03, 2018 at 03:29:12PM +0000, Slava Ovsiienko wrote:
> > Mellanox mlx5 PMD supports Flow Counters via Verbs library.
> > The current implementation is based on the Mellanox proprietary Verbs
> > library included in MLNX OFED packages. The Flow Counter support is
> > recently added into linux-rdma release (v19), so the mlx5 PMD update
> > is needed to provide Counter feature on the base of linux-rdma.
> >
> > mlx5 PMD can be compiled with MLNX OFED or linux-rdma v19+ and
> provide
> > flow counters for both.
> >
> > Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
> > ---
> >  drivers/net/mlx5/Makefile          | 10 ++++++++
> >  drivers/net/mlx5/mlx5.c            |  6 +++++
> >  drivers/net/mlx5/mlx5_flow.c       |  9 ++++++-
> >  drivers/net/mlx5/mlx5_flow.h       |  4 +++
> >  drivers/net/mlx5/mlx5_flow_verbs.c | 52
> ++++++++++++++++++++++++++++++--------
> >  drivers/net/mlx5/mlx5_glue.c       | 41
> ++++++++++++++++++++++++++++++
> >  drivers/net/mlx5/mlx5_glue.h       | 16 ++++++++++++
> >  7 files changed, 127 insertions(+), 11 deletions(-)
> >
> > diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile
> > index ca1de9f..e3d2156 100644
> > --- a/drivers/net/mlx5/Makefile
> > +++ b/drivers/net/mlx5/Makefile
> > @@ -162,6 +162,16 @@ mlx5_autoconf.h.new:
> $(RTE_SDK)/buildtools/auto-config-h.sh
> >  		type 'struct ibv_counter_set_init_attr' \
> >  		$(AUTOCONF_OUTPUT)
> >  	$Q sh -- '$<' '$@' \
> > +		HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT \
> > +		infiniband/verbs.h \
> > +		type 'struct ibv_counters_init_attr' \
> > +		$(AUTOCONF_OUTPUT)
> > +	$Q sh -- '$<' '$@' \
> > +		HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45 \
> > +		infiniband/verbs.h \
> > +		type 'struct ibv_counters_init_attr' \
> > +		$(AUTOCONF_OUTPUT)
> > +	$Q sh -- '$<' '$@' \
> 
> I still don't understand what is different between the two. These are exactly
> same checking, then why do you need to have two different macros? From
> this script, HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT is same as
> HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45, isn't it?

We have three options:
- no counter support in kernel at all
- "old" counter support (ibv_counter_set_init_attr is defined in verbs.h)
- "new" counter support (ibv_counters_init_attr  is defined in verbs.h)

Three options require at least two compilations flags. The meanings are chosen:
HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT means there is counter support (of any type)
HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45 differentiates the support type

This approach allows to avoid clumsy constructions in code like this:
#if __defined(HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT \
|| __defined(HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45)

If there is no counter support in kernel at all
HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT is NOT defined
HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45 is NOT defined
if kernel provides "old counters"
HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT is defined
HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45 is NOT defined
if kernel provides "new counters"
HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT is defined
HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45 is defined

> 
> And if you make some changes in Makefile, you should also make
> corresponding changes for the meson build.
> 
> >  		HAVE_RDMA_NL_NLDEV \
> >  		rdma/rdma_netlink.h \
> >  		enum RDMA_NL_NLDEV \
> > diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index
> > 4be6a1c..81f6ba1 100644
> > --- a/drivers/net/mlx5/mlx5.c
> > +++ b/drivers/net/mlx5/mlx5.c
> > @@ -739,8 +739,10 @@
> >  	unsigned int mprq_min_stride_num_n = 0;
> >  	unsigned int mprq_max_stride_num_n = 0;  #ifdef
> > HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
> > +#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
> >  	struct ibv_counter_set_description cs_desc = { .counter_type = 0 };
> > #endif
> > +#endif
> >  	struct ether_addr mac;
> >  	char name[RTE_ETH_NAME_MAX_LEN];
> >  	int own_domain_id = 0;
> > @@ -1009,11 +1011,15 @@
> >  	DRV_LOG(DEBUG, "checksum offloading is %ssupported",
> >  		(config.hw_csum ? "" : "not "));
> >  #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
> > +#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
> >  	config.flow_counter_en = !!attr.max_counter_sets;
> >  	mlx5_glue->describe_counter_set(ctx, 0, &cs_desc);
> >  	DRV_LOG(DEBUG, "counter type = %d, num of cs = %ld, attributes =
> %d",
> >  		cs_desc.counter_type, cs_desc.num_of_cs,
> >  		cs_desc.attributes);
> > +#else
> > +	config.flow_counter_en = 1;
> > +#endif
> >  #endif
> >  	config.ind_table_max_size =
> >  		attr.rss_caps.max_rwq_indirection_table_size;
> > diff --git a/drivers/net/mlx5/mlx5_flow.c
> > b/drivers/net/mlx5/mlx5_flow.c index 8007bf1..652580c 100644
> > --- a/drivers/net/mlx5/mlx5_flow.c
> > +++ b/drivers/net/mlx5/mlx5_flow.c
> > @@ -2306,6 +2306,13 @@ struct rte_flow *
> >  	if (flow->actions & MLX5_ACTION_COUNT) {
> >  		struct rte_flow_query_count *qc = data;
> >  		uint64_t counters[2] = {0, 0};
> > +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
> > +		int err = mlx5_glue->query_counter_set(
> > +				flow->counter->cs,
> > +				counters,
> > +				RTE_DIM(counters),
> > +
> 	IBV_READ_COUNTERS_ATTR_PREFER_CACHED);
> > +#else
> >  		struct ibv_query_counter_set_attr query_cs_attr = {
> >  			.cs = flow->counter->cs,
> >  			.query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
> @@ -2316,7 +2323,7 @@
> > struct rte_flow *
> >  		};
> >  		int err = mlx5_glue->query_counter_set(&query_cs_attr,
> >  						       &query_out);
> > -
> > +#endif
> >  		if (err)
> >  			return rte_flow_error_set
> >  				(error, err,
> > diff --git a/drivers/net/mlx5/mlx5_flow.h
> > b/drivers/net/mlx5/mlx5_flow.h index 10d700a..a3b82dd 100644
> > --- a/drivers/net/mlx5/mlx5_flow.h
> > +++ b/drivers/net/mlx5/mlx5_flow.h
> > @@ -222,7 +222,11 @@ struct mlx5_flow_counter {
> >  	uint32_t shared:1; /**< Share counter ID with other flow rules. */
> >  	uint32_t ref_cnt:31; /**< Reference counter. */
> >  	uint32_t id; /**< Counter ID. */
> > +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
> > +	struct ibv_counters *cs; /**< Holds the counters for the rule. */
> > +#else
> >  	struct ibv_counter_set *cs; /**< Holds the counters for the rule. */
> > +#endif
> >  	uint64_t hits; /**< Number of packets matched by the rule. */
> >  	uint64_t bytes; /**< Number of bytes matched by the rule. */  };
> > diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c
> > b/drivers/net/mlx5/mlx5_flow_verbs.c
> > index 05ab5fd..1c8bdba 100644
> > --- a/drivers/net/mlx5/mlx5_flow_verbs.c
> > +++ b/drivers/net/mlx5/mlx5_flow_verbs.c
> > @@ -48,27 +48,32 @@
> >  static struct mlx5_flow_counter *
> >  flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t shared,
> > uint32_t id)  {
> > +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
> >  	struct priv *priv = dev->data->dev_private;
> >  	struct mlx5_flow_counter *cnt;
> >
> > -	LIST_FOREACH(cnt, &priv->flow_counters, next) {
> > -		if (!cnt->shared || cnt->shared != shared)
> > -			continue;
> > -		if (cnt->id != id)
> > -			continue;
> > -		cnt->ref_cnt++;
> > -		return cnt;
> > +	if (shared) {
> > +		LIST_FOREACH(cnt, &priv->flow_counters, next)
> > +		if (cnt->shared && cnt->id == id) {
> > +			cnt->ref_cnt++;
> > +			return cnt;
> > +		}
> >  	}
> > -#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
> >
> >  	struct mlx5_flow_counter tmpl = {
> >  		.shared = shared,
> >  		.id = id,
> > +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
> > +		.cs = mlx5_glue->create_counter_set
> > +			(priv->ctx,
> > +			 &(struct ibv_counters_init_attr){0}), #else
> >  		.cs = mlx5_glue->create_counter_set
> >  			(priv->ctx,
> >  			 &(struct ibv_counter_set_init_attr){
> >  				 .counter_set_id = id,
> >  			 }),
> > +#endif
> >  		.hits = 0,
> >  		.bytes = 0,
> >  	};
> > @@ -77,17 +82,40 @@
> >  		rte_errno = errno;
> >  		return NULL;
> >  	}
> > +
> > +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
> > +	struct ibv_counter_attach_attr attach_attr = {0};
> > +	int ret;
> > +
> > +	attach_attr.counter_desc = IBV_COUNTER_PACKETS;
> > +	attach_attr.index = 0;
> > +	ret = ibv_attach_counters_point_flow(tmpl.cs, &attach_attr, NULL);
> > +	if (!ret) {
> > +		attach_attr.counter_desc = IBV_COUNTER_BYTES;
> > +		attach_attr.index = 1;
> > +		ret = ibv_attach_counters_point_flow(tmpl.cs,
> > +						     &attach_attr,
> > +						     NULL);
> > +	}
> > +	if (ret) {
> > +		claim_zero(mlx5_glue->destroy_counter_set(tmpl.cs));
> > +		rte_errno = ret;
> > +		return NULL;
> > +	}
> > +#endif
> >  	cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
> >  	if (!cnt) {
> > +		claim_zero(mlx5_glue->destroy_counter_set(tmpl.cs));
> >  		rte_errno = ENOMEM;
> >  		return NULL;
> >  	}
> >  	*cnt = tmpl;
> >  	LIST_INSERT_HEAD(&priv->flow_counters, cnt, next);
> >  	return cnt;
> > -#endif
> > +#else
> >  	rte_errno = ENOTSUP;
> >  	return NULL;
> > +#endif
> >  }
> >
> >  /**
> > @@ -947,7 +975,7 @@
> >  		flow->counter = flow_verbs_counter_new(dev, count-
> >shared,
> >  						       count->id);
> >  		if (!flow->counter)
> > -			return rte_flow_error_set(error, ENOTSUP,
> > +			return rte_flow_error_set(error, rte_errno,
> >
> RTE_FLOW_ERROR_TYPE_ACTION,
> >  						  action,
> >  						  "cannot get counter"
> > @@ -955,7 +983,11 @@
> >  	}
> >  	*action_flags |= MLX5_ACTION_COUNT;
> >  #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
> > +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
> > +	counter.counters = flow->counter->cs; #else
> >  	counter.counter_set_handle = flow->counter->cs->handle;
> > +#endif
> >  	flow_verbs_spec_add(dev_flow, &counter, size);  #endif
> >  	return 0;
> > diff --git a/drivers/net/mlx5/mlx5_glue.c
> > b/drivers/net/mlx5/mlx5_glue.c index 48590df..785234c 100644
> > --- a/drivers/net/mlx5/mlx5_glue.c
> > +++ b/drivers/net/mlx5/mlx5_glue.c
> > @@ -211,6 +211,39 @@
> >  	return ibv_dereg_mr(mr);
> >  }
> >
> > +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
> > +static struct ibv_counters *
> > +mlx5_glue_create_counters(struct ibv_context *context,
> > +			  struct ibv_counters_init_attr *init_attr) {
> > +	return ibv_create_counters(context, init_attr); }
> > +
> > +static int
> > +mlx5_glue_destroy_counters(struct ibv_counters *counters) {
> > +	return ibv_destroy_counters(counters); }
> > +
> > +static int
> > +mlx5_glue_attach_counters(struct ibv_counters *counters,
> > +		     struct ibv_counter_attach_attr *attr,
> > +		     struct ibv_flow *flow)
> > +{
> > +	return ibv_attach_counters_point_flow(counters, attr, flow); }
> > +
> > +static int
> > +mlx5_glue_query_counters(struct ibv_counters *counters,
> > +			 uint64_t *counters_value,
> > +			 uint32_t ncounters,
> > +			 uint32_t flags)
> > +{
> > +	return ibv_read_counters(counters, counters_value, ncounters,
> > +flags); } #endif
> > +
> > +#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
> >  static struct ibv_counter_set *
> >  mlx5_glue_create_counter_set(struct ibv_context *context,
> >  			     struct ibv_counter_set_init_attr *init_attr) @@ -
> 262,6 +295,7
> > @@
> >  	return ibv_query_counter_set(query_attr, cs_data);  #endif  }
> > +#endif
> >
> >  static void
> >  mlx5_glue_ack_async_event(struct ibv_async_event *event) @@ -420,10
> > +454,17 @@
> >  	.modify_qp = mlx5_glue_modify_qp,
> >  	.reg_mr = mlx5_glue_reg_mr,
> >  	.dereg_mr = mlx5_glue_dereg_mr,
> > +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
> > +	.create_counter_set = mlx5_glue_create_counters,
> > +	.destroy_counter_set = mlx5_glue_destroy_counters,
> > +	.attach_counter_set = mlx5_glue_attach_counters,
> > +	.query_counter_set = mlx5_glue_query_counters, #else
> >  	.create_counter_set = mlx5_glue_create_counter_set,
> >  	.destroy_counter_set = mlx5_glue_destroy_counter_set,
> >  	.describe_counter_set = mlx5_glue_describe_counter_set,
> >  	.query_counter_set = mlx5_glue_query_counter_set,
> > +#endif
> >  	.ack_async_event = mlx5_glue_ack_async_event,
> >  	.get_async_event = mlx5_glue_get_async_event,
> >  	.port_state_str = mlx5_glue_port_state_str, diff --git
> > a/drivers/net/mlx5/mlx5_glue.h b/drivers/net/mlx5/mlx5_glue.h index
> > f6e4e38..504d487 100644
> > --- a/drivers/net/mlx5/mlx5_glue.h
> > +++ b/drivers/net/mlx5/mlx5_glue.h
> > @@ -96,6 +96,21 @@ struct mlx5_glue {
> >  	struct ibv_mr *(*reg_mr)(struct ibv_pd *pd, void *addr,
> >  				 size_t length, int access);
> >  	int (*dereg_mr)(struct ibv_mr *mr);
> > +#ifdef	HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45
> > +	struct ibv_counters *(*create_counter_set)
> > +		(struct ibv_context *context,
> > +		 struct ibv_counters_init_attr *init_attr);
> > +	int (*destroy_counter_set)(struct ibv_counters *cs);
> > +	int (*attach_counter_set)
> > +		(struct ibv_counters *cs,
> > +		 struct ibv_counter_attach_attr *attr,
> > +		 struct ibv_flow *flow);
> > +	int (*query_counter_set)
> > +		(struct ibv_counters *cs,
> > +		 uint64_t *counters_value,
> > +		 uint32_t ncounters,
> > +		 uint32_t flags);
> > +#else
> >  	struct ibv_counter_set *(*create_counter_set)
> >  		(struct ibv_context *context,
> >  		 struct ibv_counter_set_init_attr *init_attr); @@ -106,6
> +121,7 @@
> > struct mlx5_glue {
> >  		 struct ibv_counter_set_description *cs_desc);
> >  	int (*query_counter_set)(struct ibv_query_counter_set_attr
> *query_attr,
> >  				 struct ibv_counter_set_data *cs_data);
> > +#endif
> >  	void (*ack_async_event)(struct ibv_async_event *event);
> >  	int (*get_async_event)(struct ibv_context *context,
> >  			       struct ibv_async_event *event);
> > --
> > 1.8.3.1
> >

^ permalink raw reply	[flat|nested] 39+ messages in thread

* Re: [dpdk-dev] [PATCH] net/mlx5: flow counters support on the Linux-rdma v19 base
  2018-10-11 14:51     ` Slava Ovsiienko
@ 2018-10-14  5:32       ` Shahaf Shuler
  0 siblings, 0 replies; 39+ messages in thread
From: Shahaf Shuler @ 2018-10-14  5:32 UTC (permalink / raw)
  To: Slava Ovsiienko, Yongseok Koh; +Cc: dev

Thursday, October 11, 2018 5:52 PM, Slava Ovsiienko:
> Subject: RE: [dpdk-dev] [PATCH] net/mlx5: flow counters support on the
> Linux-rdma v19 base
> 
> > -----Original Message-----
> > From: Yongseok Koh
> > Sent: Thursday, October 4, 2018 2:48
> > To: Slava Ovsiienko <viacheslavo@mellanox.com>
> > Cc: dev@dpdk.org; Shahaf Shuler <shahafs@mellanox.com>
> > Subject: Re: [dpdk-dev] [PATCH] net/mlx5: flow counters support on the
> > Linux-rdma v19 base
> >
> > On Wed, Oct 03, 2018 at 03:29:12PM +0000, Slava Ovsiienko wrote:
> > > Mellanox mlx5 PMD supports Flow Counters via Verbs library.
> > > The current implementation is based on the Mellanox proprietary
> > > Verbs library included in MLNX OFED packages. The Flow Counter
> > > support is recently added into linux-rdma release (v19), so the mlx5
> > > PMD update is needed to provide Counter feature on the base of linux-
> rdma.
> > >
> > > mlx5 PMD can be compiled with MLNX OFED or linux-rdma v19+ and
> > provide
> > > flow counters for both.
> > >
> > > Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
> > > ---
> > >  drivers/net/mlx5/Makefile          | 10 ++++++++
> > >  drivers/net/mlx5/mlx5.c            |  6 +++++
> > >  drivers/net/mlx5/mlx5_flow.c       |  9 ++++++-
> > >  drivers/net/mlx5/mlx5_flow.h       |  4 +++
> > >  drivers/net/mlx5/mlx5_flow_verbs.c | 52
> > ++++++++++++++++++++++++++++++--------
> > >  drivers/net/mlx5/mlx5_glue.c       | 41
> > ++++++++++++++++++++++++++++++
> > >  drivers/net/mlx5/mlx5_glue.h       | 16 ++++++++++++
> > >  7 files changed, 127 insertions(+), 11 deletions(-)
> > >
> > > diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile
> > > index ca1de9f..e3d2156 100644
> > > --- a/drivers/net/mlx5/Makefile
> > > +++ b/drivers/net/mlx5/Makefile
> > > @@ -162,6 +162,16 @@ mlx5_autoconf.h.new:
> > $(RTE_SDK)/buildtools/auto-config-h.sh
> > >  		type 'struct ibv_counter_set_init_attr' \
> > >  		$(AUTOCONF_OUTPUT)
> > >  	$Q sh -- '$<' '$@' \
> > > +		HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT \
> > > +		infiniband/verbs.h \
> > > +		type 'struct ibv_counters_init_attr' \
> > > +		$(AUTOCONF_OUTPUT)
> > > +	$Q sh -- '$<' '$@' \
> > > +		HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45 \
> > > +		infiniband/verbs.h \
> > > +		type 'struct ibv_counters_init_attr' \
> > > +		$(AUTOCONF_OUTPUT)
> > > +	$Q sh -- '$<' '$@' \
> >
> > I still don't understand what is different between the two. These are
> > exactly same checking, then why do you need to have two different
> > macros? From this script, HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT is
> same
> > as HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45, isn't it?
> 
> We have three options:
> - no counter support in kernel at all
> - "old" counter support (ibv_counter_set_init_attr is defined in verbs.h)
> - "new" counter support (ibv_counters_init_attr  is defined in verbs.h)
> 
> Three options require at least two compilations flags. The meanings are
> chosen:
> HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT means there is counter
> support (of any type)
> HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45 differentiates the
> support type
> 
> This approach allows to avoid clumsy constructions in code like this:
> #if __defined(HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT \
> || __defined(HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45)

This is not needed anyway. 
The "old counters" exists only in MLNX_OFED not upstream. 
Once the "new counters" were implemented upstream the next OFED completely replaced the old implementation. 
Meaning, there is no driver having them both. 

Hence, we can avoid this complex construction and just hold a single macro flag for the new counters. 

^ permalink raw reply	[flat|nested] 39+ messages in thread

* [dpdk-dev] [PATCH v2] net/mlx5: flow counters support on the Linux-rdma v19 base
  2018-10-03 15:29 ` [dpdk-dev] [PATCH] net/mlx5: flow counters support on the Linux-rdma " Slava Ovsiienko
  2018-10-03 23:48   ` Yongseok Koh
@ 2018-10-17 13:53   ` Viacheslav Ovsiienko
  2018-10-18  6:34     ` Shahaf Shuler
  2018-10-19 15:21     ` [dpdk-dev] [PATCH v3 0/6] net/mlx5: flow counters support for Linux-rdma v19 Viacheslav Ovsiienko
  1 sibling, 2 replies; 39+ messages in thread
From: Viacheslav Ovsiienko @ 2018-10-17 13:53 UTC (permalink / raw)
  To: shahafs, yskoh; +Cc: dev, Viacheslav Ovsiienko

Mellanox mlx5 PMD supports Flow Counters via Verbs library.
The current implementation is based on the Mellanox proprietary
Verbs library included in MLNX OFED packages. The Flow Counter
support is recently added into linux-rdma release (v19),
so the mlx5 PMD update is needed to provide Counter feature
on the base of linux-rdma.

mlx5 PMD can be compiled with MLNX OFED or linux-rdma v19+
and provide flow counters for both.

Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>

---
v2:
- rebased on top of master-net-mlx branch
- new compilation flags are introduced:
    - HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V42, kernel/verbs
      library provides the flow counter support in style of
      MLNX_OFED_4.2 to MLNX_OFED_4.4
    - HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45, kernel/verbs
      library provides the flow counter support in style of
      MLNX_OFED_4.5 or higher

v1:
- http://patches.dpdk.org/patch/45972/
---
 drivers/net/mlx5/Makefile          |  7 ++-
 drivers/net/mlx5/meson.build       |  4 +-
 drivers/net/mlx5/mlx5.c            | 13 +++++-
 drivers/net/mlx5/mlx5_flow.c       | 27 +++++++++---
 drivers/net/mlx5/mlx5_flow.h       |  5 +++
 drivers/net/mlx5/mlx5_flow_verbs.c | 81 +++++++++++++++++++++++++++--------
 drivers/net/mlx5/mlx5_glue.c       | 88 +++++++++++++++++++++++++++++++-------
 drivers/net/mlx5/mlx5_glue.h       | 22 +++++++++-
 8 files changed, 202 insertions(+), 45 deletions(-)

diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile
index 1e9c0b4..fecb57c 100644
--- a/drivers/net/mlx5/Makefile
+++ b/drivers/net/mlx5/Makefile
@@ -157,11 +157,16 @@ mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
 		enum ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT \
 		$(AUTOCONF_OUTPUT)
 	$Q sh -- '$<' '$@' \
-		HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT \
+		HAVE_IBV_DEVICE_COUNTERS_SET_V42 \
 		infiniband/verbs.h \
 		type 'struct ibv_counter_set_init_attr' \
 		$(AUTOCONF_OUTPUT)
 	$Q sh -- '$<' '$@' \
+		HAVE_IBV_DEVICE_COUNTERS_SET_V45 \
+		infiniband/verbs.h \
+		type 'struct ibv_counters_init_attr' \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
 		HAVE_RDMA_NL_NLDEV \
 		rdma/rdma_netlink.h \
 		enum RDMA_NL_NLDEV \
diff --git a/drivers/net/mlx5/meson.build b/drivers/net/mlx5/meson.build
index c192d44..e8cbe3e 100644
--- a/drivers/net/mlx5/meson.build
+++ b/drivers/net/mlx5/meson.build
@@ -79,8 +79,10 @@ if build
 	has_member_args = [
 		[ 'HAVE_IBV_MLX5_MOD_SWP', 'infiniband/mlx5dv.h',
 		'struct mlx5dv_sw_parsing_caps', 'sw_parsing_offloads' ],
-		[ 'HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT', 'infiniband/verbs.h',
+		[ 'HAVE_IBV_DEVICE_COUNTERS_SET_V42', 'infiniband/verbs.h',
 		'struct ibv_counter_set_init_attr', 'counter_set_id' ],
+		[ 'HAVE_IBV_DEVICE_COUNTERS_SET_V45', 'infiniband/verbs.h',
+		'struct ibv_counters_init_attr', 'comp_mask' ],
 	]
 	# input array for meson symbol search:
 	# [ "MACRO to define if found", "header for the search",
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 795a219..7dbe4bc 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -739,7 +739,7 @@
 	unsigned int mprq_max_stride_size_n = 0;
 	unsigned int mprq_min_stride_num_n = 0;
 	unsigned int mprq_max_stride_num_n = 0;
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 	struct ibv_counter_set_description cs_desc = { .counter_type = 0 };
 #endif
 	struct ether_addr mac;
@@ -1009,13 +1009,22 @@
 	config.hw_csum = !!(attr.device_cap_flags_ex & IBV_DEVICE_RAW_IP_CSUM);
 	DRV_LOG(DEBUG, "checksum offloading is %ssupported",
 		(config.hw_csum ? "" : "not "));
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+#if !defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) && \
+	!defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
+	DRV_LOG(DEBUG, "no flow counters support found, OFED 4.2+ "
+		       "is required for counters support");
+#endif
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 	config.flow_counter_en = !!attr.max_counter_sets;
 	mlx5_glue->describe_counter_set(ctx, 0, &cs_desc);
 	DRV_LOG(DEBUG, "counter type = %d, num of cs = %ld, attributes = %d",
 		cs_desc.counter_type, cs_desc.num_of_cs,
 		cs_desc.attributes);
 #endif
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V45
+	DRV_LOG(DEBUG, "Flow counters support OFED 4.5+/upstream assumed");
+	config.flow_counter_en = 1;
+#endif
 	config.ind_table_max_size =
 		attr.rss_caps.max_rwq_indirection_table_size;
 	/*
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index bd70fce..4df114d 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -2352,15 +2352,17 @@ struct rte_flow *
  * @return
  *   0 on success, a negative errno value otherwise and rte_errno is set.
  */
+#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
+	defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 static int
-mlx5_flow_query_count(struct rte_flow *flow __rte_unused,
-		      void *data __rte_unused,
+mlx5_flow_query_count(struct rte_flow *flow,
+		      void *data,
 		      struct rte_flow_error *error)
 {
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
 	if (flow->actions & MLX5_FLOW_ACTION_COUNT) {
 		struct rte_flow_query_count *qc = data;
 		uint64_t counters[2] = {0, 0};
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 		struct ibv_query_counter_set_attr query_cs_attr = {
 			.cs = flow->counter->cs,
 			.query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
@@ -2371,7 +2373,14 @@ struct rte_flow *
 		};
 		int err = mlx5_glue->query_counter_set(&query_cs_attr,
 						       &query_out);
-
+#endif
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V45
+		int err = mlx5_glue->query_counter_set(
+				flow->counter->cs,
+				counters,
+				RTE_DIM(counters),
+				IBV_READ_COUNTERS_ATTR_PREFER_CACHED);
+#endif
 		if (err)
 			return rte_flow_error_set
 				(error, err,
@@ -2392,12 +2401,20 @@ struct rte_flow *
 				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 				  NULL,
 				  "flow does not have counter");
-#endif
+}
+#else
+static int
+mlx5_flow_query_count(struct rte_flow *flow __rte_unused,
+		      void *data __rte_unused,
+		      struct rte_flow_error *error)
+{
 	return rte_flow_error_set(error, ENOTSUP,
 				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 				  NULL,
 				  "counters are not available");
 }
+#endif /* HAVE_IBV_DEVICE_COUNTERS_SET_Vxx */
+
 
 /**
  * Query a flows.
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 094f666..d2856c3 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -224,7 +224,12 @@ struct mlx5_flow_counter {
 	uint32_t shared:1; /**< Share counter ID with other flow rules. */
 	uint32_t ref_cnt:31; /**< Reference counter. */
 	uint32_t id; /**< Counter ID. */
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 	struct ibv_counter_set *cs; /**< Holds the counters for the rule. */
+#endif
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V45
+	struct ibv_counters *cs; /**< Holds the counters for the rule. */
+#endif
 	uint64_t hits; /**< Number of packets matched by the rule. */
 	uint64_t bytes; /**< Number of bytes matched by the rule. */
 };
diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c b/drivers/net/mlx5/mlx5_flow_verbs.c
index 65c849c..6d5f900 100644
--- a/drivers/net/mlx5/mlx5_flow_verbs.c
+++ b/drivers/net/mlx5/mlx5_flow_verbs.c
@@ -46,30 +46,37 @@
  * @return
  *   A pointer to the counter, NULL otherwise and rte_errno is set.
  */
+#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
+	defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 static struct mlx5_flow_counter *
 flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id)
 {
 	struct priv *priv = dev->data->dev_private;
 	struct mlx5_flow_counter *cnt;
 
-	LIST_FOREACH(cnt, &priv->flow_counters, next) {
-		if (!cnt->shared || cnt->shared != shared)
-			continue;
-		if (cnt->id != id)
-			continue;
-		cnt->ref_cnt++;
-		return cnt;
+	if (shared) {
+		LIST_FOREACH(cnt, &priv->flow_counters, next)
+		if (cnt->shared && cnt->id == id) {
+			cnt->ref_cnt++;
+			return cnt;
+		}
 	}
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
 
 	struct mlx5_flow_counter tmpl = {
 		.shared = shared,
 		.id = id,
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 		.cs = mlx5_glue->create_counter_set
 			(priv->ctx,
 			 &(struct ibv_counter_set_init_attr){
 				 .counter_set_id = id,
 			 }),
+#endif
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V45
+		.cs = mlx5_glue->create_counter_set
+			(priv->ctx,
+			 &(struct ibv_counters_init_attr){0}),
+#endif
 		.hits = 0,
 		.bytes = 0,
 	};
@@ -78,17 +85,36 @@
 		rte_errno = errno;
 		return NULL;
 	}
+
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V45
+	struct ibv_counter_attach_attr attach_attr = {0};
+	int ret;
+
+	attach_attr.counter_desc = IBV_COUNTER_PACKETS;
+	attach_attr.index = 0;
+	ret = ibv_attach_counters_point_flow(tmpl.cs, &attach_attr, NULL);
+	if (!ret) {
+		attach_attr.counter_desc = IBV_COUNTER_BYTES;
+		attach_attr.index = 1;
+		ret = ibv_attach_counters_point_flow(tmpl.cs,
+						     &attach_attr,
+						     NULL);
+	}
+	if (ret) {
+		claim_zero(mlx5_glue->destroy_counter_set(tmpl.cs));
+		rte_errno = ret;
+		return NULL;
+	}
+#endif
 	cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
 	if (!cnt) {
+		claim_zero(mlx5_glue->destroy_counter_set(tmpl.cs));
 		rte_errno = ENOMEM;
 		return NULL;
 	}
 	*cnt = tmpl;
 	LIST_INSERT_HEAD(&priv->flow_counters, cnt, next);
 	return cnt;
-#endif
-	rte_errno = ENOTSUP;
-	return NULL;
 }
 
 /**
@@ -106,6 +132,7 @@
 		rte_free(counter);
 	}
 }
+#endif /* HAVE_IBV_DEVICE_COUNTERS_SET_Vxx */
 
 /**
  * Add a verbs item specification into @p flow.
@@ -927,6 +954,8 @@
  * @return
  *   0 On success else a negative errno value is returned and rte_errno is set.
  */
+#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
+	defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 static int
 flow_verbs_translate_action_count(struct rte_eth_dev *dev,
 				  const struct rte_flow_action *action,
@@ -936,13 +965,11 @@
 {
 	const struct rte_flow_action_count *count = action->conf;
 	struct rte_flow *flow = dev_flow->flow;
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
 	unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
 	struct ibv_flow_spec_counter_action counter = {
 		.type = IBV_FLOW_SPEC_ACTION_COUNT,
 		.size = size,
 	};
-#endif
 
 	if (!flow->counter) {
 		flow->counter = flow_verbs_counter_new(dev, count->shared,
@@ -955,12 +982,16 @@
 						  " context.");
 	}
 	*action_flags |= MLX5_FLOW_ACTION_COUNT;
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 	counter.counter_set_handle = flow->counter->cs->handle;
-	flow_verbs_spec_add(dev_flow, &counter, size);
 #endif
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V45
+	counter.counters = flow->counter->cs;
+#endif
+	flow_verbs_spec_add(dev_flow, &counter, size);
 	return 0;
 }
+#endif /* HAVE_IBV_DEVICE_COUNTERS_SET_Vxx */
 
 /**
  * Internal validation function. For validating both actions and items.
@@ -1157,12 +1188,15 @@
 				return ret;
 			action_flags |= MLX5_FLOW_ACTION_RSS;
 			break;
+#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
+	defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 		case RTE_FLOW_ACTION_TYPE_COUNT:
 			ret = mlx5_flow_validate_action_count(dev, attr, error);
 			if (ret < 0)
 				return ret;
 			action_flags |= MLX5_FLOW_ACTION_COUNT;
 			break;
+#endif
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ACTION,
@@ -1219,11 +1253,12 @@
 		case RTE_FLOW_ACTION_TYPE_RSS:
 			detected_actions |= MLX5_FLOW_ACTION_RSS;
 			break;
+#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
+	defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 		case RTE_FLOW_ACTION_TYPE_COUNT:
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
 			size += sizeof(struct ibv_flow_spec_counter_action);
-#endif
 			detected_actions |= MLX5_FLOW_ACTION_COUNT;
+#endif
 			break;
 		default:
 			break;
@@ -1406,7 +1441,6 @@
 	if (priority == MLX5_FLOW_PRIO_RSVD)
 		priority = priv->config.flow_prio - 1;
 	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
-		int ret;
 		switch (actions->type) {
 		case RTE_FLOW_ACTION_TYPE_VOID:
 			break;
@@ -1434,7 +1468,11 @@
 							&action_flags,
 							dev_flow);
 			break;
-		case RTE_FLOW_ACTION_TYPE_COUNT:
+#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
+	defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
+		case RTE_FLOW_ACTION_TYPE_COUNT: {
+			int ret;
+
 			ret = flow_verbs_translate_action_count(dev,
 								actions,
 								&action_flags,
@@ -1443,6 +1481,8 @@
 			if (ret < 0)
 				return ret;
 			break;
+		}
+#endif
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ACTION,
@@ -1538,10 +1578,13 @@
 			verbs->hrxq = NULL;
 		}
 	}
+#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
+	defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 	if (flow->counter) {
 		flow_verbs_counter_release(flow->counter);
 		flow->counter = NULL;
 	}
+#endif
 }
 
 /**
diff --git a/drivers/net/mlx5/mlx5_glue.c b/drivers/net/mlx5/mlx5_glue.c
index 48590df..d77adb8 100644
--- a/drivers/net/mlx5/mlx5_glue.c
+++ b/drivers/net/mlx5/mlx5_glue.c
@@ -211,28 +211,84 @@
 	return ibv_dereg_mr(mr);
 }
 
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V45
+static struct ibv_counters *
+mlx5_glue_create_counters(struct ibv_context *context,
+			  struct ibv_counters_init_attr *init_attr)
+{
+	return ibv_create_counters(context, init_attr);
+}
+
+static int
+mlx5_glue_destroy_counters(struct ibv_counters *counters)
+{
+	return ibv_destroy_counters(counters);
+}
+
+static int
+mlx5_glue_attach_counters(struct ibv_counters *counters,
+		     struct ibv_counter_attach_attr *attr,
+		     struct ibv_flow *flow)
+{
+	return ibv_attach_counters_point_flow(counters, attr, flow);
+}
+
+static int
+mlx5_glue_query_counters(struct ibv_counters *counters,
+			 uint64_t *counters_value,
+			 uint32_t ncounters,
+			 uint32_t flags)
+{
+	return ibv_read_counters(counters, counters_value, ncounters, flags);
+}
+#endif /* HAVE_IBV_DEVICE_COUNTERS_SET_V45 */
+
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
+static struct ibv_counter_set *
+mlx5_glue_create_counter_set(struct ibv_context *context,
+			     struct ibv_counter_set_init_attr *init_attr)
+{
+	return ibv_create_counter_set(context, init_attr);
+}
+
+static int
+mlx5_glue_destroy_counter_set(struct ibv_counter_set *cs)
+{
+	return ibv_destroy_counter_set(cs);
+}
+
+static int
+mlx5_glue_describe_counter_set(struct ibv_context *context,
+			       uint16_t counter_set_id,
+			       struct ibv_counter_set_description *cs_desc)
+{
+	return ibv_describe_counter_set(context, counter_set_id, cs_desc);
+}
+
+static int
+mlx5_glue_query_counter_set(struct ibv_query_counter_set_attr *query_attr,
+			    struct ibv_counter_set_data *cs_data)
+{
+	return ibv_query_counter_set(query_attr, cs_data);
+}
+#endif /* HAVE_IBV_DEVICE_COUNTERS_SET_V42 */
+
+#if !defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) && \
+	!defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 static struct ibv_counter_set *
 mlx5_glue_create_counter_set(struct ibv_context *context,
 			     struct ibv_counter_set_init_attr *init_attr)
 {
-#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
 	(void)context;
 	(void)init_attr;
 	return NULL;
-#else
-	return ibv_create_counter_set(context, init_attr);
-#endif
 }
 
 static int
 mlx5_glue_destroy_counter_set(struct ibv_counter_set *cs)
 {
-#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
 	(void)cs;
 	return ENOTSUP;
-#else
-	return ibv_destroy_counter_set(cs);
-#endif
 }
 
 static int
@@ -240,28 +296,21 @@
 			       uint16_t counter_set_id,
 			       struct ibv_counter_set_description *cs_desc)
 {
-#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
 	(void)context;
 	(void)counter_set_id;
 	(void)cs_desc;
 	return ENOTSUP;
-#else
-	return ibv_describe_counter_set(context, counter_set_id, cs_desc);
-#endif
 }
 
 static int
 mlx5_glue_query_counter_set(struct ibv_query_counter_set_attr *query_attr,
 			    struct ibv_counter_set_data *cs_data)
 {
-#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
 	(void)query_attr;
 	(void)cs_data;
 	return ENOTSUP;
-#else
-	return ibv_query_counter_set(query_attr, cs_data);
-#endif
 }
+#endif /* HAVE_IBV_DEVICE_COUNTERS_SET_Vxx */
 
 static void
 mlx5_glue_ack_async_event(struct ibv_async_event *event)
@@ -420,10 +469,17 @@
 	.modify_qp = mlx5_glue_modify_qp,
 	.reg_mr = mlx5_glue_reg_mr,
 	.dereg_mr = mlx5_glue_dereg_mr,
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V45
+	.create_counter_set = mlx5_glue_create_counters,
+	.destroy_counter_set = mlx5_glue_destroy_counters,
+	.attach_counter_set = mlx5_glue_attach_counters,
+	.query_counter_set = mlx5_glue_query_counters,
+#else
 	.create_counter_set = mlx5_glue_create_counter_set,
 	.destroy_counter_set = mlx5_glue_destroy_counter_set,
 	.describe_counter_set = mlx5_glue_describe_counter_set,
 	.query_counter_set = mlx5_glue_query_counter_set,
+#endif
 	.ack_async_event = mlx5_glue_ack_async_event,
 	.get_async_event = mlx5_glue_get_async_event,
 	.port_state_str = mlx5_glue_port_state_str,
diff --git a/drivers/net/mlx5/mlx5_glue.h b/drivers/net/mlx5/mlx5_glue.h
index f6e4e38..b9033de 100644
--- a/drivers/net/mlx5/mlx5_glue.h
+++ b/drivers/net/mlx5/mlx5_glue.h
@@ -23,7 +23,11 @@
 #define MLX5_GLUE_VERSION ""
 #endif
 
-#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V45
+struct ibv_counters;
+struct ibv_counters_attach_data;
+struct ibv_counters_init_attr;
+#else
 struct ibv_counter_set;
 struct ibv_counter_set_data;
 struct ibv_counter_set_description;
@@ -96,6 +100,21 @@ struct mlx5_glue {
 	struct ibv_mr *(*reg_mr)(struct ibv_pd *pd, void *addr,
 				 size_t length, int access);
 	int (*dereg_mr)(struct ibv_mr *mr);
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V45
+	struct ibv_counters *(*create_counter_set)
+		(struct ibv_context *context,
+		 struct ibv_counters_init_attr *init_attr);
+	int (*destroy_counter_set)(struct ibv_counters *cs);
+	int (*attach_counter_set)
+		(struct ibv_counters *cs,
+		 struct ibv_counter_attach_attr *attr,
+		 struct ibv_flow *flow);
+	int (*query_counter_set)
+		(struct ibv_counters *cs,
+		 uint64_t *counters_value,
+		 uint32_t ncounters,
+		 uint32_t flags);
+#else
 	struct ibv_counter_set *(*create_counter_set)
 		(struct ibv_context *context,
 		 struct ibv_counter_set_init_attr *init_attr);
@@ -106,6 +125,7 @@ struct mlx5_glue {
 		 struct ibv_counter_set_description *cs_desc);
 	int (*query_counter_set)(struct ibv_query_counter_set_attr *query_attr,
 				 struct ibv_counter_set_data *cs_data);
+#endif
 	void (*ack_async_event)(struct ibv_async_event *event);
 	int (*get_async_event)(struct ibv_context *context,
 			       struct ibv_async_event *event);
-- 
1.8.3.1

^ permalink raw reply	[flat|nested] 39+ messages in thread

* Re: [dpdk-dev] [PATCH v2] net/mlx5: flow counters support on the Linux-rdma v19 base
  2018-10-17 13:53   ` [dpdk-dev] [PATCH v2] " Viacheslav Ovsiienko
@ 2018-10-18  6:34     ` Shahaf Shuler
  2018-10-19 15:21     ` [dpdk-dev] [PATCH v3 0/6] net/mlx5: flow counters support for Linux-rdma v19 Viacheslav Ovsiienko
  1 sibling, 0 replies; 39+ messages in thread
From: Shahaf Shuler @ 2018-10-18  6:34 UTC (permalink / raw)
  To: Slava Ovsiienko, Yongseok Koh; +Cc: dev

Hi Slava,

Please see some comments below.
Also - please rebase on top of series https://patches.dpdk.org/project/dpdk/list/?series=1961
It moves the verbs flow query to the mlx5_flow_verbs.c

Wednesday, October 17, 2018 4:54 PM, Slava Ovsiienko:
> Subject: [PATCH v2] net/mlx5: flow counters support on the Linux-rdma v19
> base
> 
> Mellanox mlx5 PMD supports Flow Counters via Verbs library.
> The current implementation is based on the Mellanox proprietary Verbs
> library included in MLNX OFED packages. The Flow Counter support is
> recently added into linux-rdma release (v19), so the mlx5 PMD update is
> needed to provide Counter feature on the base of linux-rdma.

Counter -> counter

> 
> mlx5 PMD can be compiled with MLNX OFED or linux-rdma v19+ and provide
> flow counters for both.
> 
> Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
> 
> ---
> v2:
> - rebased on top of master-net-mlx branch
> - new compilation flags are introduced:
>     - HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V42, kernel/verbs
>       library provides the flow counter support in style of
>       MLNX_OFED_4.2 to MLNX_OFED_4.4
>     - HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45, kernel/verbs
>       library provides the flow counter support in style of
>       MLNX_OFED_4.5 or higher
> 
> v1:
> - http://patches.dpdk.org/patch/45972/
> ---
>  drivers/net/mlx5/Makefile          |  7 ++-
>  drivers/net/mlx5/meson.build       |  4 +-
>  drivers/net/mlx5/mlx5.c            | 13 +++++-
>  drivers/net/mlx5/mlx5_flow.c       | 27 +++++++++---
>  drivers/net/mlx5/mlx5_flow.h       |  5 +++
>  drivers/net/mlx5/mlx5_flow_verbs.c | 81
> +++++++++++++++++++++++++++--------
>  drivers/net/mlx5/mlx5_glue.c       | 88
> +++++++++++++++++++++++++++++++-------
>  drivers/net/mlx5/mlx5_glue.h       | 22 +++++++++-
>  8 files changed, 202 insertions(+), 45 deletions(-)
> 
> diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile index
> 1e9c0b4..fecb57c 100644
> --- a/drivers/net/mlx5/Makefile
> +++ b/drivers/net/mlx5/Makefile
> @@ -157,11 +157,16 @@ mlx5_autoconf.h.new:
> $(RTE_SDK)/buildtools/auto-config-h.sh
>  		enum ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT \
>  		$(AUTOCONF_OUTPUT)
>  	$Q sh -- '$<' '$@' \
> -		HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT \
> +		HAVE_IBV_DEVICE_COUNTERS_SET_V42 \

Better to have this change of MACRO name as a separate commit. 

>  		infiniband/verbs.h \
>  		type 'struct ibv_counter_set_init_attr' \
>  		$(AUTOCONF_OUTPUT)
>  	$Q sh -- '$<' '$@' \
> +		HAVE_IBV_DEVICE_COUNTERS_SET_V45 \
> +		infiniband/verbs.h \
> +		type 'struct ibv_counters_init_attr' \
> +		$(AUTOCONF_OUTPUT)
> +	$Q sh -- '$<' '$@' \
>  		HAVE_RDMA_NL_NLDEV \
>  		rdma/rdma_netlink.h \
>  		enum RDMA_NL_NLDEV \
> diff --git a/drivers/net/mlx5/meson.build b/drivers/net/mlx5/meson.build
> index c192d44..e8cbe3e 100644
> --- a/drivers/net/mlx5/meson.build
> +++ b/drivers/net/mlx5/meson.build
> @@ -79,8 +79,10 @@ if build
>  	has_member_args = [
>  		[ 'HAVE_IBV_MLX5_MOD_SWP', 'infiniband/mlx5dv.h',
>  		'struct mlx5dv_sw_parsing_caps', 'sw_parsing_offloads' ],
> -		[ 'HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT',
> 'infiniband/verbs.h',
> +		[ 'HAVE_IBV_DEVICE_COUNTERS_SET_V42',
> 'infiniband/verbs.h',
>  		'struct ibv_counter_set_init_attr', 'counter_set_id' ],
> +		[ 'HAVE_IBV_DEVICE_COUNTERS_SET_V45',
> 'infiniband/verbs.h',
> +		'struct ibv_counters_init_attr', 'comp_mask' ],
>  	]
>  	# input array for meson symbol search:
>  	# [ "MACRO to define if found", "header for the search", diff --git
> a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index
> 795a219..7dbe4bc 100644
> --- a/drivers/net/mlx5/mlx5.c
> +++ b/drivers/net/mlx5/mlx5.c
> @@ -739,7 +739,7 @@
>  	unsigned int mprq_max_stride_size_n = 0;
>  	unsigned int mprq_min_stride_num_n = 0;
>  	unsigned int mprq_max_stride_num_n = 0; -#ifdef
> HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
> +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
>  	struct ibv_counter_set_description cs_desc = { .counter_type = 0 };
> #endif
>  	struct ether_addr mac;
> @@ -1009,13 +1009,22 @@
>  	config.hw_csum = !!(attr.device_cap_flags_ex &
> IBV_DEVICE_RAW_IP_CSUM);
>  	DRV_LOG(DEBUG, "checksum offloading is %ssupported",
>  		(config.hw_csum ? "" : "not "));
> -#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
> +#if !defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) && \
> +	!defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)

Like we discussed over the phone. Since this condition (has any of the flow counter) repeats multiple times on this commit, it is better to define it as a macro.
Something like
#define MLX5_VERBS_HAS_FLOW_COUNTER (defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45))

> +	DRV_LOG(DEBUG, "no flow counters support found, OFED 4.2+ "
> +		       "is required for counters support"); #endif #ifdef
> +HAVE_IBV_DEVICE_COUNTERS_SET_V42
>  	config.flow_counter_en = !!attr.max_counter_sets;
>  	mlx5_glue->describe_counter_set(ctx, 0, &cs_desc);
>  	DRV_LOG(DEBUG, "counter type = %d, num of cs = %ld, attributes =
> %d",
>  		cs_desc.counter_type, cs_desc.num_of_cs,
>  		cs_desc.attributes);

I never liked this log, it is too developer specific. Also the flwo_counter_en seems redundant, assuming we have the APIs we can just try to create a counter and if it fails then there is not counter support on the kernel level (this is actually the logic on the new counter approach).  
How about the following logic (following previous commit):
1. remove the flow_counter_en flag (separate commit)
2. the logic here will be:
#ifdef MLX5_VERBS_HAS_FLOW_COUNTER
	DRV_LOG(DEBUG, "Flow counters are supported")  //the user doesn't really care if those are the new/old
#else
	DRV_LOG(DEBUG, "Flow counters are not supported")
#endif

?

>  #endif
> +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V45
> +	DRV_LOG(DEBUG, "Flow counters support OFED 4.5+/upstream
> assumed");
> +	config.flow_counter_en = 1;
> +#endif
>  	config.ind_table_max_size =
>  		attr.rss_caps.max_rwq_indirection_table_size;
>  	/*
> diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
> index bd70fce..4df114d 100644
> --- a/drivers/net/mlx5/mlx5_flow.c
> +++ b/drivers/net/mlx5/mlx5_flow.c
> @@ -2352,15 +2352,17 @@ struct rte_flow *
>   * @return
>   *   0 on success, a negative errno value otherwise and rte_errno is set.
>   */
> +#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
> +	defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)

Better to insert the ifdef inside the function, so that you will not need to declare it two times. 
Then it will be like:
mlx5_flow_query_count(...) {
#ifdef MLX5_VERBS_HAS_FLOW_COUNTER
	Function logic
#else
	Return not supported
}

>  static int
> -mlx5_flow_query_count(struct rte_flow *flow __rte_unused,
> -		      void *data __rte_unused,
> +mlx5_flow_query_count(struct rte_flow *flow,
> +		      void *data,
>  		      struct rte_flow_error *error)
>  {
> -#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
>  	if (flow->actions & MLX5_FLOW_ACTION_COUNT) {
>  		struct rte_flow_query_count *qc = data;
>  		uint64_t counters[2] = {0, 0};

Better to have the logic from here..

> +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
>  		struct ibv_query_counter_set_attr query_cs_attr = {
>  			.cs = flow->counter->cs,
>  			.query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
> @@ -2371,7 +2373,14 @@ struct rte_flow *
>  		};
>  		int err = mlx5_glue->query_counter_set(&query_cs_attr,
>  						       &query_out);
> -
> +#endif
> +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V45
> +		int err = mlx5_glue->query_counter_set(
> +				flow->counter->cs,
> +				counters,
> +				RTE_DIM(counters),
> +
> 	IBV_READ_COUNTERS_ATTR_PREFER_CACHED);
> +#endif

Till here in a separate function. 
Also use:
#Ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
		Logic
#else ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V45
		Logic
#endif

It is safer and the code is more clear. 


>  		if (err)
>  			return rte_flow_error_set
>  				(error, err,
> @@ -2392,12 +2401,20 @@ struct rte_flow *
>  				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
>  				  NULL,
>  				  "flow does not have counter");
> -#endif
> +}
> +#else
> +static int
> +mlx5_flow_query_count(struct rte_flow *flow __rte_unused,
> +		      void *data __rte_unused,
> +		      struct rte_flow_error *error)
> +{
>  	return rte_flow_error_set(error, ENOTSUP,
>  				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
>  				  NULL,
>  				  "counters are not available");
>  }
> +#endif /* HAVE_IBV_DEVICE_COUNTERS_SET_Vxx */
> +
> 
>  /**
>   * Query a flows.
> diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
> index 094f666..d2856c3 100644
> --- a/drivers/net/mlx5/mlx5_flow.h
> +++ b/drivers/net/mlx5/mlx5_flow.h
> @@ -224,7 +224,12 @@ struct mlx5_flow_counter {
>  	uint32_t shared:1; /**< Share counter ID with other flow rules. */
>  	uint32_t ref_cnt:31; /**< Reference counter. */
>  	uint32_t id; /**< Counter ID. */
> +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
>  	struct ibv_counter_set *cs; /**< Holds the counters for the rule. */
> +#endif
> +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V45
> +	struct ibv_counters *cs; /**< Holds the counters for the rule. */
> +#endif

Ditto. 
If -> else if -> endif

>  	uint64_t hits; /**< Number of packets matched by the rule. */
>  	uint64_t bytes; /**< Number of bytes matched by the rule. */  }; diff
> --git a/drivers/net/mlx5/mlx5_flow_verbs.c
> b/drivers/net/mlx5/mlx5_flow_verbs.c
> index 65c849c..6d5f900 100644
> --- a/drivers/net/mlx5/mlx5_flow_verbs.c
> +++ b/drivers/net/mlx5/mlx5_flow_verbs.c
> @@ -46,30 +46,37 @@
>   * @return
>   *   A pointer to the counter, NULL otherwise and rte_errno is set.
>   */
> +#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
> +	defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)

Like before, move the macro to the function body. 
And need to return NULL and errno = ENOTSUP in case there is no support. 

>  static struct mlx5_flow_counter *
>  flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t shared,
> uint32_t id)  {
>  	struct priv *priv = dev->data->dev_private;
>  	struct mlx5_flow_counter *cnt;
> 
> -	LIST_FOREACH(cnt, &priv->flow_counters, next) {
> -		if (!cnt->shared || cnt->shared != shared)
> -			continue;
> -		if (cnt->id != id)
> -			continue;
> -		cnt->ref_cnt++;
> -		return cnt;
> +	if (shared) {
> +		LIST_FOREACH(cnt, &priv->flow_counters, next)
> +		if (cnt->shared && cnt->id == id) {
> +			cnt->ref_cnt++;
> +			return cnt;
> +		}

This logic change should be on a separate commit as it is not part of the new counter support. 

>  	}
> -#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
> 
>  	struct mlx5_flow_counter tmpl = {
>  		.shared = shared,
>  		.id = id,

Better to have all the logic of the counter creation along with the macros for the diff implementation inside a separate function. 

> +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
>  		.cs = mlx5_glue->create_counter_set
>  			(priv->ctx,
>  			 &(struct ibv_counter_set_init_attr){
>  				 .counter_set_id = id,
>  			 }),
> +#endif
> +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V45
> +		.cs = mlx5_glue->create_counter_set
> +			(priv->ctx,
> +			 &(struct ibv_counters_init_attr){0}), #endif
>  		.hits = 0,
>  		.bytes = 0,
>  	};
> @@ -78,17 +85,36 @@
>  		rte_errno = errno;
>  		return NULL;
>  	}
> +
> +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V45
> +	struct ibv_counter_attach_attr attach_attr = {0};
> +	int ret;
> +
> +	attach_attr.counter_desc = IBV_COUNTER_PACKETS;
> +	attach_attr.index = 0;
> +	ret = ibv_attach_counters_point_flow(tmpl.cs, &attach_attr, NULL);
> +	if (!ret) {
> +		attach_attr.counter_desc = IBV_COUNTER_BYTES;
> +		attach_attr.index = 1;
> +		ret = ibv_attach_counters_point_flow(tmpl.cs,
> +						     &attach_attr,
> +						     NULL);
> +	}
> +	if (ret) {
> +		claim_zero(mlx5_glue->destroy_counter_set(tmpl.cs));
> +		rte_errno = ret;
> +		return NULL;
> +	}
> +#endif
>  	cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
>  	if (!cnt) {
> +		claim_zero(mlx5_glue->destroy_counter_set(tmpl.cs));

Bug fix? 
If yes then separate commit. For bug fixes it is super important so that we can have it backported to the stable releases. 

>  		rte_errno = ENOMEM;
>  		return NULL;
>  	}
>  	*cnt = tmpl;
>  	LIST_INSERT_HEAD(&priv->flow_counters, cnt, next);
>  	return cnt;
> -#endif
> -	rte_errno = ENOTSUP;
> -	return NULL;
>  }
> 
>  /**
> @@ -106,6 +132,7 @@
>  		rte_free(counter);
>  	}
>  }
> +#endif /* HAVE_IBV_DEVICE_COUNTERS_SET_Vxx */
> 
>  /**
>   * Add a verbs item specification into @p flow.
> @@ -927,6 +954,8 @@
>   * @return
>   *   0 On success else a negative errno value is returned and rte_errno is set.
>   */
> +#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
> +	defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)

Ditto. 
Macro inside and on the else part return notsup. 

>  static int
>  flow_verbs_translate_action_count(struct rte_eth_dev *dev,
>  				  const struct rte_flow_action *action, @@ -
> 936,13 +965,11 @@  {
>  	const struct rte_flow_action_count *count = action->conf;
>  	struct rte_flow *flow = dev_flow->flow; -#ifdef
> HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
>  	unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
>  	struct ibv_flow_spec_counter_action counter = {
>  		.type = IBV_FLOW_SPEC_ACTION_COUNT,
>  		.size = size,
>  	};
> -#endif
> 
>  	if (!flow->counter) {
>  		flow->counter = flow_verbs_counter_new(dev, count-
> >shared, @@ -955,12 +982,16 @@
>  						  " context.");
>  	}
>  	*action_flags |= MLX5_FLOW_ACTION_COUNT; -#ifdef
> HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
> +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
>  	counter.counter_set_handle = flow->counter->cs->handle;
> -	flow_verbs_spec_add(dev_flow, &counter, size);
>  #endif
> +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V45
> +	counter.counters = flow->counter->cs;
> +#endif
> +	flow_verbs_spec_add(dev_flow, &counter, size);
>  	return 0;
>  }
> +#endif /* HAVE_IBV_DEVICE_COUNTERS_SET_Vxx */
> 
>  /**
>   * Internal validation function. For validating both actions and items.
> @@ -1157,12 +1188,15 @@
>  				return ret;
>  			action_flags |= MLX5_FLOW_ACTION_RSS;
>  			break;
> +#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
> +	defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)

This is not needed. It is OK to validate the count action, the code will fail to create the counters after with errno of not supported.
On the current logic you just ignore the counter action request instead of rejecting it as not supported. 

>  		case RTE_FLOW_ACTION_TYPE_COUNT:
>  			ret = mlx5_flow_validate_action_count(dev, attr,
> error);
>  			if (ret < 0)
>  				return ret;
>  			action_flags |= MLX5_FLOW_ACTION_COUNT;
>  			break;
> +#endif
>  		default:
>  			return rte_flow_error_set(error, ENOTSUP,
> 
> RTE_FLOW_ERROR_TYPE_ACTION,
> @@ -1219,11 +1253,12 @@
>  		case RTE_FLOW_ACTION_TYPE_RSS:
>  			detected_actions |= MLX5_FLOW_ACTION_RSS;
>  			break;
> +#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
> +	defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
>  		case RTE_FLOW_ACTION_TYPE_COUNT:
> -#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
>  			size += sizeof(struct ibv_flow_spec_counter_action);
> -#endif
>  			detected_actions |= MLX5_FLOW_ACTION_COUNT;
> +#endif

Keep the ifdef as before (only on the size+=). The counter creation will fail on the translate. 

>  			break;
>  		default:
>  			break;
> @@ -1406,7 +1441,6 @@
>  	if (priority == MLX5_FLOW_PRIO_RSVD)
>  		priority = priv->config.flow_prio - 1;
>  	for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
> -		int ret;
>  		switch (actions->type) {
>  		case RTE_FLOW_ACTION_TYPE_VOID:
>  			break;
> @@ -1434,7 +1468,11 @@
>  							&action_flags,
>  							dev_flow);
>  			break;
> -		case RTE_FLOW_ACTION_TYPE_COUNT:
> +#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
> +	defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)

Same. No need for this macro, you have one inside the translate function. 

> +		case RTE_FLOW_ACTION_TYPE_COUNT: {
> +			int ret;
> +
>  			ret = flow_verbs_translate_action_count(dev,
>  								actions,
> 
> 	&action_flags,
> @@ -1443,6 +1481,8 @@
>  			if (ret < 0)
>  				return ret;
>  			break;
> +		}
> +#endif
>  		default:
>  			return rte_flow_error_set(error, ENOTSUP,
> 
> RTE_FLOW_ERROR_TYPE_ACTION,
> @@ -1538,10 +1578,13 @@
>  			verbs->hrxq = NULL;
>  		}
>  	}
> +#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
> +	defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)

Why the macro is needed? If no counter support flow->counter will always be NULL. 

>  	if (flow->counter) {
>  		flow_verbs_counter_release(flow->counter);
>  		flow->counter = NULL;
>  	}
> +#endif
>  }
> 
>  /**
> diff --git a/drivers/net/mlx5/mlx5_glue.c b/drivers/net/mlx5/mlx5_glue.c
> index 48590df..d77adb8 100644
> --- a/drivers/net/mlx5/mlx5_glue.c
> +++ b/drivers/net/mlx5/mlx5_glue.c

For the whole glue part, you didn't address my comments from the V1. Attaching those again:
" Pay attention to how the old counter supported was added to the mlx5_glue lib.
There are no ifdefs of the function pointer declaration nor on the function declaration. This was because it was causing link issues between this lib to verbs. 

Same should be for the new counters. Need to declare all needed structs if not declared on the mlx5_glue.h file, and always the function pointer and the declaration. 
The ifdef is only inside the function implementation."

> @@ -211,28 +211,84 @@
>  	return ibv_dereg_mr(mr);
>  }
> 
> +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V45 static struct ibv_counters *
> +mlx5_glue_create_counters(struct ibv_context *context,
> +			  struct ibv_counters_init_attr *init_attr) {
> +	return ibv_create_counters(context, init_attr); }
> +
> +static int
> +mlx5_glue_destroy_counters(struct ibv_counters *counters) {
> +	return ibv_destroy_counters(counters); }
> +
> +static int
> +mlx5_glue_attach_counters(struct ibv_counters *counters,
> +		     struct ibv_counter_attach_attr *attr,
> +		     struct ibv_flow *flow)
> +{
> +	return ibv_attach_counters_point_flow(counters, attr, flow); }
> +
> +static int
> +mlx5_glue_query_counters(struct ibv_counters *counters,
> +			 uint64_t *counters_value,
> +			 uint32_t ncounters,
> +			 uint32_t flags)
> +{
> +	return ibv_read_counters(counters, counters_value, ncounters,
> flags);
> +} #endif /* HAVE_IBV_DEVICE_COUNTERS_SET_V45 */
> +
> +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42 static struct
> ibv_counter_set *
> +mlx5_glue_create_counter_set(struct ibv_context *context,
> +			     struct ibv_counter_set_init_attr *init_attr) {
> +	return ibv_create_counter_set(context, init_attr); }
> +
> +static int
> +mlx5_glue_destroy_counter_set(struct ibv_counter_set *cs) {
> +	return ibv_destroy_counter_set(cs);
> +}
> +
> +static int
> +mlx5_glue_describe_counter_set(struct ibv_context *context,
> +			       uint16_t counter_set_id,
> +			       struct ibv_counter_set_description *cs_desc) {
> +	return ibv_describe_counter_set(context, counter_set_id, cs_desc);
> }
> +
> +static int
> +mlx5_glue_query_counter_set(struct ibv_query_counter_set_attr
> *query_attr,
> +			    struct ibv_counter_set_data *cs_data) {
> +	return ibv_query_counter_set(query_attr, cs_data); } #endif /*
> +HAVE_IBV_DEVICE_COUNTERS_SET_V42 */
> +
> +#if !defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) && \
> +	!defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
>  static struct ibv_counter_set *
>  mlx5_glue_create_counter_set(struct ibv_context *context,
>  			     struct ibv_counter_set_init_attr *init_attr)  { -
> #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
>  	(void)context;
>  	(void)init_attr;
>  	return NULL;
> -#else
> -	return ibv_create_counter_set(context, init_attr);
> -#endif
>  }
> 
>  static int
>  mlx5_glue_destroy_counter_set(struct ibv_counter_set *cs)  { -#ifndef
> HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
>  	(void)cs;
>  	return ENOTSUP;
> -#else
> -	return ibv_destroy_counter_set(cs);
> -#endif
>  }
> 
>  static int
> @@ -240,28 +296,21 @@
>  			       uint16_t counter_set_id,
>  			       struct ibv_counter_set_description *cs_desc)  { -
> #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
>  	(void)context;
>  	(void)counter_set_id;
>  	(void)cs_desc;
>  	return ENOTSUP;
> -#else
> -	return ibv_describe_counter_set(context, counter_set_id, cs_desc);
> -#endif
>  }
> 
>  static int
>  mlx5_glue_query_counter_set(struct ibv_query_counter_set_attr
> *query_attr,
>  			    struct ibv_counter_set_data *cs_data)  { -#ifndef
> HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
>  	(void)query_attr;
>  	(void)cs_data;
>  	return ENOTSUP;
> -#else
> -	return ibv_query_counter_set(query_attr, cs_data);
> -#endif
>  }
> +#endif /* HAVE_IBV_DEVICE_COUNTERS_SET_Vxx */
> 
>  static void
>  mlx5_glue_ack_async_event(struct ibv_async_event *event) @@ -420,10
> +469,17 @@
>  	.modify_qp = mlx5_glue_modify_qp,
>  	.reg_mr = mlx5_glue_reg_mr,
>  	.dereg_mr = mlx5_glue_dereg_mr,
> +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V45
> +	.create_counter_set = mlx5_glue_create_counters,
> +	.destroy_counter_set = mlx5_glue_destroy_counters,
> +	.attach_counter_set = mlx5_glue_attach_counters,
> +	.query_counter_set = mlx5_glue_query_counters, #else
>  	.create_counter_set = mlx5_glue_create_counter_set,
>  	.destroy_counter_set = mlx5_glue_destroy_counter_set,
>  	.describe_counter_set = mlx5_glue_describe_counter_set,
>  	.query_counter_set = mlx5_glue_query_counter_set,
> +#endif
>  	.ack_async_event = mlx5_glue_ack_async_event,
>  	.get_async_event = mlx5_glue_get_async_event,
>  	.port_state_str = mlx5_glue_port_state_str, diff --git
> a/drivers/net/mlx5/mlx5_glue.h b/drivers/net/mlx5/mlx5_glue.h index
> f6e4e38..b9033de 100644
> --- a/drivers/net/mlx5/mlx5_glue.h
> +++ b/drivers/net/mlx5/mlx5_glue.h
> @@ -23,7 +23,11 @@
>  #define MLX5_GLUE_VERSION ""
>  #endif
> 
> -#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
> +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V45 struct ibv_counters; struct
> +ibv_counters_attach_data; struct ibv_counters_init_attr; #else
>  struct ibv_counter_set;
>  struct ibv_counter_set_data;
>  struct ibv_counter_set_description;
> @@ -96,6 +100,21 @@ struct mlx5_glue {
>  	struct ibv_mr *(*reg_mr)(struct ibv_pd *pd, void *addr,
>  				 size_t length, int access);
>  	int (*dereg_mr)(struct ibv_mr *mr);
> +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V45
> +	struct ibv_counters *(*create_counter_set)
> +		(struct ibv_context *context,
> +		 struct ibv_counters_init_attr *init_attr);
> +	int (*destroy_counter_set)(struct ibv_counters *cs);
> +	int (*attach_counter_set)
> +		(struct ibv_counters *cs,
> +		 struct ibv_counter_attach_attr *attr,
> +		 struct ibv_flow *flow);
> +	int (*query_counter_set)
> +		(struct ibv_counters *cs,
> +		 uint64_t *counters_value,
> +		 uint32_t ncounters,
> +		 uint32_t flags);
> +#else
>  	struct ibv_counter_set *(*create_counter_set)
>  		(struct ibv_context *context,
>  		 struct ibv_counter_set_init_attr *init_attr); @@ -106,6
> +125,7 @@ struct mlx5_glue {
>  		 struct ibv_counter_set_description *cs_desc);
>  	int (*query_counter_set)(struct ibv_query_counter_set_attr
> *query_attr,
>  				 struct ibv_counter_set_data *cs_data);
> +#endif
>  	void (*ack_async_event)(struct ibv_async_event *event);
>  	int (*get_async_event)(struct ibv_context *context,
>  			       struct ibv_async_event *event);
> --
> 1.8.3.1

^ permalink raw reply	[flat|nested] 39+ messages in thread

* [dpdk-dev] [PATCH v3 0/6] net/mlx5: flow counters support for Linux-rdma v19
  2018-10-17 13:53   ` [dpdk-dev] [PATCH v2] " Viacheslav Ovsiienko
  2018-10-18  6:34     ` Shahaf Shuler
@ 2018-10-19 15:21     ` Viacheslav Ovsiienko
  2018-10-19 15:21       ` [dpdk-dev] [PATCH v3 1/6] net/mlx5: flow counters object create function bugfix Viacheslav Ovsiienko
                         ` (8 more replies)
  1 sibling, 9 replies; 39+ messages in thread
From: Viacheslav Ovsiienko @ 2018-10-19 15:21 UTC (permalink / raw)
  To: shahafs, yskoh; +Cc: dev, Viacheslav Ovsiienko

Mellanox mlx5 PMD supports Flow counters via Verbs library.
The current implementation is based on the Mellanox proprietary
Verbs library included in MLNX OFED packages. The Flow counter
support is recently added into linux-rdma release (v19),
so the mlx5 PMD update is needed to provide Counter feature
on the base of linux-rdma.

mlx5 PMD can be compiled with MLNX OFED or linux-rdma v19+
and provide flow counters for both.

Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>

---
v3:
- mlx5 glue issue resolved correctly
- patch is reorganized info small isolated parts
  
v2:
- http://patches.dpdk.org/patch/46989/
- rebased on top of master-net-mlx branch
- new compilation flags are introduced:
  - HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V42, kernel/verbs
    library provides the flow counter support in style of
    MLNX_OFED_4.2 to MLNX_OFED_4.4
  - HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45, kernel/verbs
    library provides the flow counter support in style of
    MLNX_OFED_4.5 or higher

v1:
- http://patches.dpdk.org/patch/45972/
*** BLURB HERE ***

Viacheslav Ovsiienko (6):
  net/mlx5: flow counters object create function bugfix
  net/mlx5: flow counters new configuration flags
  net/mlx5: flow counters simplifying runtime support check
  net/mlx5: flow counters mlx5 glue library update
  net/mlx5: flow counters query function move and rename
  net/mlx5: flow counters Verbs interface functions update

 drivers/net/mlx5/Makefile          |   9 +-
 drivers/net/mlx5/meson.build       |   6 +-
 drivers/net/mlx5/mlx5.c            |  17 ++-
 drivers/net/mlx5/mlx5.h            |   1 -
 drivers/net/mlx5/mlx5_flow.c       |  16 +--
 drivers/net/mlx5/mlx5_flow.h       |   6 +
 drivers/net/mlx5/mlx5_flow_verbs.c | 230 ++++++++++++++++++++++++-------------
 drivers/net/mlx5/mlx5_glue.c       |  68 ++++++++++-
 drivers/net/mlx5/mlx5_glue.h       |  19 ++-
 9 files changed, 269 insertions(+), 103 deletions(-)

-- 
1.8.3.1

^ permalink raw reply	[flat|nested] 39+ messages in thread

* [dpdk-dev] [PATCH v3 1/6] net/mlx5: flow counters object create function bugfix
  2018-10-19 15:21     ` [dpdk-dev] [PATCH v3 0/6] net/mlx5: flow counters support for Linux-rdma v19 Viacheslav Ovsiienko
@ 2018-10-19 15:21       ` Viacheslav Ovsiienko
  2018-10-21  9:20         ` Shahaf Shuler
  2018-10-19 15:21       ` [dpdk-dev] [PATCH v3 2/6] net/mlx5: flow counters new configuration flags Viacheslav Ovsiienko
                         ` (7 subsequent siblings)
  8 siblings, 1 reply; 39+ messages in thread
From: Viacheslav Ovsiienko @ 2018-10-19 15:21 UTC (permalink / raw)
  To: shahafs, yskoh; +Cc: dev, Viacheslav Ovsiienko

The first part of patchset provides the separate commit for
bugfix. Flow counter object was not freed in case of memory
allocation error. The call of counter Verbs object deallocating
function is added. The initial value of reference counter is
set to one in order to provide the correct counter object
freeing in the flow_verbs_counter_release() function.

Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
---
 drivers/net/mlx5/mlx5_flow_verbs.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c b/drivers/net/mlx5/mlx5_flow_verbs.c
index 4ae974b..6ddb13b 100644
--- a/drivers/net/mlx5/mlx5_flow_verbs.c
+++ b/drivers/net/mlx5/mlx5_flow_verbs.c
@@ -72,6 +72,7 @@
 			 }),
 		.hits = 0,
 		.bytes = 0,
+		.ref_cnt = 1,
 	};
 
 	if (!tmpl.cs) {
@@ -80,6 +81,7 @@
 	}
 	cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
 	if (!cnt) {
+		claim_zero(mlx5_glue->destroy_counter_set(tmpl.cs));
 		rte_errno = ENOMEM;
 		return NULL;
 	}
-- 
1.8.3.1

^ permalink raw reply	[flat|nested] 39+ messages in thread

* [dpdk-dev] [PATCH v3 2/6] net/mlx5: flow counters new configuration flags
  2018-10-19 15:21     ` [dpdk-dev] [PATCH v3 0/6] net/mlx5: flow counters support for Linux-rdma v19 Viacheslav Ovsiienko
  2018-10-19 15:21       ` [dpdk-dev] [PATCH v3 1/6] net/mlx5: flow counters object create function bugfix Viacheslav Ovsiienko
@ 2018-10-19 15:21       ` Viacheslav Ovsiienko
  2018-10-21  9:20         ` Shahaf Shuler
  2018-10-19 15:21       ` [dpdk-dev] [PATCH v3 3/6] net/mlx5: flow counters simplifying runtime support check Viacheslav Ovsiienko
                         ` (6 subsequent siblings)
  8 siblings, 1 reply; 39+ messages in thread
From: Viacheslav Ovsiienko @ 2018-10-19 15:21 UTC (permalink / raw)
  To: shahafs, yskoh; +Cc: dev, Viacheslav Ovsiienko

In this part of patchset some modifications in compile configuration
flags are done. The HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT is replaced
with HAVE_IBV_DEVICE_COUNTERS_SET_V42. At this stage it is just
flag renaming. The new HAVE_IBV_DEVICE_COUNTERS_SET_V45 flag is
introduced. Both makefile and meson.build are changed, the flag
modifications are grouped, no more build system files modifications
are expected in this patchset, ones are grouped in this part.
HAVE_IBV_DEVICE_COUNTERS_SET_V45 is just introduced, no code
dependence in this part of pathset.

HAVE_IBV_DEVICE_COUNTERS_SET_V42 - is defined if system supports
the "old" flow counters functionality, MLNX_OFED version from
4.2 to 4.4 is required.

HAVE_IBV_DEVICE_COUNTERS_SET_V45 - is defined if system supports
the "new" flow counters functionality, MLNX_OVED 4.5 (or higher)
or Linux rdma-core v19 (or higher) is required.

Neither HAVE_IBV_DEVICE_COUNTERS_SET_V42 not
HAVE_IBV_DEVICE_COUNTERS_SET_V45 is defined if there is no
counters support.

Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
---
 drivers/net/mlx5/Makefile          |  9 +++++++--
 drivers/net/mlx5/meson.build       |  6 ++++--
 drivers/net/mlx5/mlx5.c            |  4 ++--
 drivers/net/mlx5/mlx5_flow_verbs.c | 10 +++++-----
 drivers/net/mlx5/mlx5_glue.c       |  8 ++++----
 drivers/net/mlx5/mlx5_glue.h       |  2 +-
 6 files changed, 23 insertions(+), 16 deletions(-)

diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile
index 1e9c0b4..28547d3 100644
--- a/drivers/net/mlx5/Makefile
+++ b/drivers/net/mlx5/Makefile
@@ -157,9 +157,14 @@ mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
 		enum ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT \
 		$(AUTOCONF_OUTPUT)
 	$Q sh -- '$<' '$@' \
-		HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT \
+		HAVE_IBV_DEVICE_COUNTERS_SET_V42 \
+ 		infiniband/verbs.h \
+ 		type 'struct ibv_counter_set_init_attr' \
+ 		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
+		HAVE_IBV_DEVICE_COUNTERS_SET_V45 \
 		infiniband/verbs.h \
-		type 'struct ibv_counter_set_init_attr' \
+		type 'struct ibv_counters_init_attr' \
 		$(AUTOCONF_OUTPUT)
 	$Q sh -- '$<' '$@' \
 		HAVE_RDMA_NL_NLDEV \
diff --git a/drivers/net/mlx5/meson.build b/drivers/net/mlx5/meson.build
index c192d44..1c4ed30 100644
--- a/drivers/net/mlx5/meson.build
+++ b/drivers/net/mlx5/meson.build
@@ -79,8 +79,10 @@ if build
 	has_member_args = [
 		[ 'HAVE_IBV_MLX5_MOD_SWP', 'infiniband/mlx5dv.h',
 		'struct mlx5dv_sw_parsing_caps', 'sw_parsing_offloads' ],
-		[ 'HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT', 'infiniband/verbs.h',
-		'struct ibv_counter_set_init_attr', 'counter_set_id' ],
+		[ 'HAVE_IBV_DEVICE_COUNTERS_SET_V42', 'infiniband/verbs.h',
+ 		'struct ibv_counter_set_init_attr', 'counter_set_id' ],
+		[ 'HAVE_IBV_DEVICE_COUNTERS_SET_V45', 'infiniband/verbs.h',
+		'struct ibv_counters_init_attr', 'comp_mask' ],
 	]
 	# input array for meson symbol search:
 	# [ "MACRO to define if found", "header for the search",
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 13f2fd4..bb19085 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -739,7 +739,7 @@
 	unsigned int mprq_max_stride_size_n = 0;
 	unsigned int mprq_min_stride_num_n = 0;
 	unsigned int mprq_max_stride_num_n = 0;
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 	struct ibv_counter_set_description cs_desc = { .counter_type = 0 };
 #endif
 	struct ether_addr mac;
@@ -1009,7 +1009,7 @@
 	config.hw_csum = !!(attr.device_cap_flags_ex & IBV_DEVICE_RAW_IP_CSUM);
 	DRV_LOG(DEBUG, "checksum offloading is %ssupported",
 		(config.hw_csum ? "" : "not "));
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 	config.flow_counter_en = !!attr.max_counter_sets;
 	mlx5_glue->describe_counter_set(ctx, 0, &cs_desc);
 	DRV_LOG(DEBUG, "counter type = %d, num of cs = %ld, attributes = %d",
diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c b/drivers/net/mlx5/mlx5_flow_verbs.c
index 6ddb13b..3d6fedb 100644
--- a/drivers/net/mlx5/mlx5_flow_verbs.c
+++ b/drivers/net/mlx5/mlx5_flow_verbs.c
@@ -60,7 +60,7 @@
 		cnt->ref_cnt++;
 		return cnt;
 	}
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 
 	struct mlx5_flow_counter tmpl = {
 		.shared = shared,
@@ -938,7 +938,7 @@
 {
 	const struct rte_flow_action_count *count = action->conf;
 	struct rte_flow *flow = dev_flow->flow;
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 	unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
 	struct ibv_flow_spec_counter_action counter = {
 		.type = IBV_FLOW_SPEC_ACTION_COUNT,
@@ -957,7 +957,7 @@
 						  " context.");
 	}
 	*action_flags |= MLX5_FLOW_ACTION_COUNT;
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 	counter.counter_set_handle = flow->counter->cs->handle;
 	flow_verbs_spec_add(dev_flow, &counter, size);
 #endif
@@ -1222,7 +1222,7 @@
 			detected_actions |= MLX5_FLOW_ACTION_RSS;
 			break;
 		case RTE_FLOW_ACTION_TYPE_COUNT:
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 			size += sizeof(struct ibv_flow_spec_counter_action);
 #endif
 			detected_actions |= MLX5_FLOW_ACTION_COUNT;
@@ -1665,7 +1665,7 @@
 		       void *data __rte_unused,
 		       struct rte_flow_error *error)
 {
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 	if (flow->actions & MLX5_FLOW_ACTION_COUNT) {
 		struct rte_flow_query_count *qc = data;
 		uint64_t counters[2] = {0, 0};
diff --git a/drivers/net/mlx5/mlx5_glue.c b/drivers/net/mlx5/mlx5_glue.c
index 48590df..889e074 100644
--- a/drivers/net/mlx5/mlx5_glue.c
+++ b/drivers/net/mlx5/mlx5_glue.c
@@ -215,7 +215,7 @@
 mlx5_glue_create_counter_set(struct ibv_context *context,
 			     struct ibv_counter_set_init_attr *init_attr)
 {
-#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 	(void)context;
 	(void)init_attr;
 	return NULL;
@@ -227,7 +227,7 @@
 static int
 mlx5_glue_destroy_counter_set(struct ibv_counter_set *cs)
 {
-#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 	(void)cs;
 	return ENOTSUP;
 #else
@@ -240,7 +240,7 @@
 			       uint16_t counter_set_id,
 			       struct ibv_counter_set_description *cs_desc)
 {
-#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 	(void)context;
 	(void)counter_set_id;
 	(void)cs_desc;
@@ -254,7 +254,7 @@
 mlx5_glue_query_counter_set(struct ibv_query_counter_set_attr *query_attr,
 			    struct ibv_counter_set_data *cs_data)
 {
-#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 	(void)query_attr;
 	(void)cs_data;
 	return ENOTSUP;
diff --git a/drivers/net/mlx5/mlx5_glue.h b/drivers/net/mlx5/mlx5_glue.h
index f6e4e38..adee972 100644
--- a/drivers/net/mlx5/mlx5_glue.h
+++ b/drivers/net/mlx5/mlx5_glue.h
@@ -23,7 +23,7 @@
 #define MLX5_GLUE_VERSION ""
 #endif
 
-#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 struct ibv_counter_set;
 struct ibv_counter_set_data;
 struct ibv_counter_set_description;
-- 
1.8.3.1

^ permalink raw reply	[flat|nested] 39+ messages in thread

* [dpdk-dev] [PATCH v3 3/6] net/mlx5: flow counters simplifying runtime support check
  2018-10-19 15:21     ` [dpdk-dev] [PATCH v3 0/6] net/mlx5: flow counters support for Linux-rdma v19 Viacheslav Ovsiienko
  2018-10-19 15:21       ` [dpdk-dev] [PATCH v3 1/6] net/mlx5: flow counters object create function bugfix Viacheslav Ovsiienko
  2018-10-19 15:21       ` [dpdk-dev] [PATCH v3 2/6] net/mlx5: flow counters new configuration flags Viacheslav Ovsiienko
@ 2018-10-19 15:21       ` Viacheslav Ovsiienko
  2018-10-21  9:20         ` Shahaf Shuler
  2018-10-19 15:21       ` [dpdk-dev] [PATCH v3 4/6] net/mlx5: flow counters mlx5 glue library update Viacheslav Ovsiienko
                         ` (5 subsequent siblings)
  8 siblings, 1 reply; 39+ messages in thread
From: Viacheslav Ovsiienko @ 2018-10-19 15:21 UTC (permalink / raw)
  To: shahafs, yskoh; +Cc: dev, Viacheslav Ovsiienko

This part of patchset removes the redundant check of counters
support in runtime. The flag flow_counter_en is eliminated
from the code. The Verbs create counter function just returns
an error if no counter support presented in kernel.

Some log messages regarding the counter support type and
presence are added.

mlx5_flow_validate_action_count() is also updated due to
flow_counter_en flag removal.

Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
---
 drivers/net/mlx5/mlx5.c      | 15 ++++++++++-----
 drivers/net/mlx5/mlx5.h      |  1 -
 drivers/net/mlx5/mlx5_flow.c | 16 +++++++++-------
 3 files changed, 19 insertions(+), 13 deletions(-)

diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index bb19085..8a33639 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1009,12 +1009,17 @@
 	config.hw_csum = !!(attr.device_cap_flags_ex & IBV_DEVICE_RAW_IP_CSUM);
 	DRV_LOG(DEBUG, "checksum offloading is %ssupported",
 		(config.hw_csum ? "" : "not "));
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
-	config.flow_counter_en = !!attr.max_counter_sets;
+#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 	mlx5_glue->describe_counter_set(ctx, 0, &cs_desc);
-	DRV_LOG(DEBUG, "counter type = %d, num of cs = %ld, attributes = %d",
-		cs_desc.counter_type, cs_desc.num_of_cs,
-		cs_desc.attributes);
+	DRV_LOG(DEBUG, "Flow counters are supported (4.2), "
+		       "type = %d, num of cs = %ld, attr = %d",
+		       cs_desc.counter_type,
+		       cs_desc.num_of_cs,
+		       cs_desc.attributes);
+#elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
+	DRV_LOG(DEBUG, "Flow counters are supported (4.5)");
+#else
+	DRV_LOG(DEBUG, "Flow counters are not supported");
 #endif
 	config.ind_table_max_size =
 		attr.rss_caps.max_rwq_indirection_table_size;
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index d14239c..74d87c0 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -114,7 +114,6 @@ struct mlx5_dev_config {
 	unsigned int tunnel_en:1;
 	/* Whether tunnel stateless offloads are supported. */
 	unsigned int mpls_en:1; /* MPLS over GRE/UDP is enabled. */
-	unsigned int flow_counter_en:1; /* Whether flow counter is supported. */
 	unsigned int cqe_comp:1; /* CQE compression is enabled. */
 	unsigned int tso:1; /* Whether TSO is supported. */
 	unsigned int tx_vec_en:1; /* Tx vector is enabled. */
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index fcabab0..c15722d 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -921,22 +921,24 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
  *   0 on success, a negative errno value otherwise and rte_ernno is set.
  */
 int
-mlx5_flow_validate_action_count(struct rte_eth_dev *dev,
+mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused,
 				const struct rte_flow_attr *attr,
 				struct rte_flow_error *error)
 {
-	struct priv *priv = dev->data->dev_private;
-
-	if (!priv->config.flow_counter_en)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
-					  "flow counters are not supported.");
+#if !defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) && \
+	!defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
+	(void)attr;
+	return rte_flow_error_set(error, ENOTSUP,
+				  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+				  "flow counters are not supported.");
+#else
 	if (attr->egress)
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
 					  "count action not supported for "
 					  "egress");
 	return 0;
+#endif
 }
 
 /**
-- 
1.8.3.1

^ permalink raw reply	[flat|nested] 39+ messages in thread

* [dpdk-dev] [PATCH v3 4/6] net/mlx5: flow counters mlx5 glue library update
  2018-10-19 15:21     ` [dpdk-dev] [PATCH v3 0/6] net/mlx5: flow counters support for Linux-rdma v19 Viacheslav Ovsiienko
                         ` (2 preceding siblings ...)
  2018-10-19 15:21       ` [dpdk-dev] [PATCH v3 3/6] net/mlx5: flow counters simplifying runtime support check Viacheslav Ovsiienko
@ 2018-10-19 15:21       ` Viacheslav Ovsiienko
  2018-10-21  9:20         ` Shahaf Shuler
  2018-10-19 15:21       ` [dpdk-dev] [PATCH v3 5/6] net/mlx5: flow counters query function move and rename Viacheslav Ovsiienko
                         ` (4 subsequent siblings)
  8 siblings, 1 reply; 39+ messages in thread
From: Viacheslav Ovsiienko @ 2018-10-19 15:21 UTC (permalink / raw)
  To: shahafs, yskoh; +Cc: dev, Viacheslav Ovsiienko

This part of patchset updates the mlx5 glue library, new
counter support function pointers are added to the glue linking
structure mlx5_glue. This structure now contains the pointers
to the both versions of counter supporting functions due to
compatibility issues. Depending on configuration flags the
functions perform actual Verbs library calls or return an error
with meaning "feature is not supported" (NULL or ENOTSUP).

Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
---
 drivers/net/mlx5/mlx5_glue.c | 60 ++++++++++++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_glue.h | 17 +++++++++++++
 2 files changed, 77 insertions(+)

diff --git a/drivers/net/mlx5/mlx5_glue.c b/drivers/net/mlx5/mlx5_glue.c
index 889e074..1afb114 100644
--- a/drivers/net/mlx5/mlx5_glue.c
+++ b/drivers/net/mlx5/mlx5_glue.c
@@ -263,6 +263,62 @@
 #endif
 }
 
+static struct ibv_counters *
+mlx5_glue_create_counters(struct ibv_context *context,
+			  struct ibv_counters_init_attr *init_attr)
+{
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
+	(void)context;
+	(void)init_attr;
+	return NULL;
+#else
+	return ibv_create_counters(context, init_attr);
+#endif
+}
+
+static int
+mlx5_glue_destroy_counters(struct ibv_counters *counters)
+{
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
+	(void)counters;
+	return ENOTSUP;
+#else
+	return ibv_destroy_counters(counters);
+#endif
+}
+
+static int
+mlx5_glue_attach_counters(struct ibv_counters *counters,
+			  struct ibv_counter_attach_attr *attr,
+			  struct ibv_flow *flow)
+{
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
+	(void)counters;
+	(void)attr;
+	(void)flow;
+	return ENOTSUP;
+#else
+	return ibv_attach_counters_point_flow(counters, attr, flow);
+#endif
+}
+
+static int
+mlx5_glue_query_counters(struct ibv_counters *counters,
+			 uint64_t *counters_value,
+			 uint32_t ncounters,
+			 uint32_t flags)
+{
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
+	(void)counters;
+	(void)counters_value;
+	(void)ncounters;
+	(void)flags;
+	return ENOTSUP;
+#else
+	return ibv_read_counters(counters, counters_value, ncounters, flags);
+#endif
+}
+
 static void
 mlx5_glue_ack_async_event(struct ibv_async_event *event)
 {
@@ -424,6 +480,10 @@
 	.destroy_counter_set = mlx5_glue_destroy_counter_set,
 	.describe_counter_set = mlx5_glue_describe_counter_set,
 	.query_counter_set = mlx5_glue_query_counter_set,
+	.create_counters = mlx5_glue_create_counters,
+	.destroy_counters = mlx5_glue_destroy_counters,
+	.attach_counters = mlx5_glue_attach_counters,
+	.query_counters = mlx5_glue_query_counters,
 	.ack_async_event = mlx5_glue_ack_async_event,
 	.get_async_event = mlx5_glue_get_async_event,
 	.port_state_str = mlx5_glue_port_state_str,
diff --git a/drivers/net/mlx5/mlx5_glue.h b/drivers/net/mlx5/mlx5_glue.h
index adee972..44bfefe 100644
--- a/drivers/net/mlx5/mlx5_glue.h
+++ b/drivers/net/mlx5/mlx5_glue.h
@@ -31,6 +31,12 @@
 struct ibv_query_counter_set_attr;
 #endif
 
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
+struct ibv_counters;
+struct ibv_counters_init_attr;
+struct ibv_counter_attach_attr;
+#endif
+
 #ifndef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
 struct mlx5dv_qp_init_attr;
 #endif
@@ -106,6 +112,17 @@ struct mlx5_glue {
 		 struct ibv_counter_set_description *cs_desc);
 	int (*query_counter_set)(struct ibv_query_counter_set_attr *query_attr,
 				 struct ibv_counter_set_data *cs_data);
+	struct ibv_counters *(*create_counters)
+		(struct ibv_context *context,
+		 struct ibv_counters_init_attr *init_attr);
+	int (*destroy_counters)(struct ibv_counters *counters);
+	int (*attach_counters)(struct ibv_counters *counters,
+			       struct ibv_counter_attach_attr *attr,
+			       struct ibv_flow *flow);
+	int (*query_counters)(struct ibv_counters *counters,
+			      uint64_t *counters_value,
+			      uint32_t ncounters,
+			      uint32_t flags);
 	void (*ack_async_event)(struct ibv_async_event *event);
 	int (*get_async_event)(struct ibv_context *context,
 			       struct ibv_async_event *event);
-- 
1.8.3.1

^ permalink raw reply	[flat|nested] 39+ messages in thread

* [dpdk-dev] [PATCH v3 5/6] net/mlx5: flow counters query function move and rename
  2018-10-19 15:21     ` [dpdk-dev] [PATCH v3 0/6] net/mlx5: flow counters support for Linux-rdma v19 Viacheslav Ovsiienko
                         ` (3 preceding siblings ...)
  2018-10-19 15:21       ` [dpdk-dev] [PATCH v3 4/6] net/mlx5: flow counters mlx5 glue library update Viacheslav Ovsiienko
@ 2018-10-19 15:21       ` Viacheslav Ovsiienko
  2018-10-21  9:20         ` Shahaf Shuler
  2018-10-19 15:21       ` [dpdk-dev] [PATCH v3 6/6] net/mlx5: flow counters Verbs interface functions update Viacheslav Ovsiienko
                         ` (3 subsequent siblings)
  8 siblings, 1 reply; 39+ messages in thread
From: Viacheslav Ovsiienko @ 2018-10-19 15:21 UTC (permalink / raw)
  To: shahafs, yskoh; +Cc: dev, Viacheslav Ovsiienko

The flow_verbs_query_count() is moved into the the group of counter
managing functions and renamed to flow_verbs_counter_query() in order
to be in unified fashion with others.

Also minor function modification is made to avoid unreachable code
warnings if there is no counter support at compile time.

Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
---
 drivers/net/mlx5/mlx5_flow_verbs.c | 111 +++++++++++++++++++------------------
 1 file changed, 56 insertions(+), 55 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c b/drivers/net/mlx5/mlx5_flow_verbs.c
index 3d6fedb..f720c35 100644
--- a/drivers/net/mlx5/mlx5_flow_verbs.c
+++ b/drivers/net/mlx5/mlx5_flow_verbs.c
@@ -110,6 +110,61 @@
 }
 
 /**
+ * Query a flow counter via Verbs library call.
+ *
+ * @see rte_flow_query()
+ * @see rte_flow_ops
+ */
+static int
+flow_verbs_counter_query(struct rte_eth_dev *dev __rte_unused,
+			 struct rte_flow *flow __rte_unused,
+			 void *data __rte_unused,
+			 struct rte_flow_error *error)
+{
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
+	if (flow->actions & MLX5_FLOW_ACTION_COUNT) {
+		struct rte_flow_query_count *qc = data;
+		uint64_t counters[2] = {0, 0};
+		struct ibv_query_counter_set_attr query_cs_attr = {
+			.cs = flow->counter->cs,
+			.query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
+		};
+		struct ibv_counter_set_data query_out = {
+			.out = counters,
+			.outlen = 2 * sizeof(uint64_t),
+		};
+		int err = mlx5_glue->query_counter_set(&query_cs_attr,
+						       &query_out);
+
+		if (err)
+			return rte_flow_error_set
+				(error, err,
+				 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				 NULL,
+				 "cannot read counter");
+		qc->hits_set = 1;
+		qc->bytes_set = 1;
+		qc->hits = counters[0] - flow->counter->hits;
+		qc->bytes = counters[1] - flow->counter->bytes;
+		if (qc->reset) {
+			flow->counter->hits = counters[0];
+			flow->counter->bytes = counters[1];
+		}
+		return 0;
+	}
+	return rte_flow_error_set(error, EINVAL,
+				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				  NULL,
+				  "flow does not have counter");
+#else
+	return rte_flow_error_set(error, ENOTSUP,
+				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				  NULL,
+				  "counters are not available");
+#endif
+}
+
+/**
  * Add a verbs item specification into @p flow.
  *
  * @param[in, out] flow
@@ -1654,60 +1709,6 @@
 }
 
 /**
- * Query a flows.
- *
- * @see rte_flow_query()
- * @see rte_flow_ops
- */
-static int
-flow_verbs_query_count(struct rte_eth_dev *dev __rte_unused,
-		       struct rte_flow *flow __rte_unused,
-		       void *data __rte_unused,
-		       struct rte_flow_error *error)
-{
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
-	if (flow->actions & MLX5_FLOW_ACTION_COUNT) {
-		struct rte_flow_query_count *qc = data;
-		uint64_t counters[2] = {0, 0};
-		struct ibv_query_counter_set_attr query_cs_attr = {
-			.cs = flow->counter->cs,
-			.query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
-		};
-		struct ibv_counter_set_data query_out = {
-			.out = counters,
-			.outlen = 2 * sizeof(uint64_t),
-		};
-		int err = mlx5_glue->query_counter_set(&query_cs_attr,
-						       &query_out);
-
-		if (err)
-			return rte_flow_error_set
-				(error, err,
-				 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-				 NULL,
-				 "cannot read counter");
-		qc->hits_set = 1;
-		qc->bytes_set = 1;
-		qc->hits = counters[0] - flow->counter->hits;
-		qc->bytes = counters[1] - flow->counter->bytes;
-		if (qc->reset) {
-			flow->counter->hits = counters[0];
-			flow->counter->bytes = counters[1];
-		}
-		return 0;
-	}
-	return rte_flow_error_set(error, EINVAL,
-				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-				  NULL,
-				  "flow does not have counter");
-#endif
-	return rte_flow_error_set(error, ENOTSUP,
-				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-				  NULL,
-				  "counters are not available");
-}
-
-/**
  * Query a flow.
  *
  * @see rte_flow_query()
@@ -1727,7 +1728,7 @@
 		case RTE_FLOW_ACTION_TYPE_VOID:
 			break;
 		case RTE_FLOW_ACTION_TYPE_COUNT:
-			ret = flow_verbs_query_count(dev, flow, data, error);
+			ret = flow_verbs_counter_query(dev, flow, data, error);
 			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
-- 
1.8.3.1

^ permalink raw reply	[flat|nested] 39+ messages in thread

* [dpdk-dev] [PATCH v3 6/6] net/mlx5: flow counters Verbs interface functions update
  2018-10-19 15:21     ` [dpdk-dev] [PATCH v3 0/6] net/mlx5: flow counters support for Linux-rdma v19 Viacheslav Ovsiienko
                         ` (4 preceding siblings ...)
  2018-10-19 15:21       ` [dpdk-dev] [PATCH v3 5/6] net/mlx5: flow counters query function move and rename Viacheslav Ovsiienko
@ 2018-10-19 15:21       ` Viacheslav Ovsiienko
  2018-10-21  9:20         ` Shahaf Shuler
  2018-10-21  9:21       ` [dpdk-dev] [PATCH v3 0/6] net/mlx5: flow counters support for Linux-rdma v19 Shahaf Shuler
                         ` (2 subsequent siblings)
  8 siblings, 1 reply; 39+ messages in thread
From: Viacheslav Ovsiienko @ 2018-10-19 15:21 UTC (permalink / raw)
  To: shahafs, yskoh; +Cc: dev, Viacheslav Ovsiienko

This part of patchset updates the functions performing the Verbs
library calls in order to support different versions of library.
The functions:
  - flow_verbs_counter_new()
  - flow_verbs_counter_release()
  - flow_verbs_counter_query()
now have the several compilations branches, depending on the
counter support found in the system at compile time.

The flow_verbs_counter_create() function is introduced as
helper for flow_verbs_counter_new(), actually this helper
create the counters with Verbs.

Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
---
 drivers/net/mlx5/mlx5_flow.h       |   6 ++
 drivers/net/mlx5/mlx5_flow_verbs.c | 129 ++++++++++++++++++++++++++++---------
 2 files changed, 104 insertions(+), 31 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 69f55cf..44c7515 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -224,7 +224,13 @@ struct mlx5_flow_counter {
 	uint32_t shared:1; /**< Share counter ID with other flow rules. */
 	uint32_t ref_cnt:31; /**< Reference counter. */
 	uint32_t id; /**< Counter ID. */
+#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 	struct ibv_counter_set *cs; /**< Holds the counters for the rule. */
+#elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
+	struct ibv_counters *cs; /**< Holds the counters for the rule. */
+#else
+	void *cs;
+#endif
 	uint64_t hits; /**< Number of packets matched by the rule. */
 	uint64_t bytes; /**< Number of bytes matched by the rule. */
 };
diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c b/drivers/net/mlx5/mlx5_flow_verbs.c
index f720c35..b657933 100644
--- a/drivers/net/mlx5/mlx5_flow_verbs.c
+++ b/drivers/net/mlx5/mlx5_flow_verbs.c
@@ -33,6 +33,59 @@
 #include "mlx5_glue.h"
 #include "mlx5_flow.h"
 
+
+#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
+	defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
+/**
+ * Create Verbs flow counter with Verbs library.
+ *
+ * @param[in] dev
+ *   Pointer to the Ethernet device structure.
+ * @param[in, out] counter
+ *   PMD flow counter object, contains the counter id,
+ *   handle of created Verbs flow counter is returned in cs field.
+ *
+ * @return
+ *   counter->cs contains a handle of created Verbs counter,
+ *   NULL if error occurred and rte_errno is set.
+ */
+static void
+flow_verbs_counter_create(struct rte_eth_dev *dev,
+			  struct mlx5_flow_counter *counter)
+{
+#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
+	struct priv *priv = dev->data->dev_private;
+	struct ibv_counter_set_init_attr init = {
+			 .counter_set_id = counter->id};
+
+	counter->cs = mlx5_glue->create_counter_set(priv->ctx, &init);
+#elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
+	struct priv *priv = dev->data->dev_private;
+	struct ibv_counters_init_attr init = {0};
+	struct ibv_counter_attach_attr attach = {0};
+	int ret;
+
+	counter->cs = mlx5_glue->create_counters(priv->ctx, &init);
+	if (!counter->cs)
+		return;
+	attach.counter_desc = IBV_COUNTER_PACKETS;
+	attach.index = 0;
+	ret = mlx5_glue->attach_counters(counter->cs, &attach, NULL);
+	if (!ret) {
+		attach.counter_desc = IBV_COUNTER_BYTES;
+		attach.index = 1;
+		ret = mlx5_glue->attach_counters
+					(counter->cs, &attach, NULL);
+	}
+	if (ret) {
+		claim_zero(mlx5_glue->destroy_counters(counter->cs));
+		counter->cs = NULL;
+		rte_errno = ret;
+	}
+#endif
+}
+#endif
+
 /**
  * Get a flow counter.
  *
@@ -49,6 +102,8 @@
 static struct mlx5_flow_counter *
 flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t shared, uint32_t id)
 {
+#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
+	defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 	struct priv *priv = dev->data->dev_private;
 	struct mlx5_flow_counter *cnt;
 
@@ -60,37 +115,32 @@
 		cnt->ref_cnt++;
 		return cnt;
 	}
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
-
-	struct mlx5_flow_counter tmpl = {
-		.shared = shared,
-		.id = id,
-		.cs = mlx5_glue->create_counter_set
-			(priv->ctx,
-			 &(struct ibv_counter_set_init_attr){
-				 .counter_set_id = id,
-			 }),
-		.hits = 0,
-		.bytes = 0,
-		.ref_cnt = 1,
-	};
-
-	if (!tmpl.cs) {
-		rte_errno = errno;
-		return NULL;
-	}
 	cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
 	if (!cnt) {
-		claim_zero(mlx5_glue->destroy_counter_set(tmpl.cs));
 		rte_errno = ENOMEM;
 		return NULL;
 	}
-	*cnt = tmpl;
-	LIST_INSERT_HEAD(&priv->flow_counters, cnt, next);
-	return cnt;
-#endif
+	cnt->id = id;
+	cnt->shared = shared;
+	cnt->ref_cnt = 1;
+	cnt->hits = 0;
+	cnt->bytes = 0;
+	/* Create counter with Verbs. */
+	flow_verbs_counter_create(dev, cnt);
+	if (cnt->cs) {
+		LIST_INSERT_HEAD(&priv->flow_counters, cnt, next);
+		return cnt;
+	}
+	/* Some error occurred in Verbs library, rte_errno is set. */
+	rte_free(cnt);
+	return NULL;
+#else
+	(void)dev;
+	(void)shared;
+	(void)id;
 	rte_errno = ENOTSUP;
 	return NULL;
+#endif
 }
 
 /**
@@ -103,7 +153,11 @@
 flow_verbs_counter_release(struct mlx5_flow_counter *counter)
 {
 	if (--counter->ref_cnt == 0) {
+#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 		claim_zero(mlx5_glue->destroy_counter_set(counter->cs));
+#elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
+		claim_zero(mlx5_glue->destroy_counters(counter->cs));
+#endif
 		LIST_REMOVE(counter, next);
 		rte_free(counter);
 	}
@@ -117,14 +171,15 @@
  */
 static int
 flow_verbs_counter_query(struct rte_eth_dev *dev __rte_unused,
-			 struct rte_flow *flow __rte_unused,
-			 void *data __rte_unused,
+			 struct rte_flow *flow, void *data,
 			 struct rte_flow_error *error)
 {
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
+#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
+	defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 	if (flow->actions & MLX5_FLOW_ACTION_COUNT) {
 		struct rte_flow_query_count *qc = data;
 		uint64_t counters[2] = {0, 0};
+#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 		struct ibv_query_counter_set_attr query_cs_attr = {
 			.cs = flow->counter->cs,
 			.query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
@@ -135,7 +190,12 @@
 		};
 		int err = mlx5_glue->query_counter_set(&query_cs_attr,
 						       &query_out);
-
+#elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
+		int err = mlx5_glue->query_counters(
+				flow->counter->cs, counters,
+				RTE_DIM(counters),
+				IBV_READ_COUNTERS_ATTR_PREFER_CACHED);
+#endif
 		if (err)
 			return rte_flow_error_set
 				(error, err,
@@ -157,6 +217,8 @@
 				  NULL,
 				  "flow does not have counter");
 #else
+	(void)flow;
+	(void)data;
 	return rte_flow_error_set(error, ENOTSUP,
 				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 				  NULL,
@@ -993,7 +1055,8 @@
 {
 	const struct rte_flow_action_count *count = action->conf;
 	struct rte_flow *flow = dev_flow->flow;
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
+#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
+	defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 	unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
 	struct ibv_flow_spec_counter_action counter = {
 		.type = IBV_FLOW_SPEC_ACTION_COUNT,
@@ -1012,9 +1075,12 @@
 						  " context.");
 	}
 	*action_flags |= MLX5_FLOW_ACTION_COUNT;
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
+#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 	counter.counter_set_handle = flow->counter->cs->handle;
 	flow_verbs_spec_add(dev_flow, &counter, size);
+#elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
+	counter.counters = flow->counter->cs;
+	flow_verbs_spec_add(dev_flow, &counter, size);
 #endif
 	return 0;
 }
@@ -1277,7 +1343,8 @@
 			detected_actions |= MLX5_FLOW_ACTION_RSS;
 			break;
 		case RTE_FLOW_ACTION_TYPE_COUNT:
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
+#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
+	defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 			size += sizeof(struct ibv_flow_spec_counter_action);
 #endif
 			detected_actions |= MLX5_FLOW_ACTION_COUNT;
-- 
1.8.3.1

^ permalink raw reply	[flat|nested] 39+ messages in thread

* Re: [dpdk-dev] [PATCH v3 1/6] net/mlx5: flow counters object create function bugfix
  2018-10-19 15:21       ` [dpdk-dev] [PATCH v3 1/6] net/mlx5: flow counters object create function bugfix Viacheslav Ovsiienko
@ 2018-10-21  9:20         ` Shahaf Shuler
  0 siblings, 0 replies; 39+ messages in thread
From: Shahaf Shuler @ 2018-10-21  9:20 UTC (permalink / raw)
  To: Slava Ovsiienko, Yongseok Koh; +Cc: dev

Friday, October 19, 2018 6:21 PM, Slava Ovsiienko:
> Subject: [PATCH v3 1/6] net/mlx5: flow counters object create function
> bugfix

How about: "net/mlx5: fix flow counter creation" ?

> 
> The first part of patchset provides the separate commit for bugfix. 

No need for this intro (exists on all patches). After the merge, in the git log no one knows what is a "patch series". 

Flow
> counter object was not freed in case of memory allocation error. The call of
> counter Verbs object deallocating function is added. The initial value of
> reference counter is set to one in order to provide the correct counter object
> freeing in the flow_verbs_counter_release() function.
> 

Missing Cc: stable
Also missing the commit which introduced the issue. On your case need to add:
Fixes: 84c406e74524 ("net/mlx5: add flow translate function")

> Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
> ---
>  drivers/net/mlx5/mlx5_flow_verbs.c | 2 ++
>  1 file changed, 2 insertions(+)
> 
> diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c
> b/drivers/net/mlx5/mlx5_flow_verbs.c
> index 4ae974b..6ddb13b 100644
> --- a/drivers/net/mlx5/mlx5_flow_verbs.c
> +++ b/drivers/net/mlx5/mlx5_flow_verbs.c
> @@ -72,6 +72,7 @@
>  			 }),
>  		.hits = 0,
>  		.bytes = 0,
> +		.ref_cnt = 1,
>  	};
> 
>  	if (!tmpl.cs) {
> @@ -80,6 +81,7 @@
>  	}
>  	cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
>  	if (!cnt) {
> +		claim_zero(mlx5_glue->destroy_counter_set(tmpl.cs));
>  		rte_errno = ENOMEM;
>  		return NULL;
>  	}
> --
> 1.8.3.1

^ permalink raw reply	[flat|nested] 39+ messages in thread

* Re: [dpdk-dev] [PATCH v3 2/6] net/mlx5: flow counters new configuration flags
  2018-10-19 15:21       ` [dpdk-dev] [PATCH v3 2/6] net/mlx5: flow counters new configuration flags Viacheslav Ovsiienko
@ 2018-10-21  9:20         ` Shahaf Shuler
  0 siblings, 0 replies; 39+ messages in thread
From: Shahaf Shuler @ 2018-10-21  9:20 UTC (permalink / raw)
  To: Slava Ovsiienko, Yongseok Koh; +Cc: dev

Friday, October 19, 2018 6:21 PM, Slava Ovsiienko:
> Subject: [PATCH v3 2/6] net/mlx5: flow counters new configuration flags

How about "net/mlx5: rename flow counter configuration macro" ?

> 
> In this part of patchset some modifications in compile configuration flags are
> done. 

Ditto. 

The HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT is replaced with
> HAVE_IBV_DEVICE_COUNTERS_SET_V42. At this stage it is just flag
> renaming. 

Up till here great. You can also add it is a preparation to the new counter support were we will have 2 different configuration macros for the counters. 
The patch should only rename the existing macro. The subsequent patch should add the new macro for the new counter support along with the implementation. 
The below part should be on the commit log of that patch. 

The new HAVE_IBV_DEVICE_COUNTERS_SET_V45 flag is
> introduced. Both makefile and meson.build are changed, the flag
> modifications are grouped, no more build system files modifications are
> expected in this patchset, ones are grouped in this part.
> HAVE_IBV_DEVICE_COUNTERS_SET_V45 is just introduced, no code
> dependence in this part of pathset.
> 
> HAVE_IBV_DEVICE_COUNTERS_SET_V42 - is defined if system supports the
> "old" flow counters functionality, MLNX_OFED version from
> 4.2 to 4.4 is required.
> 
> HAVE_IBV_DEVICE_COUNTERS_SET_V45 - is defined if system supports the
> "new" flow counters functionality, MLNX_OVED 4.5 (or higher) or Linux
> rdma-core v19 (or higher) is required.
> 
> Neither HAVE_IBV_DEVICE_COUNTERS_SET_V42 not
> HAVE_IBV_DEVICE_COUNTERS_SET_V45 is defined if there is no counters
> support.
> 
> Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
> ---
>  drivers/net/mlx5/Makefile          |  9 +++++++--
>  drivers/net/mlx5/meson.build       |  6 ++++--
>  drivers/net/mlx5/mlx5.c            |  4 ++--
>  drivers/net/mlx5/mlx5_flow_verbs.c | 10 +++++-----
>  drivers/net/mlx5/mlx5_glue.c       |  8 ++++----
>  drivers/net/mlx5/mlx5_glue.h       |  2 +-
>  6 files changed, 23 insertions(+), 16 deletions(-)
> 
> diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile index
> 1e9c0b4..28547d3 100644
> --- a/drivers/net/mlx5/Makefile
> +++ b/drivers/net/mlx5/Makefile
> @@ -157,9 +157,14 @@ mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-
> config-h.sh
>  		enum ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT \
>  		$(AUTOCONF_OUTPUT)
>  	$Q sh -- '$<' '$@' \
> -		HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT \
> +		HAVE_IBV_DEVICE_COUNTERS_SET_V42 \

>From this line

> + 		infiniband/verbs.h \
> + 		type 'struct ibv_counter_set_init_attr' \
> + 		$(AUTOCONF_OUTPUT)

Till this one you have an extra space at the start of the line. Need to have all indentation with tabs. 

> +	$Q sh -- '$<' '$@' \
> +		HAVE_IBV_DEVICE_COUNTERS_SET_V45 \
>  		infiniband/verbs.h \
> -		type 'struct ibv_counter_set_init_attr' \
> +		type 'struct ibv_counters_init_attr' \
>  		$(AUTOCONF_OUTPUT)
>  	$Q sh -- '$<' '$@' \
>  		HAVE_RDMA_NL_NLDEV \
> diff --git a/drivers/net/mlx5/meson.build b/drivers/net/mlx5/meson.build
> index c192d44..1c4ed30 100644
> --- a/drivers/net/mlx5/meson.build
> +++ b/drivers/net/mlx5/meson.build
> @@ -79,8 +79,10 @@ if build
>  	has_member_args = [
>  		[ 'HAVE_IBV_MLX5_MOD_SWP', 'infiniband/mlx5dv.h',
>  		'struct mlx5dv_sw_parsing_caps', 'sw_parsing_offloads' ],
> -		[ 'HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT',
> 'infiniband/verbs.h',
> -		'struct ibv_counter_set_init_attr', 'counter_set_id' ],
> +		[ 'HAVE_IBV_DEVICE_COUNTERS_SET_V42',
> 'infiniband/verbs.h',
> + 		'struct ibv_counter_set_init_attr', 'counter_set_id' ],

Same about the extra space. 

> +		[ 'HAVE_IBV_DEVICE_COUNTERS_SET_V45',
> 'infiniband/verbs.h',
> +		'struct ibv_counters_init_attr', 'comp_mask' ],
>  	]
>  	# input array for meson symbol search:
>  	# [ "MACRO to define if found", "header for the search", diff --git
> a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index
> 13f2fd4..bb19085 100644
> --- a/drivers/net/mlx5/mlx5.c
> +++ b/drivers/net/mlx5/mlx5.c
> @@ -739,7 +739,7 @@
>  	unsigned int mprq_max_stride_size_n = 0;
>  	unsigned int mprq_min_stride_num_n = 0;
>  	unsigned int mprq_max_stride_num_n = 0; -#ifdef
> HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
> +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
>  	struct ibv_counter_set_description cs_desc = { .counter_type = 0 };
> #endif
>  	struct ether_addr mac;
> @@ -1009,7 +1009,7 @@
>  	config.hw_csum = !!(attr.device_cap_flags_ex &
> IBV_DEVICE_RAW_IP_CSUM);
>  	DRV_LOG(DEBUG, "checksum offloading is %ssupported",
>  		(config.hw_csum ? "" : "not "));
> -#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
> +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
>  	config.flow_counter_en = !!attr.max_counter_sets;
>  	mlx5_glue->describe_counter_set(ctx, 0, &cs_desc);
>  	DRV_LOG(DEBUG, "counter type = %d, num of cs = %ld, attributes =
> %d", diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c
> b/drivers/net/mlx5/mlx5_flow_verbs.c
> index 6ddb13b..3d6fedb 100644
> --- a/drivers/net/mlx5/mlx5_flow_verbs.c
> +++ b/drivers/net/mlx5/mlx5_flow_verbs.c
> @@ -60,7 +60,7 @@
>  		cnt->ref_cnt++;
>  		return cnt;
>  	}
> -#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
> +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
> 
>  	struct mlx5_flow_counter tmpl = {
>  		.shared = shared,
> @@ -938,7 +938,7 @@
>  {
>  	const struct rte_flow_action_count *count = action->conf;
>  	struct rte_flow *flow = dev_flow->flow; -#ifdef
> HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
> +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
>  	unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
>  	struct ibv_flow_spec_counter_action counter = {
>  		.type = IBV_FLOW_SPEC_ACTION_COUNT,
> @@ -957,7 +957,7 @@
>  						  " context.");
>  	}
>  	*action_flags |= MLX5_FLOW_ACTION_COUNT; -#ifdef
> HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
> +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
>  	counter.counter_set_handle = flow->counter->cs->handle;
>  	flow_verbs_spec_add(dev_flow, &counter, size);  #endif @@ -
> 1222,7 +1222,7 @@
>  			detected_actions |= MLX5_FLOW_ACTION_RSS;
>  			break;
>  		case RTE_FLOW_ACTION_TYPE_COUNT:
> -#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
> +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
>  			size += sizeof(struct ibv_flow_spec_counter_action);
> #endif
>  			detected_actions |= MLX5_FLOW_ACTION_COUNT;
> @@ -1665,7 +1665,7 @@
>  		       void *data __rte_unused,
>  		       struct rte_flow_error *error)
>  {
> -#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
> +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
>  	if (flow->actions & MLX5_FLOW_ACTION_COUNT) {
>  		struct rte_flow_query_count *qc = data;
>  		uint64_t counters[2] = {0, 0};
> diff --git a/drivers/net/mlx5/mlx5_glue.c b/drivers/net/mlx5/mlx5_glue.c
> index 48590df..889e074 100644
> --- a/drivers/net/mlx5/mlx5_glue.c
> +++ b/drivers/net/mlx5/mlx5_glue.c
> @@ -215,7 +215,7 @@
>  mlx5_glue_create_counter_set(struct ibv_context *context,
>  			     struct ibv_counter_set_init_attr *init_attr)  { -
> #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
> +#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
>  	(void)context;
>  	(void)init_attr;
>  	return NULL;
> @@ -227,7 +227,7 @@
>  static int
>  mlx5_glue_destroy_counter_set(struct ibv_counter_set *cs)  { -#ifndef
> HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
> +#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
>  	(void)cs;
>  	return ENOTSUP;
>  #else
> @@ -240,7 +240,7 @@
>  			       uint16_t counter_set_id,
>  			       struct ibv_counter_set_description *cs_desc)  { -
> #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
> +#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
>  	(void)context;
>  	(void)counter_set_id;
>  	(void)cs_desc;
> @@ -254,7 +254,7 @@
>  mlx5_glue_query_counter_set(struct ibv_query_counter_set_attr
> *query_attr,
>  			    struct ibv_counter_set_data *cs_data)  { -#ifndef
> HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
> +#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
>  	(void)query_attr;
>  	(void)cs_data;
>  	return ENOTSUP;
> diff --git a/drivers/net/mlx5/mlx5_glue.h b/drivers/net/mlx5/mlx5_glue.h
> index f6e4e38..adee972 100644
> --- a/drivers/net/mlx5/mlx5_glue.h
> +++ b/drivers/net/mlx5/mlx5_glue.h
> @@ -23,7 +23,7 @@
>  #define MLX5_GLUE_VERSION ""
>  #endif
> 
> -#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
> +#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
>  struct ibv_counter_set;
>  struct ibv_counter_set_data;
>  struct ibv_counter_set_description;
> --
> 1.8.3.1

^ permalink raw reply	[flat|nested] 39+ messages in thread

* Re: [dpdk-dev] [PATCH v3 3/6] net/mlx5: flow counters simplifying runtime support check
  2018-10-19 15:21       ` [dpdk-dev] [PATCH v3 3/6] net/mlx5: flow counters simplifying runtime support check Viacheslav Ovsiienko
@ 2018-10-21  9:20         ` Shahaf Shuler
  0 siblings, 0 replies; 39+ messages in thread
From: Shahaf Shuler @ 2018-10-21  9:20 UTC (permalink / raw)
  To: Slava Ovsiienko, Yongseok Koh; +Cc: dev

Friday, October 19, 2018 6:21 PM, Slava Ovsiienko:
> Subject: [PATCH v3 3/6] net/mlx5: flow counters simplifying runtime support
> check

How about "net/mlx5: simplify flow counters support check

> 
> This part of patchset removes the redundant check of counters support in
> runtime. The flag flow_counter_en is eliminated from the code. The Verbs
> create counter function just returns an error if no counter support presented
> in kernel.
> 
> Some log messages regarding the counter support type and presence are
> added.
> 
> mlx5_flow_validate_action_count() is also updated due to flow_counter_en
> flag removal.

In continue to previous patch comments, this patch should only make the needed preparation for the new counters. 
The new counters macro along with implementation should be on a separate commit. 

> 
> Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
> ---
>  drivers/net/mlx5/mlx5.c      | 15 ++++++++++-----
>  drivers/net/mlx5/mlx5.h      |  1 -
>  drivers/net/mlx5/mlx5_flow.c | 16 +++++++++-------
>  3 files changed, 19 insertions(+), 13 deletions(-)
> 
> diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index
> bb19085..8a33639 100644
> --- a/drivers/net/mlx5/mlx5.c
> +++ b/drivers/net/mlx5/mlx5.c
> @@ -1009,12 +1009,17 @@
>  	config.hw_csum = !!(attr.device_cap_flags_ex &
> IBV_DEVICE_RAW_IP_CSUM);
>  	DRV_LOG(DEBUG, "checksum offloading is %ssupported",
>  		(config.hw_csum ? "" : "not "));
> -#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
> -	config.flow_counter_en = !!attr.max_counter_sets;
> +#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
>  	mlx5_glue->describe_counter_set(ctx, 0, &cs_desc);
> -	DRV_LOG(DEBUG, "counter type = %d, num of cs = %ld, attributes =
> %d",
> -		cs_desc.counter_type, cs_desc.num_of_cs,
> -		cs_desc.attributes);
> +	DRV_LOG(DEBUG, "Flow counters are supported (4.2), "
> +		       "type = %d, num of cs = %ld, attr = %d",
> +		       cs_desc.counter_type,
> +		       cs_desc.num_of_cs,
> +		       cs_desc.attributes);

I would refine on saying the counters are supported (since you cannot be sure). 
I would recommend to have a message log only if the macro is not defined to say "flow counters are not supported". This is the only thing we know for sure. 

> +#elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
> +	DRV_LOG(DEBUG, "Flow counters are supported (4.5)"); #else
> +	DRV_LOG(DEBUG, "Flow counters are not supported");
>  #endif
>  	config.ind_table_max_size =
>  		attr.rss_caps.max_rwq_indirection_table_size;
> diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index
> d14239c..74d87c0 100644
> --- a/drivers/net/mlx5/mlx5.h
> +++ b/drivers/net/mlx5/mlx5.h
> @@ -114,7 +114,6 @@ struct mlx5_dev_config {
>  	unsigned int tunnel_en:1;
>  	/* Whether tunnel stateless offloads are supported. */
>  	unsigned int mpls_en:1; /* MPLS over GRE/UDP is enabled. */
> -	unsigned int flow_counter_en:1; /* Whether flow counter is
> supported. */
>  	unsigned int cqe_comp:1; /* CQE compression is enabled. */
>  	unsigned int tso:1; /* Whether TSO is supported. */
>  	unsigned int tx_vec_en:1; /* Tx vector is enabled. */ diff --git
> a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c index
> fcabab0..c15722d 100644
> --- a/drivers/net/mlx5/mlx5_flow.c
> +++ b/drivers/net/mlx5/mlx5_flow.c
> @@ -921,22 +921,24 @@ uint32_t mlx5_flow_adjust_priority(struct
> rte_eth_dev *dev, int32_t priority,
>   *   0 on success, a negative errno value otherwise and rte_ernno is set.
>   */
>  int
> -mlx5_flow_validate_action_count(struct rte_eth_dev *dev,
> +mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused,
>  				const struct rte_flow_attr *attr,
>  				struct rte_flow_error *error)
>  {
> -	struct priv *priv = dev->data->dev_private;
> -
> -	if (!priv->config.flow_counter_en)
> -		return rte_flow_error_set(error, ENOTSUP,
> -					  RTE_FLOW_ERROR_TYPE_ACTION,
> NULL,
> -					  "flow counters are not supported.");
> +#if !defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) && \
> +	!defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
> +	(void)attr;
> +	return rte_flow_error_set(error, ENOTSUP,
> +				  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
> +				  "flow counters are not supported."); #else

This ifdef addition is not needed. Just let It fall on the counter creation (like you commit log says). 

>  	if (attr->egress)
>  		return rte_flow_error_set(error, ENOTSUP,
> 
> RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
>  					  "count action not supported for "
>  					  "egress");
>  	return 0;
> +#endif
>  }
> 
>  /**
> --
> 1.8.3.1

^ permalink raw reply	[flat|nested] 39+ messages in thread

* Re: [dpdk-dev] [PATCH v3 4/6] net/mlx5: flow counters mlx5 glue library update
  2018-10-19 15:21       ` [dpdk-dev] [PATCH v3 4/6] net/mlx5: flow counters mlx5 glue library update Viacheslav Ovsiienko
@ 2018-10-21  9:20         ` Shahaf Shuler
  0 siblings, 0 replies; 39+ messages in thread
From: Shahaf Shuler @ 2018-10-21  9:20 UTC (permalink / raw)
  To: Slava Ovsiienko, Yongseok Koh; +Cc: dev

Friday, October 19, 2018 6:21 PM, Slava Ovsiienko:
> Subject: [PATCH v3 4/6] net/mlx5: flow counters mlx5 glue library update
> 

How about: "net/mlx5: add new flow counter verbs API to glue library

> This part of patchset updates the mlx5 glue library, new counter support
> function pointers are added to the glue linking structure mlx5_glue. This
> structure now contains the pointers to the both versions of counter
> supporting functions due to compatibility issues. Depending on configuration
> flags the functions perform actual Verbs library calls or return an error with
> meaning "feature is not supported" (NULL or ENOTSUP).
> 

>From previous patch comments, this patch should declare the new macro (V45)

> Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
> ---
>  drivers/net/mlx5/mlx5_glue.c | 60
> ++++++++++++++++++++++++++++++++++++++++++++
>  drivers/net/mlx5/mlx5_glue.h | 17 +++++++++++++
>  2 files changed, 77 insertions(+)
> 
> diff --git a/drivers/net/mlx5/mlx5_glue.c b/drivers/net/mlx5/mlx5_glue.c
> index 889e074..1afb114 100644
> --- a/drivers/net/mlx5/mlx5_glue.c
> +++ b/drivers/net/mlx5/mlx5_glue.c
> @@ -263,6 +263,62 @@
>  #endif
>  }
> 
> +static struct ibv_counters *
> +mlx5_glue_create_counters(struct ibv_context *context,
> +			  struct ibv_counters_init_attr *init_attr) { #ifndef
> +HAVE_IBV_DEVICE_COUNTERS_SET_V45
> +	(void)context;
> +	(void)init_attr;
> +	return NULL;
> +#else
> +	return ibv_create_counters(context, init_attr); #endif }
> +
> +static int
> +mlx5_glue_destroy_counters(struct ibv_counters *counters) { #ifndef
> +HAVE_IBV_DEVICE_COUNTERS_SET_V45
> +	(void)counters;
> +	return ENOTSUP;
> +#else
> +	return ibv_destroy_counters(counters); #endif }
> +
> +static int
> +mlx5_glue_attach_counters(struct ibv_counters *counters,
> +			  struct ibv_counter_attach_attr *attr,
> +			  struct ibv_flow *flow)
> +{
> +#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
> +	(void)counters;
> +	(void)attr;
> +	(void)flow;
> +	return ENOTSUP;
> +#else
> +	return ibv_attach_counters_point_flow(counters, attr, flow); #endif
> }
> +
> +static int
> +mlx5_glue_query_counters(struct ibv_counters *counters,
> +			 uint64_t *counters_value,
> +			 uint32_t ncounters,
> +			 uint32_t flags)
> +{
> +#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
> +	(void)counters;
> +	(void)counters_value;
> +	(void)ncounters;
> +	(void)flags;
> +	return ENOTSUP;
> +#else
> +	return ibv_read_counters(counters, counters_value, ncounters,
> flags);
> +#endif }
> +
>  static void
>  mlx5_glue_ack_async_event(struct ibv_async_event *event)  { @@ -424,6
> +480,10 @@
>  	.destroy_counter_set = mlx5_glue_destroy_counter_set,
>  	.describe_counter_set = mlx5_glue_describe_counter_set,
>  	.query_counter_set = mlx5_glue_query_counter_set,
> +	.create_counters = mlx5_glue_create_counters,
> +	.destroy_counters = mlx5_glue_destroy_counters,
> +	.attach_counters = mlx5_glue_attach_counters,
> +	.query_counters = mlx5_glue_query_counters,
>  	.ack_async_event = mlx5_glue_ack_async_event,
>  	.get_async_event = mlx5_glue_get_async_event,
>  	.port_state_str = mlx5_glue_port_state_str, diff --git
> a/drivers/net/mlx5/mlx5_glue.h b/drivers/net/mlx5/mlx5_glue.h index
> adee972..44bfefe 100644
> --- a/drivers/net/mlx5/mlx5_glue.h
> +++ b/drivers/net/mlx5/mlx5_glue.h
> @@ -31,6 +31,12 @@
>  struct ibv_query_counter_set_attr;
>  #endif
> 
> +#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45 struct ibv_counters; struct
> +ibv_counters_init_attr; struct ibv_counter_attach_attr; #endif
> +
>  #ifndef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
>  struct mlx5dv_qp_init_attr;
>  #endif
> @@ -106,6 +112,17 @@ struct mlx5_glue {
>  		 struct ibv_counter_set_description *cs_desc);
>  	int (*query_counter_set)(struct ibv_query_counter_set_attr
> *query_attr,
>  				 struct ibv_counter_set_data *cs_data);
> +	struct ibv_counters *(*create_counters)
> +		(struct ibv_context *context,
> +		 struct ibv_counters_init_attr *init_attr);
> +	int (*destroy_counters)(struct ibv_counters *counters);
> +	int (*attach_counters)(struct ibv_counters *counters,
> +			       struct ibv_counter_attach_attr *attr,
> +			       struct ibv_flow *flow);
> +	int (*query_counters)(struct ibv_counters *counters,
> +			      uint64_t *counters_value,
> +			      uint32_t ncounters,
> +			      uint32_t flags);
>  	void (*ack_async_event)(struct ibv_async_event *event);
>  	int (*get_async_event)(struct ibv_context *context,
>  			       struct ibv_async_event *event);
> --
> 1.8.3.1

^ permalink raw reply	[flat|nested] 39+ messages in thread

* Re: [dpdk-dev] [PATCH v3 5/6] net/mlx5: flow counters query function move and rename
  2018-10-19 15:21       ` [dpdk-dev] [PATCH v3 5/6] net/mlx5: flow counters query function move and rename Viacheslav Ovsiienko
@ 2018-10-21  9:20         ` Shahaf Shuler
  0 siblings, 0 replies; 39+ messages in thread
From: Shahaf Shuler @ 2018-10-21  9:20 UTC (permalink / raw)
  To: Slava Ovsiienko, Yongseok Koh; +Cc: dev

Friday, October 19, 2018 6:21 PM, Slava Ovsiienko:
> Subject: [PATCH v3 5/6] net/mlx5: flow counters query function move and
> rename

How about: "net/mlx5: relocate flow counter query function" 

This patch should come before the glue library update

> 
> The flow_verbs_query_count() is moved into the the group of counter
> managing functions and renamed to flow_verbs_counter_query() in order to
> be in unified fashion with others.

The tcf engine has the same naming. However since you add new ones on the next commit it is OK. 
Will think if we want to rename tcf one later. 

> 
> Also minor function modification is made to avoid unreachable code warnings
> if there is no counter support at compile time.
> 
> Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
> ---
>  drivers/net/mlx5/mlx5_flow_verbs.c | 111 +++++++++++++++++++----------
> --------
>  1 file changed, 56 insertions(+), 55 deletions(-)
> 
> diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c
> b/drivers/net/mlx5/mlx5_flow_verbs.c
> index 3d6fedb..f720c35 100644
> --- a/drivers/net/mlx5/mlx5_flow_verbs.c
> +++ b/drivers/net/mlx5/mlx5_flow_verbs.c
> @@ -110,6 +110,61 @@
>  }
> 
>  /**
> + * Query a flow counter via Verbs library call.
> + *
> + * @see rte_flow_query()
> + * @see rte_flow_ops
> + */
> +static int
> +flow_verbs_counter_query(struct rte_eth_dev *dev __rte_unused,
> +			 struct rte_flow *flow __rte_unused,
> +			 void *data __rte_unused,
> +			 struct rte_flow_error *error)
> +{
> +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
> +	if (flow->actions & MLX5_FLOW_ACTION_COUNT) {
> +		struct rte_flow_query_count *qc = data;
> +		uint64_t counters[2] = {0, 0};
> +		struct ibv_query_counter_set_attr query_cs_attr = {
> +			.cs = flow->counter->cs,
> +			.query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
> +		};
> +		struct ibv_counter_set_data query_out = {
> +			.out = counters,
> +			.outlen = 2 * sizeof(uint64_t),
> +		};
> +		int err = mlx5_glue->query_counter_set(&query_cs_attr,
> +						       &query_out);
> +
> +		if (err)
> +			return rte_flow_error_set
> +				(error, err,
> +				 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
> +				 NULL,
> +				 "cannot read counter");
> +		qc->hits_set = 1;
> +		qc->bytes_set = 1;
> +		qc->hits = counters[0] - flow->counter->hits;
> +		qc->bytes = counters[1] - flow->counter->bytes;
> +		if (qc->reset) {
> +			flow->counter->hits = counters[0];
> +			flow->counter->bytes = counters[1];
> +		}
> +		return 0;
> +	}
> +	return rte_flow_error_set(error, EINVAL,
> +				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
> +				  NULL,
> +				  "flow does not have counter");
> +#else
> +	return rte_flow_error_set(error, ENOTSUP,
> +				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
> +				  NULL,
> +				  "counters are not available");
> +#endif
> +}
> +
> +/**
>   * Add a verbs item specification into @p flow.
>   *
>   * @param[in, out] flow
> @@ -1654,60 +1709,6 @@
>  }
> 
>  /**
> - * Query a flows.
> - *
> - * @see rte_flow_query()
> - * @see rte_flow_ops
> - */
> -static int
> -flow_verbs_query_count(struct rte_eth_dev *dev __rte_unused,
> -		       struct rte_flow *flow __rte_unused,
> -		       void *data __rte_unused,
> -		       struct rte_flow_error *error)
> -{
> -#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
> -	if (flow->actions & MLX5_FLOW_ACTION_COUNT) {
> -		struct rte_flow_query_count *qc = data;
> -		uint64_t counters[2] = {0, 0};
> -		struct ibv_query_counter_set_attr query_cs_attr = {
> -			.cs = flow->counter->cs,
> -			.query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
> -		};
> -		struct ibv_counter_set_data query_out = {
> -			.out = counters,
> -			.outlen = 2 * sizeof(uint64_t),
> -		};
> -		int err = mlx5_glue->query_counter_set(&query_cs_attr,
> -						       &query_out);
> -
> -		if (err)
> -			return rte_flow_error_set
> -				(error, err,
> -				 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
> -				 NULL,
> -				 "cannot read counter");
> -		qc->hits_set = 1;
> -		qc->bytes_set = 1;
> -		qc->hits = counters[0] - flow->counter->hits;
> -		qc->bytes = counters[1] - flow->counter->bytes;
> -		if (qc->reset) {
> -			flow->counter->hits = counters[0];
> -			flow->counter->bytes = counters[1];
> -		}
> -		return 0;
> -	}
> -	return rte_flow_error_set(error, EINVAL,
> -				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
> -				  NULL,
> -				  "flow does not have counter");
> -#endif
> -	return rte_flow_error_set(error, ENOTSUP,
> -				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
> -				  NULL,
> -				  "counters are not available");
> -}
> -
> -/**
>   * Query a flow.
>   *
>   * @see rte_flow_query()
> @@ -1727,7 +1728,7 @@
>  		case RTE_FLOW_ACTION_TYPE_VOID:
>  			break;
>  		case RTE_FLOW_ACTION_TYPE_COUNT:
> -			ret = flow_verbs_query_count(dev, flow, data,
> error);
> +			ret = flow_verbs_counter_query(dev, flow, data,
> error);
>  			break;
>  		default:
>  			return rte_flow_error_set(error, ENOTSUP,
> --
> 1.8.3.1

^ permalink raw reply	[flat|nested] 39+ messages in thread

* Re: [dpdk-dev] [PATCH v3 6/6] net/mlx5: flow counters Verbs interface functions update
  2018-10-19 15:21       ` [dpdk-dev] [PATCH v3 6/6] net/mlx5: flow counters Verbs interface functions update Viacheslav Ovsiienko
@ 2018-10-21  9:20         ` Shahaf Shuler
  0 siblings, 0 replies; 39+ messages in thread
From: Shahaf Shuler @ 2018-10-21  9:20 UTC (permalink / raw)
  To: Slava Ovsiienko, Yongseok Koh; +Cc: dev

Friday, October 19, 2018 6:21 PM, Slava Ovsiienko:
> Subject: [PATCH v3 6/6] net/mlx5: flow counters Verbs interface functions
> update

How about: "net/mlx5: support new flow counter API" 

> 
> This part of patchset updates the functions performing the Verbs library calls
> in order to support different versions of library.
> The functions:
>   - flow_verbs_counter_new()
>   - flow_verbs_counter_release()
>   - flow_verbs_counter_query()
> now have the several compilations branches, depending on the counter
> support found in the system at compile time.
> 
> The flow_verbs_counter_create() function is introduced as helper for
> flow_verbs_counter_new(), actually this helper create the counters with
> Verbs.
> 
> Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
> ---
>  drivers/net/mlx5/mlx5_flow.h       |   6 ++
>  drivers/net/mlx5/mlx5_flow_verbs.c | 129
> ++++++++++++++++++++++++++++---------
>  2 files changed, 104 insertions(+), 31 deletions(-)
> 
> diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
> index 69f55cf..44c7515 100644
> --- a/drivers/net/mlx5/mlx5_flow.h
> +++ b/drivers/net/mlx5/mlx5_flow.h
> @@ -224,7 +224,13 @@ struct mlx5_flow_counter {
>  	uint32_t shared:1; /**< Share counter ID with other flow rules. */
>  	uint32_t ref_cnt:31; /**< Reference counter. */
>  	uint32_t id; /**< Counter ID. */
> +#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
>  	struct ibv_counter_set *cs; /**< Holds the counters for the rule. */
> +#elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
> +	struct ibv_counters *cs; /**< Holds the counters for the rule. */
> +#else
> +	void *cs;

Why you need this else? 

> +#endif
>  	uint64_t hits; /**< Number of packets matched by the rule. */
>  	uint64_t bytes; /**< Number of bytes matched by the rule. */  }; diff
> --git a/drivers/net/mlx5/mlx5_flow_verbs.c
> b/drivers/net/mlx5/mlx5_flow_verbs.c
> index f720c35..b657933 100644
> --- a/drivers/net/mlx5/mlx5_flow_verbs.c
> +++ b/drivers/net/mlx5/mlx5_flow_verbs.c
> @@ -33,6 +33,59 @@
>  #include "mlx5_glue.h"
>  #include "mlx5_flow.h"
> 
> +
> +#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
> +	defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)

As I already commented on the previous version. Ifdef are always inside the function body, and not before the function declaration. 
If no support for counters the function should return  errno. 

> +/**
> + * Create Verbs flow counter with Verbs library.
> + *
> + * @param[in] dev
> + *   Pointer to the Ethernet device structure.
> + * @param[in, out] counter
> + *   PMD flow counter object, contains the counter id,
> + *   handle of created Verbs flow counter is returned in cs field.

The struct maybe will change in the future, but the doc isn't. better to say "mlx5 flow counter object" 

> + *
> + * @return
> + *   counter->cs contains a handle of created Verbs counter,
> + *   NULL if error occurred and rte_errno is set.

How can you return NULL if the function returns void? 

It seems this function needs to return an integer. 

> + */
> +static void
> +flow_verbs_counter_create(struct rte_eth_dev *dev,
> +			  struct mlx5_flow_counter *counter) { #if
> +defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
> +	struct priv *priv = dev->data->dev_private;
> +	struct ibv_counter_set_init_attr init = {
> +			 .counter_set_id = counter->id};
> +
> +	counter->cs = mlx5_glue->create_counter_set(priv->ctx, &init); #elif
> +defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
> +	struct priv *priv = dev->data->dev_private;
> +	struct ibv_counters_init_attr init = {0};
> +	struct ibv_counter_attach_attr attach = {0};
> +	int ret;
> +
> +	counter->cs = mlx5_glue->create_counters(priv->ctx, &init);
> +	if (!counter->cs)
> +		return;
> +	attach.counter_desc = IBV_COUNTER_PACKETS;
> +	attach.index = 0;
> +	ret = mlx5_glue->attach_counters(counter->cs, &attach, NULL);
> +	if (!ret) {
> +		attach.counter_desc = IBV_COUNTER_BYTES;
> +		attach.index = 1;
> +		ret = mlx5_glue->attach_counters
> +					(counter->cs, &attach, NULL);
> +	}
> +	if (ret) {
> +		claim_zero(mlx5_glue->destroy_counters(counter->cs));
> +		counter->cs = NULL;
> +		rte_errno = ret;
> +	}
> +#endif
> +}
> +#endif
> +
>  /**
>   * Get a flow counter.
>   *
> @@ -49,6 +102,8 @@
>  static struct mlx5_flow_counter *
>  flow_verbs_counter_new(struct rte_eth_dev *dev, uint32_t shared,
> uint32_t id)  {
> +#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
> +	defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)

No explicit need for this ifdef, the flow_verbs_counter_create will return error if the counters are not supported. This is a duplicate check. 

>  	struct priv *priv = dev->data->dev_private;
>  	struct mlx5_flow_counter *cnt;
> 
> @@ -60,37 +115,32 @@
>  		cnt->ref_cnt++;
>  		return cnt;
>  	}
> -#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
> -
> -	struct mlx5_flow_counter tmpl = {
> -		.shared = shared,
> -		.id = id,
> -		.cs = mlx5_glue->create_counter_set
> -			(priv->ctx,
> -			 &(struct ibv_counter_set_init_attr){
> -				 .counter_set_id = id,
> -			 }),
> -		.hits = 0,
> -		.bytes = 0,
> -		.ref_cnt = 1,
> -	};
> -
> -	if (!tmpl.cs) {
> -		rte_errno = errno;
> -		return NULL;
> -	}
>  	cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
>  	if (!cnt) {
> -		claim_zero(mlx5_glue->destroy_counter_set(tmpl.cs));
>  		rte_errno = ENOMEM;
>  		return NULL;
>  	}
> -	*cnt = tmpl;
> -	LIST_INSERT_HEAD(&priv->flow_counters, cnt, next);
> -	return cnt;
> -#endif
> +	cnt->id = id;
> +	cnt->shared = shared;
> +	cnt->ref_cnt = 1;
> +	cnt->hits = 0;
> +	cnt->bytes = 0;
> +	/* Create counter with Verbs. */
> +	flow_verbs_counter_create(dev, cnt);
> +	if (cnt->cs) {
> +		LIST_INSERT_HEAD(&priv->flow_counters, cnt, next);
> +		return cnt;
> +	}
> +	/* Some error occurred in Verbs library, rte_errno is set. */
> +	rte_free(cnt);
> +	return NULL;
> +#else
> +	(void)dev;
> +	(void)shared;
> +	(void)id;
>  	rte_errno = ENOTSUP;
>  	return NULL;
> +#endif
>  }
> 
>  /**
> @@ -103,7 +153,11 @@
>  flow_verbs_counter_release(struct mlx5_flow_counter *counter)  {
>  	if (--counter->ref_cnt == 0) {
> +#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
>  		claim_zero(mlx5_glue->destroy_counter_set(counter->cs));
> +#elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
> +		claim_zero(mlx5_glue->destroy_counters(counter->cs));
> +#endif
>  		LIST_REMOVE(counter, next);
>  		rte_free(counter);
>  	}
> @@ -117,14 +171,15 @@
>   */
>  static int
>  flow_verbs_counter_query(struct rte_eth_dev *dev __rte_unused,
> -			 struct rte_flow *flow __rte_unused,
> -			 void *data __rte_unused,
> +			 struct rte_flow *flow, void *data,
>  			 struct rte_flow_error *error)
>  {
> -#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
> +#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
> +	defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
>  	if (flow->actions & MLX5_FLOW_ACTION_COUNT) {
>  		struct rte_flow_query_count *qc = data;
>  		uint64_t counters[2] = {0, 0};
> +#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
>  		struct ibv_query_counter_set_attr query_cs_attr = {
>  			.cs = flow->counter->cs,
>  			.query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
> @@ -135,7 +190,12 @@
>  		};
>  		int err = mlx5_glue->query_counter_set(&query_cs_attr,
>  						       &query_out);
> -
> +#elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
> +		int err = mlx5_glue->query_counters(
> +				flow->counter->cs, counters,
> +				RTE_DIM(counters),
> +
> 	IBV_READ_COUNTERS_ATTR_PREFER_CACHED);
> +#endif
>  		if (err)
>  			return rte_flow_error_set
>  				(error, err,
> @@ -157,6 +217,8 @@
>  				  NULL,
>  				  "flow does not have counter");
>  #else
> +	(void)flow;
> +	(void)data;
>  	return rte_flow_error_set(error, ENOTSUP,
>  				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
>  				  NULL,
> @@ -993,7 +1055,8 @@
>  {
>  	const struct rte_flow_action_count *count = action->conf;
>  	struct rte_flow *flow = dev_flow->flow; -#ifdef
> HAVE_IBV_DEVICE_COUNTERS_SET_V42
> +#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
> +	defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
>  	unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
>  	struct ibv_flow_spec_counter_action counter = {
>  		.type = IBV_FLOW_SPEC_ACTION_COUNT,
> @@ -1012,9 +1075,12 @@
>  						  " context.");
>  	}
>  	*action_flags |= MLX5_FLOW_ACTION_COUNT; -#ifdef
> HAVE_IBV_DEVICE_COUNTERS_SET_V42
> +#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
>  	counter.counter_set_handle = flow->counter->cs->handle;
>  	flow_verbs_spec_add(dev_flow, &counter, size);
> +#elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
> +	counter.counters = flow->counter->cs;
> +	flow_verbs_spec_add(dev_flow, &counter, size);
>  #endif
>  	return 0;
>  }
> @@ -1277,7 +1343,8 @@
>  			detected_actions |= MLX5_FLOW_ACTION_RSS;
>  			break;
>  		case RTE_FLOW_ACTION_TYPE_COUNT:
> -#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
> +#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
> +	defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
>  			size += sizeof(struct ibv_flow_spec_counter_action);
> #endif
>  			detected_actions |= MLX5_FLOW_ACTION_COUNT;
> --
> 1.8.3.1

^ permalink raw reply	[flat|nested] 39+ messages in thread

* Re: [dpdk-dev] [PATCH v3 0/6] net/mlx5: flow counters support for Linux-rdma v19
  2018-10-19 15:21     ` [dpdk-dev] [PATCH v3 0/6] net/mlx5: flow counters support for Linux-rdma v19 Viacheslav Ovsiienko
                         ` (5 preceding siblings ...)
  2018-10-19 15:21       ` [dpdk-dev] [PATCH v3 6/6] net/mlx5: flow counters Verbs interface functions update Viacheslav Ovsiienko
@ 2018-10-21  9:21       ` Shahaf Shuler
  2018-10-23 10:04       ` [dpdk-dev] " Slava Ovsiienko
       [not found]       ` <1540289541-30019-1-git-send-email-viacheslavo@mellanox.com>
  8 siblings, 0 replies; 39+ messages in thread
From: Shahaf Shuler @ 2018-10-21  9:21 UTC (permalink / raw)
  To: Slava Ovsiienko, Yongseok Koh; +Cc: dev

Hi Slava, 

Friday, October 19, 2018 6:21 PM, Slava Ovsiienko:
> Subject: [PATCH v3 0/6] net/mlx5: flow counters support for Linux-rdma v19
> 
> Mellanox mlx5 PMD supports Flow counters via Verbs library.
> The current implementation is based on the Mellanox proprietary Verbs
> library included in MLNX OFED packages. The Flow counter support is
> recently added into linux-rdma release (v19), so the mlx5 PMD update is
> needed to provide Counter feature on the base of linux-rdma.
> 
> mlx5 PMD can be compiled with MLNX OFED or linux-rdma v19+ and provide
> flow counters for both.
> 
> Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>

Good rework of the series, we are close. Just several more comments and we are done. 

^ permalink raw reply	[flat|nested] 39+ messages in thread

* [dpdk-dev] net/mlx5: flow counters support for Linux-rdma v19
  2018-10-19 15:21     ` [dpdk-dev] [PATCH v3 0/6] net/mlx5: flow counters support for Linux-rdma v19 Viacheslav Ovsiienko
                         ` (6 preceding siblings ...)
  2018-10-21  9:21       ` [dpdk-dev] [PATCH v3 0/6] net/mlx5: flow counters support for Linux-rdma v19 Shahaf Shuler
@ 2018-10-23 10:04       ` Slava Ovsiienko
  2018-10-23 10:04         ` [dpdk-dev] [PATCH v4 1/8] net/mlx5: fix flow counters creation Slava Ovsiienko
                           ` (7 more replies)
       [not found]       ` <1540289541-30019-1-git-send-email-viacheslavo@mellanox.com>
  8 siblings, 8 replies; 39+ messages in thread
From: Slava Ovsiienko @ 2018-10-23 10:04 UTC (permalink / raw)
  To: Slava Ovsiienko; +Cc: dev

Mellanox mlx5 PMD supports Flow counters via Verbs library.
The current implementation is based on the Mellanox proprietary
Verbs library included in MLNX OFED packages. The Flow counter
support is recently added into linux-rdma release (v19),
so the mlx5 PMD update is needed to provide Counter feature
on the base of linux-rdma.

mlx5 PMD can be compiled with MLNX OFED or linux-rdma v19+
and provide flow counters for both.

Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>

---
v4:
- minor patcset parts reorganization
- rewritten headlines to be more clear
- ifdef blocks minor cleanups

v3:
- http://patches.dpdk.org/patch/47085/
- mlx5 glue issue resolved correctly
- patch is reorganized info small isolated parts
	  
v2:
- http://patches.dpdk.org/patch/46989/
- rebased on top of master-net-mlx branch
- new compilation flags are introduced:
- HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V42, kernel/verbs
  library provides the flow counter support in style of
  MLNX_OFED_4.2 to MLNX_OFED_4.4
- HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45, kernel/verbs
  library provides the flow counter support in style of
  MLNX_OFED_4.5 or higher

v1:
- http://patches.dpdk.org/patch/45972/

Viacheslav Ovsiienko (8):
  net/mlx5: fix flow counters creation
  net/mlx5: rename flow counter configuration macro
  net/mlx5: introduce new flow counters configuration macro
  net/mlx5: simplify flow counters support check
  net/mlx5: relocate flow counters query function
  net/mlx5: add new flow counter Verbs API to glue library
  net/mlx5: remove unnecessary structure initializers
  net/mlx5: support new flow counter API

 drivers/net/mlx5/Makefile          |   7 +-
 drivers/net/mlx5/meson.build       |   4 +-
 drivers/net/mlx5/mlx5.c            |  12 +-
 drivers/net/mlx5/mlx5.h            |   1 -
 drivers/net/mlx5/mlx5_flow.c       |   8 +-
 drivers/net/mlx5/mlx5_flow.h       |   4 +
 drivers/net/mlx5/mlx5_flow_tcf.c   |   5 -
 drivers/net/mlx5/mlx5_flow_verbs.c | 236 ++++++++++++++++++++++++-------------
 drivers/net/mlx5/mlx5_glue.c       |  68 ++++++++++-
 drivers/net/mlx5/mlx5_glue.h       |  19 ++-
 10 files changed, 254 insertions(+), 110 deletions(-)

-- 
1.8.3.1

^ permalink raw reply	[flat|nested] 39+ messages in thread

* [dpdk-dev] [PATCH v4 1/8] net/mlx5: fix flow counters creation
  2018-10-23 10:04       ` [dpdk-dev] " Slava Ovsiienko
@ 2018-10-23 10:04         ` Slava Ovsiienko
  2018-10-23 10:04         ` [dpdk-dev] [PATCH v4 2/8] net/mlx5: rename flow counter configuration macro Slava Ovsiienko
                           ` (6 subsequent siblings)
  7 siblings, 0 replies; 39+ messages in thread
From: Slava Ovsiienko @ 2018-10-23 10:04 UTC (permalink / raw)
  To: Slava Ovsiienko; +Cc: dev, stable

The Flow counter creation function contains two problems:

  - Flow counter object in Verbs is not freed in case of memory
    allocation error. The call of counter Verbs object deallocating
    function is added to fix.

  - The initial value of reference counter is set to one in order
    to provide the correct counter object freeing in the
    flow_verbs_counter_release() function. The reference counter
    field should be initialized to one.

Fixes: 60bd8c9747e8 ("net/mlx5: add count flow action")
Cc: stable@dpdk.org

Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
---
 drivers/net/mlx5/mlx5_flow_verbs.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c b/drivers/net/mlx5/mlx5_flow_verbs.c
index 4ae974b..6ddb13b 100644
--- a/drivers/net/mlx5/mlx5_flow_verbs.c
+++ b/drivers/net/mlx5/mlx5_flow_verbs.c
@@ -72,6 +72,7 @@
 			 }),
 		.hits = 0,
 		.bytes = 0,
+		.ref_cnt = 1,
 	};
 
 	if (!tmpl.cs) {
@@ -80,6 +81,7 @@
 	}
 	cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
 	if (!cnt) {
+		claim_zero(mlx5_glue->destroy_counter_set(tmpl.cs));
 		rte_errno = ENOMEM;
 		return NULL;
 	}
-- 
1.8.3.1

^ permalink raw reply	[flat|nested] 39+ messages in thread

* [dpdk-dev] [PATCH v4 2/8] net/mlx5: rename flow counter configuration macro
  2018-10-23 10:04       ` [dpdk-dev] " Slava Ovsiienko
  2018-10-23 10:04         ` [dpdk-dev] [PATCH v4 1/8] net/mlx5: fix flow counters creation Slava Ovsiienko
@ 2018-10-23 10:04         ` Slava Ovsiienko
  2018-10-23 10:04         ` [dpdk-dev] [PATCH v4 3/8] net/mlx5: introduce new flow counters " Slava Ovsiienko
                           ` (5 subsequent siblings)
  7 siblings, 0 replies; 39+ messages in thread
From: Slava Ovsiienko @ 2018-10-23 10:04 UTC (permalink / raw)
  To: Slava Ovsiienko; +Cc: dev

The HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT is replaced with
HAVE_IBV_DEVICE_COUNTERS_SET_V42. At this stage it is just
macro renaming. This macro is defined if system supports
the "old" Flow counters functionality, MLNX_OFED version
from 4.2 to 4.4 is required.

We need to do this preparation before introducing the new
configuration macro (HAVE_IBV_DEVICE_COUNTERS_SET_V45) for
the "new" Flow counters support.

Both makefile and meson.build are changed.

Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
---
 drivers/net/mlx5/Makefile          |  2 +-
 drivers/net/mlx5/meson.build       |  2 +-
 drivers/net/mlx5/mlx5.c            |  4 ++--
 drivers/net/mlx5/mlx5_flow_verbs.c | 10 +++++-----
 drivers/net/mlx5/mlx5_glue.c       |  8 ++++----
 drivers/net/mlx5/mlx5_glue.h       |  2 +-
 6 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile
index 1e9c0b4..124ee4e 100644
--- a/drivers/net/mlx5/Makefile
+++ b/drivers/net/mlx5/Makefile
@@ -157,7 +157,7 @@ mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
 		enum ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT \
 		$(AUTOCONF_OUTPUT)
 	$Q sh -- '$<' '$@' \
-		HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT \
+		HAVE_IBV_DEVICE_COUNTERS_SET_V42 \
 		infiniband/verbs.h \
 		type 'struct ibv_counter_set_init_attr' \
 		$(AUTOCONF_OUTPUT)
diff --git a/drivers/net/mlx5/meson.build b/drivers/net/mlx5/meson.build
index c192d44..e613a21 100644
--- a/drivers/net/mlx5/meson.build
+++ b/drivers/net/mlx5/meson.build
@@ -79,7 +79,7 @@ if build
 	has_member_args = [
 		[ 'HAVE_IBV_MLX5_MOD_SWP', 'infiniband/mlx5dv.h',
 		'struct mlx5dv_sw_parsing_caps', 'sw_parsing_offloads' ],
-		[ 'HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT', 'infiniband/verbs.h',
+		[ 'HAVE_IBV_DEVICE_COUNTERS_SET_V42', 'infiniband/verbs.h',
 		'struct ibv_counter_set_init_attr', 'counter_set_id' ],
 	]
 	# input array for meson symbol search:
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index cfff54e..cb86a69 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -739,7 +739,7 @@
 	unsigned int mprq_max_stride_size_n = 0;
 	unsigned int mprq_min_stride_num_n = 0;
 	unsigned int mprq_max_stride_num_n = 0;
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 	struct ibv_counter_set_description cs_desc = { .counter_type = 0 };
 #endif
 	struct ether_addr mac;
@@ -1009,7 +1009,7 @@
 	config.hw_csum = !!(attr.device_cap_flags_ex & IBV_DEVICE_RAW_IP_CSUM);
 	DRV_LOG(DEBUG, "checksum offloading is %ssupported",
 		(config.hw_csum ? "" : "not "));
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 	config.flow_counter_en = !!attr.max_counter_sets;
 	mlx5_glue->describe_counter_set(ctx, 0, &cs_desc);
 	DRV_LOG(DEBUG, "counter type = %d, num of cs = %ld, attributes = %d",
diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c b/drivers/net/mlx5/mlx5_flow_verbs.c
index 6ddb13b..3d6fedb 100644
--- a/drivers/net/mlx5/mlx5_flow_verbs.c
+++ b/drivers/net/mlx5/mlx5_flow_verbs.c
@@ -60,7 +60,7 @@
 		cnt->ref_cnt++;
 		return cnt;
 	}
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 
 	struct mlx5_flow_counter tmpl = {
 		.shared = shared,
@@ -938,7 +938,7 @@
 {
 	const struct rte_flow_action_count *count = action->conf;
 	struct rte_flow *flow = dev_flow->flow;
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 	unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
 	struct ibv_flow_spec_counter_action counter = {
 		.type = IBV_FLOW_SPEC_ACTION_COUNT,
@@ -957,7 +957,7 @@
 						  " context.");
 	}
 	*action_flags |= MLX5_FLOW_ACTION_COUNT;
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 	counter.counter_set_handle = flow->counter->cs->handle;
 	flow_verbs_spec_add(dev_flow, &counter, size);
 #endif
@@ -1222,7 +1222,7 @@
 			detected_actions |= MLX5_FLOW_ACTION_RSS;
 			break;
 		case RTE_FLOW_ACTION_TYPE_COUNT:
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 			size += sizeof(struct ibv_flow_spec_counter_action);
 #endif
 			detected_actions |= MLX5_FLOW_ACTION_COUNT;
@@ -1665,7 +1665,7 @@
 		       void *data __rte_unused,
 		       struct rte_flow_error *error)
 {
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 	if (flow->actions & MLX5_FLOW_ACTION_COUNT) {
 		struct rte_flow_query_count *qc = data;
 		uint64_t counters[2] = {0, 0};
diff --git a/drivers/net/mlx5/mlx5_glue.c b/drivers/net/mlx5/mlx5_glue.c
index 48590df..889e074 100644
--- a/drivers/net/mlx5/mlx5_glue.c
+++ b/drivers/net/mlx5/mlx5_glue.c
@@ -215,7 +215,7 @@
 mlx5_glue_create_counter_set(struct ibv_context *context,
 			     struct ibv_counter_set_init_attr *init_attr)
 {
-#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 	(void)context;
 	(void)init_attr;
 	return NULL;
@@ -227,7 +227,7 @@
 static int
 mlx5_glue_destroy_counter_set(struct ibv_counter_set *cs)
 {
-#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 	(void)cs;
 	return ENOTSUP;
 #else
@@ -240,7 +240,7 @@
 			       uint16_t counter_set_id,
 			       struct ibv_counter_set_description *cs_desc)
 {
-#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 	(void)context;
 	(void)counter_set_id;
 	(void)cs_desc;
@@ -254,7 +254,7 @@
 mlx5_glue_query_counter_set(struct ibv_query_counter_set_attr *query_attr,
 			    struct ibv_counter_set_data *cs_data)
 {
-#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 	(void)query_attr;
 	(void)cs_data;
 	return ENOTSUP;
diff --git a/drivers/net/mlx5/mlx5_glue.h b/drivers/net/mlx5/mlx5_glue.h
index f6e4e38..adee972 100644
--- a/drivers/net/mlx5/mlx5_glue.h
+++ b/drivers/net/mlx5/mlx5_glue.h
@@ -23,7 +23,7 @@
 #define MLX5_GLUE_VERSION ""
 #endif
 
-#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 struct ibv_counter_set;
 struct ibv_counter_set_data;
 struct ibv_counter_set_description;
-- 
1.8.3.1

^ permalink raw reply	[flat|nested] 39+ messages in thread

* [dpdk-dev] [PATCH v4 3/8] net/mlx5: introduce new flow counters configuration macro
  2018-10-23 10:04       ` [dpdk-dev] " Slava Ovsiienko
  2018-10-23 10:04         ` [dpdk-dev] [PATCH v4 1/8] net/mlx5: fix flow counters creation Slava Ovsiienko
  2018-10-23 10:04         ` [dpdk-dev] [PATCH v4 2/8] net/mlx5: rename flow counter configuration macro Slava Ovsiienko
@ 2018-10-23 10:04         ` Slava Ovsiienko
  2018-10-23 10:04         ` [dpdk-dev] [PATCH v4 4/8] net/mlx5: simplify flow counters support check Slava Ovsiienko
                           ` (4 subsequent siblings)
  7 siblings, 0 replies; 39+ messages in thread
From: Slava Ovsiienko @ 2018-10-23 10:04 UTC (permalink / raw)
  To: Slava Ovsiienko; +Cc: dev

The new configuration macro HAVE_IBV_DEVICE_COUNTERS_SET_V45 is
introduced. Both makefile and meson.build are changed.

Flow counter support code depends on the following configuration
macros:

- HAVE_IBV_DEVICE_COUNTERS_SET_V42 - is defined if system supports
  the "old" flow counters functionality, MLNX_OFED version from
  4.2 to 4.4 is required.

- HAVE_IBV_DEVICE_COUNTERS_SET_V45 - is defined if system supports
  the "new" flow counters functionality, MLNX_OVED 4.5 (or higher)
  or Linux rdma-core v19 (or higher) is required.

Neither HAVE_IBV_DEVICE_COUNTERS_SET_V42 nor
HAVE_IBV_DEVICE_COUNTERS_SET_V45 is defined if there is no
counters support.

Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
---
 drivers/net/mlx5/Makefile    | 5 +++++
 drivers/net/mlx5/meson.build | 2 ++
 2 files changed, 7 insertions(+)

diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile
index 124ee4e..fecb57c 100644
--- a/drivers/net/mlx5/Makefile
+++ b/drivers/net/mlx5/Makefile
@@ -162,6 +162,11 @@ mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh
 		type 'struct ibv_counter_set_init_attr' \
 		$(AUTOCONF_OUTPUT)
 	$Q sh -- '$<' '$@' \
+		HAVE_IBV_DEVICE_COUNTERS_SET_V45 \
+		infiniband/verbs.h \
+		type 'struct ibv_counters_init_attr' \
+		$(AUTOCONF_OUTPUT)
+	$Q sh -- '$<' '$@' \
 		HAVE_RDMA_NL_NLDEV \
 		rdma/rdma_netlink.h \
 		enum RDMA_NL_NLDEV \
diff --git a/drivers/net/mlx5/meson.build b/drivers/net/mlx5/meson.build
index e613a21..e8cbe3e 100644
--- a/drivers/net/mlx5/meson.build
+++ b/drivers/net/mlx5/meson.build
@@ -81,6 +81,8 @@ if build
 		'struct mlx5dv_sw_parsing_caps', 'sw_parsing_offloads' ],
 		[ 'HAVE_IBV_DEVICE_COUNTERS_SET_V42', 'infiniband/verbs.h',
 		'struct ibv_counter_set_init_attr', 'counter_set_id' ],
+		[ 'HAVE_IBV_DEVICE_COUNTERS_SET_V45', 'infiniband/verbs.h',
+		'struct ibv_counters_init_attr', 'comp_mask' ],
 	]
 	# input array for meson symbol search:
 	# [ "MACRO to define if found", "header for the search",
-- 
1.8.3.1

^ permalink raw reply	[flat|nested] 39+ messages in thread

* [dpdk-dev] [PATCH v4 4/8] net/mlx5: simplify flow counters support check
  2018-10-23 10:04       ` [dpdk-dev] " Slava Ovsiienko
                           ` (2 preceding siblings ...)
  2018-10-23 10:04         ` [dpdk-dev] [PATCH v4 3/8] net/mlx5: introduce new flow counters " Slava Ovsiienko
@ 2018-10-23 10:04         ` Slava Ovsiienko
  2018-10-23 10:04         ` [dpdk-dev] [PATCH v4 5/8] net/mlx5: relocate flow counters query function Slava Ovsiienko
                           ` (3 subsequent siblings)
  7 siblings, 0 replies; 39+ messages in thread
From: Slava Ovsiienko @ 2018-10-23 10:04 UTC (permalink / raw)
  To: Slava Ovsiienko; +Cc: dev

The redundant check of Flow counters support in runtime is removed.
The flag flow_counter_en is eliminated from the code. The Verbs
create counter function just returns an error if no counter
support presented in the system.

If there is no any of Flow counters configuration macro defined
the log message is emited, indicating the missing counter support.

mlx5_flow_validate_action_count() fuctnion is also updated due to
flow_counter_en flag removal.

Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
---
 drivers/net/mlx5/mlx5.c      | 12 +++---------
 drivers/net/mlx5/mlx5.h      |  1 -
 drivers/net/mlx5/mlx5_flow.c |  8 +-------
 3 files changed, 4 insertions(+), 17 deletions(-)

diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index cb86a69..a6cae69 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -739,9 +739,6 @@
 	unsigned int mprq_max_stride_size_n = 0;
 	unsigned int mprq_min_stride_num_n = 0;
 	unsigned int mprq_max_stride_num_n = 0;
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
-	struct ibv_counter_set_description cs_desc = { .counter_type = 0 };
-#endif
 	struct ether_addr mac;
 	char name[RTE_ETH_NAME_MAX_LEN];
 	int own_domain_id = 0;
@@ -1009,12 +1006,9 @@
 	config.hw_csum = !!(attr.device_cap_flags_ex & IBV_DEVICE_RAW_IP_CSUM);
 	DRV_LOG(DEBUG, "checksum offloading is %ssupported",
 		(config.hw_csum ? "" : "not "));
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
-	config.flow_counter_en = !!attr.max_counter_sets;
-	mlx5_glue->describe_counter_set(ctx, 0, &cs_desc);
-	DRV_LOG(DEBUG, "counter type = %d, num of cs = %ld, attributes = %d",
-		cs_desc.counter_type, cs_desc.num_of_cs,
-		cs_desc.attributes);
+#if !defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) && \
+	!defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
+	DRV_LOG(DEBUG, "counters are not supported");
 #endif
 	config.ind_table_max_size =
 		attr.rss_caps.max_rwq_indirection_table_size;
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index d14239c..74d87c0 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -114,7 +114,6 @@ struct mlx5_dev_config {
 	unsigned int tunnel_en:1;
 	/* Whether tunnel stateless offloads are supported. */
 	unsigned int mpls_en:1; /* MPLS over GRE/UDP is enabled. */
-	unsigned int flow_counter_en:1; /* Whether flow counter is supported. */
 	unsigned int cqe_comp:1; /* CQE compression is enabled. */
 	unsigned int tso:1; /* Whether TSO is supported. */
 	unsigned int tx_vec_en:1; /* Tx vector is enabled. */
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index fcabab0..14288e4 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -921,16 +921,10 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
  *   0 on success, a negative errno value otherwise and rte_ernno is set.
  */
 int
-mlx5_flow_validate_action_count(struct rte_eth_dev *dev,
+mlx5_flow_validate_action_count(struct rte_eth_dev *dev __rte_unused,
 				const struct rte_flow_attr *attr,
 				struct rte_flow_error *error)
 {
-	struct priv *priv = dev->data->dev_private;
-
-	if (!priv->config.flow_counter_en)
-		return rte_flow_error_set(error, ENOTSUP,
-					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
-					  "flow counters are not supported.");
 	if (attr->egress)
 		return rte_flow_error_set(error, ENOTSUP,
 					  RTE_FLOW_ERROR_TYPE_ATTR_EGRESS, NULL,
-- 
1.8.3.1

^ permalink raw reply	[flat|nested] 39+ messages in thread

* [dpdk-dev] [PATCH v4 5/8] net/mlx5: relocate flow counters query function
  2018-10-23 10:04       ` [dpdk-dev] " Slava Ovsiienko
                           ` (3 preceding siblings ...)
  2018-10-23 10:04         ` [dpdk-dev] [PATCH v4 4/8] net/mlx5: simplify flow counters support check Slava Ovsiienko
@ 2018-10-23 10:04         ` Slava Ovsiienko
  2018-10-23 10:04         ` [dpdk-dev] [PATCH v4 6/8] net/mlx5: add new flow counter Verbs API to glue library Slava Ovsiienko
                           ` (2 subsequent siblings)
  7 siblings, 0 replies; 39+ messages in thread
From: Slava Ovsiienko @ 2018-10-23 10:04 UTC (permalink / raw)
  To: Slava Ovsiienko; +Cc: dev

The flow_verbs_query_count() is moved into the the group of counter
managing functions and renamed to flow_verbs_counter_query() in order
to be in unified fashion with others.

Also minor function modification is made to avoid unreachable code
warnings if there is no counter support at compile time.

Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
---
 drivers/net/mlx5/mlx5_flow_verbs.c | 111 +++++++++++++++++++------------------
 1 file changed, 56 insertions(+), 55 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c b/drivers/net/mlx5/mlx5_flow_verbs.c
index 3d6fedb..f720c35 100644
--- a/drivers/net/mlx5/mlx5_flow_verbs.c
+++ b/drivers/net/mlx5/mlx5_flow_verbs.c
@@ -110,6 +110,61 @@
 }
 
 /**
+ * Query a flow counter via Verbs library call.
+ *
+ * @see rte_flow_query()
+ * @see rte_flow_ops
+ */
+static int
+flow_verbs_counter_query(struct rte_eth_dev *dev __rte_unused,
+			 struct rte_flow *flow __rte_unused,
+			 void *data __rte_unused,
+			 struct rte_flow_error *error)
+{
+#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
+	if (flow->actions & MLX5_FLOW_ACTION_COUNT) {
+		struct rte_flow_query_count *qc = data;
+		uint64_t counters[2] = {0, 0};
+		struct ibv_query_counter_set_attr query_cs_attr = {
+			.cs = flow->counter->cs,
+			.query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
+		};
+		struct ibv_counter_set_data query_out = {
+			.out = counters,
+			.outlen = 2 * sizeof(uint64_t),
+		};
+		int err = mlx5_glue->query_counter_set(&query_cs_attr,
+						       &query_out);
+
+		if (err)
+			return rte_flow_error_set
+				(error, err,
+				 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				 NULL,
+				 "cannot read counter");
+		qc->hits_set = 1;
+		qc->bytes_set = 1;
+		qc->hits = counters[0] - flow->counter->hits;
+		qc->bytes = counters[1] - flow->counter->bytes;
+		if (qc->reset) {
+			flow->counter->hits = counters[0];
+			flow->counter->bytes = counters[1];
+		}
+		return 0;
+	}
+	return rte_flow_error_set(error, EINVAL,
+				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				  NULL,
+				  "flow does not have counter");
+#else
+	return rte_flow_error_set(error, ENOTSUP,
+				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+				  NULL,
+				  "counters are not available");
+#endif
+}
+
+/**
  * Add a verbs item specification into @p flow.
  *
  * @param[in, out] flow
@@ -1654,60 +1709,6 @@
 }
 
 /**
- * Query a flows.
- *
- * @see rte_flow_query()
- * @see rte_flow_ops
- */
-static int
-flow_verbs_query_count(struct rte_eth_dev *dev __rte_unused,
-		       struct rte_flow *flow __rte_unused,
-		       void *data __rte_unused,
-		       struct rte_flow_error *error)
-{
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
-	if (flow->actions & MLX5_FLOW_ACTION_COUNT) {
-		struct rte_flow_query_count *qc = data;
-		uint64_t counters[2] = {0, 0};
-		struct ibv_query_counter_set_attr query_cs_attr = {
-			.cs = flow->counter->cs,
-			.query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
-		};
-		struct ibv_counter_set_data query_out = {
-			.out = counters,
-			.outlen = 2 * sizeof(uint64_t),
-		};
-		int err = mlx5_glue->query_counter_set(&query_cs_attr,
-						       &query_out);
-
-		if (err)
-			return rte_flow_error_set
-				(error, err,
-				 RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-				 NULL,
-				 "cannot read counter");
-		qc->hits_set = 1;
-		qc->bytes_set = 1;
-		qc->hits = counters[0] - flow->counter->hits;
-		qc->bytes = counters[1] - flow->counter->bytes;
-		if (qc->reset) {
-			flow->counter->hits = counters[0];
-			flow->counter->bytes = counters[1];
-		}
-		return 0;
-	}
-	return rte_flow_error_set(error, EINVAL,
-				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-				  NULL,
-				  "flow does not have counter");
-#endif
-	return rte_flow_error_set(error, ENOTSUP,
-				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
-				  NULL,
-				  "counters are not available");
-}
-
-/**
  * Query a flow.
  *
  * @see rte_flow_query()
@@ -1727,7 +1728,7 @@
 		case RTE_FLOW_ACTION_TYPE_VOID:
 			break;
 		case RTE_FLOW_ACTION_TYPE_COUNT:
-			ret = flow_verbs_query_count(dev, flow, data, error);
+			ret = flow_verbs_counter_query(dev, flow, data, error);
 			break;
 		default:
 			return rte_flow_error_set(error, ENOTSUP,
-- 
1.8.3.1

^ permalink raw reply	[flat|nested] 39+ messages in thread

* [dpdk-dev] [PATCH v4 6/8] net/mlx5: add new flow counter Verbs API to glue library
  2018-10-23 10:04       ` [dpdk-dev] " Slava Ovsiienko
                           ` (4 preceding siblings ...)
  2018-10-23 10:04         ` [dpdk-dev] [PATCH v4 5/8] net/mlx5: relocate flow counters query function Slava Ovsiienko
@ 2018-10-23 10:04         ` Slava Ovsiienko
  2018-10-23 10:04         ` [dpdk-dev] [PATCH v4 7/8] net/mlx5: remove unnecessary structure initializers Slava Ovsiienko
  2018-10-23 10:04         ` [dpdk-dev] [PATCH v4 8/8] net/mlx5: support new flow counter API Slava Ovsiienko
  7 siblings, 0 replies; 39+ messages in thread
From: Slava Ovsiienko @ 2018-10-23 10:04 UTC (permalink / raw)
  To: Slava Ovsiienko; +Cc: dev

This patch updates the mlx5 glue library, new counter support
Verbs function pointers are added to the glue linking structure
mlx5_glue. This structure now contains the pointers to the both
versions of counter supporting functions due to compatibility
issues. Depending on configuration macros the functions perform
actual Verbs library calls or return an error with meaning
"feature is not supported" (NULL or ENOTSUP).

Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
---
 drivers/net/mlx5/mlx5_glue.c | 60 ++++++++++++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_glue.h | 17 +++++++++++++
 2 files changed, 77 insertions(+)

diff --git a/drivers/net/mlx5/mlx5_glue.c b/drivers/net/mlx5/mlx5_glue.c
index 889e074..1afb114 100644
--- a/drivers/net/mlx5/mlx5_glue.c
+++ b/drivers/net/mlx5/mlx5_glue.c
@@ -263,6 +263,62 @@
 #endif
 }
 
+static struct ibv_counters *
+mlx5_glue_create_counters(struct ibv_context *context,
+			  struct ibv_counters_init_attr *init_attr)
+{
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
+	(void)context;
+	(void)init_attr;
+	return NULL;
+#else
+	return ibv_create_counters(context, init_attr);
+#endif
+}
+
+static int
+mlx5_glue_destroy_counters(struct ibv_counters *counters)
+{
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
+	(void)counters;
+	return ENOTSUP;
+#else
+	return ibv_destroy_counters(counters);
+#endif
+}
+
+static int
+mlx5_glue_attach_counters(struct ibv_counters *counters,
+			  struct ibv_counter_attach_attr *attr,
+			  struct ibv_flow *flow)
+{
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
+	(void)counters;
+	(void)attr;
+	(void)flow;
+	return ENOTSUP;
+#else
+	return ibv_attach_counters_point_flow(counters, attr, flow);
+#endif
+}
+
+static int
+mlx5_glue_query_counters(struct ibv_counters *counters,
+			 uint64_t *counters_value,
+			 uint32_t ncounters,
+			 uint32_t flags)
+{
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
+	(void)counters;
+	(void)counters_value;
+	(void)ncounters;
+	(void)flags;
+	return ENOTSUP;
+#else
+	return ibv_read_counters(counters, counters_value, ncounters, flags);
+#endif
+}
+
 static void
 mlx5_glue_ack_async_event(struct ibv_async_event *event)
 {
@@ -424,6 +480,10 @@
 	.destroy_counter_set = mlx5_glue_destroy_counter_set,
 	.describe_counter_set = mlx5_glue_describe_counter_set,
 	.query_counter_set = mlx5_glue_query_counter_set,
+	.create_counters = mlx5_glue_create_counters,
+	.destroy_counters = mlx5_glue_destroy_counters,
+	.attach_counters = mlx5_glue_attach_counters,
+	.query_counters = mlx5_glue_query_counters,
 	.ack_async_event = mlx5_glue_ack_async_event,
 	.get_async_event = mlx5_glue_get_async_event,
 	.port_state_str = mlx5_glue_port_state_str,
diff --git a/drivers/net/mlx5/mlx5_glue.h b/drivers/net/mlx5/mlx5_glue.h
index adee972..44bfefe 100644
--- a/drivers/net/mlx5/mlx5_glue.h
+++ b/drivers/net/mlx5/mlx5_glue.h
@@ -31,6 +31,12 @@
 struct ibv_query_counter_set_attr;
 #endif
 
+#ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
+struct ibv_counters;
+struct ibv_counters_init_attr;
+struct ibv_counter_attach_attr;
+#endif
+
 #ifndef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
 struct mlx5dv_qp_init_attr;
 #endif
@@ -106,6 +112,17 @@ struct mlx5_glue {
 		 struct ibv_counter_set_description *cs_desc);
 	int (*query_counter_set)(struct ibv_query_counter_set_attr *query_attr,
 				 struct ibv_counter_set_data *cs_data);
+	struct ibv_counters *(*create_counters)
+		(struct ibv_context *context,
+		 struct ibv_counters_init_attr *init_attr);
+	int (*destroy_counters)(struct ibv_counters *counters);
+	int (*attach_counters)(struct ibv_counters *counters,
+			       struct ibv_counter_attach_attr *attr,
+			       struct ibv_flow *flow);
+	int (*query_counters)(struct ibv_counters *counters,
+			      uint64_t *counters_value,
+			      uint32_t ncounters,
+			      uint32_t flags);
 	void (*ack_async_event)(struct ibv_async_event *event);
 	int (*get_async_event)(struct ibv_context *context,
 			       struct ibv_async_event *event);
-- 
1.8.3.1

^ permalink raw reply	[flat|nested] 39+ messages in thread

* [dpdk-dev] [PATCH v4 7/8] net/mlx5: remove unnecessary structure initializers
  2018-10-23 10:04       ` [dpdk-dev] " Slava Ovsiienko
                           ` (5 preceding siblings ...)
  2018-10-23 10:04         ` [dpdk-dev] [PATCH v4 6/8] net/mlx5: add new flow counter Verbs API to glue library Slava Ovsiienko
@ 2018-10-23 10:04         ` Slava Ovsiienko
  2018-10-23 10:04         ` [dpdk-dev] [PATCH v4 8/8] net/mlx5: support new flow counter API Slava Ovsiienko
  7 siblings, 0 replies; 39+ messages in thread
From: Slava Ovsiienko @ 2018-10-23 10:04 UTC (permalink / raw)
  To: Slava Ovsiienko; +Cc: dev

The unnecessary structure filed initializers to zero are removed.
We need to do this minor preparation before the following
mlx5 counter structure modification.

Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
---
 drivers/net/mlx5/mlx5_flow_tcf.c | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow_tcf.c b/drivers/net/mlx5/mlx5_flow_tcf.c
index a0680af..4076400 100644
--- a/drivers/net/mlx5/mlx5_flow_tcf.c
+++ b/drivers/net/mlx5/mlx5_flow_tcf.c
@@ -399,11 +399,6 @@ struct pedit_parser {
 	 */
 	struct mlx5_flow_counter tmpl = {
 		.ref_cnt = 1,
-		.shared = 0,
-		.id = 0,
-		.cs = NULL,
-		.hits = 0,
-		.bytes = 0,
 	};
 	cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
 	if (!cnt) {
-- 
1.8.3.1

^ permalink raw reply	[flat|nested] 39+ messages in thread

* [dpdk-dev] [PATCH v4 8/8] net/mlx5: support new flow counter API
  2018-10-23 10:04       ` [dpdk-dev] " Slava Ovsiienko
                           ` (6 preceding siblings ...)
  2018-10-23 10:04         ` [dpdk-dev] [PATCH v4 7/8] net/mlx5: remove unnecessary structure initializers Slava Ovsiienko
@ 2018-10-23 10:04         ` Slava Ovsiienko
  2018-10-24 16:31           ` Ferruh Yigit
  2018-10-27 10:54           ` [dpdk-dev] [PATCH] net/mlx5: fix flow counters deletion in Verbs Slava Ovsiienko
  7 siblings, 2 replies; 39+ messages in thread
From: Slava Ovsiienko @ 2018-10-23 10:04 UTC (permalink / raw)
  To: Slava Ovsiienko; +Cc: dev

This patch updates the functions performing the Verbs ibrary calls
in order to support different versions of the library.
The functions:
  - flow_verbs_counter_new()
  - flow_verbs_counter_release()
  - flow_verbs_counter_query()
now have the several compilation branches, depending on the
counters support found in the system at compile time.

The flow_verbs_counter_create() function is introduced as
helper for flow_verbs_counter_new(), actually this helper
create the counters with Verbs.

Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
---
 drivers/net/mlx5/mlx5_flow.h       |   4 ++
 drivers/net/mlx5/mlx5_flow_verbs.c | 135 ++++++++++++++++++++++++++++---------
 2 files changed, 107 insertions(+), 32 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index af0a125..69ef0ef 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -228,7 +228,11 @@ struct mlx5_flow_counter {
 	uint32_t shared:1; /**< Share counter ID with other flow rules. */
 	uint32_t ref_cnt:31; /**< Reference counter. */
 	uint32_t id; /**< Counter ID. */
+#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 	struct ibv_counter_set *cs; /**< Holds the counters for the rule. */
+#elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
+	struct ibv_counters *cs; /**< Holds the counters for the rule. */
+#endif
 	uint64_t hits; /**< Number of packets matched by the rule. */
 	uint64_t bytes; /**< Number of bytes matched by the rule. */
 };
diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c b/drivers/net/mlx5/mlx5_flow_verbs.c
index f720c35..2547fb8 100644
--- a/drivers/net/mlx5/mlx5_flow_verbs.c
+++ b/drivers/net/mlx5/mlx5_flow_verbs.c
@@ -34,6 +34,70 @@
 #include "mlx5_flow.h"
 
 /**
+ * Create Verbs flow counter with Verbs library.
+ *
+ * @param[in] dev
+ *   Pointer to the Ethernet device structure.
+ * @param[in, out] counter
+ *   mlx5 flow counter object, contains the counter id,
+ *   handle of created Verbs flow counter is returned
+ *   in cs field (if counters are supported).
+ *
+ * @return
+ *   0 On success else a negative errno value is returned
+ *   and rte_errno is set.
+ */
+static int
+flow_verbs_counter_create(struct rte_eth_dev *dev,
+			  struct mlx5_flow_counter *counter)
+{
+#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
+	struct priv *priv = dev->data->dev_private;
+	struct ibv_counter_set_init_attr init = {
+			 .counter_set_id = counter->id};
+
+	counter->cs = mlx5_glue->create_counter_set(priv->ctx, &init);
+	if (!counter->cs) {
+		rte_errno = ENOTSUP;
+		return -ENOTSUP;
+	}
+	return 0;
+#elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
+	struct priv *priv = dev->data->dev_private;
+	struct ibv_counters_init_attr init = {0};
+	struct ibv_counter_attach_attr attach = {0};
+	int ret;
+
+	counter->cs = mlx5_glue->create_counters(priv->ctx, &init);
+	if (!counter->cs) {
+		rte_errno = ENOTSUP;
+		return -ENOTSUP;
+	}
+	attach.counter_desc = IBV_COUNTER_PACKETS;
+	attach.index = 0;
+	ret = mlx5_glue->attach_counters(counter->cs, &attach, NULL);
+	if (!ret) {
+		attach.counter_desc = IBV_COUNTER_BYTES;
+		attach.index = 1;
+		ret = mlx5_glue->attach_counters
+					(counter->cs, &attach, NULL);
+	}
+	if (ret) {
+		claim_zero(mlx5_glue->destroy_counters(counter->cs));
+		counter->cs = NULL;
+		rte_errno = ret;
+		return -ret;
+	}
+	return 0;
+#else
+	(void)dev;
+	(void)counter;
+	rte_errno = ENOTSUP;
+	return -ENOTSUP;
+#endif
+}
+
+/**
  * Get a flow counter.
  *
  * @param[in] dev
@@ -51,6 +115,7 @@
 {
 	struct priv *priv = dev->data->dev_private;
 	struct mlx5_flow_counter *cnt;
+	int ret;
 
 	LIST_FOREACH(cnt, &priv->flow_counters, next) {
 		if (!cnt->shared || cnt->shared != shared)
@@ -60,36 +125,25 @@
 		cnt->ref_cnt++;
 		return cnt;
 	}
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
-
-	struct mlx5_flow_counter tmpl = {
-		.shared = shared,
-		.id = id,
-		.cs = mlx5_glue->create_counter_set
-			(priv->ctx,
-			 &(struct ibv_counter_set_init_attr){
-				 .counter_set_id = id,
-			 }),
-		.hits = 0,
-		.bytes = 0,
-		.ref_cnt = 1,
-	};
-
-	if (!tmpl.cs) {
-		rte_errno = errno;
-		return NULL;
-	}
 	cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
 	if (!cnt) {
-		claim_zero(mlx5_glue->destroy_counter_set(tmpl.cs));
 		rte_errno = ENOMEM;
 		return NULL;
 	}
-	*cnt = tmpl;
-	LIST_INSERT_HEAD(&priv->flow_counters, cnt, next);
-	return cnt;
-#endif
-	rte_errno = ENOTSUP;
+	cnt->id = id;
+	cnt->shared = shared;
+	cnt->ref_cnt = 1;
+	cnt->hits = 0;
+	cnt->bytes = 0;
+	/* Create counter with Verbs. */
+	ret = flow_verbs_counter_create(dev, cnt);
+	if (!ret) {
+		LIST_INSERT_HEAD(&priv->flow_counters, cnt, next);
+		return cnt;
+	}
+	/* Some error occurred in Verbs library. */
+	rte_free(cnt);
+	rte_errno = -ret;
 	return NULL;
 }
 
@@ -103,7 +157,11 @@
 flow_verbs_counter_release(struct mlx5_flow_counter *counter)
 {
 	if (--counter->ref_cnt == 0) {
+#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 		claim_zero(mlx5_glue->destroy_counter_set(counter->cs));
+#elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
+		claim_zero(mlx5_glue->destroy_counters(counter->cs));
+#endif
 		LIST_REMOVE(counter, next);
 		rte_free(counter);
 	}
@@ -117,14 +175,15 @@
  */
 static int
 flow_verbs_counter_query(struct rte_eth_dev *dev __rte_unused,
-			 struct rte_flow *flow __rte_unused,
-			 void *data __rte_unused,
+			 struct rte_flow *flow, void *data,
 			 struct rte_flow_error *error)
 {
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
+#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
+	defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 	if (flow->actions & MLX5_FLOW_ACTION_COUNT) {
 		struct rte_flow_query_count *qc = data;
 		uint64_t counters[2] = {0, 0};
+#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 		struct ibv_query_counter_set_attr query_cs_attr = {
 			.cs = flow->counter->cs,
 			.query_flags = IBV_COUNTER_SET_FORCE_UPDATE,
@@ -135,7 +194,12 @@
 		};
 		int err = mlx5_glue->query_counter_set(&query_cs_attr,
 						       &query_out);
-
+#elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
+		int err = mlx5_glue->query_counters
+			       (flow->counter->cs, counters,
+				RTE_DIM(counters),
+				IBV_READ_COUNTERS_ATTR_PREFER_CACHED);
+#endif
 		if (err)
 			return rte_flow_error_set
 				(error, err,
@@ -157,6 +221,8 @@
 				  NULL,
 				  "flow does not have counter");
 #else
+	(void)flow;
+	(void)data;
 	return rte_flow_error_set(error, ENOTSUP,
 				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
 				  NULL,
@@ -993,7 +1059,8 @@
 {
 	const struct rte_flow_action_count *count = action->conf;
 	struct rte_flow *flow = dev_flow->flow;
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
+#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
+	defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 	unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
 	struct ibv_flow_spec_counter_action counter = {
 		.type = IBV_FLOW_SPEC_ACTION_COUNT,
@@ -1012,9 +1079,12 @@
 						  " context.");
 	}
 	*action_flags |= MLX5_FLOW_ACTION_COUNT;
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
+#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
 	counter.counter_set_handle = flow->counter->cs->handle;
 	flow_verbs_spec_add(dev_flow, &counter, size);
+#elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
+	counter.counters = flow->counter->cs;
+	flow_verbs_spec_add(dev_flow, &counter, size);
 #endif
 	return 0;
 }
@@ -1277,7 +1347,8 @@
 			detected_actions |= MLX5_FLOW_ACTION_RSS;
 			break;
 		case RTE_FLOW_ACTION_TYPE_COUNT:
-#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
+#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
+	defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 			size += sizeof(struct ibv_flow_spec_counter_action);
 #endif
 			detected_actions |= MLX5_FLOW_ACTION_COUNT;
-- 
1.8.3.1

^ permalink raw reply	[flat|nested] 39+ messages in thread

* Re: [dpdk-dev] [PATCH v4 0/8] net/mlx5: flow counters support for Linux-rdma v19
       [not found]       ` <1540289541-30019-1-git-send-email-viacheslavo@mellanox.com>
@ 2018-10-24 11:02         ` Shahaf Shuler
  0 siblings, 0 replies; 39+ messages in thread
From: Shahaf Shuler @ 2018-10-24 11:02 UTC (permalink / raw)
  To: Slava Ovsiienko, dev

Tuesday, October 23, 2018 1:13 PM, Slava Ovsiienko:
> Subject: [PATCH v4 0/8] net/mlx5: flow counters support for Linux-rdma v19
> 
> Mellanox mlx5 PMD supports Flow counters via Verbs library.
> The current implementation is based on the Mellanox proprietary Verbs
> library included in MLNX OFED packages. The Flow counter support is
> recently added into linux-rdma release (v19), so the mlx5 PMD update is
> needed to provide Counter feature on the base of linux-rdma.
> 
> mlx5 PMD can be compiled with MLNX OFED or linux-rdma v19+ and provide
> flow counters for both.
> 
> Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
> 

Applied to next-net-mlx, thanks. 

> ---
> v4:
> - minor patcset parts reorganization
> - rewritten headlines to be more clear
> - ifdef blocks minor cleanups
> 
> v3:
> - http://patches.dpdk.org/patch/47085/
> - mlx5 glue issue resolved correctly
> - patch is reorganized info small isolated parts
> 
> v2:
> - http://patches.dpdk.org/patch/46989/
> - rebased on top of master-net-mlx branch
> - new compilation flags are introduced:
> - HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V42, kernel/verbs
>   library provides the flow counter support in style of
>   MLNX_OFED_4.2 to MLNX_OFED_4.4
> - HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT_V45, kernel/verbs
>   library provides the flow counter support in style of
>   MLNX_OFED_4.5 or higher
> 
> v1:
> - http://patches.dpdk.org/patch/45972/
> 
> Viacheslav Ovsiienko (8):
>   net/mlx5: fix flow counters creation
>   net/mlx5: rename flow counter configuration macro
>   net/mlx5: introduce new flow counters configuration macro
>   net/mlx5: simplify flow counters support check
>   net/mlx5: relocate flow counters query function
>   net/mlx5: add new flow counter Verbs API to glue library
>   net/mlx5: remove unnecessary structure initializers
>   net/mlx5: support new flow counter API
> 
>  drivers/net/mlx5/Makefile          |   7 +-
>  drivers/net/mlx5/meson.build       |   4 +-
>  drivers/net/mlx5/mlx5.c            |  12 +-
>  drivers/net/mlx5/mlx5.h            |   1 -
>  drivers/net/mlx5/mlx5_flow.c       |   8 +-
>  drivers/net/mlx5/mlx5_flow.h       |   4 +
>  drivers/net/mlx5/mlx5_flow_tcf.c   |   5 -
>  drivers/net/mlx5/mlx5_flow_verbs.c | 236 ++++++++++++++++++++++++--
> -----------
>  drivers/net/mlx5/mlx5_glue.c       |  68 ++++++++++-
>  drivers/net/mlx5/mlx5_glue.h       |  19 ++-
>  10 files changed, 254 insertions(+), 110 deletions(-)
> 
> --
> 1.8.3.1

^ permalink raw reply	[flat|nested] 39+ messages in thread

* Re: [dpdk-dev] [PATCH v4 8/8] net/mlx5: support new flow counter API
  2018-10-23 10:04         ` [dpdk-dev] [PATCH v4 8/8] net/mlx5: support new flow counter API Slava Ovsiienko
@ 2018-10-24 16:31           ` Ferruh Yigit
  2018-10-24 16:35             ` Ferruh Yigit
  2018-10-27 10:54           ` [dpdk-dev] [PATCH] net/mlx5: fix flow counters deletion in Verbs Slava Ovsiienko
  1 sibling, 1 reply; 39+ messages in thread
From: Ferruh Yigit @ 2018-10-24 16:31 UTC (permalink / raw)
  To: Slava Ovsiienko, Shahaf Shuler; +Cc: dev, Thomas Monjalon

On 10/23/2018 11:04 AM, Slava Ovsiienko wrote:
> @@ -1012,9 +1079,12 @@
>  						  " context.");
>  	}
>  	*action_flags |= MLX5_FLOW_ACTION_COUNT;
> -#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
> +#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
>  	counter.counter_set_handle = flow->counter->cs->handle;
>  	flow_verbs_spec_add(dev_flow, &counter, size);
> +#elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
> +	counter.counters = flow->counter->cs;
> +	flow_verbs_spec_add(dev_flow, &counter, size);
>  #endif
>  	return 0;
>  }

Hi Slava, Shahaf,

There is something wrong related above code.
In next-net-mlx the above code is different than the patch itself [1] and
causing build error.

This can be because of merge/conflict issues. Please fix issue on next-net-mlx,
I will drop the patches I have pulled and wait until this is fixed.

But my concern is what would be if this doesn't cause a build error!
If this is because of merge/conflict, this data is lost, we really should
consider using git merge.
If this is because of you updated the code in the tree, I think that is worse,
we shouldn't change code in the tree, please ask for changes in mail list.


[1]
 @@ -1012,10 +1077,12 @@ flow_verbs_translate_action_count(struct rte_eth_dev
*dev,


                                                   " context.");
         }
         *action_flags |= MLX5_FLOW_ACTION_COUNT;
 -#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42



 +#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)



         counter.counter_set_handle = flow->counter->cs->handle;
 -       flow_verbs_spec_add(dev_flow, &counter, size);



 +#elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)



 +       counter.counters = flow->counter->cs;



  #endif
 +       flow_verbs_spec_add(dev_flow, &counter, size);



         return 0;
  }

^ permalink raw reply	[flat|nested] 39+ messages in thread

* Re: [dpdk-dev] [PATCH v4 8/8] net/mlx5: support new flow counter API
  2018-10-24 16:31           ` Ferruh Yigit
@ 2018-10-24 16:35             ` Ferruh Yigit
  2018-10-24 17:25               ` Shahaf Shuler
  0 siblings, 1 reply; 39+ messages in thread
From: Ferruh Yigit @ 2018-10-24 16:35 UTC (permalink / raw)
  To: Slava Ovsiienko, Shahaf Shuler; +Cc: dev, Thomas Monjalon

On 10/24/2018 5:31 PM, Ferruh Yigit wrote:
> On 10/23/2018 11:04 AM, Slava Ovsiienko wrote:
>> @@ -1012,9 +1079,12 @@
>>  						  " context.");
>>  	}
>>  	*action_flags |= MLX5_FLOW_ACTION_COUNT;
>> -#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
>> +#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
>>  	counter.counter_set_handle = flow->counter->cs->handle;
>>  	flow_verbs_spec_add(dev_flow, &counter, size);
>> +#elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
>> +	counter.counters = flow->counter->cs;
>> +	flow_verbs_spec_add(dev_flow, &counter, size);
>>  #endif
>>  	return 0;
>>  }
> 
> Hi Slava, Shahaf,
> 
> There is something wrong related above code.
> In next-net-mlx the above code is different than the patch itself [1] and
> causing build error.
> 
> This can be because of merge/conflict issues. Please fix issue on next-net-mlx,
> I will drop the patches I have pulled and wait until this is fixed.
> 
> But my concern is what would be if this doesn't cause a build error!
> If this is because of merge/conflict, this data is lost, we really should
> consider using git merge.
> If this is because of you updated the code in the tree, I think that is worse,
> we shouldn't change code in the tree, please ask for changes in mail list.
> 
> 

[1]
 @@ -1012,10 +1077,12 @@ flow_verbs_translate_action_count(struct rte_eth_dev  *dev,
                                                  " context.");
         }
         *action_flags |= MLX5_FLOW_ACTION_COUNT;
 -#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_V42
 +#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
        counter.counter_set_handle = flow->counter->cs->handle;
 -       flow_verbs_spec_add(dev_flow, &counter, size);
 +#elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
 +       counter.counters = flow->counter->cs;
 #endif
 +       flow_verbs_spec_add(dev_flow, &counter, size);
         return 0;
  }

^ permalink raw reply	[flat|nested] 39+ messages in thread

* Re: [dpdk-dev] [PATCH v4 8/8] net/mlx5: support new flow counter API
  2018-10-24 16:35             ` Ferruh Yigit
@ 2018-10-24 17:25               ` Shahaf Shuler
  2018-10-25  8:59                 ` Ferruh Yigit
  0 siblings, 1 reply; 39+ messages in thread
From: Shahaf Shuler @ 2018-10-24 17:25 UTC (permalink / raw)
  To: Ferruh Yigit, Slava Ovsiienko; +Cc: dev, Thomas Monjalon

Hi Ferruh,

Wednesday, October 24, 2018 7:36 PM, Ferruh Yigit
> Subject: Re: [dpdk-dev] [PATCH v4 8/8] net/mlx5: support new flow counter
> API
> 
> On 10/24/2018 5:31 PM, Ferruh Yigit wrote:
> > On 10/23/2018 11:04 AM, Slava Ovsiienko wrote:
> >> @@ -1012,9 +1079,12 @@
> >>  						  " context.");
> >>  	}
> >>  	*action_flags |= MLX5_FLOW_ACTION_COUNT; -#ifdef
> >> HAVE_IBV_DEVICE_COUNTERS_SET_V42
> >> +#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
> >>  	counter.counter_set_handle = flow->counter->cs->handle;
> >>  	flow_verbs_spec_add(dev_flow, &counter, size);
> >> +#elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
> >> +	counter.counters = flow->counter->cs;
> >> +	flow_verbs_spec_add(dev_flow, &counter, size);
> >>  #endif
> >>  	return 0;
> >>  }
> >
> > Hi Slava, Shahaf,
> >
> > There is something wrong related above code.
> > In next-net-mlx the above code is different than the patch itself [1]
> > and causing build error.
> >
> > This can be because of merge/conflict issues. Please fix issue on
> > next-net-mlx, I will drop the patches I have pulled and wait until this is
> fixed.
> >
> > But my concern is what would be if this doesn't cause a build error!
> > If this is because of merge/conflict, this data is lost, we really
> > should consider using git merge.
> > If this is because of you updated the code in the tree, I think that
> > is worse, we shouldn't change code in the tree, please ask for changes in
> mail list.

sorry this is my bad.
I was checking the option during the code review to suggest Slava, noticed the compilation issue however forgot to reset the changes and instead amended them.

Sorry for that (and for the other compilation issues)
Won't happen again.

Next-net-mlx should be OK now and match the upstream series. 

> >
> >
> 
> [1]
>  @@ -1012,10 +1077,12 @@ flow_verbs_translate_action_count(struct
> rte_eth_dev  *dev,
>                                                   " context.");
>          }
>          *action_flags |= MLX5_FLOW_ACTION_COUNT;  -#ifdef
> HAVE_IBV_DEVICE_COUNTERS_SET_V42  +#if
> defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
>         counter.counter_set_handle = flow->counter->cs->handle;
>  -       flow_verbs_spec_add(dev_flow, &counter, size);
>  +#elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
>  +       counter.counters = flow->counter->cs;
>  #endif
>  +       flow_verbs_spec_add(dev_flow, &counter, size);
>          return 0;
>   }

^ permalink raw reply	[flat|nested] 39+ messages in thread

* Re: [dpdk-dev] [PATCH v4 8/8] net/mlx5: support new flow counter API
  2018-10-24 17:25               ` Shahaf Shuler
@ 2018-10-25  8:59                 ` Ferruh Yigit
  0 siblings, 0 replies; 39+ messages in thread
From: Ferruh Yigit @ 2018-10-25  8:59 UTC (permalink / raw)
  To: Shahaf Shuler, Slava Ovsiienko; +Cc: dev, Thomas Monjalon

On 10/24/2018 6:25 PM, Shahaf Shuler wrote:
> Hi Ferruh,
> 
> Wednesday, October 24, 2018 7:36 PM, Ferruh Yigit
>> Subject: Re: [dpdk-dev] [PATCH v4 8/8] net/mlx5: support new flow counter
>> API
>>
>> On 10/24/2018 5:31 PM, Ferruh Yigit wrote:
>>> On 10/23/2018 11:04 AM, Slava Ovsiienko wrote:
>>>> @@ -1012,9 +1079,12 @@
>>>>  						  " context.");
>>>>  	}
>>>>  	*action_flags |= MLX5_FLOW_ACTION_COUNT; -#ifdef
>>>> HAVE_IBV_DEVICE_COUNTERS_SET_V42
>>>> +#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
>>>>  	counter.counter_set_handle = flow->counter->cs->handle;
>>>>  	flow_verbs_spec_add(dev_flow, &counter, size);
>>>> +#elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
>>>> +	counter.counters = flow->counter->cs;
>>>> +	flow_verbs_spec_add(dev_flow, &counter, size);
>>>>  #endif
>>>>  	return 0;
>>>>  }
>>>
>>> Hi Slava, Shahaf,
>>>
>>> There is something wrong related above code.
>>> In next-net-mlx the above code is different than the patch itself [1]
>>> and causing build error.
>>>
>>> This can be because of merge/conflict issues. Please fix issue on
>>> next-net-mlx, I will drop the patches I have pulled and wait until this is
>> fixed.
>>>
>>> But my concern is what would be if this doesn't cause a build error!
>>> If this is because of merge/conflict, this data is lost, we really
>>> should consider using git merge.
>>> If this is because of you updated the code in the tree, I think that
>>> is worse, we shouldn't change code in the tree, please ask for changes in
>> mail list.
<...>
> 
> Next-net-mlx should be OK now and match the upstream series. 

Thanks, pulled.

^ permalink raw reply	[flat|nested] 39+ messages in thread

* [dpdk-dev] [PATCH] net/mlx5: fix flow counters deletion in Verbs
  2018-10-23 10:04         ` [dpdk-dev] [PATCH v4 8/8] net/mlx5: support new flow counter API Slava Ovsiienko
  2018-10-24 16:31           ` Ferruh Yigit
@ 2018-10-27 10:54           ` Slava Ovsiienko
  2018-10-28 12:53             ` Shahaf Shuler
  1 sibling, 1 reply; 39+ messages in thread
From: Slava Ovsiienko @ 2018-10-27 10:54 UTC (permalink / raw)
  To: Shahaf Shuler; +Cc: dev, Slava Ovsiienko, stable

The Flow counters created with Verbs are erroneously destroyed
in Flow remove function (flow_verbs_remove()). Counter Verbs
handles stored in the translated rule buffer become invalid.
If rule is reapplied with these invalid counter handles the
driver hangs.

The counter should be destroyed with Verbs in the Flow destroy
function. The Flow remove function should keep counters intact.

Fixes: 60bd8c9747e8 ("net/mlx5: add count flow action")
Cc: stable@dpdk.org

Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
---
 drivers/net/mlx5/mlx5_flow_verbs.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c b/drivers/net/mlx5/mlx5_flow_verbs.c
index 81bc39f..2e506b9 100644
--- a/drivers/net/mlx5/mlx5_flow_verbs.c
+++ b/drivers/net/mlx5/mlx5_flow_verbs.c
@@ -1669,10 +1669,6 @@
 			verbs->hrxq = NULL;
 		}
 	}
-	if (flow->counter) {
-		flow_verbs_counter_release(flow->counter);
-		flow->counter = NULL;
-	}
 }
 
 /**
@@ -1696,6 +1692,10 @@
 		LIST_REMOVE(dev_flow, next);
 		rte_free(dev_flow);
 	}
+	if (flow->counter) {
+		flow_verbs_counter_release(flow->counter);
+		flow->counter = NULL;
+	}
 }
 
 /**
-- 
1.8.3.1

^ permalink raw reply	[flat|nested] 39+ messages in thread

* Re: [dpdk-dev] [PATCH] net/mlx5: fix flow counters deletion in Verbs
  2018-10-27 10:54           ` [dpdk-dev] [PATCH] net/mlx5: fix flow counters deletion in Verbs Slava Ovsiienko
@ 2018-10-28 12:53             ` Shahaf Shuler
  0 siblings, 0 replies; 39+ messages in thread
From: Shahaf Shuler @ 2018-10-28 12:53 UTC (permalink / raw)
  To: Slava Ovsiienko; +Cc: dev, stable

Saturday, October 27, 2018 1:54 PM¸ Slava Ovsiienko:
> Subject: [PATCH] net/mlx5: fix flow counters deletion in Verbs
> 
> The Flow counters created with Verbs are erroneously destroyed in Flow
> remove function (flow_verbs_remove()). Counter Verbs handles stored in
> the translated rule buffer become invalid.
> If rule is reapplied with these invalid counter handles the driver hangs.
> 
> The counter should be destroyed with Verbs in the Flow destroy function.
> The Flow remove function should keep counters intact.
> 
> Fixes: 60bd8c9747e8 ("net/mlx5: add count flow action")
> Cc: stable@dpdk.org
> 

Very nice commit log.

> Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>

Acked-by: Shahaf Shuler <shahafs@mellanox.com>

Applied to next-net-mlx, thanks. 

> ---
>  drivers/net/mlx5/mlx5_flow_verbs.c | 8 ++++----
>  1 file changed, 4 insertions(+), 4 deletions(-)
> 
> diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c
> b/drivers/net/mlx5/mlx5_flow_verbs.c
> index 81bc39f..2e506b9 100644
> --- a/drivers/net/mlx5/mlx5_flow_verbs.c
> +++ b/drivers/net/mlx5/mlx5_flow_verbs.c
> @@ -1669,10 +1669,6 @@
>  			verbs->hrxq = NULL;
>  		}
>  	}
> -	if (flow->counter) {
> -		flow_verbs_counter_release(flow->counter);
> -		flow->counter = NULL;
> -	}
>  }
> 
>  /**
> @@ -1696,6 +1692,10 @@
>  		LIST_REMOVE(dev_flow, next);
>  		rte_free(dev_flow);
>  	}
> +	if (flow->counter) {
> +		flow_verbs_counter_release(flow->counter);
> +		flow->counter = NULL;
> +	}
>  }
> 
>  /**
> --
> 1.8.3.1

^ permalink raw reply	[flat|nested] 39+ messages in thread

end of thread, other threads:[~2018-10-28 12:53 UTC | newest]

Thread overview: 39+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2018-08-20 13:34 [dpdk-dev] [RFC] mlx5: flow counters support on the linux-rdma v19 base viacheslavo
2018-09-27 23:40 ` Yongseok Koh
2018-10-03 15:29 ` [dpdk-dev] [PATCH] net/mlx5: flow counters support on the Linux-rdma " Slava Ovsiienko
2018-10-03 23:48   ` Yongseok Koh
2018-10-09 13:45     ` Shahaf Shuler
2018-10-11 14:51     ` Slava Ovsiienko
2018-10-14  5:32       ` Shahaf Shuler
2018-10-17 13:53   ` [dpdk-dev] [PATCH v2] " Viacheslav Ovsiienko
2018-10-18  6:34     ` Shahaf Shuler
2018-10-19 15:21     ` [dpdk-dev] [PATCH v3 0/6] net/mlx5: flow counters support for Linux-rdma v19 Viacheslav Ovsiienko
2018-10-19 15:21       ` [dpdk-dev] [PATCH v3 1/6] net/mlx5: flow counters object create function bugfix Viacheslav Ovsiienko
2018-10-21  9:20         ` Shahaf Shuler
2018-10-19 15:21       ` [dpdk-dev] [PATCH v3 2/6] net/mlx5: flow counters new configuration flags Viacheslav Ovsiienko
2018-10-21  9:20         ` Shahaf Shuler
2018-10-19 15:21       ` [dpdk-dev] [PATCH v3 3/6] net/mlx5: flow counters simplifying runtime support check Viacheslav Ovsiienko
2018-10-21  9:20         ` Shahaf Shuler
2018-10-19 15:21       ` [dpdk-dev] [PATCH v3 4/6] net/mlx5: flow counters mlx5 glue library update Viacheslav Ovsiienko
2018-10-21  9:20         ` Shahaf Shuler
2018-10-19 15:21       ` [dpdk-dev] [PATCH v3 5/6] net/mlx5: flow counters query function move and rename Viacheslav Ovsiienko
2018-10-21  9:20         ` Shahaf Shuler
2018-10-19 15:21       ` [dpdk-dev] [PATCH v3 6/6] net/mlx5: flow counters Verbs interface functions update Viacheslav Ovsiienko
2018-10-21  9:20         ` Shahaf Shuler
2018-10-21  9:21       ` [dpdk-dev] [PATCH v3 0/6] net/mlx5: flow counters support for Linux-rdma v19 Shahaf Shuler
2018-10-23 10:04       ` [dpdk-dev] " Slava Ovsiienko
2018-10-23 10:04         ` [dpdk-dev] [PATCH v4 1/8] net/mlx5: fix flow counters creation Slava Ovsiienko
2018-10-23 10:04         ` [dpdk-dev] [PATCH v4 2/8] net/mlx5: rename flow counter configuration macro Slava Ovsiienko
2018-10-23 10:04         ` [dpdk-dev] [PATCH v4 3/8] net/mlx5: introduce new flow counters " Slava Ovsiienko
2018-10-23 10:04         ` [dpdk-dev] [PATCH v4 4/8] net/mlx5: simplify flow counters support check Slava Ovsiienko
2018-10-23 10:04         ` [dpdk-dev] [PATCH v4 5/8] net/mlx5: relocate flow counters query function Slava Ovsiienko
2018-10-23 10:04         ` [dpdk-dev] [PATCH v4 6/8] net/mlx5: add new flow counter Verbs API to glue library Slava Ovsiienko
2018-10-23 10:04         ` [dpdk-dev] [PATCH v4 7/8] net/mlx5: remove unnecessary structure initializers Slava Ovsiienko
2018-10-23 10:04         ` [dpdk-dev] [PATCH v4 8/8] net/mlx5: support new flow counter API Slava Ovsiienko
2018-10-24 16:31           ` Ferruh Yigit
2018-10-24 16:35             ` Ferruh Yigit
2018-10-24 17:25               ` Shahaf Shuler
2018-10-25  8:59                 ` Ferruh Yigit
2018-10-27 10:54           ` [dpdk-dev] [PATCH] net/mlx5: fix flow counters deletion in Verbs Slava Ovsiienko
2018-10-28 12:53             ` Shahaf Shuler
     [not found]       ` <1540289541-30019-1-git-send-email-viacheslavo@mellanox.com>
2018-10-24 11:02         ` [dpdk-dev] [PATCH v4 0/8] net/mlx5: flow counters support for Linux-rdma v19 Shahaf Shuler

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).