Re: [dpdk-dev] [PATCH v4 1/3] net/mlx5: prepare Direct Verbs for Direct Rule

[Yongseok Koh <yskoh@mellanox.com>]

> On Apr 12, 2019, at 4:51 PM, dwilder <dwilder@us.ibm.com> wrote:
> 
> I am seeing a build break when building on ubuntu 19.04 beta after this commit (cbb66daa3).
> 
> The build produces a number of the following errors.
> 
> drivers/net/mlx5/mlx5_glue.c:194:2: error: implicit declaration of function ‘free’
> ....
> drivers/net/mlx5/mlx5_glue.c:552:11: error: incompatible implicit declaration of built-in function ‘malloc’ [-Werror]
> ....
> 
> Adding "#include <stdlib.h>" to drivers/net/mlx5/mlx5_glue.c fixed it for me.
> 
> I do not see this problem on Ubuntu 18.04, maybe some headers have changed.  I have only tested on ppc64le.

Thanks for the notice.
That patch's been merged on to the PMD branch and will be merged on to the master soon

http://git.dpdk.org/next/dpdk-next-net-mlx/commit/?id=4d6b192ccc15a4adfe575900519ba90059e2abd2

Yongseok

> On 2019-04-04 02:54, Ori Kam wrote:
>> This is the first patch of a series that is designed to enable the
>> Direct Rules API.
>> The main difference between Direct Verbs and Direct Rules from API
>> prespective, is that in Direct Rules each action has it's own create
>> function and the object itself is of type void.
>> In this patch I'm adding functions to generate actions that currenlty
>> are done without create action, and I'm changing the action type to be
>> void *, so in next patches only the glue functions will need to change.
>> Signed-off-by: Ori Kam <orika@mellanox.com>
>> Acked-by: Shahaf Shuler <shahafs@mellanox.com>
>> ---
>> drivers/net/mlx5/Makefile       |   2 +-
>> drivers/net/mlx5/meson.build    |   2 +-
>> drivers/net/mlx5/mlx5.h         |   2 +
>> drivers/net/mlx5/mlx5_flow.h    |  17 +++-
>> drivers/net/mlx5/mlx5_flow_dv.c | 184 +++++++++++++++++++++++++++++-----------
>> drivers/net/mlx5/mlx5_glue.c    | 141 ++++++++++++++++++++++++------
>> drivers/net/mlx5/mlx5_glue.h    |  26 +++---
>> 7 files changed, 280 insertions(+), 94 deletions(-)
>> diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile
>> index ccd179c..0d20f0f 100644
>> --- a/drivers/net/mlx5/Makefile
>> +++ b/drivers/net/mlx5/Makefile
>> @@ -8,7 +8,7 @@ include $(RTE_SDK)/mk/rte.vars.mk
>> LIB = librte_pmd_mlx5.a
>> LIB_GLUE = $(LIB_GLUE_BASE).$(LIB_GLUE_VERSION)
>> LIB_GLUE_BASE = librte_pmd_mlx5_glue.so
>> -LIB_GLUE_VERSION = 19.02.0
>> +LIB_GLUE_VERSION = 19.05.0
>> # Sources.
>> SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5.c
>> diff --git a/drivers/net/mlx5/meson.build b/drivers/net/mlx5/meson.build
>> index a429655..a4c684e 100644
>> --- a/drivers/net/mlx5/meson.build
>> +++ b/drivers/net/mlx5/meson.build
>> @@ -4,7 +4,7 @@
>> pmd_dlopen = (get_option('ibverbs_link') == 'dlopen')
>> LIB_GLUE_BASE = 'librte_pmd_mlx5_glue.so'
>> -LIB_GLUE_VERSION = '19.02.0'
>> +LIB_GLUE_VERSION = '19.05.0'
>> LIB_GLUE = LIB_GLUE_BASE + '.' + LIB_GLUE_VERSION
>> if pmd_dlopen
>> 	dpdk_conf.set('RTE_IBVERBS_LINK_DLOPEN', 1)
>> diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
>> index 47a7d75..784bf9b 100644
>> --- a/drivers/net/mlx5/mlx5.h
>> +++ b/drivers/net/mlx5/mlx5.h
>> @@ -309,6 +309,8 @@ struct mlx5_priv {
>> 	LIST_HEAD(matchers, mlx5_flow_dv_matcher) matchers;
>> 	LIST_HEAD(encap_decap, mlx5_flow_dv_encap_decap_resource) encaps_decaps;
>> 	LIST_HEAD(modify_cmd, mlx5_flow_dv_modify_hdr_resource) modify_cmds;
>> +	LIST_HEAD(tag, mlx5_flow_dv_tag_resource) tags;
>> +	/* Tags resources cache. */
>> 	uint32_t link_speed_capa; /* Link speed capabilities. */
>> 	struct mlx5_xstats_ctrl xstats_ctrl; /* Extended stats control. */
>> 	struct mlx5_stats_ctrl stats_ctrl; /* Stats control. */
>> diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
>> index e1e798b..4f69ae2 100644
>> --- a/drivers/net/mlx5/mlx5_flow.h
>> +++ b/drivers/net/mlx5/mlx5_flow.h
>> @@ -214,7 +214,7 @@ struct mlx5_flow_dv_encap_decap_resource {
>> 	LIST_ENTRY(mlx5_flow_dv_encap_decap_resource) next;
>> 	/* Pointer to next element. */
>> 	rte_atomic32_t refcnt; /**< Reference counter. */
>> -	struct ibv_flow_action *verbs_action;
>> +	void *verbs_action;
>> 	/**< Verbs encap/decap action object. */
>> 	uint8_t buf[MLX5_ENCAP_MAX_LEN];
>> 	size_t size;
>> @@ -222,6 +222,16 @@ struct mlx5_flow_dv_encap_decap_resource {
>> 	uint8_t ft_type;
>> };
>> +/* Tag resource structure. */
>> +struct mlx5_flow_dv_tag_resource {
>> +	LIST_ENTRY(mlx5_flow_dv_tag_resource) next;
>> +	/* Pointer to next element. */
>> +	rte_atomic32_t refcnt; /**< Reference counter. */
>> +	void *action;
>> +	/**< Verbs tag action object. */
>> +	uint32_t tag; /**< the tag value. */
>> +};
>> +
>> /* Number of modification commands. */
>> #define MLX5_MODIFY_NUM 8
>> @@ -259,7 +269,7 @@ struct mlx5_flow_dv {
>> 	/**< Pointer to modify header resource in cache. */
>> 	struct ibv_flow *flow; /**< Installed flow. */
>> #ifdef HAVE_IBV_FLOW_DV_SUPPORT
>> -	struct mlx5dv_flow_action_attr actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS];
>> +	void *actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS];
>> 	/**< Action list. */
>> #endif
>> 	int actions_n; /**< number of actions. */
>> @@ -332,6 +342,7 @@ struct mlx5_flow_counter {
>> 	};
>> 	uint64_t hits; /**< Number of packets matched by the rule. */
>> 	uint64_t bytes; /**< Number of bytes matched by the rule. */
>> +	void *action; /**< Pointer to the dv action. */
>> };
>> /* Flow structure. */
>> @@ -339,6 +350,8 @@ struct rte_flow {
>> 	TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
>> 	enum mlx5_flow_drv_type drv_type; /**< Drvier type. */
>> 	struct mlx5_flow_counter *counter; /**< Holds flow counter. */
>> +	struct mlx5_flow_dv_tag_resource *tag_resource;
>> +	/**< pointer to the tag action. */
>> 	struct rte_flow_action_rss rss;/**< RSS context. */
>> 	uint8_t key[MLX5_RSS_HASH_KEY_LEN]; /**< RSS hash key. */
>> 	uint16_t (*queue)[]; /**< Destination queues to redirect traffic to. */
>> diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
>> index ad84dea..d9e2ac9 100644
>> --- a/drivers/net/mlx5/mlx5_flow_dv.c
>> +++ b/drivers/net/mlx5/mlx5_flow_dv.c
>> @@ -32,7 +32,6 @@
>> #include "mlx5_prm.h"
>> #include "mlx5_glue.h"
>> #include "mlx5_flow.h"
>> -
>> #ifdef HAVE_IBV_FLOW_DV_SUPPORT
>> #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
>> @@ -1537,6 +1536,11 @@ struct field_modify_info modify_tcp[] = {
>> 		.id = id,
>> 		.dcs = dcs,
>> 	};
>> +	tmpl.action = mlx5_glue->dv_create_flow_action_counter(dcs->obj, 0);
>> +	if (!tmpl.action) {
>> +		ret = errno;
>> +		goto error_exit;
>> +	}
>> 	*cnt = tmpl;
>> 	LIST_INSERT_HEAD(&priv->flow_counters, cnt, next);
>> 	return cnt;
>> @@ -2828,6 +2832,97 @@ struct field_modify_info modify_tcp[] = {
>> }
>> /**
>> + * Find existing tag resource or create and register a new one.
>> + *
>> + * @param dev[in, out]
>> + *   Pointer to rte_eth_dev structure.
>> + * @param[in, out] resource
>> + *   Pointer to tag resource.
>> + * @parm[in, out] dev_flow
>> + *   Pointer to the dev_flow.
>> + * @param[out] error
>> + *   pointer to error structure.
>> + *
>> + * @return
>> + *   0 on success otherwise -errno and errno is set.
>> + */
>> +static int
>> +flow_dv_tag_resource_register
>> +			(struct rte_eth_dev *dev,
>> +			 struct mlx5_flow_dv_tag_resource *resource,
>> +			 struct mlx5_flow *dev_flow,
>> +			 struct rte_flow_error *error)
>> +{
>> +	struct mlx5_priv *priv = dev->data->dev_private;
>> +	struct mlx5_flow_dv_tag_resource *cache_resource;
>> +
>> +	/* Lookup a matching resource from cache. */
>> +	LIST_FOREACH(cache_resource, &priv->tags, next) {
>> +		if (resource->tag == cache_resource->tag) {
>> +			DRV_LOG(DEBUG, "tag resource %p: refcnt %d++",
>> +				(void *)cache_resource,
>> +				rte_atomic32_read(&cache_resource->refcnt));
>> +			rte_atomic32_inc(&cache_resource->refcnt);
>> +			dev_flow->flow->tag_resource = cache_resource;
>> +			return 0;
>> +		}
>> +	}
>> +	/* Register new  resource. */
>> +	cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
>> +	if (!cache_resource)
>> +		return rte_flow_error_set(error, ENOMEM,
>> +					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
>> +					  "cannot allocate resource memory");
>> +	*cache_resource = *resource;
>> +	cache_resource->action = mlx5_glue->dv_create_flow_action_tag
>> +		(resource->tag);
>> +	if (!cache_resource->action) {
>> +		rte_free(cache_resource);
>> +		return rte_flow_error_set(error, ENOMEM,
>> +					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
>> +					  NULL, "cannot create action");
>> +	}
>> +	rte_atomic32_init(&cache_resource->refcnt);
>> +	rte_atomic32_inc(&cache_resource->refcnt);
>> +	LIST_INSERT_HEAD(&priv->tags, cache_resource, next);
>> +	dev_flow->flow->tag_resource = cache_resource;
>> +	DRV_LOG(DEBUG, "new tag resource %p: refcnt %d++",
>> +		(void *)cache_resource,
>> +		rte_atomic32_read(&cache_resource->refcnt));
>> +	return 0;
>> +}
>> +
>> +/**
>> + * Release the tag.
>> + *
>> + * @param dev
>> + *   Pointer to Ethernet device.
>> + * @param flow
>> + *   Pointer to mlx5_flow.
>> + *
>> + * @return
>> + *   1 while a reference on it exists, 0 when freed.
>> + */
>> +static int
>> +flow_dv_tag_release(struct rte_eth_dev *dev,
>> +		    struct mlx5_flow_dv_tag_resource *tag)
>> +{
>> +	assert(tag);
>> +	DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--",
>> +		dev->data->port_id, (void *)tag,
>> +		rte_atomic32_read(&tag->refcnt));
>> +	if (rte_atomic32_dec_and_test(&tag->refcnt)) {
>> +		claim_zero(mlx5_glue->destroy_flow_action(tag->action));
>> +		LIST_REMOVE(tag, next);
>> +		DRV_LOG(DEBUG, "port %u tag %p: removed",
>> +			dev->data->port_id, (void *)tag);
>> +		rte_free(tag);
>> +		return 0;
>> +	}
>> +	return 1;
>> +}
>> +
>> +/**
>>  * Fill the flow with DV spec.
>>  *
>>  * @param[in] dev
>> @@ -2872,6 +2967,7 @@ struct field_modify_info modify_tcp[] = {
>> 					  MLX5DV_FLOW_TABLE_TYPE_NIC_RX
>> 	};
>> 	union flow_dv_attr flow_attr = { .attr = 0 };
>> +	struct mlx5_flow_dv_tag_resource tag_resource;
>> 	if (priority == MLX5_FLOW_PRIO_RSVD)
>> 		priority = priv->config.flow_prio - 1;
>> @@ -2886,26 +2982,29 @@ struct field_modify_info modify_tcp[] = {
>> 		case RTE_FLOW_ACTION_TYPE_VOID:
>> 			break;
>> 		case RTE_FLOW_ACTION_TYPE_FLAG:
>> -			dev_flow->dv.actions[actions_n].type =
>> -				MLX5DV_FLOW_ACTION_TAG;
>> -			dev_flow->dv.actions[actions_n].tag_value =
>> +			tag_resource.tag =
>> 				mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
>> -			actions_n++;
>> +			if (!flow->tag_resource)
>> +				if (flow_dv_tag_resource_register
>> +				    (dev, &tag_resource, dev_flow, error))
>> +					return errno;
>> +			dev_flow->dv.actions[actions_n++] =
>> +				flow->tag_resource->action;
>> 			action_flags |= MLX5_FLOW_ACTION_FLAG;
>> 			break;
>> 		case RTE_FLOW_ACTION_TYPE_MARK:
>> -			dev_flow->dv.actions[actions_n].type =
>> -				MLX5DV_FLOW_ACTION_TAG;
>> -			dev_flow->dv.actions[actions_n].tag_value =
>> -				mlx5_flow_mark_set
>> -				(((const struct rte_flow_action_mark *)
>> -				  (actions->conf))->id);
>> -			actions_n++;
>> +			tag_resource.tag = mlx5_flow_mark_set
>> +			      (((const struct rte_flow_action_mark *)
>> +			       (actions->conf))->id);
>> +			if (!flow->tag_resource)
>> +				if (flow_dv_tag_resource_register
>> +				    (dev, &tag_resource, dev_flow, error))
>> +					return errno;
>> +			dev_flow->dv.actions[actions_n++] =
>> +				flow->tag_resource->action;
>> 			action_flags |= MLX5_FLOW_ACTION_MARK;
>> 			break;
>> 		case RTE_FLOW_ACTION_TYPE_DROP:
>> -			dev_flow->dv.actions[actions_n].type =
>> -				MLX5DV_FLOW_ACTION_DROP;
>> 			action_flags |= MLX5_FLOW_ACTION_DROP;
>> 			break;
>> 		case RTE_FLOW_ACTION_TYPE_QUEUE:
>> @@ -2933,17 +3032,13 @@ struct field_modify_info modify_tcp[] = {
>> 				rte_errno = ENOTSUP;
>> 				goto cnt_err;
>> 			}
>> -			flow->counter =
>> -				flow_dv_counter_new(dev,
>> -						    count->shared, count->id);
>> +			flow->counter = flow_dv_counter_new(dev, count->shared,
>> +							    count->id);
>> 			if (flow->counter == NULL)
>> 				goto cnt_err;
>> -			dev_flow->dv.actions[actions_n].type =
>> -					MLX5DV_FLOW_ACTION_COUNTERS_DEVX;
>> -			dev_flow->dv.actions[actions_n].obj =
>> -						flow->counter->dcs->obj;
>> +			dev_flow->dv.actions[actions_n++] =
>> +				flow->counter->action;
>> 			action_flags |= MLX5_FLOW_ACTION_COUNT;
>> -			++actions_n;
>> 			break;
>> cnt_err:
>> 			if (rte_errno == ENOTSUP)
>> @@ -2964,11 +3059,8 @@ struct field_modify_info modify_tcp[] = {
>> 			if (flow_dv_create_action_l2_encap(dev, actions,
>> 							   dev_flow, error))
>> 				return -rte_errno;
>> -			dev_flow->dv.actions[actions_n].type =
>> -				MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
>> -			dev_flow->dv.actions[actions_n].action =
>> +			dev_flow->dv.actions[actions_n++] =
>> 				dev_flow->dv.encap_decap->verbs_action;
>> -			actions_n++;
>> 			action_flags |= actions->type ==
>> 					RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP ?
>> 					MLX5_FLOW_ACTION_VXLAN_ENCAP :
>> @@ -2979,11 +3071,8 @@ struct field_modify_info modify_tcp[] = {
>> 			if (flow_dv_create_action_l2_decap(dev, dev_flow,
>> 							   error))
>> 				return -rte_errno;
>> -			dev_flow->dv.actions[actions_n].type =
>> -				MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
>> -			dev_flow->dv.actions[actions_n].action =
>> +			dev_flow->dv.actions[actions_n++] =
>> 				dev_flow->dv.encap_decap->verbs_action;
>> -			actions_n++;
>> 			action_flags |= actions->type ==
>> 					RTE_FLOW_ACTION_TYPE_VXLAN_DECAP ?
>> 					MLX5_FLOW_ACTION_VXLAN_DECAP :
>> @@ -2995,9 +3084,7 @@ struct field_modify_info modify_tcp[] = {
>> 				if (flow_dv_create_action_raw_encap
>> 					(dev, actions, dev_flow, attr, error))
>> 					return -rte_errno;
>> -				dev_flow->dv.actions[actions_n].type =
>> -					MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
>> -				dev_flow->dv.actions[actions_n].action =
>> +				dev_flow->dv.actions[actions_n++] =
>> 					dev_flow->dv.encap_decap->verbs_action;
>> 			} else {
>> 				/* Handle encap without preceding decap. */
>> @@ -3005,12 +3092,9 @@ struct field_modify_info modify_tcp[] = {
>> 								   dev_flow,
>> 								   error))
>> 					return -rte_errno;
>> -				dev_flow->dv.actions[actions_n].type =
>> -					MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
>> -				dev_flow->dv.actions[actions_n].action =
>> +				dev_flow->dv.actions[actions_n++] =
>> 					dev_flow->dv.encap_decap->verbs_action;
>> 			}
>> -			actions_n++;
>> 			action_flags |= MLX5_FLOW_ACTION_RAW_ENCAP;
>> 			break;
>> 		case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
>> @@ -3025,11 +3109,8 @@ struct field_modify_info modify_tcp[] = {
>> 								   dev_flow,
>> 								   error))
>> 					return -rte_errno;
>> -				dev_flow->dv.actions[actions_n].type =
>> -					MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
>> -				dev_flow->dv.actions[actions_n].action =
>> +				dev_flow->dv.actions[actions_n++] =
>> 					dev_flow->dv.encap_decap->verbs_action;
>> -				actions_n++;
>> 			}
>> 			/* If decap is followed by encap, handle it at encap. */
>> 			action_flags |= MLX5_FLOW_ACTION_RAW_DECAP;
>> @@ -3098,11 +3179,8 @@ struct field_modify_info modify_tcp[] = {
>> 								 dev_flow,
>> 								 error))
>> 					return -rte_errno;
>> -				dev_flow->dv.actions[actions_n].type =
>> -					MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
>> -				dev_flow->dv.actions[actions_n].action =
>> +				dev_flow->dv.actions[actions_n++] =
>> 					dev_flow->dv.modify_hdr->verbs_action;
>> -				actions_n++;
>> 			}
>> 			break;
>> 		default:
>> @@ -3277,9 +3355,9 @@ struct field_modify_info modify_tcp[] = {
>> 					 "cannot get drop hash queue");
>> 				goto error;
>> 			}
>> -			dv->actions[n].type = MLX5DV_FLOW_ACTION_DEST_IBV_QP;
>> -			dv->actions[n].qp = dv->hrxq->qp;
>> -			n++;
>> +			dv->actions[n++] =
>> +				mlx5_glue->dv_create_flow_action_dest_ibv_qp
>> +				(dv->hrxq->qp);
>> 		} else if (flow->actions &
>> 			   (MLX5_FLOW_ACTION_QUEUE | MLX5_FLOW_ACTION_RSS)) {
>> 			struct mlx5_hrxq *hrxq;
>> @@ -3304,9 +3382,9 @@ struct field_modify_info modify_tcp[] = {
>> 				goto error;
>> 			}
>> 			dv->hrxq = hrxq;
>> -			dv->actions[n].type = MLX5DV_FLOW_ACTION_DEST_IBV_QP;
>> -			dv->actions[n].qp = hrxq->qp;
>> -			n++;
>> +			dv->actions[n++] =
>> +				mlx5_glue->dv_create_flow_action_dest_ibv_qp
>> +				(dv->hrxq->qp);
>> 		}
>> 		dv->flow =
>> 			mlx5_glue->dv_create_flow(dv->matcher->matcher_object,
>> @@ -3484,6 +3562,10 @@ struct field_modify_info modify_tcp[] = {
>> 		flow_dv_counter_release(flow->counter);
>> 		flow->counter = NULL;
>> 	}
>> +	if (flow->tag_resource) {
>> +		flow_dv_tag_release(dev, flow->tag_resource);
>> +		flow->tag_resource = NULL;
>> +	}
>> 	while (!LIST_EMPTY(&flow->dev_flows)) {
>> 		dev_flow = LIST_FIRST(&flow->dev_flows);
>> 		LIST_REMOVE(dev_flow, next);
>> diff --git a/drivers/net/mlx5/mlx5_glue.c b/drivers/net/mlx5/mlx5_glue.c
>> index c817d86..4b5aade 100644
>> --- a/drivers/net/mlx5/mlx5_glue.c
>> +++ b/drivers/net/mlx5/mlx5_glue.c
>> @@ -175,10 +175,20 @@
>> }
>> static int
>> -mlx5_glue_destroy_flow_action(struct ibv_flow_action *action)
>> +mlx5_glue_destroy_flow_action(void *action)
>> {
>> #ifdef HAVE_IBV_FLOW_DV_SUPPORT
>> -	return ibv_destroy_flow_action(action);
>> +	struct mlx5dv_flow_action_attr *attr = action;
>> +	int res = 0;
>> +	switch (attr->type) {
>> +	case MLX5DV_FLOW_ACTION_TAG:
>> +		break;
>> +	default:
>> +		res = ibv_destroy_flow_action(attr->action);
>> +		break;
>> +	}
>> +	free(action);
>> +	return res;
>> #else
>> 	(void)action;
>> 	return ENOTSUP;
>> @@ -430,16 +440,23 @@
>> mlx5_glue_dv_create_flow(struct mlx5dv_flow_matcher *matcher,
>> 			 struct mlx5dv_flow_match_parameters *match_value,
>> 			 size_t num_actions,
>> -			 struct mlx5dv_flow_action_attr *actions_attr)
>> +			 void *actions[])
>> {
>> #ifdef HAVE_IBV_FLOW_DV_SUPPORT
>> +	struct mlx5dv_flow_action_attr actions_attr[8];
>> +
>> +	if (num_actions > 8)
>> +		return NULL;
>> +	for (size_t i = 0; i < num_actions; i++)
>> +		actions_attr[i] =
>> +			*((struct mlx5dv_flow_action_attr *)(actions[i]));
>> 	return mlx5dv_create_flow(matcher, match_value,
>> 				  num_actions, actions_attr);
>> #else
>> 	(void)matcher;
>> 	(void)match_value;
>> 	(void)num_actions;
>> -	(void)actions_attr;
>> +	(void)actions;
>> 	return NULL;
>> #endif
>> }
>> @@ -455,31 +472,45 @@
>> #endif
>> }
>> -static struct ibv_flow_action *
>> -mlx5_glue_dv_create_flow_action_packet_reformat
>> -		(struct ibv_context *ctx,
>> -		 size_t data_sz,
>> -		 void *data,
>> -		 enum mlx5dv_flow_action_packet_reformat_type reformat_type,
>> -		 enum mlx5dv_flow_table_type ft_type)
>> +static void *
>> +mlx5_glue_dv_create_flow_action_counter(void *counter_obj, uint32_t offset)
>> {
>> #ifdef HAVE_IBV_FLOW_DV_SUPPORT
>> -	return mlx5dv_create_flow_action_packet_reformat(ctx,
>> -							 data_sz,
>> -							 data,
>> -							 reformat_type,
>> -							 ft_type);
>> +	struct mlx5dv_flow_action_attr *action;
>> +
>> +	(void)offset;
>> +	action = malloc(sizeof(*action));
>> +	if (!action)
>> +		return NULL;
>> +	action->type = MLX5DV_FLOW_ACTION_COUNTERS_DEVX;
>> +	action->obj = counter_obj;
>> +	return action;
>> #else
>> -	(void)ctx;
>> -	(void)data_sz;
>> -	(void)data;
>> -	(void)reformat_type;
>> -	(void)ft_type;
>> +	(void)counter_obj;
>> +	(void)offset;
>> 	return NULL;
>> #endif
>> }
>> -static struct ibv_flow_action *
>> +static void *
>> +mlx5_glue_dv_create_flow_action_dest_ibv_qp(void *qp)
>> +{
>> +#ifdef HAVE_IBV_FLOW_DV_SUPPORT
>> +	struct mlx5dv_flow_action_attr *action;
>> +
>> +	action = malloc(sizeof(*action));
>> +	if (!action)
>> +		return NULL;
>> +	action->type = MLX5DV_FLOW_ACTION_DEST_IBV_QP;
>> +	action->obj = qp;
>> +	return action;
>> +#else
>> +	(void)qp;
>> +	return NULL;
>> +#endif
>> +}
>> +
>> +static void *
>> mlx5_glue_dv_create_flow_action_modify_header
>> 					(struct ibv_context *ctx,
>> 					 size_t actions_sz,
>> @@ -487,8 +518,15 @@
>> 					 enum mlx5dv_flow_table_type ft_type)
>> {
>> #ifdef HAVE_IBV_FLOW_DV_SUPPORT
>> -	return mlx5dv_create_flow_action_modify_header(ctx, actions_sz,
>> -						       actions, ft_type);
>> +	struct mlx5dv_flow_action_attr *action;
>> +
>> +	action = malloc(sizeof(*action));
>> +	if (!action)
>> +		return NULL;
>> +	action->type = MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
>> +	action->action = mlx5dv_create_flow_action_modify_header
>> +		(ctx, actions_sz, actions, ft_type);
>> +	return action;
>> #else
>> 	(void)ctx;
>> 	(void)actions_sz;
>> @@ -498,6 +536,50 @@
>> #endif
>> }
>> +static void *
>> +mlx5_glue_dv_create_flow_action_packet_reformat
>> +		(struct ibv_context *ctx,
>> +		 size_t data_sz,
>> +		 void *data,
>> +		 enum mlx5dv_flow_action_packet_reformat_type reformat_type,
>> +		 enum mlx5dv_flow_table_type ft_type)
>> +{
>> +#ifdef HAVE_IBV_FLOW_DV_SUPPORT
>> +	struct mlx5dv_flow_action_attr *action;
>> +
>> +	action = malloc(sizeof(*action));
>> +	if (!action)
>> +		return NULL;
>> +	action->type = MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
>> +	action->action = mlx5dv_create_flow_action_packet_reformat
>> +		(ctx, data_sz, data, reformat_type, ft_type);
>> +	return action;
>> +#else
>> +	(void)ctx;
>> +	(void)data_sz;
>> +	(void)data;
>> +	(void)reformat_type;
>> +	(void)ft_type;
>> +	return NULL;
>> +#endif
>> +}
>> +
>> +static void *
>> +mlx5_glue_dv_create_flow_action_tag(uint32_t tag)
>> +{
>> +#ifdef HAVE_IBV_FLOW_DV_SUPPORT
>> +	struct mlx5dv_flow_action_attr *action;
>> +	action = malloc(sizeof(*action));
>> +	if (!action)
>> +		return NULL;
>> +	action->type = MLX5DV_FLOW_ACTION_TAG;
>> +	action->tag_value = tag;
>> +	return action;
>> +#endif
>> +	(void)tag;
>> +	return NULL;
>> +}
>> +
>> static struct ibv_context *
>> mlx5_glue_dv_open_device(struct ibv_device *device)
>> {
>> @@ -645,10 +727,15 @@
>> 	.dv_create_flow_matcher = mlx5_glue_dv_create_flow_matcher,
>> 	.dv_destroy_flow_matcher = mlx5_glue_dv_destroy_flow_matcher,
>> 	.dv_create_flow = mlx5_glue_dv_create_flow,
>> -	.dv_create_flow_action_packet_reformat =
>> -			mlx5_glue_dv_create_flow_action_packet_reformat,
>> +	.dv_create_flow_action_counter =
>> +		mlx5_glue_dv_create_flow_action_counter,
>> +	.dv_create_flow_action_dest_ibv_qp =
>> +		mlx5_glue_dv_create_flow_action_dest_ibv_qp,
>> 	.dv_create_flow_action_modify_header =
>> -			mlx5_glue_dv_create_flow_action_modify_header,
>> +		mlx5_glue_dv_create_flow_action_modify_header,
>> +	.dv_create_flow_action_packet_reformat =
>> +		mlx5_glue_dv_create_flow_action_packet_reformat,
>> +	.dv_create_flow_action_tag =  mlx5_glue_dv_create_flow_action_tag,
>> 	.dv_open_device = mlx5_glue_dv_open_device,
>> 	.devx_obj_create = mlx5_glue_devx_obj_create,
>> 	.devx_obj_destroy = mlx5_glue_devx_obj_destroy,
>> diff --git a/drivers/net/mlx5/mlx5_glue.h b/drivers/net/mlx5/mlx5_glue.h
>> index b118960..32487ea 100644
>> --- a/drivers/net/mlx5/mlx5_glue.h
>> +++ b/drivers/net/mlx5/mlx5_glue.h
>> @@ -55,6 +55,10 @@
>> enum mlx5dv_flow_table_type { flow_table_type = 0, };
>> #endif
>> +#ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
>> +#define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0
>> +#endif
>> +
>> #ifndef HAVE_IBV_DEVX_OBJ
>> struct mlx5dv_devx_obj;
>> #endif
>> @@ -98,7 +102,7 @@ struct mlx5_glue {
>> 	struct ibv_flow *(*create_flow)(struct ibv_qp *qp,
>> 					struct ibv_flow_attr *flow);
>> 	int (*destroy_flow)(struct ibv_flow *flow_id);
>> -	int (*destroy_flow_action)(struct ibv_flow_action *action);
>> +	int (*destroy_flow_action)(void *action);
>> 	struct ibv_qp *(*create_qp)(struct ibv_pd *pd,
>> 				    struct ibv_qp_init_attr *qp_init_attr);
>> 	struct ibv_qp *(*create_qp_ex)
>> @@ -160,19 +164,17 @@ struct mlx5_glue {
>> 	int (*dv_destroy_flow_matcher)(struct mlx5dv_flow_matcher *matcher);
>> 	struct ibv_flow *(*dv_create_flow)(struct mlx5dv_flow_matcher *matcher,
>> 			  struct mlx5dv_flow_match_parameters *match_value,
>> -			  size_t num_actions,
>> -			  struct mlx5dv_flow_action_attr *actions_attr);
>> -	struct ibv_flow_action *(*dv_create_flow_action_packet_reformat)
>> -		(struct ibv_context *ctx,
>> -		 size_t data_sz,
>> -		 void *data,
>> +			  size_t num_actions, void *actions[]);
>> +	void *(*dv_create_flow_action_counter)(void *obj, uint32_t  offset);
>> +	void *(*dv_create_flow_action_dest_ibv_qp)(void *qp);
>> +	void *(*dv_create_flow_action_modify_header)
>> +		(struct ibv_context *ctx, size_t actions_sz, uint64_t actions[],
>> +		 enum mlx5dv_flow_table_type ft_type);
>> +	void *(*dv_create_flow_action_packet_reformat)
>> +		(struct ibv_context *ctx, size_t data_sz, void *data,
>> 		 enum mlx5dv_flow_action_packet_reformat_type reformat_type,
>> 		 enum mlx5dv_flow_table_type ft_type);
>> -	struct ibv_flow_action *(*dv_create_flow_action_modify_header)
>> -					(struct ibv_context *ctx,
>> -					 size_t actions_sz,
>> -					 uint64_t actions[],
>> -					 enum mlx5dv_flow_table_type ft_type);
>> +	void *(*dv_create_flow_action_tag)(uint32_t tag);
>> 	struct ibv_context *(*dv_open_device)(struct ibv_device *device);
>> 	struct mlx5dv_devx_obj *(*devx_obj_create)
>> 					(struct ibv_context *ctx,
>