Re: [dpdk-dev] [PATCH v4 1/3] net/mlx5: prepare Direct Verbs for Direct Rule

[dwilder <dwilder@us.ibm.com>]

I am seeing a build break when building on ubuntu 19.04 beta after this 
commit (cbb66daa3).

The build produces a number of the following errors.

drivers/net/mlx5/mlx5_glue.c:194:2: error: implicit declaration of 
function ‘free’
....
drivers/net/mlx5/mlx5_glue.c:552:11: error: incompatible implicit 
declaration of built-in function ‘malloc’ [-Werror]
....

Adding "#include <stdlib.h>" to drivers/net/mlx5/mlx5_glue.c fixed it 
for me.

I do not see this problem on Ubuntu 18.04, maybe some headers have 
changed.  I have only tested on ppc64le.


On 2019-04-04 02:54, Ori Kam wrote:
> This is the first patch of a series that is designed to enable the
> Direct Rules API.
> 
> The main difference between Direct Verbs and Direct Rules from API
> prespective, is that in Direct Rules each action has it's own create
> function and the object itself is of type void.
> 
> In this patch I'm adding functions to generate actions that currenlty
> are done without create action, and I'm changing the action type to be
> void *, so in next patches only the glue functions will need to change.
> 
> Signed-off-by: Ori Kam <orika@mellanox.com>
> Acked-by: Shahaf Shuler <shahafs@mellanox.com>
> ---
>  drivers/net/mlx5/Makefile       |   2 +-
>  drivers/net/mlx5/meson.build    |   2 +-
>  drivers/net/mlx5/mlx5.h         |   2 +
>  drivers/net/mlx5/mlx5_flow.h    |  17 +++-
>  drivers/net/mlx5/mlx5_flow_dv.c | 184 
> +++++++++++++++++++++++++++++-----------
>  drivers/net/mlx5/mlx5_glue.c    | 141 ++++++++++++++++++++++++------
>  drivers/net/mlx5/mlx5_glue.h    |  26 +++---
>  7 files changed, 280 insertions(+), 94 deletions(-)
> 
> diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile
> index ccd179c..0d20f0f 100644
> --- a/drivers/net/mlx5/Makefile
> +++ b/drivers/net/mlx5/Makefile
> @@ -8,7 +8,7 @@ include $(RTE_SDK)/mk/rte.vars.mk
>  LIB = librte_pmd_mlx5.a
>  LIB_GLUE = $(LIB_GLUE_BASE).$(LIB_GLUE_VERSION)
>  LIB_GLUE_BASE = librte_pmd_mlx5_glue.so
> -LIB_GLUE_VERSION = 19.02.0
> +LIB_GLUE_VERSION = 19.05.0
> 
>  # Sources.
>  SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5.c
> diff --git a/drivers/net/mlx5/meson.build 
> b/drivers/net/mlx5/meson.build
> index a429655..a4c684e 100644
> --- a/drivers/net/mlx5/meson.build
> +++ b/drivers/net/mlx5/meson.build
> @@ -4,7 +4,7 @@
> 
>  pmd_dlopen = (get_option('ibverbs_link') == 'dlopen')
>  LIB_GLUE_BASE = 'librte_pmd_mlx5_glue.so'
> -LIB_GLUE_VERSION = '19.02.0'
> +LIB_GLUE_VERSION = '19.05.0'
>  LIB_GLUE = LIB_GLUE_BASE + '.' + LIB_GLUE_VERSION
>  if pmd_dlopen
>  	dpdk_conf.set('RTE_IBVERBS_LINK_DLOPEN', 1)
> diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
> index 47a7d75..784bf9b 100644
> --- a/drivers/net/mlx5/mlx5.h
> +++ b/drivers/net/mlx5/mlx5.h
> @@ -309,6 +309,8 @@ struct mlx5_priv {
>  	LIST_HEAD(matchers, mlx5_flow_dv_matcher) matchers;
>  	LIST_HEAD(encap_decap, mlx5_flow_dv_encap_decap_resource) 
> encaps_decaps;
>  	LIST_HEAD(modify_cmd, mlx5_flow_dv_modify_hdr_resource) modify_cmds;
> +	LIST_HEAD(tag, mlx5_flow_dv_tag_resource) tags;
> +	/* Tags resources cache. */
>  	uint32_t link_speed_capa; /* Link speed capabilities. */
>  	struct mlx5_xstats_ctrl xstats_ctrl; /* Extended stats control. */
>  	struct mlx5_stats_ctrl stats_ctrl; /* Stats control. */
> diff --git a/drivers/net/mlx5/mlx5_flow.h 
> b/drivers/net/mlx5/mlx5_flow.h
> index e1e798b..4f69ae2 100644
> --- a/drivers/net/mlx5/mlx5_flow.h
> +++ b/drivers/net/mlx5/mlx5_flow.h
> @@ -214,7 +214,7 @@ struct mlx5_flow_dv_encap_decap_resource {
>  	LIST_ENTRY(mlx5_flow_dv_encap_decap_resource) next;
>  	/* Pointer to next element. */
>  	rte_atomic32_t refcnt; /**< Reference counter. */
> -	struct ibv_flow_action *verbs_action;
> +	void *verbs_action;
>  	/**< Verbs encap/decap action object. */
>  	uint8_t buf[MLX5_ENCAP_MAX_LEN];
>  	size_t size;
> @@ -222,6 +222,16 @@ struct mlx5_flow_dv_encap_decap_resource {
>  	uint8_t ft_type;
>  };
> 
> +/* Tag resource structure. */
> +struct mlx5_flow_dv_tag_resource {
> +	LIST_ENTRY(mlx5_flow_dv_tag_resource) next;
> +	/* Pointer to next element. */
> +	rte_atomic32_t refcnt; /**< Reference counter. */
> +	void *action;
> +	/**< Verbs tag action object. */
> +	uint32_t tag; /**< the tag value. */
> +};
> +
>  /* Number of modification commands. */
>  #define MLX5_MODIFY_NUM 8
> 
> @@ -259,7 +269,7 @@ struct mlx5_flow_dv {
>  	/**< Pointer to modify header resource in cache. */
>  	struct ibv_flow *flow; /**< Installed flow. */
>  #ifdef HAVE_IBV_FLOW_DV_SUPPORT
> -	struct mlx5dv_flow_action_attr 
> actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS];
> +	void *actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS];
>  	/**< Action list. */
>  #endif
>  	int actions_n; /**< number of actions. */
> @@ -332,6 +342,7 @@ struct mlx5_flow_counter {
>  	};
>  	uint64_t hits; /**< Number of packets matched by the rule. */
>  	uint64_t bytes; /**< Number of bytes matched by the rule. */
> +	void *action; /**< Pointer to the dv action. */
>  };
> 
>  /* Flow structure. */
> @@ -339,6 +350,8 @@ struct rte_flow {
>  	TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. 
> */
>  	enum mlx5_flow_drv_type drv_type; /**< Drvier type. */
>  	struct mlx5_flow_counter *counter; /**< Holds flow counter. */
> +	struct mlx5_flow_dv_tag_resource *tag_resource;
> +	/**< pointer to the tag action. */
>  	struct rte_flow_action_rss rss;/**< RSS context. */
>  	uint8_t key[MLX5_RSS_HASH_KEY_LEN]; /**< RSS hash key. */
>  	uint16_t (*queue)[]; /**< Destination queues to redirect traffic to. 
> */
> diff --git a/drivers/net/mlx5/mlx5_flow_dv.c 
> b/drivers/net/mlx5/mlx5_flow_dv.c
> index ad84dea..d9e2ac9 100644
> --- a/drivers/net/mlx5/mlx5_flow_dv.c
> +++ b/drivers/net/mlx5/mlx5_flow_dv.c
> @@ -32,7 +32,6 @@
>  #include "mlx5_prm.h"
>  #include "mlx5_glue.h"
>  #include "mlx5_flow.h"
> -
>  #ifdef HAVE_IBV_FLOW_DV_SUPPORT
> 
>  #ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
> @@ -1537,6 +1536,11 @@ struct field_modify_info modify_tcp[] = {
>  		.id = id,
>  		.dcs = dcs,
>  	};
> +	tmpl.action = mlx5_glue->dv_create_flow_action_counter(dcs->obj, 0);
> +	if (!tmpl.action) {
> +		ret = errno;
> +		goto error_exit;
> +	}
>  	*cnt = tmpl;
>  	LIST_INSERT_HEAD(&priv->flow_counters, cnt, next);
>  	return cnt;
> @@ -2828,6 +2832,97 @@ struct field_modify_info modify_tcp[] = {
>  }
> 
>  /**
> + * Find existing tag resource or create and register a new one.
> + *
> + * @param dev[in, out]
> + *   Pointer to rte_eth_dev structure.
> + * @param[in, out] resource
> + *   Pointer to tag resource.
> + * @parm[in, out] dev_flow
> + *   Pointer to the dev_flow.
> + * @param[out] error
> + *   pointer to error structure.
> + *
> + * @return
> + *   0 on success otherwise -errno and errno is set.
> + */
> +static int
> +flow_dv_tag_resource_register
> +			(struct rte_eth_dev *dev,
> +			 struct mlx5_flow_dv_tag_resource *resource,
> +			 struct mlx5_flow *dev_flow,
> +			 struct rte_flow_error *error)
> +{
> +	struct mlx5_priv *priv = dev->data->dev_private;
> +	struct mlx5_flow_dv_tag_resource *cache_resource;
> +
> +	/* Lookup a matching resource from cache. */
> +	LIST_FOREACH(cache_resource, &priv->tags, next) {
> +		if (resource->tag == cache_resource->tag) {
> +			DRV_LOG(DEBUG, "tag resource %p: refcnt %d++",
> +				(void *)cache_resource,
> +				rte_atomic32_read(&cache_resource->refcnt));
> +			rte_atomic32_inc(&cache_resource->refcnt);
> +			dev_flow->flow->tag_resource = cache_resource;
> +			return 0;
> +		}
> +	}
> +	/* Register new  resource. */
> +	cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
> +	if (!cache_resource)
> +		return rte_flow_error_set(error, ENOMEM,
> +					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
> +					  "cannot allocate resource memory");
> +	*cache_resource = *resource;
> +	cache_resource->action = mlx5_glue->dv_create_flow_action_tag
> +		(resource->tag);
> +	if (!cache_resource->action) {
> +		rte_free(cache_resource);
> +		return rte_flow_error_set(error, ENOMEM,
> +					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
> +					  NULL, "cannot create action");
> +	}
> +	rte_atomic32_init(&cache_resource->refcnt);
> +	rte_atomic32_inc(&cache_resource->refcnt);
> +	LIST_INSERT_HEAD(&priv->tags, cache_resource, next);
> +	dev_flow->flow->tag_resource = cache_resource;
> +	DRV_LOG(DEBUG, "new tag resource %p: refcnt %d++",
> +		(void *)cache_resource,
> +		rte_atomic32_read(&cache_resource->refcnt));
> +	return 0;
> +}
> +
> +/**
> + * Release the tag.
> + *
> + * @param dev
> + *   Pointer to Ethernet device.
> + * @param flow
> + *   Pointer to mlx5_flow.
> + *
> + * @return
> + *   1 while a reference on it exists, 0 when freed.
> + */
> +static int
> +flow_dv_tag_release(struct rte_eth_dev *dev,
> +		    struct mlx5_flow_dv_tag_resource *tag)
> +{
> +	assert(tag);
> +	DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--",
> +		dev->data->port_id, (void *)tag,
> +		rte_atomic32_read(&tag->refcnt));
> +	if (rte_atomic32_dec_and_test(&tag->refcnt)) {
> +		claim_zero(mlx5_glue->destroy_flow_action(tag->action));
> +		LIST_REMOVE(tag, next);
> +		DRV_LOG(DEBUG, "port %u tag %p: removed",
> +			dev->data->port_id, (void *)tag);
> +		rte_free(tag);
> +		return 0;
> +	}
> +	return 1;
> +}
> +
> +/**
>   * Fill the flow with DV spec.
>   *
>   * @param[in] dev
> @@ -2872,6 +2967,7 @@ struct field_modify_info modify_tcp[] = {
>  					  MLX5DV_FLOW_TABLE_TYPE_NIC_RX
>  	};
>  	union flow_dv_attr flow_attr = { .attr = 0 };
> +	struct mlx5_flow_dv_tag_resource tag_resource;
> 
>  	if (priority == MLX5_FLOW_PRIO_RSVD)
>  		priority = priv->config.flow_prio - 1;
> @@ -2886,26 +2982,29 @@ struct field_modify_info modify_tcp[] = {
>  		case RTE_FLOW_ACTION_TYPE_VOID:
>  			break;
>  		case RTE_FLOW_ACTION_TYPE_FLAG:
> -			dev_flow->dv.actions[actions_n].type =
> -				MLX5DV_FLOW_ACTION_TAG;
> -			dev_flow->dv.actions[actions_n].tag_value =
> +			tag_resource.tag =
>  				mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
> -			actions_n++;
> +			if (!flow->tag_resource)
> +				if (flow_dv_tag_resource_register
> +				    (dev, &tag_resource, dev_flow, error))
> +					return errno;
> +			dev_flow->dv.actions[actions_n++] =
> +				flow->tag_resource->action;
>  			action_flags |= MLX5_FLOW_ACTION_FLAG;
>  			break;
>  		case RTE_FLOW_ACTION_TYPE_MARK:
> -			dev_flow->dv.actions[actions_n].type =
> -				MLX5DV_FLOW_ACTION_TAG;
> -			dev_flow->dv.actions[actions_n].tag_value =
> -				mlx5_flow_mark_set
> -				(((const struct rte_flow_action_mark *)
> -				  (actions->conf))->id);
> -			actions_n++;
> +			tag_resource.tag = mlx5_flow_mark_set
> +			      (((const struct rte_flow_action_mark *)
> +			       (actions->conf))->id);
> +			if (!flow->tag_resource)
> +				if (flow_dv_tag_resource_register
> +				    (dev, &tag_resource, dev_flow, error))
> +					return errno;
> +			dev_flow->dv.actions[actions_n++] =
> +				flow->tag_resource->action;
>  			action_flags |= MLX5_FLOW_ACTION_MARK;
>  			break;
>  		case RTE_FLOW_ACTION_TYPE_DROP:
> -			dev_flow->dv.actions[actions_n].type =
> -				MLX5DV_FLOW_ACTION_DROP;
>  			action_flags |= MLX5_FLOW_ACTION_DROP;
>  			break;
>  		case RTE_FLOW_ACTION_TYPE_QUEUE:
> @@ -2933,17 +3032,13 @@ struct field_modify_info modify_tcp[] = {
>  				rte_errno = ENOTSUP;
>  				goto cnt_err;
>  			}
> -			flow->counter =
> -				flow_dv_counter_new(dev,
> -						    count->shared, count->id);
> +			flow->counter = flow_dv_counter_new(dev, count->shared,
> +							    count->id);
>  			if (flow->counter == NULL)
>  				goto cnt_err;
> -			dev_flow->dv.actions[actions_n].type =
> -					MLX5DV_FLOW_ACTION_COUNTERS_DEVX;
> -			dev_flow->dv.actions[actions_n].obj =
> -						flow->counter->dcs->obj;
> +			dev_flow->dv.actions[actions_n++] =
> +				flow->counter->action;
>  			action_flags |= MLX5_FLOW_ACTION_COUNT;
> -			++actions_n;
>  			break;
>  cnt_err:
>  			if (rte_errno == ENOTSUP)
> @@ -2964,11 +3059,8 @@ struct field_modify_info modify_tcp[] = {
>  			if (flow_dv_create_action_l2_encap(dev, actions,
>  							   dev_flow, error))
>  				return -rte_errno;
> -			dev_flow->dv.actions[actions_n].type =
> -				MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
> -			dev_flow->dv.actions[actions_n].action =
> +			dev_flow->dv.actions[actions_n++] =
>  				dev_flow->dv.encap_decap->verbs_action;
> -			actions_n++;
>  			action_flags |= actions->type ==
>  					RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP ?
>  					MLX5_FLOW_ACTION_VXLAN_ENCAP :
> @@ -2979,11 +3071,8 @@ struct field_modify_info modify_tcp[] = {
>  			if (flow_dv_create_action_l2_decap(dev, dev_flow,
>  							   error))
>  				return -rte_errno;
> -			dev_flow->dv.actions[actions_n].type =
> -				MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
> -			dev_flow->dv.actions[actions_n].action =
> +			dev_flow->dv.actions[actions_n++] =
>  				dev_flow->dv.encap_decap->verbs_action;
> -			actions_n++;
>  			action_flags |= actions->type ==
>  					RTE_FLOW_ACTION_TYPE_VXLAN_DECAP ?
>  					MLX5_FLOW_ACTION_VXLAN_DECAP :
> @@ -2995,9 +3084,7 @@ struct field_modify_info modify_tcp[] = {
>  				if (flow_dv_create_action_raw_encap
>  					(dev, actions, dev_flow, attr, error))
>  					return -rte_errno;
> -				dev_flow->dv.actions[actions_n].type =
> -					MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
> -				dev_flow->dv.actions[actions_n].action =
> +				dev_flow->dv.actions[actions_n++] =
>  					dev_flow->dv.encap_decap->verbs_action;
>  			} else {
>  				/* Handle encap without preceding decap. */
> @@ -3005,12 +3092,9 @@ struct field_modify_info modify_tcp[] = {
>  								   dev_flow,
>  								   error))
>  					return -rte_errno;
> -				dev_flow->dv.actions[actions_n].type =
> -					MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
> -				dev_flow->dv.actions[actions_n].action =
> +				dev_flow->dv.actions[actions_n++] =
>  					dev_flow->dv.encap_decap->verbs_action;
>  			}
> -			actions_n++;
>  			action_flags |= MLX5_FLOW_ACTION_RAW_ENCAP;
>  			break;
>  		case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
> @@ -3025,11 +3109,8 @@ struct field_modify_info modify_tcp[] = {
>  								   dev_flow,
>  								   error))
>  					return -rte_errno;
> -				dev_flow->dv.actions[actions_n].type =
> -					MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
> -				dev_flow->dv.actions[actions_n].action =
> +				dev_flow->dv.actions[actions_n++] =
>  					dev_flow->dv.encap_decap->verbs_action;
> -				actions_n++;
>  			}
>  			/* If decap is followed by encap, handle it at encap. */
>  			action_flags |= MLX5_FLOW_ACTION_RAW_DECAP;
> @@ -3098,11 +3179,8 @@ struct field_modify_info modify_tcp[] = {
>  								 dev_flow,
>  								 error))
>  					return -rte_errno;
> -				dev_flow->dv.actions[actions_n].type =
> -					MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
> -				dev_flow->dv.actions[actions_n].action =
> +				dev_flow->dv.actions[actions_n++] =
>  					dev_flow->dv.modify_hdr->verbs_action;
> -				actions_n++;
>  			}
>  			break;
>  		default:
> @@ -3277,9 +3355,9 @@ struct field_modify_info modify_tcp[] = {
>  					 "cannot get drop hash queue");
>  				goto error;
>  			}
> -			dv->actions[n].type = MLX5DV_FLOW_ACTION_DEST_IBV_QP;
> -			dv->actions[n].qp = dv->hrxq->qp;
> -			n++;
> +			dv->actions[n++] =
> +				mlx5_glue->dv_create_flow_action_dest_ibv_qp
> +				(dv->hrxq->qp);
>  		} else if (flow->actions &
>  			   (MLX5_FLOW_ACTION_QUEUE | MLX5_FLOW_ACTION_RSS)) {
>  			struct mlx5_hrxq *hrxq;
> @@ -3304,9 +3382,9 @@ struct field_modify_info modify_tcp[] = {
>  				goto error;
>  			}
>  			dv->hrxq = hrxq;
> -			dv->actions[n].type = MLX5DV_FLOW_ACTION_DEST_IBV_QP;
> -			dv->actions[n].qp = hrxq->qp;
> -			n++;
> +			dv->actions[n++] =
> +				mlx5_glue->dv_create_flow_action_dest_ibv_qp
> +				(dv->hrxq->qp);
>  		}
>  		dv->flow =
>  			mlx5_glue->dv_create_flow(dv->matcher->matcher_object,
> @@ -3484,6 +3562,10 @@ struct field_modify_info modify_tcp[] = {
>  		flow_dv_counter_release(flow->counter);
>  		flow->counter = NULL;
>  	}
> +	if (flow->tag_resource) {
> +		flow_dv_tag_release(dev, flow->tag_resource);
> +		flow->tag_resource = NULL;
> +	}
>  	while (!LIST_EMPTY(&flow->dev_flows)) {
>  		dev_flow = LIST_FIRST(&flow->dev_flows);
>  		LIST_REMOVE(dev_flow, next);
> diff --git a/drivers/net/mlx5/mlx5_glue.c 
> b/drivers/net/mlx5/mlx5_glue.c
> index c817d86..4b5aade 100644
> --- a/drivers/net/mlx5/mlx5_glue.c
> +++ b/drivers/net/mlx5/mlx5_glue.c
> @@ -175,10 +175,20 @@
>  }
> 
>  static int
> -mlx5_glue_destroy_flow_action(struct ibv_flow_action *action)
> +mlx5_glue_destroy_flow_action(void *action)
>  {
>  #ifdef HAVE_IBV_FLOW_DV_SUPPORT
> -	return ibv_destroy_flow_action(action);
> +	struct mlx5dv_flow_action_attr *attr = action;
> +	int res = 0;
> +	switch (attr->type) {
> +	case MLX5DV_FLOW_ACTION_TAG:
> +		break;
> +	default:
> +		res = ibv_destroy_flow_action(attr->action);
> +		break;
> +	}
> +	free(action);
> +	return res;
>  #else
>  	(void)action;
>  	return ENOTSUP;
> @@ -430,16 +440,23 @@
>  mlx5_glue_dv_create_flow(struct mlx5dv_flow_matcher *matcher,
>  			 struct mlx5dv_flow_match_parameters *match_value,
>  			 size_t num_actions,
> -			 struct mlx5dv_flow_action_attr *actions_attr)
> +			 void *actions[])
>  {
>  #ifdef HAVE_IBV_FLOW_DV_SUPPORT
> +	struct mlx5dv_flow_action_attr actions_attr[8];
> +
> +	if (num_actions > 8)
> +		return NULL;
> +	for (size_t i = 0; i < num_actions; i++)
> +		actions_attr[i] =
> +			*((struct mlx5dv_flow_action_attr *)(actions[i]));
>  	return mlx5dv_create_flow(matcher, match_value,
>  				  num_actions, actions_attr);
>  #else
>  	(void)matcher;
>  	(void)match_value;
>  	(void)num_actions;
> -	(void)actions_attr;
> +	(void)actions;
>  	return NULL;
>  #endif
>  }
> @@ -455,31 +472,45 @@
>  #endif
>  }
> 
> -static struct ibv_flow_action *
> -mlx5_glue_dv_create_flow_action_packet_reformat
> -		(struct ibv_context *ctx,
> -		 size_t data_sz,
> -		 void *data,
> -		 enum mlx5dv_flow_action_packet_reformat_type reformat_type,
> -		 enum mlx5dv_flow_table_type ft_type)
> +static void *
> +mlx5_glue_dv_create_flow_action_counter(void *counter_obj, uint32_t 
> offset)
>  {
>  #ifdef HAVE_IBV_FLOW_DV_SUPPORT
> -	return mlx5dv_create_flow_action_packet_reformat(ctx,
> -							 data_sz,
> -							 data,
> -							 reformat_type,
> -							 ft_type);
> +	struct mlx5dv_flow_action_attr *action;
> +
> +	(void)offset;
> +	action = malloc(sizeof(*action));
> +	if (!action)
> +		return NULL;
> +	action->type = MLX5DV_FLOW_ACTION_COUNTERS_DEVX;
> +	action->obj = counter_obj;
> +	return action;
>  #else
> -	(void)ctx;
> -	(void)data_sz;
> -	(void)data;
> -	(void)reformat_type;
> -	(void)ft_type;
> +	(void)counter_obj;
> +	(void)offset;
>  	return NULL;
>  #endif
>  }
> 
> -static struct ibv_flow_action *
> +static void *
> +mlx5_glue_dv_create_flow_action_dest_ibv_qp(void *qp)
> +{
> +#ifdef HAVE_IBV_FLOW_DV_SUPPORT
> +	struct mlx5dv_flow_action_attr *action;
> +
> +	action = malloc(sizeof(*action));
> +	if (!action)
> +		return NULL;
> +	action->type = MLX5DV_FLOW_ACTION_DEST_IBV_QP;
> +	action->obj = qp;
> +	return action;
> +#else
> +	(void)qp;
> +	return NULL;
> +#endif
> +}
> +
> +static void *
>  mlx5_glue_dv_create_flow_action_modify_header
>  					(struct ibv_context *ctx,
>  					 size_t actions_sz,
> @@ -487,8 +518,15 @@
>  					 enum mlx5dv_flow_table_type ft_type)
>  {
>  #ifdef HAVE_IBV_FLOW_DV_SUPPORT
> -	return mlx5dv_create_flow_action_modify_header(ctx, actions_sz,
> -						       actions, ft_type);
> +	struct mlx5dv_flow_action_attr *action;
> +
> +	action = malloc(sizeof(*action));
> +	if (!action)
> +		return NULL;
> +	action->type = MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
> +	action->action = mlx5dv_create_flow_action_modify_header
> +		(ctx, actions_sz, actions, ft_type);
> +	return action;
>  #else
>  	(void)ctx;
>  	(void)actions_sz;
> @@ -498,6 +536,50 @@
>  #endif
>  }
> 
> +static void *
> +mlx5_glue_dv_create_flow_action_packet_reformat
> +		(struct ibv_context *ctx,
> +		 size_t data_sz,
> +		 void *data,
> +		 enum mlx5dv_flow_action_packet_reformat_type reformat_type,
> +		 enum mlx5dv_flow_table_type ft_type)
> +{
> +#ifdef HAVE_IBV_FLOW_DV_SUPPORT
> +	struct mlx5dv_flow_action_attr *action;
> +
> +	action = malloc(sizeof(*action));
> +	if (!action)
> +		return NULL;
> +	action->type = MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
> +	action->action = mlx5dv_create_flow_action_packet_reformat
> +		(ctx, data_sz, data, reformat_type, ft_type);
> +	return action;
> +#else
> +	(void)ctx;
> +	(void)data_sz;
> +	(void)data;
> +	(void)reformat_type;
> +	(void)ft_type;
> +	return NULL;
> +#endif
> +}
> +
> +static void *
> +mlx5_glue_dv_create_flow_action_tag(uint32_t tag)
> +{
> +#ifdef HAVE_IBV_FLOW_DV_SUPPORT
> +	struct mlx5dv_flow_action_attr *action;
> +	action = malloc(sizeof(*action));
> +	if (!action)
> +		return NULL;
> +	action->type = MLX5DV_FLOW_ACTION_TAG;
> +	action->tag_value = tag;
> +	return action;
> +#endif
> +	(void)tag;
> +	return NULL;
> +}
> +
>  static struct ibv_context *
>  mlx5_glue_dv_open_device(struct ibv_device *device)
>  {
> @@ -645,10 +727,15 @@
>  	.dv_create_flow_matcher = mlx5_glue_dv_create_flow_matcher,
>  	.dv_destroy_flow_matcher = mlx5_glue_dv_destroy_flow_matcher,
>  	.dv_create_flow = mlx5_glue_dv_create_flow,
> -	.dv_create_flow_action_packet_reformat =
> -			mlx5_glue_dv_create_flow_action_packet_reformat,
> +	.dv_create_flow_action_counter =
> +		mlx5_glue_dv_create_flow_action_counter,
> +	.dv_create_flow_action_dest_ibv_qp =
> +		mlx5_glue_dv_create_flow_action_dest_ibv_qp,
>  	.dv_create_flow_action_modify_header =
> -			mlx5_glue_dv_create_flow_action_modify_header,
> +		mlx5_glue_dv_create_flow_action_modify_header,
> +	.dv_create_flow_action_packet_reformat =
> +		mlx5_glue_dv_create_flow_action_packet_reformat,
> +	.dv_create_flow_action_tag =  mlx5_glue_dv_create_flow_action_tag,
>  	.dv_open_device = mlx5_glue_dv_open_device,
>  	.devx_obj_create = mlx5_glue_devx_obj_create,
>  	.devx_obj_destroy = mlx5_glue_devx_obj_destroy,
> diff --git a/drivers/net/mlx5/mlx5_glue.h 
> b/drivers/net/mlx5/mlx5_glue.h
> index b118960..32487ea 100644
> --- a/drivers/net/mlx5/mlx5_glue.h
> +++ b/drivers/net/mlx5/mlx5_glue.h
> @@ -55,6 +55,10 @@
>  enum mlx5dv_flow_table_type { flow_table_type = 0, };
>  #endif
> 
> +#ifndef HAVE_IBV_FLOW_DEVX_COUNTERS
> +#define MLX5DV_FLOW_ACTION_COUNTERS_DEVX 0
> +#endif
> +
>  #ifndef HAVE_IBV_DEVX_OBJ
>  struct mlx5dv_devx_obj;
>  #endif
> @@ -98,7 +102,7 @@ struct mlx5_glue {
>  	struct ibv_flow *(*create_flow)(struct ibv_qp *qp,
>  					struct ibv_flow_attr *flow);
>  	int (*destroy_flow)(struct ibv_flow *flow_id);
> -	int (*destroy_flow_action)(struct ibv_flow_action *action);
> +	int (*destroy_flow_action)(void *action);
>  	struct ibv_qp *(*create_qp)(struct ibv_pd *pd,
>  				    struct ibv_qp_init_attr *qp_init_attr);
>  	struct ibv_qp *(*create_qp_ex)
> @@ -160,19 +164,17 @@ struct mlx5_glue {
>  	int (*dv_destroy_flow_matcher)(struct mlx5dv_flow_matcher *matcher);
>  	struct ibv_flow *(*dv_create_flow)(struct mlx5dv_flow_matcher 
> *matcher,
>  			  struct mlx5dv_flow_match_parameters *match_value,
> -			  size_t num_actions,
> -			  struct mlx5dv_flow_action_attr *actions_attr);
> -	struct ibv_flow_action *(*dv_create_flow_action_packet_reformat)
> -		(struct ibv_context *ctx,
> -		 size_t data_sz,
> -		 void *data,
> +			  size_t num_actions, void *actions[]);
> +	void *(*dv_create_flow_action_counter)(void *obj, uint32_t  offset);
> +	void *(*dv_create_flow_action_dest_ibv_qp)(void *qp);
> +	void *(*dv_create_flow_action_modify_header)
> +		(struct ibv_context *ctx, size_t actions_sz, uint64_t actions[],
> +		 enum mlx5dv_flow_table_type ft_type);
> +	void *(*dv_create_flow_action_packet_reformat)
> +		(struct ibv_context *ctx, size_t data_sz, void *data,
>  		 enum mlx5dv_flow_action_packet_reformat_type reformat_type,
>  		 enum mlx5dv_flow_table_type ft_type);
> -	struct ibv_flow_action *(*dv_create_flow_action_modify_header)
> -					(struct ibv_context *ctx,
> -					 size_t actions_sz,
> -					 uint64_t actions[],
> -					 enum mlx5dv_flow_table_type ft_type);
> +	void *(*dv_create_flow_action_tag)(uint32_t tag);
>  	struct ibv_context *(*dv_open_device)(struct ibv_device *device);
>  	struct mlx5dv_devx_obj *(*devx_obj_create)
>  					(struct ibv_context *ctx,