Re: [dpdk-dev] [PATCH v2 7/9] net/mlx5: add E-Switch port ID action to Direct Verbs

[Yongseok Koh <yskoh@mellanox.com>]

On Thu, Apr 18, 2019 at 11:28:48AM +0000, Ori Kam wrote:
> This commits adds matching on source port, using DV API.
> 
> Signed-off-by: Ori Kam <orika@mellanox.com>
> ---

Acked-by: Yongseok Koh <yskoh@mellanox.com>

> v2:
> * Address ML comments.
> ---
>  drivers/net/mlx5/mlx5.h         |   2 +
>  drivers/net/mlx5/mlx5_flow.h    |  12 ++++
>  drivers/net/mlx5/mlx5_flow_dv.c | 149 ++++++++++++++++++++++++++++++++++++++++
>  drivers/net/mlx5/mlx5_glue.c    |  14 ++++
>  drivers/net/mlx5/mlx5_glue.h    |   1 +
>  5 files changed, 178 insertions(+)
> 
> diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
> index dd22bb6..4b6dbc2 100644
> --- a/drivers/net/mlx5/mlx5.h
> +++ b/drivers/net/mlx5/mlx5.h
> @@ -281,6 +281,8 @@ struct mlx5_ibv_shared {
>  	LIST_HEAD(modify_cmd, mlx5_flow_dv_modify_hdr_resource) modify_cmds;
>  	LIST_HEAD(tag, mlx5_flow_dv_tag_resource) tags;
>  	LIST_HEAD(jump, mlx5_flow_dv_jump_tbl_resource) jump_tbl;
> +	LIST_HEAD(port_id_action_list, mlx5_flow_dv_port_id_action_resource)
> +		port_id_action_list; /* List of port ID actions. */
>  	/* Shared interrupt handler section. */
>  	pthread_mutex_t intr_mutex; /* Interrupt config mutex. */
>  	uint32_t intr_cnt; /* Interrupt handler reference counter. */
> diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
> index 9d72024..c419e6b 100644
> --- a/drivers/net/mlx5/mlx5_flow.h
> +++ b/drivers/net/mlx5/mlx5_flow.h
> @@ -267,6 +267,16 @@ struct mlx5_flow_dv_jump_tbl_resource {
>  	struct mlx5_flow_tbl_resource *tbl; /**< The target table. */
>  };
>  
> +/* Port ID resource structure. */
> +struct mlx5_flow_dv_port_id_action_resource {
> +	LIST_ENTRY(mlx5_flow_dv_port_id_action_resource) next;
> +	/* Pointer to next element. */
> +	rte_atomic32_t refcnt; /**< Reference counter. */
> +	void *action;
> +	/**< Verbs tag action object. */
> +	uint32_t port_id; /**< Port ID value. */
> +};
> +
>  /*
>   * Max number of actions per DV flow.
>   * See CREATE_FLOW_MAX_FLOW_ACTIONS_SUPPORTED
> @@ -289,6 +299,8 @@ struct mlx5_flow_dv {
>  	struct ibv_flow *flow; /**< Installed flow. */
>  	struct mlx5_flow_dv_jump_tbl_resource *jump;
>  	/**< Pointer to the jump action resource. */
> +	struct mlx5_flow_dv_port_id_action_resource *port_id_action;
> +	/**< Pointer to port ID action resource. */
>  #ifdef HAVE_IBV_FLOW_DV_SUPPORT
>  	void *actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS];
>  	/**< Action list. */
> diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
> index 182abb3..9c5826c 100644
> --- a/drivers/net/mlx5/mlx5_flow_dv.c
> +++ b/drivers/net/mlx5/mlx5_flow_dv.c
> @@ -1054,6 +1054,70 @@ struct field_modify_info modify_tcp[] = {
>  }
>  
>  /**
> + * Find existing table port ID resource or create and register a new one.
> + *
> + * @param dev[in, out]
> + *   Pointer to rte_eth_dev structure.
> + * @param[in, out] resource
> + *   Pointer to port ID action resource.
> + * @parm[in, out] dev_flow
> + *   Pointer to the dev_flow.
> + * @param[out] error
> + *   pointer to error structure.
> + *
> + * @return
> + *   0 on success otherwise -errno and errno is set.
> + */
> +static int
> +flow_dv_port_id_action_resource_register
> +			(struct rte_eth_dev *dev,
> +			 struct mlx5_flow_dv_port_id_action_resource *resource,
> +			 struct mlx5_flow *dev_flow,
> +			 struct rte_flow_error *error)
> +{
> +	struct mlx5_priv *priv = dev->data->dev_private;
> +	struct mlx5_ibv_shared *sh = priv->sh;
> +	struct mlx5_flow_dv_port_id_action_resource *cache_resource;
> +
> +	/* Lookup a matching resource from cache. */
> +	LIST_FOREACH(cache_resource, &sh->port_id_action_list, next) {
> +		if (resource->port_id == cache_resource->port_id) {
> +			DRV_LOG(DEBUG, "port id action resource resource %p: "
> +				"refcnt %d++",
> +				(void *)cache_resource,
> +				rte_atomic32_read(&cache_resource->refcnt));
> +			rte_atomic32_inc(&cache_resource->refcnt);
> +			dev_flow->dv.port_id_action = cache_resource;
> +			return 0;
> +		}
> +	}
> +	/* Register new port id action resource. */
> +	cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
> +	if (!cache_resource)
> +		return rte_flow_error_set(error, ENOMEM,
> +					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
> +					  "cannot allocate resource memory");
> +	*cache_resource = *resource;
> +	cache_resource->action =
> +		mlx5_glue->dr_create_flow_action_dest_vport(priv->sh->fdb_ns,
> +							    resource->port_id);
> +	if (!cache_resource->action) {
> +		rte_free(cache_resource);
> +		return rte_flow_error_set(error, ENOMEM,
> +					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
> +					  NULL, "cannot create action");
> +	}
> +	rte_atomic32_init(&cache_resource->refcnt);
> +	rte_atomic32_inc(&cache_resource->refcnt);
> +	LIST_INSERT_HEAD(&sh->port_id_action_list, cache_resource, next);
> +	dev_flow->dv.port_id_action = cache_resource;
> +	DRV_LOG(DEBUG, "new port id action resource %p: refcnt %d++",
> +		(void *)cache_resource,
> +		rte_atomic32_read(&cache_resource->refcnt));
> +	return 0;
> +}
> +
> +/**
>   * Get the size of specific rte_flow_item_type
>   *
>   * @param[in] item_type
> @@ -3456,6 +3520,44 @@ struct field_modify_info modify_tcp[] = {
>  }
>  
>  /**
> + * Translate port ID action to vport.
> + *
> + * @param[in] dev
> + *   Pointer to rte_eth_dev structure.
> + * @param[in] action
> + *   Pointer to the port ID action.
> + * @param[out] dst_port_id
> + *   The target port ID.
> + * @param[out] error
> + *   Pointer to the error structure.
> + *
> + * @return
> + *   0 on success, a negative errno value otherwise and rte_errno is set.
> + */
> +static int
> +flow_dv_translate_action_port_id(struct rte_eth_dev *dev,
> +				 const struct rte_flow_action *action,
> +				 uint32_t *dst_port_id,
> +				 struct rte_flow_error *error)
> +{
> +	uint32_t port;
> +	uint16_t port_id;
> +	int ret;
> +	const struct rte_flow_action_port_id *conf =
> +			(const struct rte_flow_action_port_id *)action->conf;
> +
> +	port = conf->original ? dev->data->port_id : conf->id;
> +	ret = mlx5_port_to_eswitch_info(port, NULL, &port_id);
> +	if (ret)
> +		return rte_flow_error_set(error, -ret,
> +					  RTE_FLOW_ERROR_TYPE_ACTION,
> +					  NULL,
> +					  "No eswitch info was found for port");
> +	*dst_port_id = port_id;
> +	return 0;
> +}
> +
> +/**
>   * Fill the flow with DV spec.
>   *
>   * @param[in] dev
> @@ -3513,10 +3615,24 @@ struct field_modify_info modify_tcp[] = {
>  		const struct rte_flow_action_jump *jump_data;
>  		struct mlx5_flow_dv_jump_tbl_resource jump_tbl_resource;
>  		struct mlx5_flow_tbl_resource *tbl;
> +		uint32_t port_id = 0;
> +		struct mlx5_flow_dv_port_id_action_resource port_id_resource;
>  
>  		switch (actions->type) {
>  		case RTE_FLOW_ACTION_TYPE_VOID:
>  			break;
> +		case RTE_FLOW_ACTION_TYPE_PORT_ID:
> +			if (flow_dv_translate_action_port_id(dev, action,
> +							     &port_id, error))
> +				return -rte_errno;
> +			port_id_resource.port_id = port_id;
> +			if (flow_dv_port_id_action_resource_register
> +			    (dev, &port_id_resource, dev_flow, error))
> +				return -rte_errno;
> +			dev_flow->dv.actions[actions_n++] =
> +				dev_flow->dv.port_id_action->action;
> +			action_flags |= MLX5_FLOW_ACTION_PORT_ID;
> +			break;
>  		case RTE_FLOW_ACTION_TYPE_FLAG:
>  			tag_resource.tag =
>  				mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
> @@ -4116,6 +4232,37 @@ struct field_modify_info modify_tcp[] = {
>  }
>  
>  /**
> + * Release port ID action resource.
> + *
> + * @param flow
> + *   Pointer to mlx5_flow.
> + *
> + * @return
> + *   1 while a reference on it exists, 0 when freed.
> + */
> +static int
> +flow_dv_port_id_action_resource_release(struct mlx5_flow *flow)
> +{
> +	struct mlx5_flow_dv_port_id_action_resource *cache_resource =
> +		flow->dv.port_id_action;
> +
> +	assert(cache_resource->action);
> +	DRV_LOG(DEBUG, "port ID action resource %p: refcnt %d--",
> +		(void *)cache_resource,
> +		rte_atomic32_read(&cache_resource->refcnt));
> +	if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
> +		claim_zero(mlx5_glue->destroy_flow_action
> +				(cache_resource->action));
> +		LIST_REMOVE(cache_resource, next);
> +		rte_free(cache_resource);
> +		DRV_LOG(DEBUG, "port id action resource %p: removed",
> +			(void *)cache_resource);
> +		return 0;
> +	}
> +	return 1;
> +}
> +
> +/**
>   * Remove the flow from the NIC but keeps it in memory.
>   *
>   * @param[in] dev
> @@ -4182,6 +4329,8 @@ struct field_modify_info modify_tcp[] = {
>  			flow_dv_modify_hdr_resource_release(dev_flow);
>  		if (dev_flow->dv.jump)
>  			flow_dv_jump_tbl_resource_release(dev_flow);
> +		if (dev_flow->dv.port_id_action)
> +			flow_dv_port_id_action_resource_release(dev_flow);
>  		rte_free(dev_flow);
>  	}
>  }
> diff --git a/drivers/net/mlx5/mlx5_glue.c b/drivers/net/mlx5/mlx5_glue.c
> index a508faa..117190f 100644
> --- a/drivers/net/mlx5/mlx5_glue.c
> +++ b/drivers/net/mlx5/mlx5_glue.c
> @@ -382,6 +382,18 @@
>  }
>  
>  static void *
> +mlx5_glue_dr_create_flow_action_dest_vport(void *ns, uint32_t vport)
> +{
> +#ifdef HAVE_MLX5DV_DR_ESWITCH
> +	return mlx5dv_dr_create_action_dest_vport(ns, vport);
> +#else
> +	(void)ns;
> +	(void)vport;
> +	return NULL;
> +#endif
> +}
> +
> +static void *
>  mlx5_glue_dr_create_flow_tbl(void *ns, uint32_t level)
>  {
>  #ifdef HAVE_MLX5DV_DR
> @@ -847,6 +859,8 @@
>  	.cq_ex_to_cq = mlx5_glue_cq_ex_to_cq,
>  	.dr_create_flow_action_dest_flow_tbl =
>  		mlx5_glue_dr_create_flow_action_dest_flow_tbl,
> +	.dr_create_flow_action_dest_vport =
> +		mlx5_glue_dr_create_flow_action_dest_vport,
>  	.dr_create_flow_tbl = mlx5_glue_dr_create_flow_tbl,
>  	.dr_destroy_flow_tbl = mlx5_glue_dr_destroy_flow_tbl,
>  	.dr_create_ns = mlx5_glue_dr_create_ns,
> diff --git a/drivers/net/mlx5/mlx5_glue.h b/drivers/net/mlx5/mlx5_glue.h
> index 058e9b1..26f5cb3 100644
> --- a/drivers/net/mlx5/mlx5_glue.h
> +++ b/drivers/net/mlx5/mlx5_glue.h
> @@ -146,6 +146,7 @@ struct mlx5_glue {
>  	const char *(*port_state_str)(enum ibv_port_state port_state);
>  	struct ibv_cq *(*cq_ex_to_cq)(struct ibv_cq_ex *cq);
>  	void *(*dr_create_flow_action_dest_flow_tbl)(void *tbl);
> +	void *(*dr_create_flow_action_dest_vport)(void *ns, uint32_t vport);
>  	void *(*dr_create_flow_tbl)(void *ns, uint32_t level);
>  	int (*dr_destroy_flow_tbl)(void *tbl);
>  	void *(*dr_create_ns)(struct ibv_context *ctx,
> -- 
> 1.8.3.1
>