[PATCH 0/3] net/mlx5: extend send to kernel action support

[PATCH 0/3] net/mlx5: extend send to kernel action support

[Jiawei Wang]

The send to kernel action was supported in NIC and FDB tables,
Currently[1], the send to kernel action is created in NIC RX only.

With some TC rules (example: roce packets, redirects into rep ports)
and DPDK RTE rules for the rest of the traffic. Then it needs the
specific rule to re-route the packets into the kernel through the FDB
table.
    
This patchset adds the FDB and NIC-TX tables support for sending to the kernel
action, also updates the documentation.

[1]
https://patchwork.dpdk.org/project/dpdk/list/?series=25318

Jiawei Wang (3):
  net/mlx5: extend send to kernel action support
  net/mlx5: extend hws send to kernel action support
  doc: adds the description for send to kernel

 doc/guides/nics/mlx5.rst           |  7 +++++
 doc/guides/prog_guide/rte_flow.rst |  9 ++++++
 drivers/net/mlx5/linux/mlx5_os.c   | 25 ++++++++++-------
 drivers/net/mlx5/mlx5.h            |  6 ++--
 drivers/net/mlx5/mlx5_flow.h       |  1 +
 drivers/net/mlx5/mlx5_flow_dv.c    | 17 +++++++----
 drivers/net/mlx5/mlx5_flow_hw.c    | 45 +++++++++++++++++++++---------
 lib/ethdev/rte_flow.h              |  1 -
 8 files changed, 79 insertions(+), 32 deletions(-)

-- 
2.18.1

[PATCH 1/3] net/mlx5: extend send to kernel action support

[Jiawei Wang]

The send to kernel action was supported in NIC and FDB tables,
Currently, the send to kernel action is created in NIC RX only.

With some TC rules (example: roce packets, redirects into rep ports)
and DPDK RTE rules for the rest of the traffic. Then it needs the
specific rule to re-route the packets into the kernel through the FDB
table.

This patch adds the FDB and NIC-TX tables support for sending to the kernel
action.

Signed-off-by: Jiawei Wang <jiaweiw@nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_os.c | 25 +++++++++++++++----------
 drivers/net/mlx5/mlx5.h          |  4 +++-
 drivers/net/mlx5/mlx5_flow.h     |  1 +
 drivers/net/mlx5/mlx5_flow_dv.c  | 17 +++++++++++------
 4 files changed, 30 insertions(+), 17 deletions(-)

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index d8f1adfe3d..38572eb652 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -675,6 +675,9 @@ void
 mlx5_os_free_shared_dr(struct mlx5_priv *priv)
 {
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
+#ifdef HAVE_MLX5DV_DR
+	int i;
+#endif
 
 	MLX5_ASSERT(sh && sh->refcnt);
 	if (sh->refcnt > 1)
@@ -703,18 +706,20 @@ mlx5_os_free_shared_dr(struct mlx5_priv *priv)
 		mlx5_glue->destroy_flow_action(sh->pop_vlan_action);
 		sh->pop_vlan_action = NULL;
 	}
-	if (sh->send_to_kernel_action.action) {
-		void *action = sh->send_to_kernel_action.action;
+	for (i = 0; i < MLX5DR_TABLE_TYPE_MAX; i++) {
+		if (sh->send_to_kernel_action[i].action) {
+			void *action = sh->send_to_kernel_action[i].action;
 
-		mlx5_glue->destroy_flow_action(action);
-		sh->send_to_kernel_action.action = NULL;
-	}
-	if (sh->send_to_kernel_action.tbl) {
-		struct mlx5_flow_tbl_resource *tbl =
-				sh->send_to_kernel_action.tbl;
+			mlx5_glue->destroy_flow_action(action);
+			sh->send_to_kernel_action[i].action = NULL;
+		}
+		if (sh->send_to_kernel_action[i].tbl) {
+			struct mlx5_flow_tbl_resource *tbl =
+					sh->send_to_kernel_action[i].tbl;
 
-		flow_dv_tbl_resource_release(sh, tbl);
-		sh->send_to_kernel_action.tbl = NULL;
+			flow_dv_tbl_resource_release(sh, tbl);
+			sh->send_to_kernel_action[i].tbl = NULL;
+		}
 	}
 #endif /* HAVE_MLX5DV_DR */
 	if (sh->default_miss_action)
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 3785103308..6960a07d40 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -1432,7 +1432,9 @@ struct mlx5_dev_ctx_shared {
 	/* Direct Rules tables for FDB, NIC TX+RX */
 	void *dr_drop_action; /* Pointer to DR drop action, any domain. */
 	void *pop_vlan_action; /* Pointer to DR pop VLAN action. */
-	struct mlx5_send_to_kernel_action send_to_kernel_action;
+#if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
+	struct mlx5_send_to_kernel_action send_to_kernel_action[MLX5DR_TABLE_TYPE_MAX];
+#endif
 	struct mlx5_hlist *encaps_decaps; /* Encap/decap action hash list. */
 	struct mlx5_hlist *modify_cmds;
 	struct mlx5_hlist *tag_table;
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 3a97975d69..6beac3902c 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -331,6 +331,7 @@ enum mlx5_feature_name {
 
 #define MLX5_FLOW_FATE_ESWITCH_ACTIONS \
 	(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_PORT_ID | \
+	 MLX5_FLOW_ACTION_SEND_TO_KERNEL | \
 	 MLX5_FLOW_ACTION_JUMP | MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY)
 
 #define MLX5_FLOW_MODIFY_HDR_ACTIONS (MLX5_FLOW_ACTION_SET_IPV4_SRC | \
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index a8dd9920e6..fe0814a2eb 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -12724,17 +12724,22 @@ flow_dv_translate_action_sample(struct rte_eth_dev *dev,
 
 static void *
 flow_dv_translate_action_send_to_kernel(struct rte_eth_dev *dev,
+					const struct rte_flow_attr *attr,
 					struct rte_flow_error *error)
 {
 	struct mlx5_flow_tbl_resource *tbl;
 	struct mlx5_dev_ctx_shared *sh;
 	uint32_t priority;
 	void *action;
+	int ft_type;
 	int ret;
 
 	sh = MLX5_SH(dev);
-	if (sh->send_to_kernel_action.action)
-		return sh->send_to_kernel_action.action;
+	ft_type = (attr->ingress) ? MLX5DR_TABLE_TYPE_NIC_RX :
+		  ((attr->transfer) ? MLX5DR_TABLE_TYPE_FDB :
+		  MLX5DR_TABLE_TYPE_NIC_TX);
+	if (sh->send_to_kernel_action[ft_type].action)
+		return sh->send_to_kernel_action[ft_type].action;
 	priority = mlx5_get_send_to_kernel_priority(dev);
 	if (priority == (uint32_t)-1) {
 		rte_flow_error_set(error, ENOTSUP,
@@ -12742,7 +12747,7 @@ flow_dv_translate_action_send_to_kernel(struct rte_eth_dev *dev,
 				   "required priority is not available");
 		return NULL;
 	}
-	tbl = flow_dv_tbl_resource_get(dev, 0, 0, 0, false, NULL, 0, 0, 0,
+	tbl = flow_dv_tbl_resource_get(dev, 0, attr->egress, attr->transfer, false, NULL, 0, 0, 0,
 				       error);
 	if (!tbl) {
 		rte_flow_error_set(error, ENODATA,
@@ -12759,8 +12764,8 @@ flow_dv_translate_action_send_to_kernel(struct rte_eth_dev *dev,
 		goto err;
 	}
 	MLX5_ASSERT(action);
-	sh->send_to_kernel_action.action = action;
-	sh->send_to_kernel_action.tbl = tbl;
+	sh->send_to_kernel_action[ft_type].action = action;
+	sh->send_to_kernel_action[ft_type].tbl = tbl;
 	return action;
 err:
 	flow_dv_tbl_resource_release(sh, tbl);
@@ -14511,7 +14516,7 @@ flow_dv_translate(struct rte_eth_dev *dev,
 			break;
 		case RTE_FLOW_ACTION_TYPE_SEND_TO_KERNEL:
 			dev_flow->dv.actions[actions_n] =
-				flow_dv_translate_action_send_to_kernel(dev,
+				flow_dv_translate_action_send_to_kernel(dev, attr,
 							error);
 			if (!dev_flow->dv.actions[actions_n])
 				return -rte_errno;
-- 
2.18.1

[PATCH 2/3] net/mlx5: extend hws send to kernel action support

[Jiawei Wang]

The hws send to kernel action was supported in NIC and FDB tables,
Currently, the send to kernel action is created in NIC RX only.

This patch adds the FDB and NIC-TX tables support for sending to the kernel
action for HWS.

Signed-off-by: Jiawei Wang <jiaweiw@nvidia.com>
---
 drivers/net/mlx5/mlx5.h         |  2 +-
 drivers/net/mlx5/mlx5_flow_hw.c | 45 +++++++++++++++++++++++----------
 2 files changed, 33 insertions(+), 14 deletions(-)

diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 6960a07d40..d49b81a360 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -1858,7 +1858,7 @@ struct mlx5_priv {
 	/* HW steering global tag action. */
 	struct mlx5dr_action *hw_tag[2];
 	/* HW steering global send to kernel action. */
-	struct mlx5dr_action *hw_send_to_kernel;
+	struct mlx5dr_action *hw_send_to_kernel[MLX5DR_TABLE_TYPE_MAX];
 	/* HW steering create ongoing rte flow table list header. */
 	LIST_HEAD(flow_hw_tbl_ongo, rte_flow_template_table) flow_hw_tbl_ongo;
 	struct mlx5_indexed_pool *acts_ipool; /* Action data indexed pool. */
diff --git a/drivers/net/mlx5/mlx5_flow_hw.c b/drivers/net/mlx5/mlx5_flow_hw.c
index 92a5c92669..84d869a686 100644
--- a/drivers/net/mlx5/mlx5_flow_hw.c
+++ b/drivers/net/mlx5/mlx5_flow_hw.c
@@ -1439,6 +1439,7 @@ __flow_hw_actions_translate(struct rte_eth_dev *dev,
 	uint32_t ct_idx;
 	int err;
 	uint32_t target_grp = 0;
+	int table_type;
 
 	flow_hw_modify_field_init(&mhdr, at);
 	if (attr->transfer)
@@ -1635,7 +1636,10 @@ __flow_hw_actions_translate(struct rte_eth_dev *dev,
 						"Send to kernel action on root table is not supported in HW steering mode");
 			}
 			action_pos = at->actions_off[actions - at->actions];
-			acts->rule_acts[action_pos].action = priv->hw_send_to_kernel;
+			table_type = attr->ingress ? MLX5DR_TABLE_TYPE_NIC_RX :
+				     ((attr->egress) ? MLX5DR_TABLE_TYPE_NIC_TX :
+				     MLX5DR_TABLE_TYPE_FDB);
+			acts->rule_acts[action_pos].action = priv->hw_send_to_kernel[table_type];
 			break;
 		case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
 			err = flow_hw_modify_field_compile(dev, attr, action_start,
@@ -4345,8 +4349,11 @@ mlx5_flow_hw_actions_validate(struct rte_eth_dev *dev,
 	const struct rte_flow_action_count *count_mask = NULL;
 	bool fixed_cnt = false;
 	uint64_t action_flags = 0;
-	uint16_t i;
 	bool actions_end = false;
+#ifdef HAVE_MLX5DV_DR_ACTION_CREATE_DEST_ROOT_TABLE
+	int table_type;
+#endif
+	uint16_t i;
 	int ret;
 
 	/* FDB actions are only valid to proxy port. */
@@ -4397,7 +4404,10 @@ mlx5_flow_hw_actions_validate(struct rte_eth_dev *dev,
 							  RTE_FLOW_ERROR_TYPE_ACTION,
 							  action,
 							  "action not supported in guest port");
-			if (!priv->hw_send_to_kernel)
+			table_type = attr->ingress ? MLX5DR_TABLE_TYPE_NIC_RX :
+				     ((attr->egress) ? MLX5DR_TABLE_TYPE_NIC_TX :
+				     MLX5DR_TABLE_TYPE_FDB);
+			if (!priv->hw_send_to_kernel[table_type])
 				return rte_flow_error_set(error, ENOTSUP,
 							  RTE_FLOW_ERROR_TYPE_ACTION,
 							  action,
@@ -5955,13 +5965,19 @@ static void
 flow_hw_create_send_to_kernel_actions(struct mlx5_priv *priv __rte_unused)
 {
 #ifdef HAVE_MLX5DV_DR_ACTION_CREATE_DEST_ROOT_TABLE
-	priv->hw_send_to_kernel =
-			mlx5dr_action_create_dest_root(priv->dr_ctx,
-						       MLX5_HW_LOWEST_PRIO_ROOT,
-						       MLX5DR_ACTION_FLAG_HWS_RX);
-	if (!priv->hw_send_to_kernel) {
-		DRV_LOG(WARNING, "Unable to create HWS send to kernel action");
-		return;
+	int action_flag;
+	int i;
+
+	for (i = MLX5DR_TABLE_TYPE_NIC_RX; i < MLX5DR_TABLE_TYPE_MAX; i++) {
+		action_flag = mlx5_hw_act_flag[1][i];
+		priv->hw_send_to_kernel[i] =
+				mlx5dr_action_create_dest_root(priv->dr_ctx,
+							MLX5_HW_LOWEST_PRIO_ROOT,
+							action_flag);
+		if (!priv->hw_send_to_kernel[i]) {
+			DRV_LOG(WARNING, "Unable to create HWS send to kernel action");
+			return;
+		}
 	}
 #endif
 }
@@ -5969,9 +5985,12 @@ flow_hw_create_send_to_kernel_actions(struct mlx5_priv *priv __rte_unused)
 static void
 flow_hw_destroy_send_to_kernel_action(struct mlx5_priv *priv)
 {
-	if (priv->hw_send_to_kernel) {
-		mlx5dr_action_destroy(priv->hw_send_to_kernel);
-		priv->hw_send_to_kernel = NULL;
+	int i;
+	for (i = MLX5DR_TABLE_TYPE_NIC_RX; i < MLX5DR_TABLE_TYPE_MAX; i++) {
+		if (priv->hw_send_to_kernel[i]) {
+			mlx5dr_action_destroy(priv->hw_send_to_kernel[i]);
+			priv->hw_send_to_kernel[i] = NULL;
+		}
 	}
 }
 
-- 
2.18.1

[PATCH 3/3] doc: adds the description for send to kernel

[Jiawei Wang]

This patch adds the description for send to kernel action support.

Signed-off-by: Jiawei Wang <jiaweiw@nvidia.com>
Acked-by: Ori Kam <orika@nvidia.com>
---
 doc/guides/nics/mlx5.rst           | 7 +++++++
 doc/guides/prog_guide/rte_flow.rst | 9 +++++++++
 lib/ethdev/rte_flow.h              | 1 -
 3 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/doc/guides/nics/mlx5.rst b/doc/guides/nics/mlx5.rst
index eac67a7864..7bee57d9dd 100644
--- a/doc/guides/nics/mlx5.rst
+++ b/doc/guides/nics/mlx5.rst
@@ -165,6 +165,7 @@ Features
 - Sub-Function.
 - Matching on represented port.
 - Matching on aggregated affinity.
+- Send to kernel.
 
 
 Limitations
@@ -720,6 +721,12 @@ Limitations
   The flow engine of a process cannot move from active to standby mode
   if preceding active application rules are still present and vice versa.
 
+- Send to kernel action (``RTE_FLOW_ACTION_TYPE_SEND_TO_KERNEL``):
+
+  - Supports on non-root table.
+  - Supports on isolated mode.
+  - In HW steering (``dv_flow_en`` = 2):
+    - not supported on guest port.
 
 Statistics
 ----------
diff --git a/doc/guides/prog_guide/rte_flow.rst b/doc/guides/prog_guide/rte_flow.rst
index 5bc998a433..f8c130bc8b 100644
--- a/doc/guides/prog_guide/rte_flow.rst
+++ b/doc/guides/prog_guide/rte_flow.rst
@@ -3490,6 +3490,15 @@ The ``quota`` value is reduced according to ``mode`` setting.
    | ``RTE_FLOW_QUOTA_MODE_L3``      | Count packet bytes starting from L3 |
    +------------------+----------------------------------------------------+
 
+Action: ``SEND_TO_KERNEL``
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Send packets to the kernel, without going to userspace at all.
+
+The packets will be received by the kernel driver sharing the
+same device as the DPDK port on which this action is configured.
+
+
 Negative types
 ~~~~~~~~~~~~~~
 
diff --git a/lib/ethdev/rte_flow.h b/lib/ethdev/rte_flow.h
index 2ebb76dbc0..23addb4382 100644
--- a/lib/ethdev/rte_flow.h
+++ b/lib/ethdev/rte_flow.h
@@ -2931,7 +2931,6 @@ enum rte_flow_action_type {
 	 * The packets will be received by the kernel driver sharing
 	 * the same device as the DPDK port on which this action is configured.
 	 * This action mostly suits bifurcated driver model.
-	 * This is an ingress non-transfer action only.
 	 *
 	 * No associated configuration structure.
 	 */
-- 
2.18.1

RE: [PATCH 1/3] net/mlx5: extend send to kernel action support

[Suanming Mou]

> -----Original Message-----
> From: Jiawei(Jonny) Wang <jiaweiw@nvidia.com>
> Sent: Friday, September 8, 2023 5:21 PM
> To: Suanming Mou <suanmingm@nvidia.com>; Ori Kam <orika@nvidia.com>;
> NBU-Contact-Thomas Monjalon (EXTERNAL) <thomas@monjalon.net>
> Cc: dev@dpdk.org; Raslan Darawsheh <rasland@nvidia.com>
> Subject: [PATCH 1/3] net/mlx5: extend send to kernel action support
> 
> The send to kernel action was supported in NIC and FDB tables, Currently, the
> send to kernel action is created in NIC RX only.
> 
> With some TC rules (example: roce packets, redirects into rep ports) and DPDK
> RTE rules for the rest of the traffic. Then it needs the specific rule to re-route the
> packets into the kernel through the FDB table.
> 
> This patch adds the FDB and NIC-TX tables support for sending to the kernel
> action.
> 
> Signed-off-by: Jiawei Wang <jiaweiw@nvidia.com>
Acked-by: Suanming Mou <suanmingm@nvidia.com>
> ---
>  drivers/net/mlx5/linux/mlx5_os.c | 25 +++++++++++++++----------
>  drivers/net/mlx5/mlx5.h          |  4 +++-
>  drivers/net/mlx5/mlx5_flow.h     |  1 +
>  drivers/net/mlx5/mlx5_flow_dv.c  | 17 +++++++++++------
>  4 files changed, 30 insertions(+), 17 deletions(-)
> 
> diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
> index d8f1adfe3d..38572eb652 100644
> --- a/drivers/net/mlx5/linux/mlx5_os.c
> +++ b/drivers/net/mlx5/linux/mlx5_os.c
> @@ -675,6 +675,9 @@ void
>  mlx5_os_free_shared_dr(struct mlx5_priv *priv)  {
>  	struct mlx5_dev_ctx_shared *sh = priv->sh;
> +#ifdef HAVE_MLX5DV_DR
> +	int i;
> +#endif
> 
>  	MLX5_ASSERT(sh && sh->refcnt);
>  	if (sh->refcnt > 1)
> @@ -703,18 +706,20 @@ mlx5_os_free_shared_dr(struct mlx5_priv *priv)
>  		mlx5_glue->destroy_flow_action(sh->pop_vlan_action);
>  		sh->pop_vlan_action = NULL;
>  	}
> -	if (sh->send_to_kernel_action.action) {
> -		void *action = sh->send_to_kernel_action.action;
> +	for (i = 0; i < MLX5DR_TABLE_TYPE_MAX; i++) {
> +		if (sh->send_to_kernel_action[i].action) {
> +			void *action = sh->send_to_kernel_action[i].action;
> 
> -		mlx5_glue->destroy_flow_action(action);
> -		sh->send_to_kernel_action.action = NULL;
> -	}
> -	if (sh->send_to_kernel_action.tbl) {
> -		struct mlx5_flow_tbl_resource *tbl =
> -				sh->send_to_kernel_action.tbl;
> +			mlx5_glue->destroy_flow_action(action);
> +			sh->send_to_kernel_action[i].action = NULL;
> +		}
> +		if (sh->send_to_kernel_action[i].tbl) {
> +			struct mlx5_flow_tbl_resource *tbl =
> +					sh->send_to_kernel_action[i].tbl;
> 
> -		flow_dv_tbl_resource_release(sh, tbl);
> -		sh->send_to_kernel_action.tbl = NULL;
> +			flow_dv_tbl_resource_release(sh, tbl);
> +			sh->send_to_kernel_action[i].tbl = NULL;
> +		}
>  	}
>  #endif /* HAVE_MLX5DV_DR */
>  	if (sh->default_miss_action)
> diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index
> 3785103308..6960a07d40 100644
> --- a/drivers/net/mlx5/mlx5.h
> +++ b/drivers/net/mlx5/mlx5.h
> @@ -1432,7 +1432,9 @@ struct mlx5_dev_ctx_shared {
>  	/* Direct Rules tables for FDB, NIC TX+RX */
>  	void *dr_drop_action; /* Pointer to DR drop action, any domain. */
>  	void *pop_vlan_action; /* Pointer to DR pop VLAN action. */
> -	struct mlx5_send_to_kernel_action send_to_kernel_action;
> +#if defined(HAVE_IBV_FLOW_DV_SUPPORT)
> || !defined(HAVE_INFINIBAND_VERBS_H)
> +	struct mlx5_send_to_kernel_action
> +send_to_kernel_action[MLX5DR_TABLE_TYPE_MAX];
> +#endif
>  	struct mlx5_hlist *encaps_decaps; /* Encap/decap action hash list. */
>  	struct mlx5_hlist *modify_cmds;
>  	struct mlx5_hlist *tag_table;
> diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index
> 3a97975d69..6beac3902c 100644
> --- a/drivers/net/mlx5/mlx5_flow.h
> +++ b/drivers/net/mlx5/mlx5_flow.h
> @@ -331,6 +331,7 @@ enum mlx5_feature_name {
> 
>  #define MLX5_FLOW_FATE_ESWITCH_ACTIONS \
>  	(MLX5_FLOW_ACTION_DROP | MLX5_FLOW_ACTION_PORT_ID | \
> +	 MLX5_FLOW_ACTION_SEND_TO_KERNEL | \
>  	 MLX5_FLOW_ACTION_JUMP |
> MLX5_FLOW_ACTION_METER_WITH_TERMINATED_POLICY)
> 
>  #define MLX5_FLOW_MODIFY_HDR_ACTIONS
> (MLX5_FLOW_ACTION_SET_IPV4_SRC | \ diff --git
> a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index
> a8dd9920e6..fe0814a2eb 100644
> --- a/drivers/net/mlx5/mlx5_flow_dv.c
> +++ b/drivers/net/mlx5/mlx5_flow_dv.c
> @@ -12724,17 +12724,22 @@ flow_dv_translate_action_sample(struct
> rte_eth_dev *dev,
> 
>  static void *
>  flow_dv_translate_action_send_to_kernel(struct rte_eth_dev *dev,
> +					const struct rte_flow_attr *attr,
>  					struct rte_flow_error *error)
>  {
>  	struct mlx5_flow_tbl_resource *tbl;
>  	struct mlx5_dev_ctx_shared *sh;
>  	uint32_t priority;
>  	void *action;
> +	int ft_type;
>  	int ret;
> 
>  	sh = MLX5_SH(dev);
> -	if (sh->send_to_kernel_action.action)
> -		return sh->send_to_kernel_action.action;
> +	ft_type = (attr->ingress) ? MLX5DR_TABLE_TYPE_NIC_RX :
> +		  ((attr->transfer) ? MLX5DR_TABLE_TYPE_FDB :
> +		  MLX5DR_TABLE_TYPE_NIC_TX);
> +	if (sh->send_to_kernel_action[ft_type].action)
> +		return sh->send_to_kernel_action[ft_type].action;
>  	priority = mlx5_get_send_to_kernel_priority(dev);
>  	if (priority == (uint32_t)-1) {
>  		rte_flow_error_set(error, ENOTSUP,
> @@ -12742,7 +12747,7 @@ flow_dv_translate_action_send_to_kernel(struct
> rte_eth_dev *dev,
>  				   "required priority is not available");
>  		return NULL;
>  	}
> -	tbl = flow_dv_tbl_resource_get(dev, 0, 0, 0, false, NULL, 0, 0, 0,
> +	tbl = flow_dv_tbl_resource_get(dev, 0, attr->egress, attr->transfer,
> +false, NULL, 0, 0, 0,
>  				       error);
>  	if (!tbl) {
>  		rte_flow_error_set(error, ENODATA,
> @@ -12759,8 +12764,8 @@ flow_dv_translate_action_send_to_kernel(struct
> rte_eth_dev *dev,
>  		goto err;
>  	}
>  	MLX5_ASSERT(action);
> -	sh->send_to_kernel_action.action = action;
> -	sh->send_to_kernel_action.tbl = tbl;
> +	sh->send_to_kernel_action[ft_type].action = action;
> +	sh->send_to_kernel_action[ft_type].tbl = tbl;
>  	return action;
>  err:
>  	flow_dv_tbl_resource_release(sh, tbl); @@ -14511,7 +14516,7 @@
> flow_dv_translate(struct rte_eth_dev *dev,
>  			break;
>  		case RTE_FLOW_ACTION_TYPE_SEND_TO_KERNEL:
>  			dev_flow->dv.actions[actions_n] =
> -				flow_dv_translate_action_send_to_kernel(dev,
> +				flow_dv_translate_action_send_to_kernel(dev,
> attr,
>  							error);
>  			if (!dev_flow->dv.actions[actions_n])
>  				return -rte_errno;
> --
> 2.18.1

RE: [PATCH 2/3] net/mlx5: extend hws send to kernel action support

[Suanming Mou]

> -----Original Message-----
> From: Jiawei(Jonny) Wang <jiaweiw@nvidia.com>
> Sent: Friday, September 8, 2023 5:21 PM
> To: Suanming Mou <suanmingm@nvidia.com>; Ori Kam <orika@nvidia.com>;
> NBU-Contact-Thomas Monjalon (EXTERNAL) <thomas@monjalon.net>
> Cc: dev@dpdk.org; Raslan Darawsheh <rasland@nvidia.com>
> Subject: [PATCH 2/3] net/mlx5: extend hws send to kernel action support
> 
> The hws send to kernel action was supported in NIC and FDB tables, Currently,
> the send to kernel action is created in NIC RX only.
> 
> This patch adds the FDB and NIC-TX tables support for sending to the kernel
> action for HWS.
> 
> Signed-off-by: Jiawei Wang <jiaweiw@nvidia.com>

Acked-by: Suanming Mou <suanmingm@nvidia.com>

> ---
>  drivers/net/mlx5/mlx5.h         |  2 +-
>  drivers/net/mlx5/mlx5_flow_hw.c | 45 +++++++++++++++++++++++----------
>  2 files changed, 33 insertions(+), 14 deletions(-)
> 
> diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index
> 6960a07d40..d49b81a360 100644
> --- a/drivers/net/mlx5/mlx5.h
> +++ b/drivers/net/mlx5/mlx5.h
> @@ -1858,7 +1858,7 @@ struct mlx5_priv {
>  	/* HW steering global tag action. */
>  	struct mlx5dr_action *hw_tag[2];
>  	/* HW steering global send to kernel action. */
> -	struct mlx5dr_action *hw_send_to_kernel;
> +	struct mlx5dr_action *hw_send_to_kernel[MLX5DR_TABLE_TYPE_MAX];
>  	/* HW steering create ongoing rte flow table list header. */
>  	LIST_HEAD(flow_hw_tbl_ongo, rte_flow_template_table)
> flow_hw_tbl_ongo;
>  	struct mlx5_indexed_pool *acts_ipool; /* Action data indexed pool. */
> diff --git a/drivers/net/mlx5/mlx5_flow_hw.c
> b/drivers/net/mlx5/mlx5_flow_hw.c index 92a5c92669..84d869a686 100644
> --- a/drivers/net/mlx5/mlx5_flow_hw.c
> +++ b/drivers/net/mlx5/mlx5_flow_hw.c
> @@ -1439,6 +1439,7 @@ __flow_hw_actions_translate(struct rte_eth_dev
> *dev,
>  	uint32_t ct_idx;
>  	int err;
>  	uint32_t target_grp = 0;
> +	int table_type;
> 
>  	flow_hw_modify_field_init(&mhdr, at);
>  	if (attr->transfer)
> @@ -1635,7 +1636,10 @@ __flow_hw_actions_translate(struct rte_eth_dev
> *dev,
>  						"Send to kernel action on root
> table is not supported in HW steering mode");
>  			}
>  			action_pos = at->actions_off[actions - at->actions];
> -			acts->rule_acts[action_pos].action = priv-
> >hw_send_to_kernel;
> +			table_type = attr->ingress ?
> MLX5DR_TABLE_TYPE_NIC_RX :
> +				     ((attr->egress) ?
> MLX5DR_TABLE_TYPE_NIC_TX :
> +				     MLX5DR_TABLE_TYPE_FDB);
> +			acts->rule_acts[action_pos].action =
> +priv->hw_send_to_kernel[table_type];
>  			break;
>  		case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
>  			err = flow_hw_modify_field_compile(dev, attr,
> action_start, @@ -4345,8 +4349,11 @@ mlx5_flow_hw_actions_validate(struct
> rte_eth_dev *dev,
>  	const struct rte_flow_action_count *count_mask = NULL;
>  	bool fixed_cnt = false;
>  	uint64_t action_flags = 0;
> -	uint16_t i;
>  	bool actions_end = false;
> +#ifdef HAVE_MLX5DV_DR_ACTION_CREATE_DEST_ROOT_TABLE
> +	int table_type;
> +#endif
> +	uint16_t i;
>  	int ret;
> 
>  	/* FDB actions are only valid to proxy port. */ @@ -4397,7 +4404,10
> @@ mlx5_flow_hw_actions_validate(struct rte_eth_dev *dev,
> 
> RTE_FLOW_ERROR_TYPE_ACTION,
>  							  action,
>  							  "action not supported
> in guest port");
> -			if (!priv->hw_send_to_kernel)
> +			table_type = attr->ingress ?
> MLX5DR_TABLE_TYPE_NIC_RX :
> +				     ((attr->egress) ?
> MLX5DR_TABLE_TYPE_NIC_TX :
> +				     MLX5DR_TABLE_TYPE_FDB);
> +			if (!priv->hw_send_to_kernel[table_type])
>  				return rte_flow_error_set(error, ENOTSUP,
> 
> RTE_FLOW_ERROR_TYPE_ACTION,
>  							  action,
> @@ -5955,13 +5965,19 @@ static void
>  flow_hw_create_send_to_kernel_actions(struct mlx5_priv *priv __rte_unused)
> {  #ifdef HAVE_MLX5DV_DR_ACTION_CREATE_DEST_ROOT_TABLE
> -	priv->hw_send_to_kernel =
> -			mlx5dr_action_create_dest_root(priv->dr_ctx,
> -
> MLX5_HW_LOWEST_PRIO_ROOT,
> -
> MLX5DR_ACTION_FLAG_HWS_RX);
> -	if (!priv->hw_send_to_kernel) {
> -		DRV_LOG(WARNING, "Unable to create HWS send to kernel
> action");
> -		return;
> +	int action_flag;
> +	int i;
> +
> +	for (i = MLX5DR_TABLE_TYPE_NIC_RX; i < MLX5DR_TABLE_TYPE_MAX;
> i++) {
> +		action_flag = mlx5_hw_act_flag[1][i];
> +		priv->hw_send_to_kernel[i] =
> +				mlx5dr_action_create_dest_root(priv->dr_ctx,
> +
> 	MLX5_HW_LOWEST_PRIO_ROOT,
> +							action_flag);
> +		if (!priv->hw_send_to_kernel[i]) {
> +			DRV_LOG(WARNING, "Unable to create HWS send to
> kernel action");
> +			return;
> +		}
>  	}
>  #endif
>  }
> @@ -5969,9 +5985,12 @@ flow_hw_create_send_to_kernel_actions(struct
> mlx5_priv *priv __rte_unused)  static void
> flow_hw_destroy_send_to_kernel_action(struct mlx5_priv *priv)  {
> -	if (priv->hw_send_to_kernel) {
> -		mlx5dr_action_destroy(priv->hw_send_to_kernel);
> -		priv->hw_send_to_kernel = NULL;
> +	int i;
> +	for (i = MLX5DR_TABLE_TYPE_NIC_RX; i < MLX5DR_TABLE_TYPE_MAX;
> i++) {
> +		if (priv->hw_send_to_kernel[i]) {
> +			mlx5dr_action_destroy(priv->hw_send_to_kernel[i]);
> +			priv->hw_send_to_kernel[i] = NULL;
> +		}
>  	}
>  }
> 
> --
> 2.18.1

RE: [PATCH 1/3] net/mlx5: extend send to kernel action support

[Raslan Darawsheh]

Hi,

> -----Original Message-----
> From: Jiawei(Jonny) Wang <jiaweiw@nvidia.com>
> Sent: Friday, September 8, 2023 12:21 PM
> To: Suanming Mou <suanmingm@nvidia.com>; Ori Kam <orika@nvidia.com>;
> NBU-Contact-Thomas Monjalon (EXTERNAL) <thomas@monjalon.net>
> Cc: dev@dpdk.org; Raslan Darawsheh <rasland@nvidia.com>
> Subject: [PATCH 1/3] net/mlx5: extend send to kernel action support
> 
> The send to kernel action was supported in NIC and FDB tables, Currently, the
> send to kernel action is created in NIC RX only.
> 
> With some TC rules (example: roce packets, redirects into rep ports) and DPDK
> RTE rules for the rest of the traffic. Then it needs the specific rule to re-route
> the packets into the kernel through the FDB table.
> 
> This patch adds the FDB and NIC-TX tables support for sending to the kernel
> action.
> 
> Signed-off-by: Jiawei Wang <jiaweiw@nvidia.com>

Series applied to next-net-mlx,

Kindest regards,
Raslan Darawsheh