* [PATCH 01/11] net/mlx5: initial design of non template to hws
@ 2024-06-02 10:28 Maayan Kashani
[not found] ` <20240603080505.2641-1-mkashani@nvidia.com>
[not found] ` <20240603104850.9935-1-mkashani@nvidia.com>
0 siblings, 2 replies; 7+ messages in thread
From: Maayan Kashani @ 2024-06-02 10:28 UTC (permalink / raw)
To: dev
Cc: mkashani, dsosnowski, rasland, Bing Zhao, stable,
Viacheslav Ovsiienko, Ori Kam, Suanming Mou, Matan Azrad
[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain; charset="y", Size: 203254 bytes --]
Implement the frame and needed building
blocks for non template to hws API's.
Added validate, list_create and list_destroy to mlx5_flow_hw_drv_ops.
Rename old list_create/list_destroy functions to legacy_*
and added a call from verbs/dv ops to the legacy functions.
Updated rte_flow_hw as needed.
Added rte_flow_nt2hws structure for non-template rule data.
Signed-off-by: Maayan Kashani <mkashani@nvidia.com>
squash to net/mlx5: initial design of non template to hws
---
drivers/net/mlx5/linux/mlx5_os.c | 8 +-
drivers/net/mlx5/mlx5.h | 9 +
drivers/net/mlx5/mlx5_flow.c | 139 +++++---
drivers/net/mlx5/mlx5_flow.h | 74 ++++-
drivers/net/mlx5/mlx5_flow_dv.c | 192 +++++++++---
drivers/net/mlx5/mlx5_flow_hw.c | 487 +++++++++++++++++++++++++++--
drivers/net/mlx5/mlx5_flow_verbs.c | 2 +
drivers/net/mlx5/mlx5_trigger.c | 6 +
drivers/net/mlx5/windows/mlx5_os.c | 8 +-
9 files changed, 808 insertions(+), 117 deletions(-)
diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 6dd12f0f68..346e6c7bf9 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -69,7 +69,7 @@ static rte_spinlock_t mlx5_shared_data_lock = RTE_SPINLOCK_INITIALIZER;
static struct mlx5_local_data mlx5_local_data;
/* rte flow indexed pool configuration. */
-static struct mlx5_indexed_pool_config icfg[] = {
+static const struct mlx5_indexed_pool_config default_icfg[] = {
{
.size = sizeof(struct rte_flow),
.trunk_size = 64,
@@ -1068,7 +1068,9 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
struct mlx5_port_info vport_info = { .query_flags = 0 };
int nl_rdma;
int i;
+ struct mlx5_indexed_pool_config icfg[RTE_DIM(default_icfg)];
+ memcpy(icfg, default_icfg, sizeof(icfg));
/* Determine if this port representor is supposed to be spawned. */
if (switch_info->representor && dpdk_dev->devargs &&
!mlx5_representor_match(spawn, eth_da))
@@ -1539,6 +1541,10 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
icfg[i].release_mem_en = !!sh->config.reclaim_mode;
if (sh->config.reclaim_mode)
icfg[i].per_core_cache = 0;
+#ifdef HAVE_MLX5_HWS_SUPPORT
+ if (priv->sh->config.dv_flow_en == 2)
+ icfg[i].size = sizeof(struct rte_flow_hw) + sizeof(struct rte_flow_nt2hws);
+#endif
priv->flows[i] = mlx5_ipool_create(&icfg[i]);
if (!priv->flows[i])
goto error;
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 9e4a5feb49..92ad33d486 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -2293,6 +2293,15 @@ int mlx5_flow_validate(struct rte_eth_dev *dev,
const struct rte_flow_item items[],
const struct rte_flow_action actions[],
struct rte_flow_error *error);
+uint32_t
+mlx5_flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
+ const struct rte_flow_attr *attr,
+ const struct rte_flow_item items[],
+ const struct rte_flow_action actions[],
+ bool external, struct rte_flow_error *error);
+void
+mlx5_flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
+ uint32_t flow_idx);
struct rte_flow *mlx5_flow_create(struct rte_eth_dev *dev,
const struct rte_flow_attr *attr,
const struct rte_flow_item items[],
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 8eafceff37..c1bcb0a548 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -4929,15 +4929,16 @@ flow_check_hairpin_split(struct rte_eth_dev *dev,
}
/* Declare flow create/destroy prototype in advance. */
+
static uint32_t
-flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
+flow_drv_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
const struct rte_flow_attr *attr,
const struct rte_flow_item items[],
const struct rte_flow_action actions[],
bool external, struct rte_flow_error *error);
static void
-flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
+flow_drv_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
uint32_t flow_idx);
int
@@ -5058,7 +5059,7 @@ flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx)
* be applied, removed, deleted in arbitrary order
* by list traversing.
*/
- mcp_res->rix_flow = flow_list_create(dev, MLX5_FLOW_TYPE_MCP,
+ mcp_res->rix_flow = flow_drv_list_create(dev, MLX5_FLOW_TYPE_MCP,
&attr, items, actions, false, error);
if (!mcp_res->rix_flow) {
mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx);
@@ -5152,7 +5153,7 @@ flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
struct mlx5_priv *priv = dev->data->dev_private;
MLX5_ASSERT(mcp_res->rix_flow);
- flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow);
+ flow_drv_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow);
mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
}
@@ -7242,12 +7243,12 @@ flow_tunnel_from_rule(const struct mlx5_flow *flow)
* @return
* A flow index on success, 0 otherwise and rte_errno is set.
*/
-static uint32_t
-flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
- const struct rte_flow_attr *attr,
- const struct rte_flow_item items[],
- const struct rte_flow_action original_actions[],
- bool external, struct rte_flow_error *error)
+uint32_t
+flow_legacy_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
+ const struct rte_flow_attr *attr,
+ const struct rte_flow_item items[],
+ const struct rte_flow_action original_actions[],
+ bool external, struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct rte_flow *flow = NULL;
@@ -7294,8 +7295,14 @@ flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
.table_id = 0
};
int ret;
+ struct mlx5_shared_action_rss *shared_rss_action;
- MLX5_ASSERT(wks);
+ if (!wks)
+ return rte_flow_error_set(error, ENOMEM,
+ RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+ NULL,
+ "failed to push flow workspace");
+ memset(indir_actions, 0, sizeof(indir_actions));
rss_desc = &wks->rss_desc;
ret = flow_action_handles_translate(dev, original_actions,
indir_actions,
@@ -7469,11 +7476,16 @@ flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
ret = rte_errno; /* Save rte_errno before cleanup. */
flow_mreg_del_copy_action(dev, flow);
flow_drv_destroy(dev, flow);
- if (rss_desc->shared_rss)
- rte_atomic_fetch_sub_explicit(&((struct mlx5_shared_action_rss *)
+
+ if (rss_desc->shared_rss) {
+ shared_rss_action = (struct mlx5_shared_action_rss *)
mlx5_ipool_get
(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS],
- rss_desc->shared_rss))->refcnt, 1, rte_memory_order_relaxed);
+ rss_desc->shared_rss);
+ if (shared_rss_action)
+ rte_atomic_fetch_sub_explicit(&(shared_rss_action)->refcnt, 1,
+ rte_memory_order_relaxed);
+ }
mlx5_ipool_free(priv->flows[type], idx);
rte_errno = ret; /* Restore rte_errno. */
ret = rte_errno;
@@ -7528,7 +7540,7 @@ mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev)
};
struct rte_flow_error error;
- return (void *)(uintptr_t)flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
+ return (void *)(uintptr_t)flow_drv_list_create(dev, MLX5_FLOW_TYPE_CTL,
&attr, &pattern,
actions, false, &error);
}
@@ -7596,14 +7608,14 @@ mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev *dev, uint32_t sq_num)
* Creates group 0, highest priority jump flow.
* Matches txq to bypass kernel packets.
*/
- if (flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, actions,
+ if (flow_drv_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, actions,
false, &error) == 0)
return 0;
/* Create group 1, lowest priority redirect flow for txq. */
attr.group = 1;
actions[0].conf = &port;
actions[0].type = RTE_FLOW_ACTION_TYPE_PORT_ID;
- return flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern,
+ return flow_drv_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern,
actions, false, &error);
}
@@ -7737,8 +7749,9 @@ mlx5_flow_cache_flow_toggle(struct rte_eth_dev *dev, bool orig_prio)
attr = flow_info->attr;
if (orig_prio)
attr.priority = flow_info->orig_prio;
- flow_info->flow_idx_high_prio = flow_list_create(dev, MLX5_FLOW_TYPE_GEN,
- &attr, flow_info->items, flow_info->actions,
+ flow_info->flow_idx_high_prio = mlx5_flow_list_create(dev,
+ MLX5_FLOW_TYPE_GEN, &attr,
+ flow_info->items, flow_info->actions,
true, &error);
if (!flow_info->flow_idx_high_prio) {
DRV_LOG(ERR, "Priority toggle failed internally.");
@@ -7758,7 +7771,7 @@ mlx5_flow_cache_flow_toggle(struct rte_eth_dev *dev, bool orig_prio)
flow_info->flow_idx_low_prio);
if (high && low) {
RTE_SWAP(*low, *high);
- flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN,
+ flow_drv_list_destroy(dev, MLX5_FLOW_TYPE_GEN,
flow_info->flow_idx_low_prio);
flow_info->flow_idx_high_prio = 0;
}
@@ -7772,7 +7785,7 @@ mlx5_flow_cache_flow_toggle(struct rte_eth_dev *dev, bool orig_prio)
while (flow_info) {
if (flow_info->orig_prio != flow_info->attr.priority) {
if (flow_info->flow_idx_high_prio)
- flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN,
+ flow_drv_list_destroy(dev, MLX5_FLOW_TYPE_GEN,
flow_info->flow_idx_high_prio);
else
break;
@@ -7907,13 +7920,6 @@ mlx5_flow_create(struct rte_eth_dev *dev,
uint32_t prio = attr->priority;
uint32_t flow_idx;
- if (priv->sh->config.dv_flow_en == 2) {
- rte_flow_error_set(error, ENOTSUP,
- RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
- NULL,
- "Flow non-Q creation not supported");
- return NULL;
- }
/*
* If the device is not started yet, it is not allowed to created a
* flow from application. PMD default flows and traffic control flows
@@ -7934,18 +7940,44 @@ mlx5_flow_create(struct rte_eth_dev *dev,
RTE_PMD_MLX5_FLOW_ENGINE_FLAG_STANDBY_DUP_INGRESS)))
new_attr->priority += 1;
}
- flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, attr, items, actions, true, error);
+ flow_idx = flow_drv_list_create(dev, MLX5_FLOW_TYPE_GEN, attr, items, actions, true, error);
if (!flow_idx)
return NULL;
if (unlikely(mlx5_need_cache_flow(priv, attr))) {
if (mlx5_flow_cache_flow_info(dev, attr, prio, items, actions, flow_idx)) {
- flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx);
+ flow_drv_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx);
flow_idx = 0;
}
}
return (void *)(uintptr_t)flow_idx;
}
+uint32_t
+mlx5_flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
+ const struct rte_flow_attr *attr,
+ const struct rte_flow_item items[],
+ const struct rte_flow_action actions[],
+ bool external, struct rte_flow_error *error)
+{
+ return flow_drv_list_create(dev, type, attr, items, actions, external,
+ error);
+}
+
+uint32_t
+flow_drv_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
+ const struct rte_flow_attr *attr,
+ const struct rte_flow_item items[],
+ const struct rte_flow_action actions[],
+ bool external, struct rte_flow_error *error)
+{
+ const struct mlx5_flow_driver_ops *fops;
+ enum mlx5_flow_drv_type drv_type = flow_get_drv_type(dev, attr);
+
+ fops = flow_get_drv_ops(drv_type);
+ return fops->list_create(dev, type, attr, items, actions, external,
+ error);
+}
+
/**
* Destroy a flow in a list.
*
@@ -7954,15 +7986,16 @@ mlx5_flow_create(struct rte_eth_dev *dev,
* @param[in] flow_idx
* Index of flow to destroy.
*/
-static void
-flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
- uint32_t flow_idx)
+void
+flow_legacy_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
+ uint32_t flow_idx)
{
struct mlx5_priv *priv = dev->data->dev_private;
- struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], flow_idx);
+ struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], (uint32_t)flow_idx);
if (!flow)
return;
+ MLX5_ASSERT((type >= MLX5_FLOW_TYPE_CTL) && (type < MLX5_FLOW_TYPE_MAXI));
MLX5_ASSERT(flow->type == type);
/*
* Update RX queue flags only if port is started, otherwise it is
@@ -7984,6 +8017,25 @@ flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
mlx5_ipool_free(priv->flows[type], flow_idx);
}
+static void
+flow_drv_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
+ uint32_t flow_idx)
+{
+ const struct mlx5_flow_driver_ops *fops;
+ struct rte_flow_attr attr = { .transfer = 0 };
+ enum mlx5_flow_drv_type drv_type = flow_get_drv_type(dev, &attr);
+
+ fops = flow_get_drv_ops(drv_type);
+ fops->list_destroy(dev, type, flow_idx);
+}
+
+void
+mlx5_flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
+ uint32_t flow_idx)
+{
+ flow_drv_list_destroy(dev, type, flow_idx);
+}
+
/**
* Destroy all flows.
*
@@ -8013,7 +8065,7 @@ mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type,
#endif
MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) {
- flow_list_destroy(dev, type, fidx);
+ flow_drv_list_destroy(dev, type, fidx);
if (unlikely(mlx5_need_cache_flow(priv, NULL) && type == MLX5_FLOW_TYPE_GEN)) {
flow_info = LIST_FIRST(&mode_info->hot_upgrade);
while (flow_info) {
@@ -8285,7 +8337,7 @@ mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev,
actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP;
actions[0].conf = &jump;
actions[1].type = RTE_FLOW_ACTION_TYPE_END;
- flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
+ flow_idx = flow_drv_list_create(dev, MLX5_FLOW_TYPE_CTL,
&attr, items, actions, false, &error);
if (!flow_idx) {
DRV_LOG(DEBUG,
@@ -8375,7 +8427,7 @@ mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
action_rss.types = 0;
for (i = 0; i != priv->reta_idx_n; ++i)
queue[i] = (*priv->reta_idx)[i];
- flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
+ flow_idx = flow_drv_list_create(dev, MLX5_FLOW_TYPE_CTL,
&attr, items, actions, false, &error);
if (!flow_idx)
return -rte_errno;
@@ -8450,7 +8502,7 @@ mlx5_flow_lacp_miss(struct rte_eth_dev *dev)
},
};
struct rte_flow_error error;
- uint32_t flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
+ uint32_t flow_idx = flow_drv_list_create(dev, MLX5_FLOW_TYPE_CTL,
&attr, items, actions,
false, &error);
@@ -8479,7 +8531,7 @@ mlx5_flow_destroy(struct rte_eth_dev *dev,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL,
"Flow non-Q destruction not supported");
- flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN,
+ flow_drv_list_destroy(dev, MLX5_FLOW_TYPE_GEN,
(uintptr_t)(void *)flow);
if (unlikely(mlx5_need_cache_flow(priv, NULL))) {
flow_info = LIST_FIRST(&mode_info->hot_upgrade);
@@ -9791,14 +9843,14 @@ mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
if (!priv->sh->config.dv_flow_en)
break;
/* Create internal flow, validation skips copy action. */
- flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr,
+ flow_idx = flow_drv_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr,
items, actions, false, &error);
flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],
flow_idx);
if (!flow)
continue;
priv->sh->flow_mreg_c[n++] = idx;
- flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx);
+ flow_drv_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx);
}
for (; n < MLX5_MREG_C_NUM; ++n)
priv->sh->flow_mreg_c[n] = REG_NON;
@@ -12015,11 +12067,12 @@ mlx5_flow_discover_ipv6_tc_support(struct rte_eth_dev *dev)
actions[0].conf = &set_dscp;
actions[1].type = RTE_FLOW_ACTION_TYPE_END;
- flow_idx = flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, items, actions, true, &error);
+ flow_idx = mlx5_flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr, items,
+ actions, true, &error);
if (!flow_idx)
return -EOPNOTSUPP;
- flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx);
+ mlx5_flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx);
return 0;
}
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 8b4088e35e..e06e7d5cc8 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -620,8 +620,12 @@ struct mlx5_flow_dv_match_params {
/* Matcher structure. */
struct mlx5_flow_dv_matcher {
struct mlx5_list_entry entry; /**< Pointer to the next element. */
- struct mlx5_flow_tbl_resource *tbl;
- /**< Pointer to the table(group) the matcher associated with. */
+ union {
+ struct mlx5_flow_tbl_resource *tbl;
+ /**< Pointer to the table(group) the matcher associated with for DV flow. */
+ struct mlx5_flow_group *group;
+ /* Group of this matcher for HWS non template flow. */
+ };
void *matcher_object; /**< Pointer to DV matcher */
uint16_t crc; /**< CRC of key. */
uint16_t priority; /**< Priority of matcher. */
@@ -1303,10 +1307,24 @@ enum {
#pragma GCC diagnostic ignored "-Wpedantic"
#endif
+#define MLX5_DR_RULE_SIZE 72
+
+/** HWS non template flow data. */
+struct rte_flow_nt2hws {
+ /** BWC rule pointer. */
+ struct mlx5dr_bwc_rule *nt_rule;
+ /** The matcher for non template api. */
+ struct mlx5_flow_dv_matcher *matcher;
+} __rte_packed;
+
/** HWS flow struct. */
struct rte_flow_hw {
- /** The table flow allcated from. */
- struct rte_flow_template_table *table;
+ union {
+ /** The table flow allcated from. */
+ struct rte_flow_template_table *table;
+ /** Data needed for non template flows. */
+ struct rte_flow_nt2hws *nt2hws;
+ };
/** Application's private data passed to enqueued flow operation. */
void *user_data;
/** Flow index from indexed pool. */
@@ -1591,6 +1609,8 @@ struct mlx5_flow_group {
enum mlx5dr_table_type type; /* Table type. */
uint32_t group_id; /* Group id. */
uint32_t idx; /* Group memory index. */
+ /* List of all matchers created for this group in non template api */
+ struct mlx5_list *matchers;
};
@@ -2092,7 +2112,20 @@ void flow_hw_set_port_info(struct rte_eth_dev *dev);
void flow_hw_clear_port_info(struct rte_eth_dev *dev);
int flow_hw_create_vport_action(struct rte_eth_dev *dev);
void flow_hw_destroy_vport_action(struct rte_eth_dev *dev);
-
+int
+flow_hw_init(struct rte_eth_dev *dev,
+ struct rte_flow_error *error);
+
+typedef uint32_t (*mlx5_flow_list_create_t)(struct rte_eth_dev *dev,
+ enum mlx5_flow_type type,
+ const struct rte_flow_attr *attr,
+ const struct rte_flow_item items[],
+ const struct rte_flow_action actions[],
+ bool external,
+ struct rte_flow_error *error);
+typedef void (*mlx5_flow_list_destroy_t)(struct rte_eth_dev *dev,
+ enum mlx5_flow_type type,
+ uint32_t flow_idx);
typedef int (*mlx5_flow_validate_t)(struct rte_eth_dev *dev,
const struct rte_flow_attr *attr,
const struct rte_flow_item items[],
@@ -2455,6 +2488,8 @@ typedef int (*table_resize_complete_t)(struct rte_eth_dev *dev,
struct rte_flow_error *error);
struct mlx5_flow_driver_ops {
+ mlx5_flow_list_create_t list_create;
+ mlx5_flow_list_destroy_t list_destroy;
mlx5_flow_validate_t validate;
mlx5_flow_prepare_t prepare;
mlx5_flow_translate_t translate;
@@ -3071,11 +3106,14 @@ struct mlx5_list_entry *flow_dv_encap_decap_clone_cb(void *tool_ctx,
void flow_dv_encap_decap_clone_free_cb(void *tool_ctx,
struct mlx5_list_entry *entry);
-int flow_dv_matcher_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
+int flow_matcher_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
void *ctx);
-struct mlx5_list_entry *flow_dv_matcher_create_cb(void *tool_ctx, void *ctx);
-void flow_dv_matcher_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
-
+struct mlx5_list_entry *flow_matcher_create_cb(void *tool_ctx, void *ctx);
+void flow_matcher_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_matcher_clone_cb(void *tool_ctx __rte_unused,
+ struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_matcher_clone_free_cb(void *tool_ctx __rte_unused,
+ struct mlx5_list_entry *entry);
int flow_dv_port_id_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
void *cb_ctx);
struct mlx5_list_entry *flow_dv_port_id_create_cb(void *tool_ctx, void *cb_ctx);
@@ -3121,6 +3159,10 @@ void flow_dv_action_rss_l34_hash_adjust(uint64_t rss_types,
uint64_t *hash_field);
uint32_t flow_dv_action_rss_hrxq_lookup(struct rte_eth_dev *dev, uint32_t idx,
const uint64_t hash_fields);
+int flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
+ const struct rte_flow_item items[],
+ const struct rte_flow_action actions[],
+ bool external, int hairpin, struct rte_flow_error *error);
struct mlx5_list_entry *flow_hw_grp_create_cb(void *tool_ctx, void *cb_ctx);
void flow_hw_grp_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
@@ -3226,6 +3268,13 @@ int flow_dv_translate_items_hws(const struct rte_flow_item *items,
uint8_t *match_criteria,
struct rte_flow_error *error);
+int __flow_dv_translate_items_hws(const struct rte_flow_item *items,
+ struct mlx5_flow_attr *attr, void *key,
+ uint32_t key_type, uint64_t *item_flags,
+ uint8_t *match_criteria,
+ bool nt_flow,
+ struct rte_flow_error *error);
+
int mlx5_flow_pick_transfer_proxy(struct rte_eth_dev *dev,
uint16_t *proxy_port_id,
struct rte_flow_error *error);
@@ -3268,6 +3317,13 @@ int mlx5_flow_item_field_width(struct rte_eth_dev *dev,
enum rte_flow_field_id field, int inherit,
const struct rte_flow_attr *attr,
struct rte_flow_error *error);
+uint32_t flow_legacy_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
+ const struct rte_flow_attr *attr,
+ const struct rte_flow_item items[],
+ const struct rte_flow_action actions[],
+ bool external, struct rte_flow_error *error);
+void flow_legacy_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
+ uint32_t flow_idx);
static __rte_always_inline int
flow_hw_get_srh_flex_parser_byte_off_from_ctx(void *dr_ctx __rte_unused)
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index f9c56af86c..e3855731b8 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -7622,7 +7622,7 @@ mlx5_flow_validate_item_ib_bth(struct rte_eth_dev *dev,
* @return
* 0 on success, a negative errno value otherwise and rte_errno is set.
*/
-static int
+int
flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
const struct rte_flow_item items[],
const struct rte_flow_action actions[],
@@ -8076,6 +8076,7 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
&item_flags, error);
if (ret < 0)
return ret;
+ last_item = MLX5_FLOW_LAYER_ASO_CT;
break;
case MLX5_RTE_FLOW_ITEM_TYPE_TUNNEL:
/* tunnel offload item was processed before
@@ -8214,6 +8215,11 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
rw_act_num += MLX5_ACT_NUM_SET_MARK;
break;
case RTE_FLOW_ACTION_TYPE_SET_META:
+ if (priv->sh->config.dv_flow_en == 2)
+ return rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_ACTION,
+ actions,
+ "action not supported");
ret = flow_dv_validate_action_set_meta(dev, actions,
action_flags,
attr, error);
@@ -8568,6 +8574,11 @@ flow_dv_validate(struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
rw_act_num += MLX5_ACT_NUM_SET_TAG;
break;
case RTE_FLOW_ACTION_TYPE_METER:
+ if (priv->sh->config.dv_flow_en == 2)
+ return rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_ACTION,
+ actions,
+ "action not supported");
ret = mlx5_flow_validate_action_meter(dev,
action_flags,
item_flags,
@@ -10408,6 +10419,8 @@ flow_dv_match_meta_reg(void *key, enum modify_reg reg_type,
MLX5_ADDR_OF(fte_match_param, key, misc_parameters_2);
uint32_t temp;
+ if (!key)
+ return;
data &= mask;
switch (reg_type) {
case REG_A:
@@ -11505,8 +11518,8 @@ __flow_dv_adjust_buf_size(size_t *size, uint8_t match_criteria)
}
}
-static struct mlx5_list_entry *
-flow_dv_matcher_clone_cb(void *tool_ctx __rte_unused,
+struct mlx5_list_entry *
+flow_matcher_clone_cb(void *tool_ctx __rte_unused,
struct mlx5_list_entry *entry, void *cb_ctx)
{
struct mlx5_flow_cb_ctx *ctx = cb_ctx;
@@ -11528,8 +11541,8 @@ flow_dv_matcher_clone_cb(void *tool_ctx __rte_unused,
return &resource->entry;
}
-static void
-flow_dv_matcher_clone_free_cb(void *tool_ctx __rte_unused,
+void
+flow_matcher_clone_free_cb(void *tool_ctx __rte_unused,
struct mlx5_list_entry *entry)
{
mlx5_free(entry);
@@ -11602,11 +11615,11 @@ flow_dv_tbl_create_cb(void *tool_ctx, void *cb_ctx)
key.is_fdb ? "FDB" : "NIC", key.is_egress ? "egress" : "ingress",
key.level, key.id);
tbl_data->matchers = mlx5_list_create(matcher_name, sh, true,
- flow_dv_matcher_create_cb,
- flow_dv_matcher_match_cb,
- flow_dv_matcher_remove_cb,
- flow_dv_matcher_clone_cb,
- flow_dv_matcher_clone_free_cb);
+ flow_matcher_create_cb,
+ flow_matcher_match_cb,
+ flow_matcher_remove_cb,
+ flow_matcher_clone_cb,
+ flow_matcher_clone_free_cb);
if (!tbl_data->matchers) {
rte_flow_error_set(error, ENOMEM,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
@@ -11809,7 +11822,7 @@ flow_dv_tbl_resource_release(struct mlx5_dev_ctx_shared *sh,
}
int
-flow_dv_matcher_match_cb(void *tool_ctx __rte_unused,
+flow_matcher_match_cb(void *tool_ctx __rte_unused,
struct mlx5_list_entry *entry, void *cb_ctx)
{
struct mlx5_flow_cb_ctx *ctx = cb_ctx;
@@ -11824,7 +11837,7 @@ flow_dv_matcher_match_cb(void *tool_ctx __rte_unused,
}
struct mlx5_list_entry *
-flow_dv_matcher_create_cb(void *tool_ctx, void *cb_ctx)
+flow_matcher_create_cb(void *tool_ctx, void *cb_ctx)
{
struct mlx5_dev_ctx_shared *sh = tool_ctx;
struct mlx5_flow_cb_ctx *ctx = cb_ctx;
@@ -11846,23 +11859,26 @@ flow_dv_matcher_create_cb(void *tool_ctx, void *cb_ctx)
"cannot create matcher");
return NULL;
}
+ /*Consider memcpy(resource, ref, sizeof(*resource));*/
*resource = *ref;
- dv_attr.match_criteria_enable =
- flow_dv_matcher_enable(resource->mask.buf);
- __flow_dv_adjust_buf_size(&ref->mask.size,
- dv_attr.match_criteria_enable);
- dv_attr.priority = ref->priority;
- if (tbl->is_egress)
- dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
- ret = mlx5_flow_os_create_flow_matcher(sh->cdev->ctx, &dv_attr,
- tbl->tbl.obj,
- &resource->matcher_object);
- if (ret) {
- mlx5_free(resource);
- rte_flow_error_set(ctx->error, ENOMEM,
- RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
- "cannot create matcher");
- return NULL;
+ if (sh->config.dv_flow_en != 2) {
+ dv_attr.match_criteria_enable =
+ flow_dv_matcher_enable(resource->mask.buf);
+ __flow_dv_adjust_buf_size(&ref->mask.size,
+ dv_attr.match_criteria_enable);
+ dv_attr.priority = ref->priority;
+ if (tbl->is_egress)
+ dv_attr.flags |= IBV_FLOW_ATTR_FLAGS_EGRESS;
+ ret = mlx5_flow_os_create_flow_matcher(sh->cdev->ctx, &dv_attr,
+ tbl->tbl.obj,
+ &resource->matcher_object);
+ if (ret) {
+ mlx5_free(resource);
+ rte_flow_error_set(ctx->error, ENOMEM,
+ RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+ "cannot create matcher");
+ return NULL;
+ }
}
return &resource->entry;
}
@@ -14228,6 +14244,60 @@ flow_dv_translate_items(struct rte_eth_dev *dev,
return 0;
}
+/**
+ * Fill the flow matcher with DV spec for items supported in non template mode.
+ *
+ * @param[in] dev
+ * Pointer to rte_eth_dev structure.
+ * @param[in] items
+ * Pointer to the list of items.
+ * @param[in] wks
+ * Pointer to the matcher workspace.
+ * @param[in] key
+ * Pointer to the flow matcher key.
+ * @param[in] key_type
+ * Key type.
+ * @param[out] error
+ * Pointer to the error structure.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+flow_dv_translate_items_nta(struct rte_eth_dev *dev,
+ const struct rte_flow_item *items,
+ struct mlx5_dv_matcher_workspace *wks,
+ void *key, uint32_t key_type,
+ struct rte_flow_error *error)
+{
+ int item_type;
+ int ret = 0;
+ int tunnel;
+ /* Dummy structure to enable the key calculation for flex item. */
+ struct mlx5_flow_dv_match_params flex_item_key;
+
+ tunnel = !!(wks->item_flags & MLX5_FLOW_LAYER_TUNNEL);
+ item_type = items->type;
+ switch (item_type) {
+ case RTE_FLOW_ITEM_TYPE_CONNTRACK:
+ flow_dv_translate_item_aso_ct(dev, key, NULL, items);
+ wks->last_item = MLX5_FLOW_LAYER_ASO_CT;
+ break;
+ /* TODO: remove once flex item translation is added to flow_dv_translate_items. */
+ case RTE_FLOW_ITEM_TYPE_FLEX:
+ mlx5_flex_flow_translate_item(dev, key, flex_item_key.buf, items, tunnel != 0);
+ wks->last_item = tunnel ? MLX5_FLOW_ITEM_INNER_FLEX : MLX5_FLOW_ITEM_OUTER_FLEX;
+ break;
+ default:
+ ret = flow_dv_translate_items(dev, items, wks, key, key_type, error);
+ if (ret)
+ return ret;
+ break;
+ }
+ wks->item_flags |= wks->last_item;
+ return 0;
+}
+
/**
* Fill the HW steering flow with DV spec.
*
@@ -14241,6 +14311,8 @@ flow_dv_translate_items(struct rte_eth_dev *dev,
* Key type.
* @param[in, out] item_flags
* Pointer to the flow item flags.
+ * @param[in, out] nt_flow
+ * Non template flow.
* @param[out] error
* Pointer to the error structure.
*
@@ -14248,10 +14320,11 @@ flow_dv_translate_items(struct rte_eth_dev *dev,
* 0 on success, a negative errno value otherwise and rte_errno is set.
*/
int
-flow_dv_translate_items_hws(const struct rte_flow_item *items,
+__flow_dv_translate_items_hws(const struct rte_flow_item *items,
struct mlx5_flow_attr *attr, void *key,
uint32_t key_type, uint64_t *item_flags,
uint8_t *match_criteria,
+ bool nt_flow,
struct rte_flow_error *error)
{
struct mlx5_flow_workspace *flow_wks = mlx5_flow_push_thread_workspace();
@@ -14281,10 +14354,18 @@ flow_dv_translate_items_hws(const struct rte_flow_item *items,
NULL, "item not supported");
goto exit;
}
- ret = flow_dv_translate_items(&rte_eth_devices[attr->port_id],
- items, &wks, key, key_type, NULL);
- if (ret)
- goto exit;
+ /* Non template flow. */
+ if (nt_flow) {
+ ret = flow_dv_translate_items_nta(&rte_eth_devices[attr->port_id],
+ items, &wks, key, key_type, NULL);
+ if (ret)
+ goto exit;
+ } else {
+ ret = flow_dv_translate_items(&rte_eth_devices[attr->port_id],
+ items, &wks, key, key_type, NULL);
+ if (ret)
+ goto exit;
+ }
}
if (wks.item_flags & MLX5_FLOW_ITEM_INTEGRITY) {
flow_dv_translate_item_integrity_post(key,
@@ -14333,6 +14414,37 @@ flow_dv_translate_items_hws(const struct rte_flow_item *items,
return ret;
}
+/**
+ * Fill the HW steering flow with DV spec.
+ * This function assumes given flow is created from template API.
+ *
+ * @param[in] items
+ * Pointer to the list of items.
+ * @param[in] attr
+ * Pointer to the flow attributes.
+ * @param[in] key
+ * Pointer to the flow matcher key.
+ * @param[in] key_type
+ * Key type.
+ * @param[in, out] item_flags
+ * Pointer to the flow item flags.
+ * @param[out] error
+ * Pointer to the error structure.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+flow_dv_translate_items_hws(const struct rte_flow_item *items,
+ struct mlx5_flow_attr *attr, void *key,
+ uint32_t key_type, uint64_t *item_flags,
+ uint8_t *match_criteria,
+ struct rte_flow_error *error)
+{
+ return __flow_dv_translate_items_hws(items, attr, key, key_type, item_flags, match_criteria,
+ false, error);
+}
+
/**
* Fill the SW steering flow with DV spec.
*
@@ -14389,6 +14501,7 @@ flow_dv_translate_items_sws(struct rte_eth_dev *dev,
case RTE_FLOW_ITEM_TYPE_CONNTRACK:
flow_dv_translate_item_aso_ct(dev, match_mask,
match_value, items);
+ wks.last_item = MLX5_FLOW_LAYER_ASO_CT;
break;
case RTE_FLOW_ITEM_TYPE_FLEX:
flow_dv_translate_item_flex(dev, match_mask,
@@ -15670,14 +15783,21 @@ flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
}
void
-flow_dv_matcher_remove_cb(void *tool_ctx __rte_unused,
+flow_matcher_remove_cb(void *tool_ctx __rte_unused,
struct mlx5_list_entry *entry)
{
struct mlx5_flow_dv_matcher *resource = container_of(entry,
typeof(*resource),
entry);
+#ifdef HAVE_MLX5_HWS_SUPPORT
+ struct mlx5_dev_ctx_shared *sh = tool_ctx;
- claim_zero(mlx5_flow_os_destroy_flow_matcher(resource->matcher_object));
+ if (sh->config.dv_flow_en == 2)
+ claim_zero(mlx5dr_bwc_matcher_destroy((struct mlx5dr_bwc_matcher *)
+ resource->matcher_object));
+ else
+#endif
+ claim_zero(mlx5_flow_os_destroy_flow_matcher(resource->matcher_object));
mlx5_free(resource);
}
@@ -20082,6 +20202,8 @@ flow_dv_discover_priorities(struct rte_eth_dev *dev,
}
const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
+ .list_create = flow_legacy_list_create,
+ .list_destroy = flow_legacy_list_destroy,
.validate = flow_dv_validate,
.prepare = flow_dv_prepare,
.translate = flow_dv_translate,
diff --git a/drivers/net/mlx5/mlx5_flow_hw.c b/drivers/net/mlx5/mlx5_flow_hw.c
index 80efcf44fa..efd2141913 100644
--- a/drivers/net/mlx5/mlx5_flow_hw.c
+++ b/drivers/net/mlx5/mlx5_flow_hw.c
@@ -256,6 +256,9 @@ get_mlx5dr_table_type(const struct rte_flow_attr *attr)
return type;
}
+/* Non template default queue size used for inner ctrl queue. */
+#define MLX5_NT_DEFAULT_QUEUE_SIZE 32
+
struct mlx5_mirror_clone {
enum rte_flow_action_type type;
void *action_ctx;
@@ -4859,41 +4862,20 @@ flow_hw_table_update(struct rte_eth_dev *dev,
return 0;
}
-/**
- * Translates group index specified by the user in @p attr to internal
- * group index.
- *
- * Translation is done by incrementing group index, so group n becomes n + 1.
- *
- * @param[in] dev
- * Pointer to Ethernet device.
- * @param[in] cfg
- * Pointer to the template table configuration.
- * @param[in] group
- * Currently used group index (table group or jump destination).
- * @param[out] table_group
- * Pointer to output group index.
- * @param[out] error
- * Pointer to error structure.
- *
- * @return
- * 0 on success. Otherwise, returns negative error code, rte_errno is set
- * and error structure is filled.
- */
-static int
-flow_hw_translate_group(struct rte_eth_dev *dev,
- const struct mlx5_flow_template_table_cfg *cfg,
+static inline int
+__translate_group(struct rte_eth_dev *dev,
+ const struct rte_flow_attr *flow_attr,
+ bool external,
uint32_t group,
uint32_t *table_group,
struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_sh_config *config = &priv->sh->config;
- const struct rte_flow_attr *flow_attr = &cfg->attr.flow_attr;
if (config->dv_esw_en &&
priv->fdb_def_rule &&
- cfg->external &&
+ external &&
flow_attr->transfer) {
if (group > MLX5_HW_MAX_TRANSFER_GROUP)
return rte_flow_error_set(error, EINVAL,
@@ -4903,7 +4885,7 @@ flow_hw_translate_group(struct rte_eth_dev *dev,
*table_group = group + 1;
} else if (config->dv_esw_en &&
(config->repr_matching || config->dv_xmeta_en == MLX5_XMETA_MODE_META32_HWS) &&
- cfg->external &&
+ external &&
flow_attr->egress) {
/*
* On E-Switch setups, default egress flow rules are inserted to allow
@@ -4927,6 +4909,39 @@ flow_hw_translate_group(struct rte_eth_dev *dev,
return 0;
}
+/**
+ * Translates group index specified by the user in @p attr to internal
+ * group index.
+ *
+ * Translation is done by incrementing group index, so group n becomes n + 1.
+ *
+ * @param[in] dev
+ * Pointer to Ethernet device.
+ * @param[in] cfg
+ * Pointer to the template table configuration.
+ * @param[in] group
+ * Currently used group index (table group or jump destination).
+ * @param[out] table_group
+ * Pointer to output group index.
+ * @param[out] error
+ * Pointer to error structure.
+ *
+ * @return
+ * 0 on success. Otherwise, returns negative error code, rte_errno is set
+ * and error structure is filled.
+ */
+static int
+flow_hw_translate_group(struct rte_eth_dev *dev,
+ const struct mlx5_flow_template_table_cfg *cfg,
+ uint32_t group,
+ uint32_t *table_group,
+ struct rte_flow_error *error)
+{
+ const struct rte_flow_attr *flow_attr = &cfg->attr.flow_attr;
+
+ return __translate_group(dev, flow_attr, cfg->external, group, table_group, error);
+}
+
/**
* Create flow table.
*
@@ -7113,7 +7128,7 @@ flow_hw_actions_template_create(struct rte_eth_dev *dev,
unsigned int act_num;
unsigned int i;
struct rte_flow_actions_template *at = NULL;
- uint16_t pos = UINT16_MAX;
+ uint16_t pos;
uint64_t action_flags = 0;
struct rte_flow_action tmp_action[MLX5_HW_MAX_ACTS];
struct rte_flow_action tmp_mask[MLX5_HW_MAX_ACTS];
@@ -8032,6 +8047,9 @@ flow_hw_grp_create_cb(void *tool_ctx, void *cb_ctx)
struct mlx5dr_table *tbl = NULL;
struct mlx5dr_action *jump;
uint32_t idx = 0;
+ MKSTR(matcher_name, "%s_%s_%u_%u_matcher_list",
+ attr->transfer ? "FDB" : "NIC", attr->egress ? "egress" : "ingress",
+ attr->group, idx);
grp_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_HW_GRP], &idx);
if (!grp_data) {
@@ -8069,6 +8087,13 @@ flow_hw_grp_create_cb(void *tool_ctx, void *cb_ctx)
goto error;
grp_data->jump.root_action = jump;
}
+
+ grp_data->matchers = mlx5_list_create(matcher_name, sh, true,
+ flow_matcher_create_cb,
+ flow_matcher_match_cb,
+ flow_matcher_remove_cb,
+ flow_matcher_clone_cb,
+ flow_matcher_clone_free_cb);
grp_data->dev = dev;
grp_data->idx = idx;
grp_data->group_id = attr->group;
@@ -8112,6 +8137,7 @@ flow_hw_grp_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
if (grp_data->jump.root_action)
mlx5dr_action_destroy(grp_data->jump.root_action);
mlx5dr_table_destroy(grp_data->tbl);
+ mlx5_list_destroy(grp_data->matchers);
mlx5_ipool_free(sh->ipool[MLX5_IPOOL_HW_GRP], grp_data->idx);
}
@@ -10387,6 +10413,10 @@ flow_hw_configure(struct rte_eth_dev *dev,
int ret = 0;
uint32_t action_flags;
+ if (mlx5dr_rule_get_handle_size() != MLX5_DR_RULE_SIZE) {
+ rte_errno = EINVAL;
+ goto err;
+ }
if (flow_hw_validate_attributes(port_attr, nb_queue, queue_attr, error))
return -rte_errno;
/*
@@ -10531,6 +10561,8 @@ flow_hw_configure(struct rte_eth_dev *dev,
rte_atomic_fetch_add_explicit(&host_priv->shared_refcnt, 1,
rte_memory_order_relaxed);
}
+ /* Set backward compatibale mode to support non template RTE FLOW API.*/
+ dr_ctx_attr.bwc = true;
dr_ctx = mlx5dr_context_open(priv->sh->cdev->ctx, &dr_ctx_attr);
/* rte_errno has been updated by HWS layer. */
if (!dr_ctx)
@@ -11981,6 +12013,402 @@ flow_hw_get_aged_flows(struct rte_eth_dev *dev, void **contexts,
dev->data->port_id);
return flow_hw_get_q_aged_flows(dev, 0, contexts, nb_contexts, error);
}
+/**
+ * Initialization function for non template API which calls
+ * flow_hw_configure with default values.
+ *
+ * @param[in] dev
+ * Pointer to the Ethernet device structure.
+ * @param[out] error
+ * Pointer to the error structure.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+ /* Configure non queues cause 1 queue is configured by default for inner usage. */
+
+int
+flow_hw_init(struct rte_eth_dev *dev,
+ struct rte_flow_error *error)
+{
+ const struct rte_flow_port_attr port_attr = {0};
+ struct rte_flow_queue_attr queue_attr = {.size = MLX5_NT_DEFAULT_QUEUE_SIZE};
+ const struct rte_flow_queue_attr *attr_list = &queue_attr;
+
+ /**
+ * If user uses template and non template API:
+ * User will call flow_hw_configure and non template
+ * API will use the allocated actions.
+ * Init function will not call flow_hw_configure.
+ *
+ * If user uses only non template API's:
+ * Init function will call flow_hw_configure.
+ * It will not allocate memory for actions.
+ * When needed allocation, it will handle same as for SWS today,
+ * meaning using bulk allocations and resize as needed.
+ */
+ /* Configure hws with default values. */
+ DRV_LOG(DEBUG, "Apply default configuration, zero number of queues, inner control queue size is %u",
+ MLX5_NT_DEFAULT_QUEUE_SIZE);
+ return flow_hw_configure(dev, &port_attr, 0, &attr_list, error);
+}
+
+static int flow_hw_prepare(struct rte_eth_dev *dev,
+ const struct rte_flow_action actions[] __rte_unused,
+ enum mlx5_flow_type type,
+ struct rte_flow_hw **flow,
+ struct rte_flow_error *error)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ uint32_t idx = 0;
+
+ /*
+ * Notice pool idx size = (sizeof(struct rte_flow_hw)
+ * + sizeof(struct rte_flow_nt2hws)) for HWS mode.
+ */
+ *flow = mlx5_ipool_zmalloc(priv->flows[type], &idx);
+ if (!(*flow))
+ return rte_flow_error_set(error, ENOMEM,
+ RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+ "cannot allocate flow memory");
+ /* Allocating 2 structures in one pool slot, updating nt2hw pointer.*/
+ (*flow)->nt2hws = (struct rte_flow_nt2hws *)
+ ((uintptr_t)(*flow) + sizeof(struct rte_flow_hw));
+ (*flow)->idx = idx;
+ /*TODO: consider if other allocation is needed for actions translate. */
+ return 0;
+}
+
+static int flow_hw_translate_actions(struct rte_eth_dev *dev __rte_unused,
+ const struct rte_flow_attr *attr __rte_unused,
+ const struct rte_flow_action actions[] __rte_unused,
+ struct rte_flow_hw *flow __rte_unused,
+ struct rte_flow_error *error __rte_unused)
+{
+ /* TODO implement */
+ return 0;
+}
+
+static int flow_hw_register_matcher(struct rte_eth_dev *dev,
+ const struct rte_flow_attr *attr,
+ const struct rte_flow_item items[],
+ bool external,
+ struct rte_flow_hw *flow,
+ struct mlx5_flow_dv_matcher *matcher,
+ struct rte_flow_error *error)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct rte_flow_error sub_error = {
+ .type = RTE_FLOW_ERROR_TYPE_NONE,
+ .cause = NULL,
+ .message = NULL,
+ };
+ struct rte_flow_attr flow_attr = *attr;
+ struct mlx5_flow_cb_ctx ctx = {
+ .dev = dev,
+ .error = &sub_error,
+ .data = &flow_attr,
+ };
+ struct mlx5_flow_cb_ctx matcher_ctx = {
+ .error = &sub_error,
+ .data = matcher,
+ };
+ struct mlx5_list_entry *group_entry;
+ struct mlx5_list_entry *matcher_entry;
+ struct mlx5_flow_dv_matcher *resource;
+ struct mlx5_list *matchers_list;
+ struct mlx5_flow_group *flow_group;
+ uint32_t group = 0;
+ int ret;
+
+
+ matcher->crc = rte_raw_cksum((const void *)matcher->mask.buf,
+ matcher->mask.size);
+ matcher->priority = mlx5_get_matcher_priority(dev, attr,
+ matcher->priority,
+ external);
+
+ ret = __translate_group(dev, attr, external, attr->group, &group, error);
+ if (ret)
+ return ret;
+
+ /* Register the flow group. */
+ group_entry = mlx5_hlist_register(priv->sh->groups, group, &ctx);
+ if (!group_entry)
+ goto error;
+ flow_group = container_of(group_entry, struct mlx5_flow_group, entry);
+
+ matchers_list = flow_group->matchers;
+ matcher->group = flow_group;
+ matcher_entry = mlx5_list_register(matchers_list, &matcher_ctx);
+ if (!matcher_entry)
+ goto error;
+ resource = container_of(matcher_entry, typeof(*resource), entry);
+ if (!resource)
+ goto error;
+ flow->nt2hws->matcher = resource;
+
+ /* If matcher was not found and reused in list, create matcher object. */
+ if (!resource->matcher_object) {
+ resource->matcher_object = (void *)mlx5dr_bwc_matcher_create
+ (flow_group->tbl, matcher->priority, items);
+ }
+ /* If matcher create failed */
+ if (!(resource->matcher_object))
+ goto error;
+ return 0;
+
+error:
+ if (error) {
+ if (sub_error.type != RTE_FLOW_ERROR_TYPE_NONE)
+ rte_memcpy(error, &sub_error, sizeof(sub_error));
+ return rte_flow_error_set(error, ENOMEM,
+ RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+ NULL, "fail to register matcher");
+ } else {
+ return -ENOMEM;
+ }
+}
+
+static int flow_hw_apply(struct rte_eth_dev *dev __rte_unused, /* TODO: remove if not used */
+ const struct rte_flow_item items[],
+ struct mlx5dr_rule_action rule_actions[],
+ struct rte_flow_hw *flow,
+ struct rte_flow_error *error)
+{
+ struct mlx5dr_bwc_rule *rule = NULL;
+
+ rule = mlx5dr_bwc_rule_create((struct mlx5dr_bwc_matcher *)
+ flow->nt2hws->matcher->matcher_object,
+ items, rule_actions);
+ flow->nt2hws->nt_rule = rule;
+ if (!rule) {
+ if (error)
+ return rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+ NULL, "fail to create rte flow");
+ else
+ return -EINVAL;
+ }
+ return 0;
+}
+
+#ifdef HAVE_MLX5_HWS_SUPPORT
+/**
+ * Create a flow.
+ *
+ * @param[in] dev
+ * Pointer to Ethernet device.
+ * @param[in] type
+ * Flow type.
+ * @param[in] attr
+ * Flow rule attributes.
+ * @param[in] items
+ * Pattern specification (list terminated by the END pattern item).
+ * @param[in] actions
+ * Associated actions (list terminated by the END action).
+ * @param[in] external
+ * This flow rule is created by request external to PMD.
+ * @param[out] flow
+ * Flow pointer
+ * @param[out] error
+ * Perform verbose error reporting if not NULL.
+ *
+ * @return
+ * 0 on success, negative errno value otherwise and rte_errno set.
+ */
+static int flow_hw_create_flow(struct rte_eth_dev *dev,
+ enum mlx5_flow_type type,
+ const struct rte_flow_attr *attr,
+ const struct rte_flow_item items[],
+ const struct rte_flow_action actions[],
+ bool external,
+ struct rte_flow_hw **flow,
+ struct rte_flow_error *error)
+{
+ int ret;
+ struct mlx5_hw_actions hw_act;
+ struct mlx5_flow_dv_matcher matcher = {
+ .mask = {
+ .size = sizeof(matcher.mask.buf),
+ },
+ };
+ uint32_t tbl_type;
+
+ struct mlx5_flow_attr flow_attr = {
+ .port_id = dev->data->port_id,
+ .group = attr->group,
+ .priority = attr->priority,
+ .rss_level = 0,
+ /*
+ * TODO: currently only mlx5_flow_lacp_miss rule is relevant:
+ * action type=(enum rte_flow_action_type) MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS.
+ * I don't want to waist time going over all actions for this corner case.
+ * Needs to use another preparation code to update this action flags.
+ * if (action_type == (enum rte_flow_action_type)
+ * MLX5_RTE_FLOW_ACTION_TYPE_DEFAULT_MISS)
+ * act_flags |= MLX5_FLOW_ACTION_DEFAULT_MISS;
+ */
+ .act_flags = 0, /*TODO update*/
+ .tbl_type = 0,
+ };
+
+ memset(&hw_act, 0, sizeof(hw_act));
+ if (attr->transfer)
+ tbl_type = MLX5DR_TABLE_TYPE_FDB;
+ else if (attr->egress)
+ tbl_type = MLX5DR_TABLE_TYPE_NIC_TX;
+ else
+ tbl_type = MLX5DR_TABLE_TYPE_NIC_RX;
+ flow_attr.tbl_type = tbl_type;
+
+ /* Allocate needed memory. */
+ ret = flow_hw_prepare(dev, actions, type, flow, error);
+ if (ret)
+ goto error;
+
+ /* TODO TBD flow_hw_handle_tunnel_offload(). */
+
+ (*flow)->nt2hws->matcher = &matcher;
+ ret = __flow_dv_translate_items_hws(items, &flow_attr, &matcher.mask.buf,
+ MLX5_SET_MATCHER_HS_M, 0,
+ NULL, true, error);
+ if (ret)
+ goto error;
+
+ ret = flow_hw_register_matcher(dev, attr, items, external, *flow, &matcher, error);
+ if (ret)
+ goto error;
+
+ ret = flow_hw_translate_actions(dev, attr, actions, *flow, error);
+ if (ret)
+ goto error;
+
+ /*
+ * If the flow is external (from application) OR device is started,
+ * OR mreg discover, then apply immediately.
+ */
+ if (external || dev->data->dev_started ||
+ (attr->group == MLX5_FLOW_MREG_CP_TABLE_GROUP &&
+ attr->priority == MLX5_FLOW_LOWEST_PRIO_INDICATOR)) {
+ ret = flow_hw_apply(dev, items, hw_act.rule_acts, *flow, error);
+ if (ret)
+ goto error;
+ }
+ return 0;
+
+error:
+ if (error)
+ return rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+ NULL, "fail to create rte flow");
+ else
+ return -EINVAL;
+}
+#endif
+
+static void
+flow_hw_destroy(struct rte_eth_dev *dev, struct rte_flow_hw *flow)
+{
+ int ret;
+
+ if (!flow || !flow->nt2hws)
+ return;
+
+ if (flow->nt2hws->nt_rule) {
+ ret = mlx5dr_bwc_rule_destroy(flow->nt2hws->nt_rule);
+ if (likely(!ret))
+ DRV_LOG(ERR, "bwc rule destroy failed");
+ }
+
+ /* TODO: notice this function does not handle shared/static actions. */
+ hw_cmpl_flow_update_or_destroy(dev, NULL, 0, NULL);
+
+ /**
+ * TODO: TBD - Release tunnel related memory allocations(mlx5_flow_tunnel_free)
+ * – needed only if supporting tunnel offloads, notice update RX queue flags in SWS.
+ */
+
+ /**
+ * Notice matcher destroy will take place when matcher's list is destroyed
+ * , same as for DV.
+ */
+}
+
+#ifdef HAVE_MLX5_HWS_SUPPORT
+/**
+ * Destroy a flow.
+ *
+ * @param[in] dev
+ * Pointer to Ethernet device.
+ * @param[in] type
+ * Flow type.
+ * @param[in] flow_idx
+ * Index of flow to destroy.
+ */
+static void flow_hw_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
+ uint32_t flow_idx)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ /* Get flow via idx */
+ struct rte_flow_hw *flow = mlx5_ipool_get(priv->flows[type], flow_idx);
+
+ DRV_LOG(DEBUG, "Non template flow index %u destroy", flow_idx);
+ if (!flow)
+ return;
+ flow_hw_destroy(dev, flow);
+ /* Release flow memory by idx */
+ mlx5_ipool_free(priv->flows[type], flow_idx);
+}
+#endif
+
+/**
+ * Create a flow.
+ *
+ * @param[in] dev
+ * Pointer to Ethernet device.
+ * @param[in] type
+ * Flow type.
+ * @param[in] attr
+ * Flow rule attributes.
+ * @param[in] items
+ * Pattern specification (list terminated by the END pattern item).
+ * @param[in] actions
+ * Associated actions (list terminated by the END action).
+ * @param[in] external
+ * This flow rule is created by request external to PMD.
+ * @param[out] error
+ * Perform verbose error reporting if not NULL.
+ *
+ * @return
+ * A flow index on success, 0 otherwise and rte_errno is set.
+ */
+static uint32_t flow_hw_list_create(struct rte_eth_dev *dev,
+ enum mlx5_flow_type type,
+ const struct rte_flow_attr *attr,
+ const struct rte_flow_item items[],
+ const struct rte_flow_action actions[],
+ bool external,
+ struct rte_flow_error *error)
+{
+ int ret;
+ struct rte_flow_hw *flow = NULL;
+
+ /*TODO: Handle split/expand to num_flows. */
+
+ DRV_LOG(DEBUG, "Non template flow creation");
+ /* Create single flow. */
+ ret = flow_hw_create_flow(dev, type, attr, items, actions, external, &flow, error);
+ if (ret)
+ goto free;
+ if (flow)
+ return flow->idx;
+
+free:
+ if (flow)
+ flow_hw_list_destroy(dev, type, flow->idx);
+ return 0;
+}
static void
mlx5_mirror_destroy_clone(struct rte_eth_dev *dev,
@@ -12971,6 +13399,9 @@ flow_hw_update_resized(struct rte_eth_dev *dev, uint32_t queue,
}
const struct mlx5_flow_driver_ops mlx5_flow_hw_drv_ops = {
+ .list_create = flow_hw_list_create,
+ .list_destroy = flow_hw_list_destroy,
+ .validate = flow_dv_validate,
.info_get = flow_hw_info_get,
.configure = flow_hw_configure,
.pattern_validate = flow_hw_pattern_validate,
diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c b/drivers/net/mlx5/mlx5_flow_verbs.c
index fe9c818abc..6324ee3e3c 100644
--- a/drivers/net/mlx5/mlx5_flow_verbs.c
+++ b/drivers/net/mlx5/mlx5_flow_verbs.c
@@ -2183,6 +2183,8 @@ flow_verbs_sync_domain(struct rte_eth_dev *dev, uint32_t domains,
}
const struct mlx5_flow_driver_ops mlx5_flow_verbs_drv_ops = {
+ .list_create = flow_legacy_list_create,
+ .list_destroy = flow_legacy_list_destroy,
.validate = flow_verbs_validate,
.prepare = flow_verbs_prepare,
.translate = flow_verbs_translate,
diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c
index c241a1dbd7..6fa7c01cd0 100644
--- a/drivers/net/mlx5/mlx5_trigger.c
+++ b/drivers/net/mlx5/mlx5_trigger.c
@@ -1158,6 +1158,12 @@ mlx5_dev_start(struct rte_eth_dev *dev)
DRV_LOG(DEBUG, "port %u starting device", dev->data->port_id);
#ifdef HAVE_MLX5_HWS_SUPPORT
if (priv->sh->config.dv_flow_en == 2) {
+ /*If previous configuration does not exist. */
+ if (!(priv->dr_ctx)) {
+ ret = flow_hw_init(dev, NULL);
+ if (ret)
+ return ret;
+ }
/* If there is no E-Switch, then there are no start/stop order limitations. */
if (!priv->sh->config.dv_esw_en)
goto continue_dev_start;
diff --git a/drivers/net/mlx5/windows/mlx5_os.c b/drivers/net/mlx5/windows/mlx5_os.c
index b731bdff06..98022ed3c7 100644
--- a/drivers/net/mlx5/windows/mlx5_os.c
+++ b/drivers/net/mlx5/windows/mlx5_os.c
@@ -35,7 +35,7 @@ static const char *MZ_MLX5_PMD_SHARED_DATA = "mlx5_pmd_shared_data";
static rte_spinlock_t mlx5_shared_data_lock = RTE_SPINLOCK_INITIALIZER;
/* rte flow indexed pool configuration. */
-static struct mlx5_indexed_pool_config icfg[] = {
+static const struct mlx5_indexed_pool_config default_icfg[] = {
{
.size = sizeof(struct rte_flow),
.trunk_size = 64,
@@ -352,7 +352,9 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
int own_domain_id = 0;
uint16_t port_id;
int i;
+ struct mlx5_indexed_pool_config icfg[RTE_DIM(default_icfg)];
+ memcpy(icfg, default_icfg, sizeof(icfg));
/* Build device name. */
strlcpy(name, dpdk_dev->name, sizeof(name));
/* check if the device is already spawned */
@@ -538,6 +540,10 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
icfg[i].release_mem_en = !!sh->config.reclaim_mode;
if (sh->config.reclaim_mode)
icfg[i].per_core_cache = 0;
+#ifdef HAVE_MLX5_HWS_SUPPORT
+ if (priv->sh->config.dv_flow_en == 2)
+ icfg[i].size = sizeof(struct rte_flow_hw) + sizeof(struct rte_flow_nt2hws);
+#endif
priv->flows[i] = mlx5_ipool_create(&icfg[i]);
if (!priv->flows[i])
goto error;
--
2.25.1
From dceb3679ee066497fcb61ea6659d686966f36daf Mon Sep 17 00:00:00 2001
From: Bing Zhao <bingz@nvidia.com>
Date: Mon, 19 Jun 2023 13:16:28 +0800
Subject: [PATCH 02/11] net/mlx5: add dummy last action
The RTE_FLOW_ACTION_TYPE_END action needs to be translated into the
MLX5DR_ACTION_TYP_LAST for the future usage. It is only needed in
the hardware steering backward compatible API.
Signed-off-by: Bing Zhao <bingz@nvidia.com>
---
drivers/net/mlx5/mlx5.h | 1 +
drivers/net/mlx5/mlx5_flow_hw.c | 19 +++++++++++++++++++
2 files changed, 20 insertions(+)
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 92ad33d486..0d30e7ab36 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -1509,6 +1509,7 @@ struct mlx5_dev_ctx_shared {
/* Direct Rules tables for FDB, NIC TX+RX */
void *dr_drop_action; /* Pointer to DR drop action, any domain. */
void *pop_vlan_action; /* Pointer to DR pop VLAN action. */
+ void *hw_dummy_last; /* Pointer to the DR dummy end action, any domain. */
#if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
struct mlx5_send_to_kernel_action send_to_kernel_action[MLX5DR_TABLE_TYPE_MAX];
#endif
diff --git a/drivers/net/mlx5/mlx5_flow_hw.c b/drivers/net/mlx5/mlx5_flow_hw.c
index efd2141913..015622f3bf 100644
--- a/drivers/net/mlx5/mlx5_flow_hw.c
+++ b/drivers/net/mlx5/mlx5_flow_hw.c
@@ -10639,6 +10639,17 @@ flow_hw_configure(struct rte_eth_dev *dev,
if (ret)
goto err;
}
+ /*
+ * All domains will use the same dummy action, only used in the backward
+ * compatible API. Initialize it for only once. No order dependency.
+ */
+ if (!priv->sh->hw_dummy_last) {
+ priv->sh->hw_dummy_last = mlx5dr_action_create_last(priv->dr_ctx,
+ MLX5DR_ACTION_FLAG_HWS_RX);
+ if (!priv->sh->hw_dummy_last)
+ /* Do not overwrite the rte_errno. */
+ goto err;
+ }
if (!priv->shared_host)
flow_hw_create_send_to_kernel_actions(priv);
if (port_attr->nb_conn_tracks || (host_priv && host_priv->hws_ctpool)) {
@@ -10717,6 +10728,10 @@ flow_hw_configure(struct rte_eth_dev *dev,
flow_hw_ct_mng_destroy(dev, priv->ct_mng);
priv->ct_mng = NULL;
}
+ if (priv->sh->hw_dummy_last) {
+ mlx5dr_action_destroy(priv->sh->hw_dummy_last);
+ priv->sh->hw_dummy_last = NULL;
+ }
flow_hw_destroy_send_to_kernel_action(priv);
flow_hw_cleanup_ctrl_fdb_tables(dev);
flow_hw_free_vport_actions(priv);
@@ -10832,6 +10847,10 @@ flow_hw_resource_release(struct rte_eth_dev *dev)
mlx5dr_action_destroy(priv->hw_def_miss);
flow_hw_destroy_nat64_actions(priv);
flow_hw_destroy_vlan(dev);
+ if (priv->sh->hw_dummy_last) {
+ mlx5dr_action_destroy(priv->sh->hw_dummy_last);
+ priv->sh->hw_dummy_last = NULL;
+ }
flow_hw_destroy_send_to_kernel_action(priv);
flow_hw_free_vport_actions(priv);
if (priv->acts_ipool) {
--
2.25.1
From aaabbcdb925a109dc75abed20dbb0edc00f2b72e Mon Sep 17 00:00:00 2001
From: Bing Zhao <bingz@nvidia.com>
Date: Tue, 6 Jun 2023 14:25:17 +0800
Subject: [PATCH 03/11] net/mlx5: add basic actions support for non-template
API
Support JUMP / DROP / QUEUE / MARK / FLAG now.
Signed-off-by: Bing Zhao <bingz@nvidia.com>
---
drivers/net/mlx5/mlx5_flow_hw.c | 153 ++++++++++++++++++++++++++++----
1 file changed, 135 insertions(+), 18 deletions(-)
diff --git a/drivers/net/mlx5/mlx5_flow_hw.c b/drivers/net/mlx5/mlx5_flow_hw.c
index 015622f3bf..38112ef1a7 100644
--- a/drivers/net/mlx5/mlx5_flow_hw.c
+++ b/drivers/net/mlx5/mlx5_flow_hw.c
@@ -604,8 +604,7 @@ flow_hw_jump_release(struct rte_eth_dev *dev, struct mlx5_hw_jump_action *jump)
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_flow_group *grp;
- grp = container_of
- (jump, struct mlx5_flow_group, jump);
+ grp = container_of(jump, struct mlx5_flow_group, jump);
mlx5_hlist_unregister(priv->sh->flow_tbls, &grp->entry);
}
@@ -705,17 +704,9 @@ flow_hw_template_destroy_mhdr_action(struct mlx5_hw_modify_header_action *mhdr)
* Pointer to the template HW steering DR actions.
*/
static void
-__flow_hw_action_template_destroy(struct rte_eth_dev *dev,
- struct mlx5_hw_actions *acts)
+__flow_hw_actions_release(struct rte_eth_dev *dev, struct mlx5_hw_actions *acts)
{
struct mlx5_priv *priv = dev->data->dev_private;
- struct mlx5_action_construct_data *data;
-
- while (!LIST_EMPTY(&acts->act_list)) {
- data = LIST_FIRST(&acts->act_list);
- LIST_REMOVE(data, next);
- mlx5_ipool_free(priv->acts_ipool, data->idx);
- }
if (acts->mark)
if (!(rte_atomic_fetch_sub_explicit(&priv->hws_mark_refcnt, 1,
@@ -760,6 +751,32 @@ __flow_hw_action_template_destroy(struct rte_eth_dev *dev,
}
}
+/**
+ * Destroy DR actions created by action template.
+ *
+ * For DR actions created during table creation's action translate.
+ * Need to destroy the DR action when destroying the table.
+ *
+ * @param[in] dev
+ * Pointer to the rte_eth_dev structure.
+ * @param[in] acts
+ * Pointer to the template HW steering DR actions.
+ */
+static void
+__flow_hw_action_template_destroy(struct rte_eth_dev *dev, struct mlx5_hw_actions *acts)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_action_construct_data *data;
+
+ while (!LIST_EMPTY(&acts->act_list)) {
+ data = LIST_FIRST(&acts->act_list);
+ LIST_REMOVE(data, next);
+ mlx5_ipool_free(priv->acts_ipool, data->idx);
+ }
+
+ __flow_hw_actions_release(dev, acts);
+}
+
/**
* Append dynamic action to the dynamic action list.
*
@@ -12098,14 +12115,113 @@ static int flow_hw_prepare(struct rte_eth_dev *dev,
return 0;
}
-static int flow_hw_translate_actions(struct rte_eth_dev *dev __rte_unused,
- const struct rte_flow_attr *attr __rte_unused,
- const struct rte_flow_action actions[] __rte_unused,
- struct rte_flow_hw *flow __rte_unused,
- struct rte_flow_error *error __rte_unused)
+static int
+flow_hw_translate_actions(struct rte_eth_dev *dev,
+ const struct rte_flow_attr *attr,
+ const struct rte_flow_action actions[],
+ struct rte_flow_hw *flow,
+ struct mlx5_hw_actions *hw_acts,
+ bool external,
+ struct rte_flow_error *error)
{
- /* TODO implement */
+ struct mlx5_priv *priv = dev->data->dev_private;
+ enum mlx5dr_table_type type;
+ enum mlx5_hw_action_flag_type flag_type;
+ bool actions_end = false;
+ uint64_t action_flags = 0; /* to be used when needed */
+ uint32_t actions_n = 0;
+ uint32_t mark_id;
+ uint32_t jump_group;
+ bool is_mark;
+ struct mlx5_flow_template_table_cfg tbl_cfg;
+ enum mlx5_flow_fate_type fate_type = MLX5_FLOW_FATE_NONE;
+
+ if (attr->transfer)
+ type = MLX5DR_TABLE_TYPE_FDB;
+ else if (attr->egress)
+ type = MLX5DR_TABLE_TYPE_NIC_TX;
+ else
+ type = MLX5DR_TABLE_TYPE_NIC_RX;
+ /* The group in the attribute translation was done in advance. */
+ flag_type = (attr->group == 0) ? MLX5_HW_ACTION_FLAG_ROOT :
+ MLX5_HW_ACTION_FLAG_NONE_ROOT;
+ for (; !actions_end; actions++) {
+ switch (actions->type) {
+ case RTE_FLOW_ACTION_TYPE_VOID:
+ break;
+ case RTE_FLOW_ACTION_TYPE_DROP:
+ hw_acts->rule_acts[actions_n++].action = priv->hw_drop[flag_type];
+ fate_type = MLX5_FLOW_FATE_DROP;
+ break;
+ case RTE_FLOW_ACTION_TYPE_FLAG:
+ case RTE_FLOW_ACTION_TYPE_MARK:
+ is_mark = actions->type == RTE_FLOW_ACTION_TYPE_MARK;
+ mark_id = is_mark ?
+ ((const struct rte_flow_action_mark *)(actions->conf))->id :
+ MLX5_FLOW_MARK_DEFAULT;
+ hw_acts->rule_acts[actions_n].tag.value = mlx5_flow_mark_set(mark_id);
+ hw_acts->rule_acts[actions_n].action = priv->hw_tag[flag_type];
+ actions_n++;
+ action_flags |= is_mark ? MLX5_FLOW_ACTION_MARK : MLX5_FLOW_ACTION_FLAG;
+ hw_acts->mark = true;
+ rte_atomic_fetch_add_explicit(&priv->hws_mark_refcnt, 1,
+ rte_memory_order_relaxed);
+ flow_hw_rxq_flag_set(dev, true);
+ break;
+ case RTE_FLOW_ACTION_TYPE_JUMP:
+ jump_group = ((const struct rte_flow_action_jump *)actions->conf)->group;
+ tbl_cfg.attr.flow_attr = *attr;
+ tbl_cfg.external = external;
+ /* The flow_hw_jump_action_register() can be refactored. */
+ hw_acts->jump = flow_hw_jump_action_register(dev, &tbl_cfg,
+ jump_group, error);
+ if (hw_acts->jump == NULL)
+ goto clean_up;
+ hw_acts->rule_acts[actions_n++].action =
+ (flag_type == MLX5_HW_ACTION_FLAG_NONE_ROOT) ?
+ hw_acts->jump->hws_action : hw_acts->jump->root_action;
+ action_flags |= MLX5_FLOW_ACTION_JUMP;
+ fate_type = MLX5_FLOW_FATE_JUMP;
+ break;
+ case RTE_FLOW_ACTION_TYPE_QUEUE:
+ /* Right now, only Rx supports the TIR, validation is needed. */
+ hw_acts->tir = flow_hw_tir_action_register(dev,
+ mlx5_hw_act_flag[flag_type][type], actions);
+ if (hw_acts->tir == NULL) {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+ "Failed to translate queue.");
+ goto clean_up;
+ }
+ action_flags |= MLX5_FLOW_ACTION_QUEUE;
+ fate_type = MLX5_FLOW_FATE_QUEUE;
+ break;
+ case RTE_FLOW_ACTION_TYPE_END:
+ /*
+ * Using NULL action right now, maybe a new API can be used
+ * to create a dummy action with type MLX5DR_ACTION_TYP_LAST.
+ */
+ hw_acts->rule_acts[actions_n++].action = priv->sh->hw_dummy_last;
+ actions_end = true;
+ break;
+ default:
+ break;
+ }
+ }
+ if (fate_type == MLX5_FLOW_FATE_QUEUE) {
+ hw_acts->rule_acts[actions_n++].action = hw_acts->tir->action;
+ flow->hrxq = hw_acts->tir;
+ } else {
+ if (fate_type == MLX5_FLOW_FATE_JUMP)
+ flow->jump = hw_acts->jump;
+ }
+ /* Total actions number should be validated before. */
+ MLX5_ASSERT(actions_n <= MLX5_HW_MAX_ACTS);
return 0;
+clean_up:
+ /* Make sure that there is no garbage in the actions. */
+ __flow_hw_actions_release(dev, hw_acts);
+ return -rte_errno;
}
static int flow_hw_register_matcher(struct rte_eth_dev *dev,
@@ -12299,7 +12415,8 @@ static int flow_hw_create_flow(struct rte_eth_dev *dev,
if (ret)
goto error;
- ret = flow_hw_translate_actions(dev, attr, actions, *flow, error);
+ /* Note: the actions should be saved in the sub-flow rule itself for reference. */
+ ret = flow_hw_translate_actions(dev, attr, actions, *flow, &hw_act, external, error);
if (ret)
goto error;
--
2.25.1
From 5695efb6fdfa1645d164559dc5f856e1a6975a3c Mon Sep 17 00:00:00 2001
From: Bing Zhao <bingz@nvidia.com>
Date: Fri, 1 Sep 2023 12:54:02 +0800
Subject: [PATCH 04/11] net/mlx5: add default miss action support in nt2hws
mode
The validation will be handled later.
Now it only supports the SEND_TO_KERNEL on non-root tables.
Signed-off-by: Bing Zhao <bingz@nvidia.com>
---
drivers/net/mlx5/mlx5_flow_hw.c | 23 ++++++++++++++++++++---
1 file changed, 20 insertions(+), 3 deletions(-)
diff --git a/drivers/net/mlx5/mlx5_flow_hw.c b/drivers/net/mlx5/mlx5_flow_hw.c
index 38112ef1a7..e993e70494 100644
--- a/drivers/net/mlx5/mlx5_flow_hw.c
+++ b/drivers/net/mlx5/mlx5_flow_hw.c
@@ -12129,22 +12129,28 @@ flow_hw_translate_actions(struct rte_eth_dev *dev,
enum mlx5_hw_action_flag_type flag_type;
bool actions_end = false;
uint64_t action_flags = 0; /* to be used when needed */
+ int ret;
uint32_t actions_n = 0;
uint32_t mark_id;
uint32_t jump_group;
+ uint32_t src_group = 0;
bool is_mark;
struct mlx5_flow_template_table_cfg tbl_cfg;
enum mlx5_flow_fate_type fate_type = MLX5_FLOW_FATE_NONE;
+ RTE_SET_USED(action_flags);
+ /* The group in the attribute translation was done in advance. */
+ ret = __translate_group(dev, attr, external, attr->group, &src_group, error);
+ if (ret)
+ return ret;
if (attr->transfer)
type = MLX5DR_TABLE_TYPE_FDB;
else if (attr->egress)
type = MLX5DR_TABLE_TYPE_NIC_TX;
else
type = MLX5DR_TABLE_TYPE_NIC_RX;
- /* The group in the attribute translation was done in advance. */
- flag_type = (attr->group == 0) ? MLX5_HW_ACTION_FLAG_ROOT :
- MLX5_HW_ACTION_FLAG_NONE_ROOT;
+ flag_type = (src_group == 0) ? MLX5_HW_ACTION_FLAG_ROOT :
+ MLX5_HW_ACTION_FLAG_NONE_ROOT;
for (; !actions_end; actions++) {
switch (actions->type) {
case RTE_FLOW_ACTION_TYPE_VOID:
@@ -12196,6 +12202,17 @@ flow_hw_translate_actions(struct rte_eth_dev *dev,
action_flags |= MLX5_FLOW_ACTION_QUEUE;
fate_type = MLX5_FLOW_FATE_QUEUE;
break;
+ case RTE_FLOW_ACTION_TYPE_SEND_TO_KERNEL:
+ if (src_group == 0) {
+ rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+ "Send to kernel action on the root table is"
+ " not supported in HWS mode now.");
+ goto clean_up;
+ }
+ hw_acts->rule_acts[actions_n++].action = priv->hw_send_to_kernel[type];
+ action_flags |= MLX5_FLOW_ACTION_SEND_TO_KERNEL;
+ break;
case RTE_FLOW_ACTION_TYPE_END:
/*
* Using NULL action right now, maybe a new API can be used
--
2.25.1
From 1c5c1c10eacd7eb5bae70f9a58081e4895696b56 Mon Sep 17 00:00:00 2001
From: Maayan Kashani <mkashani@nvidia.com>
Date: Wed, 22 Nov 2023 09:44:17 +0200
Subject: [PATCH 05/11] net/mlx5: add ASO actions support to non-template mode
This patch adds counter, connection tracking, meter
and age actions support to non-template mode.
For CT, counter and meter: if no previous allocation was
handled by hw configure routine,
Half of the maximum supported number of objects will be allocated.
For AGE action, if no counters were allocated,
allocate half of the maximum,
and then allocate same number of AGE objects.
Also extracted the shared host handling to the configure function.
And align all ASO actions to have init function
for future code improvement.
This patch does not affect SW Steering flow engine.
Signed-off-by: Maayan Kashani <mkashani@nvidia.com>
---
drivers/net/mlx5/mlx5.h | 7 ++
drivers/net/mlx5/mlx5_flow_hw.c | 185 +++++++++++++++++++++++++++-----
drivers/net/mlx5/mlx5_hws_cnt.c | 47 ++++----
drivers/net/mlx5/mlx5_hws_cnt.h | 10 +-
4 files changed, 190 insertions(+), 59 deletions(-)
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 0d30e7ab36..26ce485ce8 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -792,6 +792,13 @@ struct mlx5_dev_shared_port {
/* Only yellow color valid. */
#define MLX5_MTR_POLICY_MODE_OY 3
+/* Max number of meters allocated in non template mode. */
+#define MLX5_MTR_NT_MAX (1 << 23) /*TODO verify number */
+/* Max number of connection tracking allocated in non template mode */
+#define MLX5_CT_NT_MAX (1 << 23) /*TODO verify number */
+/* Max number of counters allocated in non template mode */
+#define MLX5_CNT_MAX (1 << 23) /*TODO verify number */
+
enum mlx5_meter_domain {
MLX5_MTR_DOMAIN_INGRESS,
MLX5_MTR_DOMAIN_EGRESS,
diff --git a/drivers/net/mlx5/mlx5_flow_hw.c b/drivers/net/mlx5/mlx5_flow_hw.c
index e993e70494..77d40bad8a 100644
--- a/drivers/net/mlx5/mlx5_flow_hw.c
+++ b/drivers/net/mlx5/mlx5_flow_hw.c
@@ -287,6 +287,11 @@ static void
flow_hw_construct_quota(struct mlx5_priv *priv,
struct mlx5dr_rule_action *rule_act, uint32_t qid);
+static int
+mlx5_flow_ct_init(struct rte_eth_dev *dev,
+ uint32_t nb_conn_tracks,
+ uint16_t nb_queue);
+
static __rte_always_inline uint32_t flow_hw_tx_tag_regc_mask(struct rte_eth_dev *dev);
static __rte_always_inline uint32_t flow_hw_tx_tag_regc_value(struct rte_eth_dev *dev);
@@ -1673,7 +1678,7 @@ flow_hw_meter_mark_alloc(struct rte_eth_dev *dev, uint32_t queue,
}
if (meter_mark->profile == NULL)
return NULL;
- aso_mtr = mlx5_ipool_malloc(priv->hws_mpool->idx_pool, &mtr_id);
+ aso_mtr = mlx5_ipool_malloc(pool->idx_pool, &mtr_id);
if (!aso_mtr)
return NULL;
/* Fill the flow meter parameters. */
@@ -2483,8 +2488,10 @@ __flow_hw_actions_translate(struct rte_eth_dev *dev,
recom_type = MLX5DR_ACTION_TYP_POP_IPV6_ROUTE_EXT;
break;
case RTE_FLOW_ACTION_TYPE_SEND_TO_KERNEL:
- flow_hw_translate_group(dev, cfg, attr->group,
+ ret = flow_hw_translate_group(dev, cfg, attr->group,
&target_grp, error);
+ if (ret)
+ return ret;
if (target_grp == 0) {
__flow_hw_action_template_destroy(dev, acts);
return rte_flow_error_set(error, ENOTSUP,
@@ -2531,8 +2538,10 @@ __flow_hw_actions_translate(struct rte_eth_dev *dev,
goto err;
break;
case RTE_FLOW_ACTION_TYPE_AGE:
- flow_hw_translate_group(dev, cfg, attr->group,
+ ret = flow_hw_translate_group(dev, cfg, attr->group,
&target_grp, error);
+ if (ret)
+ return ret;
if (target_grp == 0) {
__flow_hw_action_template_destroy(dev, acts);
return rte_flow_error_set(error, ENOTSUP,
@@ -2547,8 +2556,10 @@ __flow_hw_actions_translate(struct rte_eth_dev *dev,
goto err;
break;
case RTE_FLOW_ACTION_TYPE_COUNT:
- flow_hw_translate_group(dev, cfg, attr->group,
+ ret = flow_hw_translate_group(dev, cfg, attr->group,
&target_grp, error);
+ if (ret)
+ return ret;
if (target_grp == 0) {
__flow_hw_action_template_destroy(dev, acts);
return rte_flow_error_set(error, ENOTSUP,
@@ -9705,12 +9716,12 @@ flow_hw_ct_pool_destroy(struct rte_eth_dev *dev,
static struct mlx5_aso_ct_pool *
flow_hw_ct_pool_create(struct rte_eth_dev *dev,
- const struct rte_flow_port_attr *port_attr)
+ uint32_t nb_conn_tracks)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_aso_ct_pool *pool;
struct mlx5_devx_obj *obj;
- uint32_t nb_cts = rte_align32pow2(port_attr->nb_conn_tracks);
+ uint32_t nb_cts = rte_align32pow2(nb_conn_tracks);
uint32_t log_obj_size = rte_log2_u32(nb_cts);
struct mlx5_indexed_pool_config cfg = {
.size = sizeof(struct mlx5_aso_ct_action),
@@ -9762,7 +9773,7 @@ flow_hw_ct_pool_create(struct rte_eth_dev *dev,
pool->devx_obj = host_priv->hws_ctpool->devx_obj;
pool->cts = host_priv->hws_ctpool->cts;
MLX5_ASSERT(pool->cts);
- MLX5_ASSERT(!port_attr->nb_conn_tracks);
+ MLX5_ASSERT(!nb_conn_tracks);
}
reg_id = mlx5_flow_get_reg_id(dev, MLX5_ASO_CONNTRACK, 0, NULL);
flags |= MLX5DR_ACTION_FLAG_HWS_RX | MLX5DR_ACTION_FLAG_HWS_TX;
@@ -9782,6 +9793,46 @@ flow_hw_ct_pool_create(struct rte_eth_dev *dev,
return NULL;
}
+static int
+mlx5_flow_ct_init(struct rte_eth_dev *dev,
+ uint32_t nb_conn_tracks,
+ uint16_t nb_queue)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ uint32_t mem_size;
+ int ret = -ENOMEM;
+
+ if (!priv->shared_host) {
+ mem_size = sizeof(struct mlx5_aso_sq) * nb_queue +
+ sizeof(*priv->ct_mng);
+ priv->ct_mng = mlx5_malloc(MLX5_MEM_ZERO, mem_size,
+ RTE_CACHE_LINE_SIZE,
+ SOCKET_ID_ANY);
+ if (!priv->ct_mng)
+ goto err;
+ ret = mlx5_aso_ct_queue_init(priv->sh, priv->ct_mng,
+ nb_queue);
+ if (ret)
+ goto err;
+ }
+ priv->hws_ctpool = flow_hw_ct_pool_create(dev, nb_conn_tracks);
+ if (!priv->hws_ctpool)
+ goto err;
+ priv->sh->ct_aso_en = 1;
+ return 0;
+
+err:
+ if (priv->hws_ctpool) {
+ flow_hw_ct_pool_destroy(dev, priv->hws_ctpool);
+ priv->hws_ctpool = NULL;
+ }
+ if (priv->ct_mng) {
+ flow_hw_ct_mng_destroy(dev, priv->ct_mng);
+ priv->ct_mng = NULL;
+ }
+ return ret;
+}
+
static void
flow_hw_destroy_vlan(struct rte_eth_dev *dev)
{
@@ -10429,6 +10480,7 @@ flow_hw_configure(struct rte_eth_dev *dev,
bool is_proxy = !!(priv->sh->config.dv_esw_en && priv->master);
int ret = 0;
uint32_t action_flags;
+ bool strict_queue = false;
if (mlx5dr_rule_get_handle_size() != MLX5_DR_RULE_SIZE) {
rte_errno = EINVAL;
@@ -10670,25 +10722,13 @@ flow_hw_configure(struct rte_eth_dev *dev,
if (!priv->shared_host)
flow_hw_create_send_to_kernel_actions(priv);
if (port_attr->nb_conn_tracks || (host_priv && host_priv->hws_ctpool)) {
- if (!priv->shared_host) {
- mem_size = sizeof(struct mlx5_aso_sq) * nb_q_updated +
- sizeof(*priv->ct_mng);
- priv->ct_mng = mlx5_malloc(MLX5_MEM_ZERO, mem_size,
- RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
- if (!priv->ct_mng)
- goto err;
- if (mlx5_aso_ct_queue_init(priv->sh, priv->ct_mng, nb_q_updated))
- goto err;
- }
- priv->hws_ctpool = flow_hw_ct_pool_create(dev, port_attr);
- if (!priv->hws_ctpool)
+ if (mlx5_flow_ct_init(dev, port_attr->nb_conn_tracks, nb_q_updated))
goto err;
- priv->sh->ct_aso_en = 1;
}
if (port_attr->nb_counters || (host_priv && host_priv->hws_cpool)) {
- priv->hws_cpool = mlx5_hws_cnt_pool_create(dev, port_attr,
- nb_queue);
- if (priv->hws_cpool == NULL)
+ if (mlx5_hws_cnt_pool_create(dev, port_attr->nb_counters,
+ nb_queue,
+ (host_priv ? host_priv->hws_cpool : NULL)))
goto err;
}
if (port_attr->nb_aging_objects) {
@@ -10705,12 +10745,17 @@ flow_hw_configure(struct rte_eth_dev *dev,
rte_errno = EINVAL;
goto err;
}
- ret = mlx5_hws_age_pool_init(dev, port_attr, nb_queue);
- if (ret < 0) {
- rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
- NULL, "Failed to init age pool.");
+ if (port_attr->flags & RTE_FLOW_PORT_FLAG_SHARE_INDIRECT) {
+ DRV_LOG(ERR, "Aging is not supported "
+ "in cross vHCA sharing mode");
+ ret = -ENOTSUP;
goto err;
}
+ strict_queue = !!(port_attr->flags & RTE_FLOW_PORT_FLAG_STRICT_QUEUE);
+ ret = mlx5_hws_age_pool_init(dev, port_attr->nb_aging_objects,
+ nb_queue, strict_queue);
+ if (ret < 0)
+ goto err;
}
ret = flow_hw_create_vlan(dev);
if (ret) {
@@ -12322,7 +12367,78 @@ static int flow_hw_register_matcher(struct rte_eth_dev *dev,
}
}
-static int flow_hw_apply(struct rte_eth_dev *dev __rte_unused, /* TODO: remove if not used */
+static int flow_hw_allocate_actions(struct rte_eth_dev *dev,
+ const struct rte_flow_action actions[],
+ struct rte_flow_error *error)
+{
+ bool actions_end = false;
+ struct mlx5_priv *priv = dev->data->dev_private;
+ int ret;
+
+ for (; !actions_end; actions++) {
+ switch ((int)actions->type) {
+ case RTE_FLOW_ACTION_TYPE_AGE:
+ /* If no age objects were previously allocated. */
+ if (!priv->hws_age_req) {
+ /* If no counters were previously allocated. */
+ if (!priv->hws_cpool) {
+ ret = mlx5_hws_cnt_pool_create(dev, MLX5_CNT_MAX,
+ priv->nb_queue, NULL);
+ if (ret)
+ goto err;
+ }
+ if (priv->hws_cpool) {
+ /* Allocate same number of counters. */
+ ret = mlx5_hws_age_pool_init(dev,
+ priv->hws_cpool->cfg.request_num,
+ priv->nb_queue, false);
+ if (ret)
+ goto err;
+ }
+ }
+ break;
+ case RTE_FLOW_ACTION_TYPE_COUNT:
+ /* If no counters were previously allocated. */
+ if (!priv->hws_cpool) {
+ ret = mlx5_hws_cnt_pool_create(dev, MLX5_CNT_MAX,
+ priv->nb_queue, NULL);
+ if (ret)
+ goto err;
+ }
+ break;
+ case RTE_FLOW_ACTION_TYPE_CONNTRACK:
+ /* If no CT were previously allocated. */
+ if (!priv->hws_ctpool) {
+ ret = mlx5_flow_ct_init(dev, MLX5_CT_NT_MAX, priv->nb_queue);
+ if (ret)
+ goto err;
+ }
+ break;
+ case RTE_FLOW_ACTION_TYPE_METER_MARK:
+ /* If no meters were previously allocated. */
+ if (!priv->hws_mpool) {
+ ret = mlx5_flow_meter_init(dev, MLX5_MTR_NT_MAX, 0, 0,
+ priv->nb_queue);
+ if (ret)
+ goto err;
+ }
+ break;
+ case RTE_FLOW_ACTION_TYPE_END:
+ actions_end = true;
+ break;
+ default:
+ break;
+ }
+ }
+ return 0;
+err:
+ return rte_flow_error_set(error, ret,
+ RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+ NULL, "fail to allocate actions");
+}
+
+/* TODO: remove dev if not used */
+static int flow_hw_apply(struct rte_eth_dev *dev __rte_unused,
const struct rte_flow_item items[],
struct mlx5dr_rule_action rule_actions[],
struct rte_flow_hw *flow,
@@ -12432,11 +12548,24 @@ static int flow_hw_create_flow(struct rte_eth_dev *dev,
if (ret)
goto error;
+ /*
+ * ASO allocation – iterating on actions list to allocate missing resources.
+ * In the future when validate function in hws will be added,
+ * The output actions bit mask instead of
+ * looping on the actions array twice.
+ */
+ ret = flow_hw_allocate_actions(dev, actions, error);
+
/* Note: the actions should be saved in the sub-flow rule itself for reference. */
ret = flow_hw_translate_actions(dev, attr, actions, *flow, &hw_act, external, error);
if (ret)
goto error;
+ /*
+ * TODO: check regarding release: CT index is not saved per rule,
+ * the index is in the conf of given action.
+ */
+
/*
* If the flow is external (from application) OR device is started,
* OR mreg discover, then apply immediately.
diff --git a/drivers/net/mlx5/mlx5_hws_cnt.c b/drivers/net/mlx5/mlx5_hws_cnt.c
index 1b625e07bd..36d422bdfa 100644
--- a/drivers/net/mlx5/mlx5_hws_cnt.c
+++ b/drivers/net/mlx5/mlx5_hws_cnt.c
@@ -443,7 +443,7 @@ mlx5_hws_cnt_pool_init(struct mlx5_dev_ctx_shared *sh,
(uint32_t)cnt_num, SOCKET_ID_ANY,
RING_F_MP_HTS_ENQ | RING_F_SC_DEQ | RING_F_EXACT_SZ);
if (cntp->wait_reset_list == NULL) {
- DRV_LOG(ERR, "failed to create free list ring");
+ DRV_LOG(ERR, "failed to create wait reset list ring");
goto error;
}
snprintf(mz_name, sizeof(mz_name), "%s_U_RING", pcfg->name);
@@ -631,16 +631,17 @@ mlx5_hws_cnt_pool_action_create(struct mlx5_priv *priv,
return ret;
}
-struct mlx5_hws_cnt_pool *
+int
mlx5_hws_cnt_pool_create(struct rte_eth_dev *dev,
- const struct rte_flow_port_attr *pattr, uint16_t nb_queue)
+ uint32_t nb_counters, uint16_t nb_queue,
+ struct mlx5_hws_cnt_pool *chost)
{
struct mlx5_hws_cnt_pool *cpool = NULL;
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_hws_cache_param cparam = {0};
struct mlx5_hws_cnt_pool_cfg pcfg = {0};
char *mp_name;
- int ret = 0;
+ int ret = -1;
size_t sz;
mp_name = mlx5_malloc(MLX5_MEM_ZERO, RTE_MEMZONE_NAMESIZE, 0, SOCKET_ID_ANY);
@@ -648,13 +649,9 @@ mlx5_hws_cnt_pool_create(struct rte_eth_dev *dev,
goto error;
snprintf(mp_name, RTE_MEMZONE_NAMESIZE, "MLX5_HWS_CNT_P_%x", dev->data->port_id);
pcfg.name = mp_name;
- pcfg.request_num = pattr->nb_counters;
+ pcfg.request_num = nb_counters;
pcfg.alloc_factor = HWS_CNT_ALLOC_FACTOR_DEFAULT;
- if (pattr->flags & RTE_FLOW_PORT_FLAG_SHARE_INDIRECT) {
- struct mlx5_priv *host_priv =
- priv->shared_host->data->dev_private;
- struct mlx5_hws_cnt_pool *chost = host_priv->hws_cpool;
-
+ if (chost) {
pcfg.host_cpool = chost;
cpool = mlx5_hws_cnt_pool_init(priv->sh, &pcfg, &cparam);
if (cpool == NULL)
@@ -662,13 +659,13 @@ mlx5_hws_cnt_pool_create(struct rte_eth_dev *dev,
ret = mlx5_hws_cnt_pool_action_create(priv, cpool);
if (ret != 0)
goto error;
- return cpool;
+ goto success;
}
/* init cnt service if not. */
if (priv->sh->cnt_svc == NULL) {
ret = mlx5_hws_cnt_svc_init(priv->sh);
- if (ret != 0)
- return NULL;
+ if (ret)
+ return ret;
}
cparam.fetch_sz = HWS_CNT_CACHE_FETCH_DEFAULT;
cparam.preload_sz = HWS_CNT_CACHE_PRELOAD_DEFAULT;
@@ -701,10 +698,13 @@ mlx5_hws_cnt_pool_create(struct rte_eth_dev *dev,
rte_spinlock_lock(&priv->sh->cpool_lock);
LIST_INSERT_HEAD(&priv->sh->hws_cpool_list, cpool, next);
rte_spinlock_unlock(&priv->sh->cpool_lock);
- return cpool;
+success:
+ priv->hws_cpool = cpool;
+ return 0;
error:
mlx5_hws_cnt_pool_destroy(priv->sh, cpool);
- return NULL;
+ priv->hws_cpool = NULL;
+ return ret;
}
void
@@ -1217,8 +1217,9 @@ mlx5_hws_age_info_destroy(struct mlx5_priv *priv)
*/
int
mlx5_hws_age_pool_init(struct rte_eth_dev *dev,
- const struct rte_flow_port_attr *attr,
- uint16_t nb_queues)
+ uint32_t nb_aging_objects,
+ uint16_t nb_queues,
+ bool strict_queue)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_age_info *age_info = GET_PORT_AGE_INFO(priv);
@@ -1233,28 +1234,20 @@ mlx5_hws_age_pool_init(struct rte_eth_dev *dev,
.free = mlx5_free,
.type = "mlx5_hws_age_pool",
};
- bool strict_queue = false;
uint32_t nb_alloc_cnts;
uint32_t rsize;
uint32_t nb_ages_updated;
int ret;
- strict_queue = !!(attr->flags & RTE_FLOW_PORT_FLAG_STRICT_QUEUE);
MLX5_ASSERT(priv->hws_cpool);
- if (attr->flags & RTE_FLOW_PORT_FLAG_SHARE_INDIRECT) {
- DRV_LOG(ERR, "Aging sn not supported "
- "in cross vHCA sharing mode");
- rte_errno = ENOTSUP;
- return -ENOTSUP;
- }
nb_alloc_cnts = mlx5_hws_cnt_pool_get_size(priv->hws_cpool);
if (strict_queue) {
rsize = mlx5_hws_aged_out_q_ring_size_get(nb_alloc_cnts,
nb_queues);
- nb_ages_updated = rsize * nb_queues + attr->nb_aging_objects;
+ nb_ages_updated = rsize * nb_queues + nb_aging_objects;
} else {
rsize = mlx5_hws_aged_out_ring_size_get(nb_alloc_cnts);
- nb_ages_updated = rsize + attr->nb_aging_objects;
+ nb_ages_updated = rsize + nb_aging_objects;
}
ret = mlx5_hws_age_info_init(dev, nb_queues, strict_queue, rsize);
if (ret < 0)
diff --git a/drivers/net/mlx5/mlx5_hws_cnt.h b/drivers/net/mlx5/mlx5_hws_cnt.h
index db4e99e37c..996ac8dd9a 100644
--- a/drivers/net/mlx5/mlx5_hws_cnt.h
+++ b/drivers/net/mlx5/mlx5_hws_cnt.h
@@ -712,9 +712,10 @@ mlx5_hws_cnt_service_thread_create(struct mlx5_dev_ctx_shared *sh);
void
mlx5_hws_cnt_service_thread_destroy(struct mlx5_dev_ctx_shared *sh);
-struct mlx5_hws_cnt_pool *
+int
mlx5_hws_cnt_pool_create(struct rte_eth_dev *dev,
- const struct rte_flow_port_attr *pattr, uint16_t nb_queue);
+ uint32_t nb_counters, uint16_t nb_queue,
+ struct mlx5_hws_cnt_pool *chost);
void
mlx5_hws_cnt_pool_destroy(struct mlx5_dev_ctx_shared *sh,
@@ -744,8 +745,9 @@ mlx5_hws_age_context_get(struct mlx5_priv *priv, uint32_t idx);
int
mlx5_hws_age_pool_init(struct rte_eth_dev *dev,
- const struct rte_flow_port_attr *attr,
- uint16_t nb_queues);
+ uint32_t nb_aging_objects,
+ uint16_t nb_queues,
+ bool strict_queue);
void
mlx5_hws_age_pool_destroy(struct mlx5_priv *priv);
--
2.25.1
From df1a2502e07b2b2aff2dad089547146f077050dd Mon Sep 17 00:00:00 2001
From: Maayan Kashani <mkashani@nvidia.com>
Date: Sun, 31 Dec 2023 18:44:29 +0200
Subject: [PATCH 06/11] net/mlx5: fix segfault on counter pool destroy
If the counter pool was not added to list,
and an error state was reached,
on attempt to destroy the counter pool,
segmentation fault was received during list remove action.
Added a check to verify the list is not empty before trying to
remove the cpool from the list.
Invalid state, leading to segfault,
can also be reached in the following scenario:
1. mlx5_hws_cnt_pool_init() does a zmalloc and initializes most
of the fields of cpool, but does not initialize the next field.
2. mlx5_hws_cnt_pool_dcs_alloc() attempts to bulk allocate flow counters.
If this fails, we skip straight to 4.
In HW, this can fail simply if FW doesn't support bulk flow
counter allocation.
3. Right before the goto error, we insert the cpool to the hws_cpool_list.
This is where the next field is initialized.
4. mlx5_hws_cnt_pool_destroy() assumes the cpool's next field
is initialized and SEGVs if not.
So, added a guard against cases where the entry was uninitialized
(checking le_prev field is not NULL).
Fixes: 6ac2104ac125 ("net/mlx5: fix counter query during port close")
Cc: stable@dpdk.org
Signed-off-by: Maayan Kashani <mkashani@nvidia.com>
---
drivers/net/mlx5/mlx5_hws_cnt.c | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/drivers/net/mlx5/mlx5_hws_cnt.c b/drivers/net/mlx5/mlx5_hws_cnt.c
index 36d422bdfa..a46a4bd94e 100644
--- a/drivers/net/mlx5/mlx5_hws_cnt.c
+++ b/drivers/net/mlx5/mlx5_hws_cnt.c
@@ -718,7 +718,9 @@ mlx5_hws_cnt_pool_destroy(struct mlx5_dev_ctx_shared *sh,
* Maybe blocked for at most 200ms here.
*/
rte_spinlock_lock(&sh->cpool_lock);
- LIST_REMOVE(cpool, next);
+ /* Try to remove cpool before it was added to list caused segfault. */
+ if (!LIST_EMPTY(&sh->hws_cpool_list) && cpool->next.le_prev)
+ LIST_REMOVE(cpool, next);
rte_spinlock_unlock(&sh->cpool_lock);
if (cpool->cfg.host_cpool == NULL) {
if (--sh->cnt_svc->refcnt == 0)
--
2.25.1
From 05d3f58305e87a0f79c2879d8edc94acf6e8f89b Mon Sep 17 00:00:00 2001
From: Maayan Kashani <mkashani@nvidia.com>
Date: Wed, 13 Dec 2023 14:59:45 +0200
Subject: [PATCH 07/11] net/mlx5: abstract action handling and enable
reconfigure
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
reuse HWS actions translation code in order to create
the actions for lower layer.
All actions are handled as non masked.
Therefore, will be translated in the rule construct
phase shared with the template code
and not during the action template translate.
Currently there is no option to reconfigure in template API’s.
To support non template API and template API simultaneously,
There is a need to enable the user to reconfigure after default
configuration was applied by the driver on port start.
Fix flow create ptr
In SWS the return flow pointer is the flow index,
in HWS the return flow pointer is the flow data pointer.
Currently the return value of non template code did not
Enable destroy or quary.
Signed-off-by: Maayan Kashani <mkashani@nvidia.com>
---
drivers/net/mlx5/mlx5.h | 25 +-
drivers/net/mlx5/mlx5_flow.c | 31 +-
drivers/net/mlx5/mlx5_flow.h | 14 +-
drivers/net/mlx5/mlx5_flow_dv.c | 18 +-
drivers/net/mlx5/mlx5_flow_hw.c | 811 ++++++++++++++++----------------
5 files changed, 463 insertions(+), 436 deletions(-)
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 26ce485ce8..67986a00b4 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -793,11 +793,11 @@ struct mlx5_dev_shared_port {
#define MLX5_MTR_POLICY_MODE_OY 3
/* Max number of meters allocated in non template mode. */
-#define MLX5_MTR_NT_MAX (1 << 23) /*TODO verify number */
+#define MLX5_MTR_NT_MAX (1 << 23)
/* Max number of connection tracking allocated in non template mode */
-#define MLX5_CT_NT_MAX (1 << 23) /*TODO verify number */
+#define MLX5_CT_NT_MAX (1 << 23)
/* Max number of counters allocated in non template mode */
-#define MLX5_CNT_MAX (1 << 23) /*TODO verify number */
+#define MLX5_CNT_MAX (1 << 23)
enum mlx5_meter_domain {
MLX5_MTR_DOMAIN_INGRESS,
@@ -1516,7 +1516,6 @@ struct mlx5_dev_ctx_shared {
/* Direct Rules tables for FDB, NIC TX+RX */
void *dr_drop_action; /* Pointer to DR drop action, any domain. */
void *pop_vlan_action; /* Pointer to DR pop VLAN action. */
- void *hw_dummy_last; /* Pointer to the DR dummy end action, any domain. */
#if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
struct mlx5_send_to_kernel_action send_to_kernel_action[MLX5DR_TABLE_TYPE_MAX];
#endif
@@ -1797,6 +1796,14 @@ struct mlx5_hw_ctrl_flow {
struct mlx5_hw_ctrl_flow_info info;
};
+/* HW Steering port configuration passed to rte_flow_configure(). */
+struct mlx5_flow_hw_attr {
+ struct rte_flow_port_attr port_attr;
+ uint16_t nb_queue;
+ struct rte_flow_queue_attr *queue_attr;
+ bool nt_mode;
+};
+
/*
* Flow rule structure for flow engine mode control, focus on group 0.
* Apply to all supported domains.
@@ -1818,12 +1825,6 @@ struct rte_pmd_mlx5_flow_engine_mode_info {
/* The list is maintained in insertion order. */
LIST_HEAD(hot_up_info, mlx5_dv_flow_info) hot_upgrade;
};
-/* HW Steering port configuration passed to rte_flow_configure(). */
-struct mlx5_flow_hw_attr {
- struct rte_flow_port_attr port_attr;
- uint16_t nb_queue;
- struct rte_flow_queue_attr *queue_attr;
-};
struct mlx5_flow_hw_ctrl_rx;
@@ -2301,7 +2302,7 @@ int mlx5_flow_validate(struct rte_eth_dev *dev,
const struct rte_flow_item items[],
const struct rte_flow_action actions[],
struct rte_flow_error *error);
-uint32_t
+uintptr_t
mlx5_flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
const struct rte_flow_attr *attr,
const struct rte_flow_item items[],
@@ -2309,7 +2310,7 @@ mlx5_flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
bool external, struct rte_flow_error *error);
void
mlx5_flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
- uint32_t flow_idx);
+ uintptr_t flow_idx);
struct rte_flow *mlx5_flow_create(struct rte_eth_dev *dev,
const struct rte_flow_attr *attr,
const struct rte_flow_item items[],
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index c1bcb0a548..68f20300a5 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -4930,7 +4930,7 @@ flow_check_hairpin_split(struct rte_eth_dev *dev,
/* Declare flow create/destroy prototype in advance. */
-static uint32_t
+static uintptr_t
flow_drv_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
const struct rte_flow_attr *attr,
const struct rte_flow_item items[],
@@ -4939,7 +4939,7 @@ flow_drv_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
static void
flow_drv_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
- uint32_t flow_idx);
+ uintptr_t flow_idx);
int
flow_dv_mreg_match_cb(void *tool_ctx __rte_unused,
@@ -7243,7 +7243,7 @@ flow_tunnel_from_rule(const struct mlx5_flow *flow)
* @return
* A flow index on success, 0 otherwise and rte_errno is set.
*/
-uint32_t
+uintptr_t
flow_legacy_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
const struct rte_flow_attr *attr,
const struct rte_flow_item items[],
@@ -7918,7 +7918,7 @@ mlx5_flow_create(struct rte_eth_dev *dev,
struct mlx5_priv *priv = dev->data->dev_private;
struct rte_flow_attr *new_attr = (void *)(uintptr_t)attr;
uint32_t prio = attr->priority;
- uint32_t flow_idx;
+ uintptr_t flow_idx;
/*
* If the device is not started yet, it is not allowed to created a
@@ -7952,7 +7952,7 @@ mlx5_flow_create(struct rte_eth_dev *dev,
return (void *)(uintptr_t)flow_idx;
}
-uint32_t
+uintptr_t
mlx5_flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
const struct rte_flow_attr *attr,
const struct rte_flow_item items[],
@@ -7963,7 +7963,7 @@ mlx5_flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
error);
}
-uint32_t
+uintptr_t
flow_drv_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
const struct rte_flow_attr *attr,
const struct rte_flow_item items[],
@@ -7988,7 +7988,7 @@ flow_drv_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
*/
void
flow_legacy_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
- uint32_t flow_idx)
+ uintptr_t flow_idx)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct rte_flow *flow = mlx5_ipool_get(priv->flows[type], (uint32_t)flow_idx);
@@ -8019,7 +8019,7 @@ flow_legacy_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
static void
flow_drv_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
- uint32_t flow_idx)
+ uintptr_t flow_idx)
{
const struct mlx5_flow_driver_ops *fops;
struct rte_flow_attr attr = { .transfer = 0 };
@@ -8031,7 +8031,7 @@ flow_drv_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
void
mlx5_flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
- uint32_t flow_idx)
+ uintptr_t flow_idx)
{
flow_drv_list_destroy(dev, type, flow_idx);
}
@@ -8060,12 +8060,14 @@ mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type,
if (priv->sh->config.dv_flow_en == 2 &&
type == MLX5_FLOW_TYPE_GEN) {
flow_hw_q_flow_flush(dev, NULL);
- return;
}
#endif
-
MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) {
- flow_drv_list_destroy(dev, type, fidx);
+ if (priv->sh->config.dv_flow_en == 2) {
+ flow_drv_list_destroy(dev, type, (uintptr_t)flow);
+ } else {
+ flow_drv_list_destroy(dev, type, fidx);
+ }
if (unlikely(mlx5_need_cache_flow(priv, NULL) && type == MLX5_FLOW_TYPE_GEN)) {
flow_info = LIST_FIRST(&mode_info->hot_upgrade);
while (flow_info) {
@@ -8526,11 +8528,6 @@ mlx5_flow_destroy(struct rte_eth_dev *dev,
struct rte_pmd_mlx5_flow_engine_mode_info *mode_info = &priv->mode_info;
struct mlx5_dv_flow_info *flow_info;
- if (priv->sh->config.dv_flow_en == 2)
- return rte_flow_error_set(error, ENOTSUP,
- RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
- NULL,
- "Flow non-Q destruction not supported");
flow_drv_list_destroy(dev, MLX5_FLOW_TYPE_GEN,
(uintptr_t)(void *)flow);
if (unlikely(mlx5_need_cache_flow(priv, NULL))) {
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index e06e7d5cc8..5bcfc1d88a 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -1315,6 +1315,8 @@ struct rte_flow_nt2hws {
struct mlx5dr_bwc_rule *nt_rule;
/** The matcher for non template api. */
struct mlx5_flow_dv_matcher *matcher;
+ /**< Auxiliary data stored per flow. */
+ struct rte_flow_hw_aux *flow_aux;
} __rte_packed;
/** HWS flow struct. */
@@ -1349,12 +1351,14 @@ struct rte_flow_hw {
struct mlx5_hrxq *hrxq;
};
+ /** Equals true if it is non template rule. */
+ bool nt_rule;
/**
* Padding for alignment to 56 bytes.
* Since mlx5dr rule is 72 bytes, whole flow is contained within 128 B (2 cache lines).
* This space is reserved for future additions to flow struct.
*/
- uint8_t padding[10];
+ uint8_t padding[9];
/** HWS layer data struct. */
uint8_t rule[];
} __rte_packed;
@@ -2116,7 +2120,7 @@ int
flow_hw_init(struct rte_eth_dev *dev,
struct rte_flow_error *error);
-typedef uint32_t (*mlx5_flow_list_create_t)(struct rte_eth_dev *dev,
+typedef uintptr_t (*mlx5_flow_list_create_t)(struct rte_eth_dev *dev,
enum mlx5_flow_type type,
const struct rte_flow_attr *attr,
const struct rte_flow_item items[],
@@ -2125,7 +2129,7 @@ typedef uint32_t (*mlx5_flow_list_create_t)(struct rte_eth_dev *dev,
struct rte_flow_error *error);
typedef void (*mlx5_flow_list_destroy_t)(struct rte_eth_dev *dev,
enum mlx5_flow_type type,
- uint32_t flow_idx);
+ uintptr_t flow_idx);
typedef int (*mlx5_flow_validate_t)(struct rte_eth_dev *dev,
const struct rte_flow_attr *attr,
const struct rte_flow_item items[],
@@ -3317,13 +3321,13 @@ int mlx5_flow_item_field_width(struct rte_eth_dev *dev,
enum rte_flow_field_id field, int inherit,
const struct rte_flow_attr *attr,
struct rte_flow_error *error);
-uint32_t flow_legacy_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
+uintptr_t flow_legacy_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
const struct rte_flow_attr *attr,
const struct rte_flow_item items[],
const struct rte_flow_action actions[],
bool external, struct rte_flow_error *error);
void flow_legacy_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
- uint32_t flow_idx);
+ uintptr_t flow_idx);
static __rte_always_inline int
flow_hw_get_srh_flex_parser_byte_off_from_ctx(void *dr_ctx __rte_unused)
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index e3855731b8..a4fde4125e 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -11843,12 +11843,14 @@ flow_matcher_create_cb(void *tool_ctx, void *cb_ctx)
struct mlx5_flow_cb_ctx *ctx = cb_ctx;
struct mlx5_flow_dv_matcher *ref = ctx->data;
struct mlx5_flow_dv_matcher *resource;
+#ifdef HAVE_MLX5_HWS_SUPPORT
+ const struct rte_flow_item *items;
+#endif
struct mlx5dv_flow_matcher_attr dv_attr = {
.type = IBV_FLOW_ATTR_NORMAL,
.match_mask = (void *)&ref->mask,
};
- struct mlx5_flow_tbl_data_entry *tbl = container_of(ref->tbl,
- typeof(*tbl), tbl);
+ struct mlx5_flow_tbl_data_entry *tbl;
int ret;
resource = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*resource), 0,
@@ -11859,9 +11861,9 @@ flow_matcher_create_cb(void *tool_ctx, void *cb_ctx)
"cannot create matcher");
return NULL;
}
- /*Consider memcpy(resource, ref, sizeof(*resource));*/
*resource = *ref;
if (sh->config.dv_flow_en != 2) {
+ tbl = container_of(ref->tbl, typeof(*tbl), tbl);
dv_attr.match_criteria_enable =
flow_dv_matcher_enable(resource->mask.buf);
__flow_dv_adjust_buf_size(&ref->mask.size,
@@ -11879,6 +11881,16 @@ flow_matcher_create_cb(void *tool_ctx, void *cb_ctx)
"cannot create matcher");
return NULL;
}
+ } else {
+#ifdef HAVE_MLX5_HWS_SUPPORT
+ items = *((const struct rte_flow_item **)(ctx->data2));
+ resource->matcher_object = mlx5dr_bwc_matcher_create
+ (resource->group->tbl, resource->priority, items);
+ if (!(resource->matcher_object))
+ return NULL;
+#else
+ return NULL;
+#endif
}
return &resource->entry;
}
diff --git a/drivers/net/mlx5/mlx5_flow_hw.c b/drivers/net/mlx5/mlx5_flow_hw.c
index 77d40bad8a..d768968676 100644
--- a/drivers/net/mlx5/mlx5_flow_hw.c
+++ b/drivers/net/mlx5/mlx5_flow_hw.c
@@ -135,7 +135,7 @@ mlx5_flow_hw_aux(uint16_t port_id, struct rte_flow_hw *flow)
return RTE_PTR_ADD(flow, offset);
} else {
- return &table->flow_aux[flow->idx - 1];
+ return ((flow->nt_rule) ? flow->nt2hws->flow_aux : &table->flow_aux[flow->idx - 1]);
}
}
@@ -622,6 +622,10 @@ flow_hw_jump_release(struct rte_eth_dev *dev, struct mlx5_hw_jump_action *jump)
* DR action flags.
* @param[in] action
* rte flow action.
+ * @param[in] item_flags
+ * Item flags for non template rule.
+ * @param[in] is_template
+ * True if it is a template rule.
*
* @return
* Table on success, NULL otherwise and rte_errno is set.
@@ -629,7 +633,9 @@ flow_hw_jump_release(struct rte_eth_dev *dev, struct mlx5_hw_jump_action *jump)
static inline struct mlx5_hrxq*
flow_hw_tir_action_register(struct rte_eth_dev *dev,
uint32_t hws_flags,
- const struct rte_flow_action *action)
+ const struct rte_flow_action *action,
+ uint64_t item_flags,
+ bool is_template)
{
struct mlx5_flow_rss_desc rss_desc = {
.hws_flags = hws_flags,
@@ -652,7 +658,10 @@ flow_hw_tir_action_register(struct rte_eth_dev *dev,
rss_desc.key_len = MLX5_RSS_HASH_KEY_LEN;
rss_desc.types = !rss->types ? RTE_ETH_RSS_IP : rss->types;
rss_desc.symmetric_hash_function = MLX5_RSS_IS_SYMM(rss->func);
- flow_hw_hashfields_set(&rss_desc, &rss_desc.hash_fields);
+ if (is_template)
+ flow_hw_hashfields_set(&rss_desc, &rss_desc.hash_fields);
+ else
+ flow_dv_hashfields_set(item_flags, &rss_desc, &rss_desc.hash_fields);
flow_dv_action_rss_l34_hash_adjust(rss->types,
&rss_desc.hash_fields);
if (rss->level > 1) {
@@ -2217,6 +2226,8 @@ mlx5_create_ipv6_ext_reformat(struct rte_eth_dev *dev,
* Pointer to the template HW steering DR actions.
* @param[in] at
* Action template.
+ * @param[in] nt_mode
+ * Non template rule translate.
* @param[out] error
* Pointer to error structure.
*
@@ -2224,11 +2235,12 @@ mlx5_create_ipv6_ext_reformat(struct rte_eth_dev *dev,
* 0 on success, a negative errno otherwise and rte_errno is set.
*/
static int
-__flow_hw_actions_translate(struct rte_eth_dev *dev,
+__flow_hw_translate_actions_template(struct rte_eth_dev *dev,
const struct mlx5_flow_template_table_cfg *cfg,
struct mlx5_hw_actions *acts,
struct rte_flow_actions_template *at,
struct mlx5_tbl_multi_pattern_ctx *mp_ctx,
+ bool nt_mode __rte_unused,
struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
@@ -2375,7 +2387,7 @@ __flow_hw_actions_translate(struct rte_eth_dev *dev,
acts->tir = flow_hw_tir_action_register
(dev,
mlx5_hw_act_flag[!!attr->group][type],
- actions);
+ actions, 0, true);
if (!acts->tir)
goto err;
acts->rule_acts[dr_pos].action =
@@ -2391,7 +2403,7 @@ __flow_hw_actions_translate(struct rte_eth_dev *dev,
acts->tir = flow_hw_tir_action_register
(dev,
mlx5_hw_act_flag[!!attr->group][type],
- actions);
+ actions, 0, true);
if (!acts->tir)
goto err;
acts->rule_acts[dr_pos].action =
@@ -2679,6 +2691,40 @@ __flow_hw_actions_translate(struct rte_eth_dev *dev,
"fail to create rte table");
}
+/**
+ * Translate rte_flow actions to DR action.
+ *
+ * As the action template has already indicated the actions. Translate
+ * the rte_flow actions to DR action if possible. So in flow create
+ * stage we will save cycles from handing the actions' organizing.
+ * For the actions with limited information, need to add these to a
+ * list.
+ *
+ * @param[in] dev
+ * Pointer to the rte_eth_dev structure.
+ * @param[in] cfg
+ * Pointer to the table configuration.
+ * @param[in/out] acts
+ * Pointer to the template HW steering DR actions.
+ * @param[in] at
+ * Action template.
+ * @param[out] error
+ * Pointer to error structure.
+ *
+ * @return
+ * 0 on success, a negative errno otherwise and rte_errno is set.
+ */
+static int
+flow_hw_translate_actions_template(struct rte_eth_dev *dev,
+ const struct mlx5_flow_template_table_cfg *cfg,
+ struct mlx5_hw_actions *acts,
+ struct rte_flow_actions_template *at,
+ struct mlx5_tbl_multi_pattern_ctx *mp_ctx,
+ struct rte_flow_error *error)
+{
+ return __flow_hw_translate_actions_template(dev, cfg, acts, at, mp_ctx, false, error);
+}
+
static __rte_always_inline struct mlx5dr_rule_action *
flow_hw_get_dr_action_buffer(struct mlx5_priv *priv,
struct rte_flow_template_table *table,
@@ -2721,7 +2767,7 @@ flow_hw_populate_rule_acts_caches(struct rte_eth_dev *dev,
* 0 on success, negative value otherwise and rte_errno is set.
*/
static int
-flow_hw_actions_translate(struct rte_eth_dev *dev,
+flow_hw_translate_all_actions_templates(struct rte_eth_dev *dev,
struct rte_flow_template_table *tbl,
struct rte_flow_error *error)
{
@@ -2729,7 +2775,7 @@ flow_hw_actions_translate(struct rte_eth_dev *dev,
uint32_t i;
for (i = 0; i < tbl->nb_action_templates; i++) {
- if (__flow_hw_actions_translate(dev, &tbl->cfg,
+ if (flow_hw_translate_actions_template(dev, &tbl->cfg,
&tbl->ats[i].acts,
tbl->ats[i].action_template,
&tbl->mpctx, error))
@@ -2819,10 +2865,10 @@ flow_hw_construct_quota(struct mlx5_priv *priv,
* The flow creation queue index.
* @param[in] action
* Pointer to the shared indirect rte_flow action.
- * @param[in] table
+ * @param[in] table
* Pointer to the flow table.
- * @param[in] it_idx
- * Item template index the action template refer to.
+ * @param[in] item_flags
+ * Item flags.
* @param[in] action_flags
* Actions bit-map detected in this template.
* @param[in, out] flow
@@ -2836,8 +2882,8 @@ flow_hw_construct_quota(struct mlx5_priv *priv,
static __rte_always_inline int
flow_hw_shared_action_construct(struct rte_eth_dev *dev, uint32_t queue,
const struct rte_flow_action *action,
- struct rte_flow_template_table *table,
- const uint8_t it_idx, uint64_t action_flags,
+ struct rte_flow_template_table *table __rte_unused,
+ const uint64_t item_flags, uint64_t action_flags,
struct rte_flow_hw *flow,
struct mlx5dr_rule_action *rule_act)
{
@@ -2853,7 +2899,6 @@ flow_hw_shared_action_construct(struct rte_eth_dev *dev, uint32_t queue,
uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
uint32_t idx = act_idx &
((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
- uint64_t item_flags;
cnt_id_t age_cnt;
memset(&act_data, 0, sizeof(act_data));
@@ -2872,7 +2917,6 @@ flow_hw_shared_action_construct(struct rte_eth_dev *dev, uint32_t queue,
act_data.shared_rss.symmetric_hash_function =
MLX5_RSS_IS_SYMM(shared_rss->origin.func);
- item_flags = table->its[it_idx]->item_flags;
if (flow_hw_shared_action_get
(dev, &act_data, item_flags, rule_act))
return -1;
@@ -3108,8 +3152,10 @@ flow_hw_release_actions(struct rte_eth_dev *dev,
* Pointer to container for temporarily constructed actions' parameters.
* @param[in] hw_acts
* Pointer to translated actions from template.
- * @param[in] it_idx
- * Item template index the action template refer to.
+ * @param[in] items_flags
+ * Item flags.
+ * @param[in] table
+ * Pointer to the template table.
* @param[in] actions
* Array of rte_flow action need to be checked.
* @param[in] rule_acts
@@ -3125,7 +3171,8 @@ flow_hw_actions_construct(struct rte_eth_dev *dev,
struct rte_flow_hw *flow,
struct mlx5_flow_hw_action_params *ap,
const struct mlx5_hw_action_template *hw_at,
- const uint8_t it_idx,
+ uint64_t item_flags,
+ struct rte_flow_template_table *table,
const struct rte_flow_action actions[],
struct mlx5dr_rule_action *rule_acts,
uint32_t queue,
@@ -3133,7 +3180,6 @@ flow_hw_actions_construct(struct rte_eth_dev *dev,
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_aso_mtr_pool *pool = priv->hws_mpool;
- struct rte_flow_template_table *table = flow->table;
struct mlx5_action_construct_data *act_data;
const struct rte_flow_actions_template *at = hw_at->action_template;
const struct mlx5_hw_actions *hw_acts = &hw_at->acts;
@@ -3186,7 +3232,6 @@ flow_hw_actions_construct(struct rte_eth_dev *dev,
LIST_FOREACH(act_data, &hw_acts->act_list, next) {
uint32_t jump_group;
uint32_t tag;
- uint64_t item_flags;
struct mlx5_hw_jump_action *jump;
struct mlx5_hrxq *hrxq;
uint32_t ct_idx;
@@ -3213,8 +3258,8 @@ flow_hw_actions_construct(struct rte_eth_dev *dev,
break;
case RTE_FLOW_ACTION_TYPE_INDIRECT:
if (flow_hw_shared_action_construct
- (dev, queue, action, table, it_idx,
- at->action_flags, flow,
+ (dev, queue, action, table,
+ item_flags, at->action_flags, flow,
&rule_acts[act_data->action_dst]))
goto error;
break;
@@ -3246,7 +3291,9 @@ flow_hw_actions_construct(struct rte_eth_dev *dev,
case RTE_FLOW_ACTION_TYPE_QUEUE:
hrxq = flow_hw_tir_action_register(dev,
ft_flag,
- action);
+ action,
+ item_flags,
+ !flow->nt_rule);
if (!hrxq)
goto error;
rule_acts[act_data->action_dst].action = hrxq->action;
@@ -3254,7 +3301,6 @@ flow_hw_actions_construct(struct rte_eth_dev *dev,
flow->flags |= MLX5_FLOW_HW_FLOW_FLAG_FATE_HRXQ;
break;
case MLX5_RTE_FLOW_ACTION_TYPE_RSS:
- item_flags = table->its[it_idx]->item_flags;
if (flow_hw_shared_action_get
(dev, act_data, item_flags,
&rule_acts[act_data->action_dst]))
@@ -3633,7 +3679,8 @@ flow_hw_async_flow_create(struct rte_eth_dev *dev,
*/
if (flow_hw_actions_construct(dev, flow, &ap,
&table->ats[action_template_index],
- pattern_template_index, actions,
+ table->its[pattern_template_index]->item_flags,
+ flow->table, actions,
rule_acts, queue, error))
goto error;
rule_items = flow_hw_get_rule_items(dev, table, items,
@@ -3773,7 +3820,8 @@ flow_hw_async_flow_create_by_index(struct rte_eth_dev *dev,
*/
if (flow_hw_actions_construct(dev, flow, &ap,
&table->ats[action_template_index],
- 0, actions, rule_acts, queue, error)) {
+ table->its[0]->item_flags, table,
+ actions, rule_acts, queue, error)) {
rte_errno = EINVAL;
goto error;
}
@@ -3902,7 +3950,8 @@ flow_hw_async_flow_update(struct rte_eth_dev *dev,
*/
if (flow_hw_actions_construct(dev, nf, &ap,
&table->ats[action_template_index],
- nf->mt_idx, actions,
+ table->its[nf->mt_idx]->item_flags,
+ table, actions,
rule_acts, queue, error)) {
rte_errno = EINVAL;
goto error;
@@ -4173,15 +4222,17 @@ hw_cmpl_flow_update_or_destroy(struct rte_eth_dev *dev,
rte_memcpy(flow, upd_flow, offsetof(struct rte_flow_hw, rule));
aux->orig = aux->upd;
flow->operation_type = MLX5_FLOW_HW_FLOW_OP_TYPE_CREATE;
- if (table->resource)
+ if (!flow->nt_rule && table->resource)
mlx5_ipool_free(table->resource, res_idx);
}
}
if (flow->operation_type == MLX5_FLOW_HW_FLOW_OP_TYPE_DESTROY ||
flow->operation_type == MLX5_FLOW_HW_FLOW_OP_TYPE_RSZ_TBL_DESTROY) {
- if (table->resource)
- mlx5_ipool_free(table->resource, res_idx);
- mlx5_ipool_free(table->flow, flow->idx);
+ if (!flow->nt_rule) {
+ if (table->resource)
+ mlx5_ipool_free(table->resource, res_idx);
+ mlx5_ipool_free(table->flow, flow->idx);
+ }
}
}
@@ -4585,7 +4636,7 @@ mlx5_hw_build_template_table(struct rte_eth_dev *dev,
/* do NOT translate table action if `dev` was not started */
if (!dev->data->dev_started)
continue;
- ret = __flow_hw_actions_translate(dev, &tbl->cfg,
+ ret = flow_hw_translate_actions_template(dev, &tbl->cfg,
&tbl->ats[i].acts,
action_templates[i],
&tbl->mpctx, error);
@@ -4882,7 +4933,7 @@ flow_hw_table_update(struct rte_eth_dev *dev,
struct rte_flow_template_table *tbl;
while ((tbl = LIST_FIRST(&priv->flow_hw_tbl_ongo)) != NULL) {
- if (flow_hw_actions_translate(dev, tbl, error))
+ if (flow_hw_translate_all_actions_templates(dev, tbl, error))
return -1;
LIST_REMOVE(tbl, next);
LIST_INSERT_HEAD(&priv->flow_hw_tbl, tbl, next);
@@ -6757,17 +6808,20 @@ flow_hw_template_actions_list(struct rte_flow_actions_template *at,
* Pointer to the rte_eth_dev structure.
* @param[in] at
* Pointer to flow actions template to be updated.
+ * @param[out] action_types
+ * Action types array to be filled.
+ * @param[out] tmpl_flags
+ * Template DR flags to be filled.
*
* @return
- * DR action template pointer on success and action offsets in @p at are updated.
- * NULL otherwise.
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
*/
-static struct mlx5dr_action_template *
-flow_hw_dr_actions_template_create(struct rte_eth_dev *dev,
- struct rte_flow_actions_template *at)
+static int
+flow_hw_parse_flow_actions_to_dr_actions(struct rte_eth_dev *dev,
+ struct rte_flow_actions_template *at,
+ enum mlx5dr_action_type action_types[MLX5_HW_MAX_ACTS],
+ uint32_t *tmpl_flags __rte_unused)
{
- struct mlx5dr_action_template *dr_template;
- enum mlx5dr_action_type action_types[MLX5_HW_MAX_ACTS] = { MLX5DR_ACTION_TYP_LAST };
unsigned int i;
uint16_t curr_off;
enum mlx5dr_action_type reformat_act_type = MLX5DR_ACTION_TYP_REFORMAT_TNL_L2_TO_L2;
@@ -6792,14 +6846,14 @@ flow_hw_dr_actions_template_create(struct rte_eth_dev *dev,
ret = flow_hw_template_actions_list(at, i, action_types,
&curr_off, &cnt_off);
if (ret)
- return NULL;
+ return ret;
break;
case RTE_FLOW_ACTION_TYPE_INDIRECT:
ret = flow_hw_dr_actions_template_handle_shared
(at->masks[i].type, i, action_types,
&curr_off, &cnt_off, at);
if (ret)
- return NULL;
+ return ret;
break;
case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
@@ -6899,26 +6953,20 @@ flow_hw_dr_actions_template_create(struct rte_eth_dev *dev,
at->recom_off = recom_off;
action_types[recom_off] = recom_type;
}
- dr_template = mlx5dr_action_template_create(action_types, 0);
- if (dr_template) {
- at->dr_actions_num = curr_off;
- } else {
- DRV_LOG(ERR, "Failed to create DR action template: %d", rte_errno);
- return NULL;
- }
+ at->dr_actions_num = curr_off;
+
/* Create srh flex parser for remove anchor. */
if ((recom_type == MLX5DR_ACTION_TYP_POP_IPV6_ROUTE_EXT ||
recom_type == MLX5DR_ACTION_TYP_PUSH_IPV6_ROUTE_EXT) &&
- mlx5_alloc_srh_flex_parser(dev)) {
+ (ret = mlx5_alloc_srh_flex_parser(dev))) {
DRV_LOG(ERR, "Failed to create srv6 flex parser");
- claim_zero(mlx5dr_action_template_destroy(dr_template));
- return NULL;
+ return ret;
}
- return dr_template;
+ return 0;
err_actions_num:
DRV_LOG(ERR, "Number of HW actions (%u) exceeded maximum (%u) allowed in template",
curr_off, MLX5_HW_MAX_ACTS);
- return NULL;
+ return -EINVAL;
}
static void
@@ -7138,6 +7186,8 @@ static const struct rte_flow_action quota_color_inc_mask = {
* Associated actions (list terminated by the END action).
* @param[in] masks
* List of actions that marks which of the action's member is constant.
+ * @param[in] nt_mode
+ * Non template mode.
* @param[out] error
* Pointer to error structure.
*
@@ -7145,10 +7195,11 @@ static const struct rte_flow_action quota_color_inc_mask = {
* Action template pointer on success, NULL otherwise and rte_errno is set.
*/
static struct rte_flow_actions_template *
-flow_hw_actions_template_create(struct rte_eth_dev *dev,
+__flow_hw_actions_template_create(struct rte_eth_dev *dev,
const struct rte_flow_actions_template_attr *attr,
const struct rte_flow_action actions[],
const struct rte_flow_action masks[],
+ bool nt_mode,
struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
@@ -7169,6 +7220,9 @@ flow_hw_actions_template_create(struct rte_eth_dev *dev,
struct rte_flow_action mf_masks[MLX5_HW_MAX_ACTS];
uint32_t expand_mf_num = 0;
uint16_t src_off[MLX5_HW_MAX_ACTS] = {0, };
+ enum mlx5dr_action_type action_types[MLX5_HW_MAX_ACTS] = { MLX5DR_ACTION_TYP_LAST };
+ uint32_t tmpl_flags = 0;
+ int ret;
if (mlx5_flow_hw_actions_validate(dev, attr, actions, masks,
&action_flags, error))
@@ -7328,17 +7382,23 @@ flow_hw_actions_template_create(struct rte_eth_dev *dev,
break;
}
}
- at->tmpl = flow_hw_dr_actions_template_create(dev, at);
- if (!at->tmpl)
+ ret = flow_hw_parse_flow_actions_to_dr_actions(dev, at, action_types, &tmpl_flags);
+ if (ret)
goto error;
at->action_flags = action_flags;
+ /* In non template mode there is no need to create the dr template. */
+ if (nt_mode)
+ return at;
+ at->tmpl = mlx5dr_action_template_create(action_types, tmpl_flags);
+ if (!at->tmpl) {
+ DRV_LOG(ERR, "Failed to create DR action template: %d", rte_errno);
+ goto error;
+ }
rte_atomic_fetch_add_explicit(&at->refcnt, 1, rte_memory_order_relaxed);
LIST_INSERT_HEAD(&priv->flow_hw_at, at, next);
return at;
error:
if (at) {
- if (at->tmpl)
- mlx5dr_action_template_destroy(at->tmpl);
mlx5_free(at);
}
rte_flow_error_set(error, rte_errno,
@@ -7347,6 +7407,33 @@ flow_hw_actions_template_create(struct rte_eth_dev *dev,
return NULL;
}
+/**
+ * Create flow action template.
+ *
+ * @param[in] dev
+ * Pointer to the rte_eth_dev structure.
+ * @param[in] attr
+ * Pointer to the action template attributes.
+ * @param[in] actions
+ * Associated actions (list terminated by the END action).
+ * @param[in] masks
+ * List of actions that marks which of the action's member is constant.
+ * @param[out] error
+ * Pointer to error structure.
+ *
+ * @return
+ * Action template pointer on success, NULL otherwise and rte_errno is set.
+ */
+static struct rte_flow_actions_template *
+flow_hw_actions_template_create(struct rte_eth_dev *dev,
+ const struct rte_flow_actions_template_attr *attr,
+ const struct rte_flow_action actions[],
+ const struct rte_flow_action masks[],
+ struct rte_flow_error *error)
+{
+ return __flow_hw_actions_template_create(dev, attr, actions, masks, false, error);
+}
+
/**
* Destroy flow action template.
*
@@ -8164,8 +8251,8 @@ flow_hw_grp_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
mlx5dr_action_destroy(grp_data->jump.hws_action);
if (grp_data->jump.root_action)
mlx5dr_action_destroy(grp_data->jump.root_action);
- mlx5dr_table_destroy(grp_data->tbl);
mlx5_list_destroy(grp_data->matchers);
+ mlx5dr_table_destroy(grp_data->tbl);
mlx5_ipool_free(sh->ipool[MLX5_IPOOL_HW_GRP], grp_data->idx);
}
@@ -10256,6 +10343,8 @@ mlx5_flow_hw_cleanup_ctrl_rx_templates(struct rte_eth_dev *dev)
* Number of queue.
* @param[in] queue_attr
* Array that holds attributes for each flow queue.
+ * @param[in] nt_mode
+ * Non template mode.
*
* @return
* Pointer to copied HWS configuration is returned on success.
@@ -10265,6 +10354,7 @@ static struct mlx5_flow_hw_attr *
flow_hw_alloc_copy_config(const struct rte_flow_port_attr *port_attr,
const uint16_t nb_queue,
const struct rte_flow_queue_attr *queue_attr[],
+ bool nt_mode,
struct rte_flow_error *error)
{
struct mlx5_flow_hw_attr *hw_attr;
@@ -10284,6 +10374,7 @@ flow_hw_alloc_copy_config(const struct rte_flow_port_attr *port_attr,
hw_attr->queue_attr = (struct rte_flow_queue_attr *)(hw_attr + 1);
for (i = 0; i < nb_queue; ++i)
memcpy(&hw_attr->queue_attr[i], queue_attr[i], sizeof(hw_attr->queue_attr[i]));
+ hw_attr->nt_mode = nt_mode;
return hw_attr;
}
@@ -10388,6 +10479,103 @@ flow_hw_action_template_drop_init(struct rte_eth_dev *dev,
return 0;
}
+static void
+__flow_hw_resource_release(struct rte_eth_dev *dev, bool ctx_close)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct rte_flow_template_table *tbl;
+ struct rte_flow_pattern_template *it;
+ struct rte_flow_actions_template *at;
+ struct mlx5_flow_group *grp;
+ uint32_t i;
+
+ flow_hw_rxq_flag_set(dev, false);
+ flow_hw_flush_all_ctrl_flows(dev);
+ flow_hw_cleanup_ctrl_fdb_tables(dev);
+ flow_hw_cleanup_tx_repr_tagging(dev);
+ flow_hw_cleanup_ctrl_rx_tables(dev);
+ flow_hw_action_template_drop_release(dev);
+ while (!LIST_EMPTY(&priv->flow_hw_grp)) {
+ grp = LIST_FIRST(&priv->flow_hw_grp);
+ flow_hw_group_unset_miss_group(dev, grp, NULL);
+ }
+ while (!LIST_EMPTY(&priv->flow_hw_tbl_ongo)) {
+ tbl = LIST_FIRST(&priv->flow_hw_tbl_ongo);
+ flow_hw_table_destroy(dev, tbl, NULL);
+ }
+ while (!LIST_EMPTY(&priv->flow_hw_tbl)) {
+ tbl = LIST_FIRST(&priv->flow_hw_tbl);
+ flow_hw_table_destroy(dev, tbl, NULL);
+ }
+ while (!LIST_EMPTY(&priv->flow_hw_itt)) {
+ it = LIST_FIRST(&priv->flow_hw_itt);
+ flow_hw_pattern_template_destroy(dev, it, NULL);
+ }
+ while (!LIST_EMPTY(&priv->flow_hw_at)) {
+ at = LIST_FIRST(&priv->flow_hw_at);
+ flow_hw_actions_template_destroy(dev, at, NULL);
+ }
+ for (i = 0; i < MLX5_HW_ACTION_FLAG_MAX; i++) {
+ if (priv->hw_drop[i])
+ mlx5dr_action_destroy(priv->hw_drop[i]);
+ if (priv->hw_tag[i])
+ mlx5dr_action_destroy(priv->hw_tag[i]);
+ }
+ if (priv->hw_def_miss)
+ mlx5dr_action_destroy(priv->hw_def_miss);
+ flow_hw_destroy_nat64_actions(priv);
+ flow_hw_destroy_vlan(dev);
+ flow_hw_destroy_send_to_kernel_action(priv);
+ flow_hw_free_vport_actions(priv);
+ if (priv->acts_ipool) {
+ mlx5_ipool_destroy(priv->acts_ipool);
+ priv->acts_ipool = NULL;
+ }
+ if (priv->hws_age_req)
+ mlx5_hws_age_pool_destroy(priv);
+ if (!priv->shared_host && priv->hws_cpool) {
+ mlx5_hws_cnt_pool_destroy(priv->sh, priv->hws_cpool);
+ priv->hws_cpool = NULL;
+ }
+ if (priv->hws_ctpool) {
+ flow_hw_ct_pool_destroy(dev, priv->hws_ctpool);
+ priv->hws_ctpool = NULL;
+ }
+ if (priv->ct_mng) {
+ flow_hw_ct_mng_destroy(dev, priv->ct_mng);
+ priv->ct_mng = NULL;
+ }
+ mlx5_flow_quota_destroy(dev);
+ if (priv->hw_q) {
+ for (i = 0; i < priv->nb_queue; i++) {
+ struct mlx5_hw_q *hwq = &priv->hw_q[i];
+ rte_ring_free(hwq->indir_iq);
+ rte_ring_free(hwq->indir_cq);
+ rte_ring_free(hwq->flow_transfer_pending);
+ rte_ring_free(hwq->flow_transfer_completed);
+ }
+ mlx5_free(priv->hw_q);
+ priv->hw_q = NULL;
+ }
+ if (ctx_close) {
+ if (priv->dr_ctx) {
+ claim_zero(mlx5dr_context_close(priv->dr_ctx));
+ priv->dr_ctx = NULL;
+ }
+ }
+ if (priv->shared_host) {
+ struct mlx5_priv *host_priv = priv->shared_host->data->dev_private;
+ rte_atomic_fetch_sub_explicit(&host_priv->shared_refcnt, 1,
+ rte_memory_order_relaxed);
+ priv->shared_host = NULL;
+ }
+ if (priv->hw_attr) {
+ mlx5_free(priv->hw_attr);
+ priv->hw_attr = NULL;
+ }
+ priv->nb_queue = 0;
+}
+
static __rte_always_inline struct rte_ring *
mlx5_hwq_ring_create(uint16_t port_id, uint32_t queue, uint32_t size, const char *str)
{
@@ -10441,6 +10629,8 @@ flow_hw_validate_attributes(const struct rte_flow_port_attr *port_attr,
* Number of queue.
* @param[in] queue_attr
* Array that holds attributes for each flow queue.
+ * @param[in] nt_mode
+ * Non-template mode.
* @param[out] error
* Pointer to error structure.
*
@@ -10448,15 +10638,15 @@ flow_hw_validate_attributes(const struct rte_flow_port_attr *port_attr,
* 0 on success, a negative errno value otherwise and rte_errno is set.
*/
static int
-flow_hw_configure(struct rte_eth_dev *dev,
+__flow_hw_configure(struct rte_eth_dev *dev,
const struct rte_flow_port_attr *port_attr,
uint16_t nb_queue,
const struct rte_flow_queue_attr *queue_attr[],
+ bool nt_mode,
struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_priv *host_priv = NULL;
- struct mlx5dr_context *dr_ctx = NULL;
struct mlx5dr_context_attr dr_ctx_attr = {0};
struct mlx5_hw_q *hw_q;
struct mlx5_hw_q_job *job = NULL;
@@ -10489,8 +10679,9 @@ flow_hw_configure(struct rte_eth_dev *dev,
if (flow_hw_validate_attributes(port_attr, nb_queue, queue_attr, error))
return -rte_errno;
/*
- * Calling rte_flow_configure() again is allowed if and only if
- * provided configuration matches the initially provided one.
+ * Calling rte_flow_configure() again is allowed if
+ * provided configuration matches the initially provided one,
+ * or previous configuration was default non template one.
*/
if (priv->dr_ctx) {
MLX5_ASSERT(priv->hw_attr != NULL);
@@ -10502,15 +10693,20 @@ flow_hw_configure(struct rte_eth_dev *dev,
goto err;
}
}
- if (flow_hw_compare_config(priv->hw_attr, port_attr, nb_queue, queue_attr))
- return 0;
- else
- return rte_flow_error_set(error, ENOTSUP,
- RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
- "Changing HWS configuration attributes "
- "is not supported");
+ /* If previous configuration was not default non template mode config. */
+ if (!(priv->hw_attr->nt_mode)) {
+ if (flow_hw_compare_config(priv->hw_attr, port_attr, nb_queue, queue_attr))
+ return 0;
+ else
+ return rte_flow_error_set(error, ENOTSUP,
+ RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+ "Changing HWS configuration attributes "
+ "is not supported");
+ }
+ /* Reconfiguration, need to release all resources from previous allocation. */
+ __flow_hw_resource_release(dev, true);
}
- priv->hw_attr = flow_hw_alloc_copy_config(port_attr, nb_queue, queue_attr, error);
+ priv->hw_attr = flow_hw_alloc_copy_config(port_attr, nb_queue, queue_attr, nt_mode, error);
if (!priv->hw_attr) {
ret = -rte_errno;
goto err;
@@ -10632,11 +10828,10 @@ flow_hw_configure(struct rte_eth_dev *dev,
}
/* Set backward compatibale mode to support non template RTE FLOW API.*/
dr_ctx_attr.bwc = true;
- dr_ctx = mlx5dr_context_open(priv->sh->cdev->ctx, &dr_ctx_attr);
+ priv->dr_ctx = mlx5dr_context_open(priv->sh->cdev->ctx, &dr_ctx_attr);
/* rte_errno has been updated by HWS layer. */
- if (!dr_ctx)
+ if (!priv->dr_ctx)
goto err;
- priv->dr_ctx = dr_ctx;
priv->nb_queue = nb_q_updated;
rte_spinlock_init(&priv->hw_ctrl_lock);
LIST_INIT(&priv->hw_ctrl_flows);
@@ -10705,19 +10900,10 @@ flow_hw_configure(struct rte_eth_dev *dev,
goto err;
}
ret = flow_hw_create_ctrl_tables(dev, error);
- if (ret)
- goto err;
- }
- /*
- * All domains will use the same dummy action, only used in the backward
- * compatible API. Initialize it for only once. No order dependency.
- */
- if (!priv->sh->hw_dummy_last) {
- priv->sh->hw_dummy_last = mlx5dr_action_create_last(priv->dr_ctx,
- MLX5DR_ACTION_FLAG_HWS_RX);
- if (!priv->sh->hw_dummy_last)
- /* Do not overwrite the rte_errno. */
+ if (ret) {
+ rte_errno = -ret;
goto err;
+ }
}
if (!priv->shared_host)
flow_hw_create_send_to_kernel_actions(priv);
@@ -10773,77 +10959,7 @@ flow_hw_configure(struct rte_eth_dev *dev,
dev->flow_fp_ops = &mlx5_flow_hw_fp_ops;
return 0;
err:
- priv->hws_strict_queue = 0;
- flow_hw_destroy_nat64_actions(priv);
- flow_hw_destroy_vlan(dev);
- if (priv->hws_age_req)
- mlx5_hws_age_pool_destroy(priv);
- if (priv->hws_cpool) {
- mlx5_hws_cnt_pool_destroy(priv->sh, priv->hws_cpool);
- priv->hws_cpool = NULL;
- }
- if (priv->hws_ctpool) {
- flow_hw_ct_pool_destroy(dev, priv->hws_ctpool);
- priv->hws_ctpool = NULL;
- }
- if (priv->ct_mng) {
- flow_hw_ct_mng_destroy(dev, priv->ct_mng);
- priv->ct_mng = NULL;
- }
- if (priv->sh->hw_dummy_last) {
- mlx5dr_action_destroy(priv->sh->hw_dummy_last);
- priv->sh->hw_dummy_last = NULL;
- }
- flow_hw_destroy_send_to_kernel_action(priv);
- flow_hw_cleanup_ctrl_fdb_tables(dev);
- flow_hw_free_vport_actions(priv);
- if (priv->hw_def_miss) {
- mlx5dr_action_destroy(priv->hw_def_miss);
- priv->hw_def_miss = NULL;
- }
- flow_hw_cleanup_tx_repr_tagging(dev);
- for (i = 0; i < MLX5_HW_ACTION_FLAG_MAX; i++) {
- if (priv->hw_drop[i]) {
- mlx5dr_action_destroy(priv->hw_drop[i]);
- priv->hw_drop[i] = NULL;
- }
- if (priv->hw_tag[i]) {
- mlx5dr_action_destroy(priv->hw_tag[i]);
- priv->hw_tag[i] = NULL;
- }
- }
- mlx5_flow_meter_uninit(dev);
- mlx5_flow_quota_destroy(dev);
- flow_hw_cleanup_ctrl_rx_tables(dev);
- flow_hw_action_template_drop_release(dev);
- if (dr_ctx) {
- claim_zero(mlx5dr_context_close(dr_ctx));
- priv->dr_ctx = NULL;
- }
- if (priv->shared_host) {
- struct mlx5_priv *host_priv = priv->shared_host->data->dev_private;
-
- rte_atomic_fetch_sub_explicit(&host_priv->shared_refcnt, 1,
- rte_memory_order_relaxed);
- priv->shared_host = NULL;
- }
- if (priv->hw_q) {
- for (i = 0; i < nb_q_updated; i++) {
- rte_ring_free(priv->hw_q[i].indir_iq);
- rte_ring_free(priv->hw_q[i].indir_cq);
- rte_ring_free(priv->hw_q[i].flow_transfer_pending);
- rte_ring_free(priv->hw_q[i].flow_transfer_completed);
- }
- mlx5_free(priv->hw_q);
- priv->hw_q = NULL;
- }
- if (priv->acts_ipool) {
- mlx5_ipool_destroy(priv->acts_ipool);
- priv->acts_ipool = NULL;
- }
- mlx5_free(priv->hw_attr);
- priv->hw_attr = NULL;
- priv->nb_queue = 0;
+ __flow_hw_resource_release(dev, true);
if (_queue_attr)
mlx5_free(_queue_attr);
/* Do not overwrite the internal errno information. */
@@ -10854,6 +10970,33 @@ flow_hw_configure(struct rte_eth_dev *dev,
"fail to configure port");
}
+/**
+ * Configure port HWS resources.
+ *
+ * @param[in] dev
+ * Pointer to the rte_eth_dev structure.
+ * @param[in] port_attr
+ * Port configuration attributes.
+ * @param[in] nb_queue
+ * Number of queue.
+ * @param[in] queue_attr
+ * Array that holds attributes for each flow queue.
+ * @param[out] error
+ * Pointer to error structure.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+static int
+flow_hw_configure(struct rte_eth_dev *dev,
+ const struct rte_flow_port_attr *port_attr,
+ uint16_t nb_queue,
+ const struct rte_flow_queue_attr *queue_attr[],
+ struct rte_flow_error *error)
+{
+ return __flow_hw_configure(dev, port_attr, nb_queue, queue_attr, false, error);
+}
+
/**
* Release HWS resources.
*
@@ -10864,93 +11007,10 @@ void
flow_hw_resource_release(struct rte_eth_dev *dev)
{
struct mlx5_priv *priv = dev->data->dev_private;
- struct rte_flow_template_table *tbl;
- struct rte_flow_pattern_template *it;
- struct rte_flow_actions_template *at;
- struct mlx5_flow_group *grp;
- uint32_t i;
if (!priv->dr_ctx)
return;
- dev->flow_fp_ops = &rte_flow_fp_default_ops;
- flow_hw_rxq_flag_set(dev, false);
- flow_hw_flush_all_ctrl_flows(dev);
- flow_hw_cleanup_ctrl_fdb_tables(dev);
- flow_hw_cleanup_tx_repr_tagging(dev);
- flow_hw_cleanup_ctrl_rx_tables(dev);
- flow_hw_action_template_drop_release(dev);
- while (!LIST_EMPTY(&priv->flow_hw_grp)) {
- grp = LIST_FIRST(&priv->flow_hw_grp);
- flow_hw_group_unset_miss_group(dev, grp, NULL);
- }
- while (!LIST_EMPTY(&priv->flow_hw_tbl_ongo)) {
- tbl = LIST_FIRST(&priv->flow_hw_tbl_ongo);
- flow_hw_table_destroy(dev, tbl, NULL);
- }
- while (!LIST_EMPTY(&priv->flow_hw_tbl)) {
- tbl = LIST_FIRST(&priv->flow_hw_tbl);
- flow_hw_table_destroy(dev, tbl, NULL);
- }
- while (!LIST_EMPTY(&priv->flow_hw_itt)) {
- it = LIST_FIRST(&priv->flow_hw_itt);
- flow_hw_pattern_template_destroy(dev, it, NULL);
- }
- while (!LIST_EMPTY(&priv->flow_hw_at)) {
- at = LIST_FIRST(&priv->flow_hw_at);
- flow_hw_actions_template_destroy(dev, at, NULL);
- }
- for (i = 0; i < MLX5_HW_ACTION_FLAG_MAX; i++) {
- if (priv->hw_drop[i])
- mlx5dr_action_destroy(priv->hw_drop[i]);
- if (priv->hw_tag[i])
- mlx5dr_action_destroy(priv->hw_tag[i]);
- }
- if (priv->hw_def_miss)
- mlx5dr_action_destroy(priv->hw_def_miss);
- flow_hw_destroy_nat64_actions(priv);
- flow_hw_destroy_vlan(dev);
- if (priv->sh->hw_dummy_last) {
- mlx5dr_action_destroy(priv->sh->hw_dummy_last);
- priv->sh->hw_dummy_last = NULL;
- }
- flow_hw_destroy_send_to_kernel_action(priv);
- flow_hw_free_vport_actions(priv);
- if (priv->acts_ipool) {
- mlx5_ipool_destroy(priv->acts_ipool);
- priv->acts_ipool = NULL;
- }
- if (priv->hws_age_req)
- mlx5_hws_age_pool_destroy(priv);
- if (priv->hws_cpool) {
- mlx5_hws_cnt_pool_destroy(priv->sh, priv->hws_cpool);
- priv->hws_cpool = NULL;
- }
- if (priv->hws_ctpool) {
- flow_hw_ct_pool_destroy(dev, priv->hws_ctpool);
- priv->hws_ctpool = NULL;
- }
- if (priv->ct_mng) {
- flow_hw_ct_mng_destroy(dev, priv->ct_mng);
- priv->ct_mng = NULL;
- }
- mlx5_flow_quota_destroy(dev);
- for (i = 0; i < priv->nb_queue; i++) {
- rte_ring_free(priv->hw_q[i].indir_iq);
- rte_ring_free(priv->hw_q[i].indir_cq);
- rte_ring_free(priv->hw_q[i].flow_transfer_pending);
- rte_ring_free(priv->hw_q[i].flow_transfer_completed);
- }
- mlx5_free(priv->hw_q);
- priv->hw_q = NULL;
- if (priv->shared_host) {
- struct mlx5_priv *host_priv = priv->shared_host->data->dev_private;
- rte_atomic_fetch_sub_explicit(&host_priv->shared_refcnt, 1,
- rte_memory_order_relaxed);
- priv->shared_host = NULL;
- }
- mlx5_free(priv->hw_attr);
- priv->hw_attr = NULL;
- priv->nb_queue = 0;
+ __flow_hw_resource_release(dev, false);
}
/* Sets vport tag and mask, for given port, used in HWS rules. */
@@ -12113,7 +12173,7 @@ flow_hw_init(struct rte_eth_dev *dev,
struct rte_flow_error *error)
{
const struct rte_flow_port_attr port_attr = {0};
- struct rte_flow_queue_attr queue_attr = {.size = MLX5_NT_DEFAULT_QUEUE_SIZE};
+ const struct rte_flow_queue_attr queue_attr = {.size = MLX5_NT_DEFAULT_QUEUE_SIZE};
const struct rte_flow_queue_attr *attr_list = &queue_attr;
/**
@@ -12131,7 +12191,7 @@ flow_hw_init(struct rte_eth_dev *dev,
/* Configure hws with default values. */
DRV_LOG(DEBUG, "Apply default configuration, zero number of queues, inner control queue size is %u",
MLX5_NT_DEFAULT_QUEUE_SIZE);
- return flow_hw_configure(dev, &port_attr, 0, &attr_list, error);
+ return __flow_hw_configure(dev, &port_attr, 0, &attr_list, true, error);
}
static int flow_hw_prepare(struct rte_eth_dev *dev,
@@ -12156,134 +12216,99 @@ static int flow_hw_prepare(struct rte_eth_dev *dev,
(*flow)->nt2hws = (struct rte_flow_nt2hws *)
((uintptr_t)(*flow) + sizeof(struct rte_flow_hw));
(*flow)->idx = idx;
+ (*flow)->nt2hws->flow_aux = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct rte_flow_hw_aux),
+ RTE_CACHE_LINE_SIZE, rte_dev_numa_node(dev->device));
+ if (!(*flow)->nt2hws->flow_aux)
+ return rte_flow_error_set(error, ENOMEM,
+ RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+ "cannot allocate flow aux memory");
/*TODO: consider if other allocation is needed for actions translate. */
return 0;
}
static int
-flow_hw_translate_actions(struct rte_eth_dev *dev,
+flow_hw_translate_flow_actions(struct rte_eth_dev *dev,
const struct rte_flow_attr *attr,
const struct rte_flow_action actions[],
struct rte_flow_hw *flow,
struct mlx5_hw_actions *hw_acts,
+ uint64_t item_flags,
bool external,
struct rte_flow_error *error)
{
- struct mlx5_priv *priv = dev->data->dev_private;
- enum mlx5dr_table_type type;
- enum mlx5_hw_action_flag_type flag_type;
- bool actions_end = false;
- uint64_t action_flags = 0; /* to be used when needed */
- int ret;
- uint32_t actions_n = 0;
- uint32_t mark_id;
- uint32_t jump_group;
+ int ret = 0;
uint32_t src_group = 0;
- bool is_mark;
- struct mlx5_flow_template_table_cfg tbl_cfg;
- enum mlx5_flow_fate_type fate_type = MLX5_FLOW_FATE_NONE;
-
+ enum mlx5dr_table_type table_type;
+ struct rte_flow_template_table *table = NULL;
+ struct mlx5_flow_group grp;
+ uint64_t action_flags = 0;
+ struct rte_flow_actions_template *at = NULL;
+ struct rte_flow_actions_template_attr template_attr = {
+ .egress = attr->egress,
+ .ingress = attr->ingress,
+ .transfer = attr->transfer,
+ };
+ struct rte_flow_action masks[MLX5_HW_MAX_ACTS];
+ struct mlx5_flow_hw_action_params ap;
+ memset(&masks, 0, sizeof(masks));
+ int i = -1;
+ do {
+ i++;
+ masks[i].type = actions[i].type;
+ } while (masks[i].type != RTE_FLOW_ACTION_TYPE_END);
RTE_SET_USED(action_flags);
/* The group in the attribute translation was done in advance. */
ret = __translate_group(dev, attr, external, attr->group, &src_group, error);
if (ret)
return ret;
if (attr->transfer)
- type = MLX5DR_TABLE_TYPE_FDB;
+ table_type = MLX5DR_TABLE_TYPE_FDB;
else if (attr->egress)
- type = MLX5DR_TABLE_TYPE_NIC_TX;
+ table_type = MLX5DR_TABLE_TYPE_NIC_TX;
else
- type = MLX5DR_TABLE_TYPE_NIC_RX;
- flag_type = (src_group == 0) ? MLX5_HW_ACTION_FLAG_ROOT :
- MLX5_HW_ACTION_FLAG_NONE_ROOT;
- for (; !actions_end; actions++) {
- switch (actions->type) {
- case RTE_FLOW_ACTION_TYPE_VOID:
- break;
- case RTE_FLOW_ACTION_TYPE_DROP:
- hw_acts->rule_acts[actions_n++].action = priv->hw_drop[flag_type];
- fate_type = MLX5_FLOW_FATE_DROP;
- break;
- case RTE_FLOW_ACTION_TYPE_FLAG:
- case RTE_FLOW_ACTION_TYPE_MARK:
- is_mark = actions->type == RTE_FLOW_ACTION_TYPE_MARK;
- mark_id = is_mark ?
- ((const struct rte_flow_action_mark *)(actions->conf))->id :
- MLX5_FLOW_MARK_DEFAULT;
- hw_acts->rule_acts[actions_n].tag.value = mlx5_flow_mark_set(mark_id);
- hw_acts->rule_acts[actions_n].action = priv->hw_tag[flag_type];
- actions_n++;
- action_flags |= is_mark ? MLX5_FLOW_ACTION_MARK : MLX5_FLOW_ACTION_FLAG;
- hw_acts->mark = true;
- rte_atomic_fetch_add_explicit(&priv->hws_mark_refcnt, 1,
- rte_memory_order_relaxed);
- flow_hw_rxq_flag_set(dev, true);
- break;
- case RTE_FLOW_ACTION_TYPE_JUMP:
- jump_group = ((const struct rte_flow_action_jump *)actions->conf)->group;
- tbl_cfg.attr.flow_attr = *attr;
- tbl_cfg.external = external;
- /* The flow_hw_jump_action_register() can be refactored. */
- hw_acts->jump = flow_hw_jump_action_register(dev, &tbl_cfg,
- jump_group, error);
- if (hw_acts->jump == NULL)
- goto clean_up;
- hw_acts->rule_acts[actions_n++].action =
- (flag_type == MLX5_HW_ACTION_FLAG_NONE_ROOT) ?
- hw_acts->jump->hws_action : hw_acts->jump->root_action;
- action_flags |= MLX5_FLOW_ACTION_JUMP;
- fate_type = MLX5_FLOW_FATE_JUMP;
- break;
- case RTE_FLOW_ACTION_TYPE_QUEUE:
- /* Right now, only Rx supports the TIR, validation is needed. */
- hw_acts->tir = flow_hw_tir_action_register(dev,
- mlx5_hw_act_flag[flag_type][type], actions);
- if (hw_acts->tir == NULL) {
- rte_flow_error_set(error, EINVAL,
- RTE_FLOW_ERROR_TYPE_ACTION, NULL,
- "Failed to translate queue.");
- goto clean_up;
- }
- action_flags |= MLX5_FLOW_ACTION_QUEUE;
- fate_type = MLX5_FLOW_FATE_QUEUE;
- break;
- case RTE_FLOW_ACTION_TYPE_SEND_TO_KERNEL:
- if (src_group == 0) {
- rte_flow_error_set(error, ENOTSUP,
- RTE_FLOW_ERROR_TYPE_ACTION, NULL,
- "Send to kernel action on the root table is"
- " not supported in HWS mode now.");
- goto clean_up;
- }
- hw_acts->rule_acts[actions_n++].action = priv->hw_send_to_kernel[type];
- action_flags |= MLX5_FLOW_ACTION_SEND_TO_KERNEL;
- break;
- case RTE_FLOW_ACTION_TYPE_END:
- /*
- * Using NULL action right now, maybe a new API can be used
- * to create a dummy action with type MLX5DR_ACTION_TYP_LAST.
- */
- hw_acts->rule_acts[actions_n++].action = priv->sh->hw_dummy_last;
- actions_end = true;
- break;
- default:
- break;
- }
- }
- if (fate_type == MLX5_FLOW_FATE_QUEUE) {
- hw_acts->rule_acts[actions_n++].action = hw_acts->tir->action;
- flow->hrxq = hw_acts->tir;
- } else {
- if (fate_type == MLX5_FLOW_FATE_JUMP)
- flow->jump = hw_acts->jump;
+ table_type = MLX5DR_TABLE_TYPE_NIC_RX;
+ /* TODO: consider add flag if using only non template mode to reduce table struct size. */
+ table = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*table), 0, SOCKET_ID_ANY);
+ /* TODO: consider sending only relevant fields to construct. */
+ if (!table)
+ return rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_ACTION,
+ actions, "Failed to allocate dummy table");
+ /* Notice All actions will be unmasked. */
+ at = __flow_hw_actions_template_create(dev, &template_attr, actions, masks, true, error);
+ if (!at) {
+ ret = -rte_errno;
+ goto end;
}
- /* Total actions number should be validated before. */
- MLX5_ASSERT(actions_n <= MLX5_HW_MAX_ACTS);
- return 0;
+ if (ret)
+ goto clean_up;
+ grp.group_id = src_group;
+ table->grp = &grp;
+ table->type = table_type;
+ table->cfg.external = external;
+ table->nb_action_templates = 1;
+ memcpy(&table->cfg.attr.flow_attr, attr, sizeof(*attr));
+ table->ats[0].action_template = at;
+ ret = __flow_hw_translate_actions_template(dev, &table->cfg, hw_acts, at,
+ &table->mpctx, true, error);
+ if (ret)
+ goto clean_up;
+ table->ats[0].acts = *hw_acts;
+ ret = flow_hw_actions_construct(dev, flow, &ap,
+ &table->ats[0], item_flags, table,
+ actions, hw_acts->rule_acts, 0, error);
+ if (ret)
+ goto clean_up;
+
+ goto end;
clean_up:
/* Make sure that there is no garbage in the actions. */
__flow_hw_actions_release(dev, hw_acts);
- return -rte_errno;
+end:
+ if (table)
+ mlx5_free(table);
+ if (at)
+ mlx5_free(at);
+ return ret;
}
static int flow_hw_register_matcher(struct rte_eth_dev *dev,
@@ -12306,9 +12331,11 @@ static int flow_hw_register_matcher(struct rte_eth_dev *dev,
.error = &sub_error,
.data = &flow_attr,
};
+ void *items_ptr = &items;
struct mlx5_flow_cb_ctx matcher_ctx = {
.error = &sub_error,
.data = matcher,
+ .data2 = items_ptr,
};
struct mlx5_list_entry *group_entry;
struct mlx5_list_entry *matcher_entry;
@@ -12344,30 +12371,18 @@ static int flow_hw_register_matcher(struct rte_eth_dev *dev,
if (!resource)
goto error;
flow->nt2hws->matcher = resource;
-
- /* If matcher was not found and reused in list, create matcher object. */
- if (!resource->matcher_object) {
- resource->matcher_object = (void *)mlx5dr_bwc_matcher_create
- (flow_group->tbl, matcher->priority, items);
- }
- /* If matcher create failed */
- if (!(resource->matcher_object))
- goto error;
return 0;
error:
- if (error) {
+ if (error)
if (sub_error.type != RTE_FLOW_ERROR_TYPE_NONE)
rte_memcpy(error, &sub_error, sizeof(sub_error));
- return rte_flow_error_set(error, ENOMEM,
- RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
- NULL, "fail to register matcher");
- } else {
- return -ENOMEM;
- }
+ return rte_flow_error_set(error, ENOMEM,
+ RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+ NULL, "fail to register matcher");
}
-static int flow_hw_allocate_actions(struct rte_eth_dev *dev,
+static int flow_hw_ensure_action_pools_allocated(struct rte_eth_dev *dev,
const struct rte_flow_action actions[],
struct rte_flow_error *error)
{
@@ -12383,7 +12398,7 @@ static int flow_hw_allocate_actions(struct rte_eth_dev *dev,
/* If no counters were previously allocated. */
if (!priv->hws_cpool) {
ret = mlx5_hws_cnt_pool_create(dev, MLX5_CNT_MAX,
- priv->nb_queue, NULL);
+ priv->nb_queue, NULL);
if (ret)
goto err;
}
@@ -12401,7 +12416,7 @@ static int flow_hw_allocate_actions(struct rte_eth_dev *dev,
/* If no counters were previously allocated. */
if (!priv->hws_cpool) {
ret = mlx5_hws_cnt_pool_create(dev, MLX5_CNT_MAX,
- priv->nb_queue, NULL);
+ priv->nb_queue, NULL);
if (ret)
goto err;
}
@@ -12451,12 +12466,9 @@ static int flow_hw_apply(struct rte_eth_dev *dev __rte_unused,
items, rule_actions);
flow->nt2hws->nt_rule = rule;
if (!rule) {
- if (error)
- return rte_flow_error_set(error, EINVAL,
- RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
- NULL, "fail to create rte flow");
- else
- return -EINVAL;
+ return rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+ NULL, "fail to create rte flow");
}
return 0;
}
@@ -12521,6 +12533,8 @@ static int flow_hw_create_flow(struct rte_eth_dev *dev,
.tbl_type = 0,
};
+ uint64_t item_flags = 0;
+
memset(&hw_act, 0, sizeof(hw_act));
if (attr->transfer)
tbl_type = MLX5DR_TABLE_TYPE_FDB;
@@ -12536,11 +12550,12 @@ static int flow_hw_create_flow(struct rte_eth_dev *dev,
goto error;
/* TODO TBD flow_hw_handle_tunnel_offload(). */
-
+ (*flow)->nt_rule = true;
(*flow)->nt2hws->matcher = &matcher;
- ret = __flow_dv_translate_items_hws(items, &flow_attr, &matcher.mask.buf,
- MLX5_SET_MATCHER_HS_M, 0,
- NULL, true, error);
+ ret = flow_dv_translate_items_hws(items, &flow_attr, &matcher.mask.buf,
+ MLX5_SET_MATCHER_HS_M, &item_flags,
+ NULL, error);
+
if (ret)
goto error;
@@ -12554,10 +12569,13 @@ static int flow_hw_create_flow(struct rte_eth_dev *dev,
* The output actions bit mask instead of
* looping on the actions array twice.
*/
- ret = flow_hw_allocate_actions(dev, actions, error);
+ ret = flow_hw_ensure_action_pools_allocated(dev, actions, error);
+ if (ret)
+ goto error;
/* Note: the actions should be saved in the sub-flow rule itself for reference. */
- ret = flow_hw_translate_actions(dev, attr, actions, *flow, &hw_act, external, error);
+ ret = flow_hw_translate_flow_actions(dev, attr, actions, *flow, &hw_act,
+ item_flags, external, error);
if (ret)
goto error;
@@ -12580,12 +12598,7 @@ static int flow_hw_create_flow(struct rte_eth_dev *dev,
return 0;
error:
- if (error)
- return rte_flow_error_set(error, EINVAL,
- RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
- NULL, "fail to create rte flow");
- else
- return -EINVAL;
+ return ret;
}
#endif
@@ -12599,12 +12612,12 @@ flow_hw_destroy(struct rte_eth_dev *dev, struct rte_flow_hw *flow)
if (flow->nt2hws->nt_rule) {
ret = mlx5dr_bwc_rule_destroy(flow->nt2hws->nt_rule);
- if (likely(!ret))
+ if (ret)
DRV_LOG(ERR, "bwc rule destroy failed");
}
-
+ flow->operation_type = MLX5_FLOW_HW_FLOW_OP_TYPE_DESTROY;
/* TODO: notice this function does not handle shared/static actions. */
- hw_cmpl_flow_update_or_destroy(dev, NULL, 0, NULL);
+ hw_cmpl_flow_update_or_destroy(dev, flow, 0, NULL);
/**
* TODO: TBD - Release tunnel related memory allocations(mlx5_flow_tunnel_free)
@@ -12615,6 +12628,8 @@ flow_hw_destroy(struct rte_eth_dev *dev, struct rte_flow_hw *flow)
* Notice matcher destroy will take place when matcher's list is destroyed
* , same as for DV.
*/
+ if (flow->nt2hws->flow_aux)
+ mlx5_free(flow->nt2hws->flow_aux);
}
#ifdef HAVE_MLX5_HWS_SUPPORT
@@ -12625,22 +12640,21 @@ flow_hw_destroy(struct rte_eth_dev *dev, struct rte_flow_hw *flow)
* Pointer to Ethernet device.
* @param[in] type
* Flow type.
- * @param[in] flow_idx
- * Index of flow to destroy.
+ * @param[in] flow_addr
+ * Address of flow to destroy.
*/
static void flow_hw_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
- uint32_t flow_idx)
+ uintptr_t flow_addr)
{
struct mlx5_priv *priv = dev->data->dev_private;
/* Get flow via idx */
- struct rte_flow_hw *flow = mlx5_ipool_get(priv->flows[type], flow_idx);
+ struct rte_flow_hw *flow = (struct rte_flow_hw *)flow_addr;
- DRV_LOG(DEBUG, "Non template flow index %u destroy", flow_idx);
if (!flow)
return;
flow_hw_destroy(dev, flow);
/* Release flow memory by idx */
- mlx5_ipool_free(priv->flows[type], flow_idx);
+ mlx5_ipool_free(priv->flows[type], flow->idx);
}
#endif
@@ -12663,9 +12677,9 @@ static void flow_hw_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type ty
* Perform verbose error reporting if not NULL.
*
* @return
- * A flow index on success, 0 otherwise and rte_errno is set.
+ * A flow addr on success, 0 otherwise and rte_errno is set.
*/
-static uint32_t flow_hw_list_create(struct rte_eth_dev *dev,
+static uintptr_t flow_hw_list_create(struct rte_eth_dev *dev,
enum mlx5_flow_type type,
const struct rte_flow_attr *attr,
const struct rte_flow_item items[],
@@ -12678,17 +12692,16 @@ static uint32_t flow_hw_list_create(struct rte_eth_dev *dev,
/*TODO: Handle split/expand to num_flows. */
- DRV_LOG(DEBUG, "Non template flow creation");
/* Create single flow. */
ret = flow_hw_create_flow(dev, type, attr, items, actions, external, &flow, error);
if (ret)
goto free;
if (flow)
- return flow->idx;
+ return (uintptr_t)flow;
free:
if (flow)
- flow_hw_list_destroy(dev, type, flow->idx);
+ flow_hw_list_destroy(dev, type, (uintptr_t)flow);
return 0;
}
@@ -12834,7 +12847,7 @@ mirror_format_tir(struct rte_eth_dev *dev,
table_type = get_mlx5dr_table_type(&table_cfg->attr.flow_attr);
hws_flags = mlx5_hw_act_flag[MLX5_HW_ACTION_FLAG_NONE_ROOT][table_type];
- tir_ctx = flow_hw_tir_action_register(dev, hws_flags, action);
+ tir_ctx = flow_hw_tir_action_register(dev, hws_flags, action, 0, true);
if (!tir_ctx)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION,
--
2.25.1
From 5f2411e5bbeeb8866f4a01de5e58a5c47bf87a88 Mon Sep 17 00:00:00 2001
From: Maayan Kashani <mkashani@nvidia.com>
Date: Mon, 26 Feb 2024 14:28:25 +0200
Subject: [PATCH 08/11] common/mlx5: read connection tracking attributes
Need to read maximum connection tracking
objects from HCA attributes.
Signed-off-by: Maayan Kashani <mkashani@nvidia.com>
---
drivers/common/mlx5/mlx5_devx_cmds.c | 3 +++
drivers/common/mlx5/mlx5_devx_cmds.h | 1 +
2 files changed, 4 insertions(+)
diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c
index 9952733c90..0e428842ff 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.c
+++ b/drivers/common/mlx5/mlx5_devx_cmds.c
@@ -1170,6 +1170,9 @@ mlx5_devx_cmd_query_hca_attr(void *ctx,
(rc & MLX5_CROSS_VHCA_ALLOWED_OBJS_TIR) &&
(rc & MLX5_CROSS_VHCA_ALLOWED_OBJS_FT) &&
(rc & MLX5_CROSS_VHCA_ALLOWED_OBJS_RTC);
+ if (attr->ct_offload)
+ attr->log_max_conn_track_offload = MLX5_GET(cmd_hca_cap_2, hcattr,
+ log_max_conn_track_offload);
}
if (attr->log_min_stride_wqe_sz == 0)
attr->log_min_stride_wqe_sz = MLX5_MPRQ_LOG_MIN_STRIDE_WQE_SIZE;
diff --git a/drivers/common/mlx5/mlx5_devx_cmds.h b/drivers/common/mlx5/mlx5_devx_cmds.h
index c79f8dc48d..f7383f4cfe 100644
--- a/drivers/common/mlx5/mlx5_devx_cmds.h
+++ b/drivers/common/mlx5/mlx5_devx_cmds.h
@@ -326,6 +326,7 @@ struct mlx5_hca_attr {
uint32_t wqe_based_flow_table_sup:1;
uint8_t max_header_modify_pattern_length;
uint64_t system_image_guid;
+ uint32_t log_max_conn_track_offload:5;
};
/* LAG Context. */
--
2.25.1
From 52f092243ef81427fca2fda82845077b9f8b8e3c Mon Sep 17 00:00:00 2001
From: Maayan Kashani <mkashani@nvidia.com>
Date: Sun, 28 Jan 2024 14:09:21 +0200
Subject: [PATCH 09/11] net/mlx5: support bulk actions in non template mode
Add support for encap/decap/modify header
action for non template API.
Save 1 action per bulk according to action data.
Reuse action if possible.
Store actions same as for SWS today, use same key structure.
Signed-off-by: Maayan Kashani <mkashani@nvidia.com>
---
drivers/net/mlx5/mlx5_flow.h | 44 ++++--
drivers/net/mlx5/mlx5_flow_dv.c | 268 ++++++++++++++++++++------------
drivers/net/mlx5/mlx5_flow_hw.c | 184 ++++++++++++++++++++--
3 files changed, 368 insertions(+), 128 deletions(-)
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 5bcfc1d88a..7ccc3cb7cd 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -666,6 +666,10 @@ struct mlx5_flow_dv_modify_hdr_resource {
struct mlx5_list_entry entry;
void *action; /**< Modify header action object. */
uint32_t idx;
+#ifdef HAVE_MLX5_HWS_SUPPORT
+ void *mh_dr_pattern; /**< Modify header DR pattern(HWS only). */
+#endif
+ uint64_t flags; /**< Flags for RDMA API(HWS only). */
/* Key area for hash list matching: */
uint8_t ft_type; /**< Flow table type, Rx or Tx. */
uint8_t actions_num; /**< Number of modification actions. */
@@ -1317,7 +1321,11 @@ struct rte_flow_nt2hws {
struct mlx5_flow_dv_matcher *matcher;
/**< Auxiliary data stored per flow. */
struct rte_flow_hw_aux *flow_aux;
-} __rte_packed;
+ /** Modify header pointer. */
+ struct mlx5_flow_dv_modify_hdr_resource *modify_hdr;
+ /** Encap/decap index. */
+ uint32_t rix_encap_decap;
+};
/** HWS flow struct. */
struct rte_flow_hw {
@@ -3079,14 +3087,14 @@ struct mlx5_list_entry *flow_dv_tag_clone_cb(void *tool_ctx,
void *cb_ctx);
void flow_dv_tag_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry);
-int flow_dv_modify_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
+int flow_modify_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
void *cb_ctx);
-struct mlx5_list_entry *flow_dv_modify_create_cb(void *tool_ctx, void *ctx);
-void flow_dv_modify_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
-struct mlx5_list_entry *flow_dv_modify_clone_cb(void *tool_ctx,
+struct mlx5_list_entry *flow_modify_create_cb(void *tool_ctx, void *ctx);
+void flow_modify_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_modify_clone_cb(void *tool_ctx,
struct mlx5_list_entry *oentry,
void *ctx);
-void flow_dv_modify_clone_free_cb(void *tool_ctx,
+void flow_modify_clone_free_cb(void *tool_ctx,
struct mlx5_list_entry *entry);
struct mlx5_list_entry *flow_dv_mreg_create_cb(void *tool_ctx, void *ctx);
@@ -3098,18 +3106,30 @@ struct mlx5_list_entry *flow_dv_mreg_clone_cb(void *tool_ctx,
void *ctx);
void flow_dv_mreg_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry);
-int flow_dv_encap_decap_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
+int flow_encap_decap_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
void *cb_ctx);
-struct mlx5_list_entry *flow_dv_encap_decap_create_cb(void *tool_ctx,
+struct mlx5_list_entry *flow_encap_decap_create_cb(void *tool_ctx,
void *cb_ctx);
-void flow_dv_encap_decap_remove_cb(void *tool_ctx,
+void flow_encap_decap_remove_cb(void *tool_ctx,
struct mlx5_list_entry *entry);
-struct mlx5_list_entry *flow_dv_encap_decap_clone_cb(void *tool_ctx,
+struct mlx5_list_entry *flow_encap_decap_clone_cb(void *tool_ctx,
struct mlx5_list_entry *entry,
void *cb_ctx);
-void flow_dv_encap_decap_clone_free_cb(void *tool_ctx,
+void flow_encap_decap_clone_free_cb(void *tool_ctx,
struct mlx5_list_entry *entry);
-
+int __flow_encap_decap_resource_register
+ (struct rte_eth_dev *dev,
+ struct mlx5_flow_dv_encap_decap_resource *resource,
+ bool is_root,
+ struct mlx5_flow_dv_encap_decap_resource **encap_decap,
+ struct rte_flow_error *error);
+int __flow_modify_hdr_resource_register
+ (struct rte_eth_dev *dev,
+ struct mlx5_flow_dv_modify_hdr_resource *resource,
+ struct mlx5_flow_dv_modify_hdr_resource **modify,
+ struct rte_flow_error *error);
+int flow_encap_decap_resource_release(struct rte_eth_dev *dev,
+ uint32_t encap_decap_idx);
int flow_matcher_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
void *ctx);
struct mlx5_list_entry *flow_matcher_create_cb(void *tool_ctx, void *ctx);
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index a4fde4125e..3611ffa4a1 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -97,10 +97,6 @@ union flow_dv_attr {
uint32_t attr;
};
-static int
-flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
- uint32_t encap_decap_idx);
-
static int
flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
uint32_t port_id);
@@ -4272,7 +4268,7 @@ flow_dv_validate_item_aggr_affinity(struct rte_eth_dev *dev,
}
int
-flow_dv_encap_decap_match_cb(void *tool_ctx __rte_unused,
+flow_encap_decap_match_cb(void *tool_ctx __rte_unused,
struct mlx5_list_entry *entry, void *cb_ctx)
{
struct mlx5_flow_cb_ctx *ctx = cb_ctx;
@@ -4293,7 +4289,7 @@ flow_dv_encap_decap_match_cb(void *tool_ctx __rte_unused,
}
struct mlx5_list_entry *
-flow_dv_encap_decap_create_cb(void *tool_ctx, void *cb_ctx)
+flow_encap_decap_create_cb(void *tool_ctx, void *cb_ctx)
{
struct mlx5_dev_ctx_shared *sh = tool_ctx;
struct mlx5_flow_cb_ctx *ctx = cb_ctx;
@@ -4301,14 +4297,11 @@ flow_dv_encap_decap_create_cb(void *tool_ctx, void *cb_ctx)
struct mlx5_flow_dv_encap_decap_resource *ctx_resource = ctx->data;
struct mlx5_flow_dv_encap_decap_resource *resource;
uint32_t idx;
- int ret;
+ int ret = 0;
+#ifdef HAVE_MLX5_HWS_SUPPORT
+ struct mlx5dr_action_reformat_header hdr;
+#endif
- if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
- domain = sh->fdb_domain;
- else if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
- domain = sh->rx_domain;
- else
- domain = sh->tx_domain;
/* Register new encap/decap resource. */
resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], &idx);
if (!resource) {
@@ -4318,10 +4311,29 @@ flow_dv_encap_decap_create_cb(void *tool_ctx, void *cb_ctx)
return NULL;
}
*resource = *ctx_resource;
- resource->idx = idx;
- ret = mlx5_flow_os_create_flow_action_packet_reformat(sh->cdev->ctx,
- domain, resource,
- &resource->action);
+ if (sh->config.dv_flow_en == 2) {
+#ifdef HAVE_MLX5_HWS_SUPPORT
+ hdr.sz = ctx_resource->size;
+ hdr.data = ctx_resource->buf;
+ resource->action = mlx5dr_action_create_reformat
+ (ctx->data2, (enum mlx5dr_action_type)ctx_resource->reformat_type, 1,
+ &hdr, 1, ctx_resource->flags);
+ if (!resource->action)
+ ret = -1;
+#else
+ ret = -1;
+#endif
+ } else {
+ if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
+ domain = sh->fdb_domain;
+ else if (ctx_resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_RX)
+ domain = sh->rx_domain;
+ else
+ domain = sh->tx_domain;
+ ret = mlx5_flow_os_create_flow_action_packet_reformat(sh->cdev->ctx,
+ domain, resource,
+ &resource->action);
+ }
if (ret) {
mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx);
rte_flow_error_set(ctx->error, ENOMEM,
@@ -4329,12 +4341,12 @@ flow_dv_encap_decap_create_cb(void *tool_ctx, void *cb_ctx)
NULL, "cannot create action");
return NULL;
}
-
+ resource->idx = idx;
return &resource->entry;
}
struct mlx5_list_entry *
-flow_dv_encap_decap_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
+flow_encap_decap_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
void *cb_ctx)
{
struct mlx5_dev_ctx_shared *sh = tool_ctx;
@@ -4356,7 +4368,7 @@ flow_dv_encap_decap_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
}
void
-flow_dv_encap_decap_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
+flow_encap_decap_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
{
struct mlx5_dev_ctx_shared *sh = tool_ctx;
struct mlx5_flow_dv_encap_decap_resource *res =
@@ -4365,26 +4377,11 @@ flow_dv_encap_decap_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], res->idx);
}
-/**
- * Find existing encap/decap resource or create and register a new one.
- *
- * @param[in, out] dev
- * Pointer to rte_eth_dev structure.
- * @param[in, out] resource
- * Pointer to encap/decap resource.
- * @parm[in, out] dev_flow
- * Pointer to the dev_flow.
- * @param[out] error
- * pointer to error structure.
- *
- * @return
- * 0 on success otherwise -errno and errno is set.
- */
-static int
-flow_dv_encap_decap_resource_register
- (struct rte_eth_dev *dev,
+int
+__flow_encap_decap_resource_register(struct rte_eth_dev *dev,
struct mlx5_flow_dv_encap_decap_resource *resource,
- struct mlx5_flow *dev_flow,
+ bool is_root,
+ struct mlx5_flow_dv_encap_decap_resource **encap_decap,
struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
@@ -4407,13 +4404,14 @@ flow_dv_encap_decap_resource_register
{
.ft_type = resource->ft_type,
.refmt_type = resource->reformat_type,
- .is_root = !!dev_flow->dv.group,
+ .is_root = is_root,
.reserve = 0,
}
};
struct mlx5_flow_cb_ctx ctx = {
.error = error,
.data = resource,
+ .data2 = priv->dr_ctx,
};
struct mlx5_hlist *encaps_decaps;
uint64_t key64;
@@ -4422,15 +4420,14 @@ flow_dv_encap_decap_resource_register
"encaps_decaps",
MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ,
true, true, sh,
- flow_dv_encap_decap_create_cb,
- flow_dv_encap_decap_match_cb,
- flow_dv_encap_decap_remove_cb,
- flow_dv_encap_decap_clone_cb,
- flow_dv_encap_decap_clone_free_cb,
+ flow_encap_decap_create_cb,
+ flow_encap_decap_match_cb,
+ flow_encap_decap_remove_cb,
+ flow_encap_decap_clone_cb,
+ flow_encap_decap_clone_free_cb,
error);
if (unlikely(!encaps_decaps))
return -rte_errno;
- resource->flags = dev_flow->dv.group ? 0 : 1;
key64 = __rte_raw_cksum(&encap_decap_key.v32,
sizeof(encap_decap_key.v32), 0);
if (resource->reformat_type !=
@@ -4440,9 +4437,40 @@ flow_dv_encap_decap_resource_register
entry = mlx5_hlist_register(encaps_decaps, key64, &ctx);
if (!entry)
return -rte_errno;
- resource = container_of(entry, typeof(*resource), entry);
- dev_flow->dv.encap_decap = resource;
- dev_flow->handle->dvh.rix_encap_decap = resource->idx;
+ *encap_decap = container_of(entry, typeof(*resource), entry);
+ return 0;
+}
+
+/**
+ * Find existing encap/decap resource or create and register a new one.
+ *
+ * @param[in, out] dev
+ * Pointer to rte_eth_dev structure.
+ * @param[in, out] resource
+ * Pointer to encap/decap resource.
+ * @param[in, out] dev_flow
+ * Pointer to the dev_flow.
+ * @param[out] error
+ * pointer to error structure.
+ *
+ * @return
+ * 0 on success otherwise -errno and errno is set.
+ */
+static int
+flow_dv_encap_decap_resource_register
+ (struct rte_eth_dev *dev,
+ struct mlx5_flow_dv_encap_decap_resource *resource,
+ struct mlx5_flow *dev_flow,
+ struct rte_flow_error *error)
+{
+ int ret;
+
+ resource->flags = dev_flow->dv.group ? 0 : 1;
+ ret = __flow_encap_decap_resource_register(dev, resource, !!dev_flow->dv.group,
+ &dev_flow->dv.encap_decap, error);
+ if (ret)
+ return ret;
+ dev_flow->handle->dvh.rix_encap_decap = dev_flow->dv.encap_decap->idx;
return 0;
}
@@ -6122,7 +6150,7 @@ flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags,
}
int
-flow_dv_modify_match_cb(void *tool_ctx __rte_unused,
+flow_modify_match_cb(void *tool_ctx __rte_unused,
struct mlx5_list_entry *entry, void *cb_ctx)
{
struct mlx5_flow_cb_ctx *ctx = cb_ctx;
@@ -6178,7 +6206,7 @@ flow_dv_modify_ipool_get(struct mlx5_dev_ctx_shared *sh, uint8_t index)
}
struct mlx5_list_entry *
-flow_dv_modify_create_cb(void *tool_ctx, void *cb_ctx)
+flow_modify_create_cb(void *tool_ctx, void *cb_ctx)
{
struct mlx5_dev_ctx_shared *sh = tool_ctx;
struct mlx5_flow_cb_ctx *ctx = cb_ctx;
@@ -6187,11 +6215,13 @@ flow_dv_modify_create_cb(void *tool_ctx, void *cb_ctx)
struct mlx5_flow_dv_modify_hdr_resource *ref = ctx->data;
struct mlx5_indexed_pool *ipool = flow_dv_modify_ipool_get(sh,
ref->actions_num - 1);
- int ret;
+ int ret = 0;
uint32_t data_len = ref->actions_num * sizeof(ref->actions[0]);
uint32_t key_len = sizeof(*ref) - offsetof(typeof(*ref), ft_type);
uint32_t idx;
+ struct mlx5_tbl_multi_pattern_ctx *mpctx;
+ typeof(mpctx->mh) *mh_dr_pattern = ref->mh_dr_pattern;
if (unlikely(!ipool)) {
rte_flow_error_set(ctx->error, ENOMEM,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
@@ -6205,18 +6235,30 @@ flow_dv_modify_create_cb(void *tool_ctx, void *cb_ctx)
"cannot allocate resource memory");
return NULL;
}
- rte_memcpy(RTE_PTR_ADD(entry, offsetof(typeof(*entry), ft_type)),
- RTE_PTR_ADD(ref, offsetof(typeof(*ref), ft_type)),
- key_len + data_len);
- if (entry->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
- ns = sh->fdb_domain;
- else if (entry->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
- ns = sh->tx_domain;
- else
- ns = sh->rx_domain;
- ret = mlx5_flow_os_create_flow_action_modify_header
- (sh->cdev->ctx, ns, entry,
- data_len, &entry->action);
+ rte_memcpy(&entry->ft_type,
+ RTE_PTR_ADD(ref, offsetof(typeof(*ref), ft_type)),
+ key_len + data_len);
+ if (sh->config.dv_flow_en == 2) {
+#ifdef HAVE_MLX5_HWS_SUPPORT
+ entry->action = mlx5dr_action_create_modify_header(ctx->data2,
+ mh_dr_pattern->elements_num,
+ mh_dr_pattern->pattern, 0, ref->flags);
+ if (!entry->action)
+ ret = -1;
+#else
+ ret = -1;
+#endif
+ } else {
+ if (entry->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
+ ns = sh->fdb_domain;
+ else if (entry->ft_type == MLX5DV_FLOW_TABLE_TYPE_NIC_TX)
+ ns = sh->tx_domain;
+ else
+ ns = sh->rx_domain;
+ ret = mlx5_flow_os_create_flow_action_modify_header
+ (sh->cdev->ctx, ns, entry,
+ data_len, &entry->action);
+ }
if (ret) {
mlx5_ipool_free(sh->mdh_ipools[ref->actions_num - 1], idx);
rte_flow_error_set(ctx->error, ENOMEM,
@@ -6229,7 +6271,7 @@ flow_dv_modify_create_cb(void *tool_ctx, void *cb_ctx)
}
struct mlx5_list_entry *
-flow_dv_modify_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
+flow_modify_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
void *cb_ctx)
{
struct mlx5_dev_ctx_shared *sh = tool_ctx;
@@ -6253,7 +6295,7 @@ flow_dv_modify_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
}
void
-flow_dv_modify_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
+flow_modify_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
{
struct mlx5_dev_ctx_shared *sh = tool_ctx;
struct mlx5_flow_dv_modify_hdr_resource *res =
@@ -6521,27 +6563,11 @@ flow_dv_validate_action_sample(uint64_t *action_flags,
return 0;
}
-/**
- * Find existing modify-header resource or create and register a new one.
- *
- * @param dev[in, out]
- * Pointer to rte_eth_dev structure.
- * @param[in, out] resource
- * Pointer to modify-header resource.
- * @parm[in, out] dev_flow
- * Pointer to the dev_flow.
- * @param[out] error
- * pointer to error structure.
- *
- * @return
- * 0 on success otherwise -errno and errno is set.
- */
-static int
-flow_dv_modify_hdr_resource_register
- (struct rte_eth_dev *dev,
- struct mlx5_flow_dv_modify_hdr_resource *resource,
- struct mlx5_flow *dev_flow,
- struct rte_flow_error *error)
+int
+__flow_modify_hdr_resource_register(struct rte_eth_dev *dev,
+ struct mlx5_flow_dv_modify_hdr_resource *resource,
+ struct mlx5_flow_dv_modify_hdr_resource **modify,
+ struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_dev_ctx_shared *sh = priv->sh;
@@ -6552,6 +6578,7 @@ flow_dv_modify_hdr_resource_register
struct mlx5_flow_cb_ctx ctx = {
.error = error,
.data = resource,
+ .data2 = priv->dr_ctx,
};
struct mlx5_hlist *modify_cmds;
uint64_t key64;
@@ -6560,15 +6587,14 @@ flow_dv_modify_hdr_resource_register
"hdr_modify",
MLX5_FLOW_HDR_MODIFY_HTABLE_SZ,
true, false, sh,
- flow_dv_modify_create_cb,
- flow_dv_modify_match_cb,
- flow_dv_modify_remove_cb,
- flow_dv_modify_clone_cb,
- flow_dv_modify_clone_free_cb,
+ flow_modify_create_cb,
+ flow_modify_match_cb,
+ flow_modify_remove_cb,
+ flow_modify_clone_cb,
+ flow_modify_clone_free_cb,
error);
if (unlikely(!modify_cmds))
return -rte_errno;
- resource->root = !dev_flow->dv.group;
if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
resource->root))
return rte_flow_error_set(error, EOVERFLOW,
@@ -6578,11 +6604,37 @@ flow_dv_modify_hdr_resource_register
entry = mlx5_hlist_register(modify_cmds, key64, &ctx);
if (!entry)
return -rte_errno;
- resource = container_of(entry, typeof(*resource), entry);
- dev_flow->handle->dvh.modify_hdr = resource;
+ *modify = container_of(entry, typeof(*resource), entry);
return 0;
}
+/**
+ * Find existing modify-header resource or create and register a new one.
+ *
+ * @param dev[in, out]
+ * Pointer to rte_eth_dev structure.
+ * @param[in, out] resource
+ * Pointer to modify-header resource.
+ * @param[in, out] dev_flow
+ * Pointer to the dev_flow.
+ * @param[out] error
+ * pointer to error structure.
+ *
+ * @return
+ * 0 on success otherwise -errno and errno is set.
+ */
+static int
+flow_dv_modify_hdr_resource_register
+ (struct rte_eth_dev *dev,
+ struct mlx5_flow_dv_modify_hdr_resource *resource,
+ struct mlx5_flow *dev_flow,
+ struct rte_flow_error *error)
+{
+ resource->root = !dev_flow->dv.group;
+ return __flow_modify_hdr_resource_register(dev, resource,
+ &dev_flow->handle->dvh.modify_hdr, error);
+}
+
/**
* Get DV flow counter by index.
*
@@ -12403,7 +12455,7 @@ flow_dv_sample_sub_actions_release(struct rte_eth_dev *dev,
act_res->rix_hrxq = 0;
}
if (act_res->rix_encap_decap) {
- flow_dv_encap_decap_resource_release(dev,
+ flow_encap_decap_resource_release(dev,
act_res->rix_encap_decap);
act_res->rix_encap_decap = 0;
}
@@ -15840,13 +15892,18 @@ flow_dv_matcher_release(struct rte_eth_dev *dev,
}
void
-flow_dv_encap_decap_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
+flow_encap_decap_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
{
struct mlx5_dev_ctx_shared *sh = tool_ctx;
struct mlx5_flow_dv_encap_decap_resource *res =
container_of(entry, typeof(*res), entry);
- claim_zero(mlx5_flow_os_destroy_flow_action(res->action));
+#ifdef HAVE_MLX5_HWS_SUPPORT
+ if (sh->config.dv_flow_en == 2)
+ claim_zero(mlx5dr_action_destroy(res->action));
+ else
+#endif
+ claim_zero(mlx5_flow_os_destroy_flow_action(res->action));
mlx5_ipool_free(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], res->idx);
}
@@ -15861,8 +15918,8 @@ flow_dv_encap_decap_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
* @return
* 1 while a reference on it exists, 0 when freed.
*/
-static int
-flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
+int
+flow_encap_decap_resource_release(struct rte_eth_dev *dev,
uint32_t encap_decap_idx)
{
struct mlx5_priv *priv = dev->data->dev_private;
@@ -15902,13 +15959,18 @@ flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
}
void
-flow_dv_modify_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
+flow_modify_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
{
struct mlx5_flow_dv_modify_hdr_resource *res =
container_of(entry, typeof(*res), entry);
struct mlx5_dev_ctx_shared *sh = tool_ctx;
- claim_zero(mlx5_flow_os_destroy_flow_action(res->action));
+#ifdef HAVE_MLX5_HWS_SUPPORT
+ if (sh->config.dv_flow_en == 2)
+ claim_zero(mlx5dr_action_destroy(res->action));
+ else
+#endif
+ claim_zero(mlx5_flow_os_destroy_flow_action(res->action));
mlx5_ipool_free(sh->mdh_ipools[res->actions_num - 1], res->idx);
}
@@ -16277,7 +16339,7 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
if (dev_handle->dvh.rix_dest_array)
flow_dv_dest_array_resource_release(dev, dev_handle);
if (dev_handle->dvh.rix_encap_decap)
- flow_dv_encap_decap_resource_release(dev,
+ flow_encap_decap_resource_release(dev,
dev_handle->dvh.rix_encap_decap);
if (dev_handle->dvh.modify_hdr)
flow_dv_modify_hdr_resource_release(dev, dev_handle);
diff --git a/drivers/net/mlx5/mlx5_flow_hw.c b/drivers/net/mlx5/mlx5_flow_hw.c
index d768968676..41f20ed222 100644
--- a/drivers/net/mlx5/mlx5_flow_hw.c
+++ b/drivers/net/mlx5/mlx5_flow_hw.c
@@ -9,6 +9,7 @@
#include <mlx5_malloc.h>
#include "mlx5.h"
+#include "mlx5_common.h"
#include "mlx5_defs.h"
#include "mlx5_flow.h"
#include "mlx5_rx.h"
@@ -224,6 +225,22 @@ mlx5_multi_pattern_reformat_to_index(enum mlx5dr_action_type type)
return -1;
}
+/* Include only supported reformat actions for BWC non template API. */
+static __rte_always_inline int
+mlx5_bwc_multi_pattern_reformat_to_index(enum mlx5dr_action_type type)
+{
+ switch (type) {
+ case MLX5DR_ACTION_TYP_REFORMAT_TNL_L2_TO_L2:
+ case MLX5DR_ACTION_TYP_REFORMAT_L2_TO_TNL_L2:
+ case MLX5DR_ACTION_TYP_REFORMAT_TNL_L3_TO_L2:
+ case MLX5DR_ACTION_TYP_REFORMAT_L2_TO_TNL_L3:
+ return mlx5_multi_pattern_reformat_to_index(type);
+ default:
+ break;
+ }
+ return -1;
+}
+
static __rte_always_inline enum mlx5dr_action_type
mlx5_multi_pattern_reformat_index_to_type(uint32_t ix)
{
@@ -1317,11 +1334,13 @@ flow_hw_converted_mhdr_cmds_append(struct mlx5_hw_modify_header_action *mhdr,
static __rte_always_inline void
flow_hw_modify_field_init(struct mlx5_hw_modify_header_action *mhdr,
- struct rte_flow_actions_template *at)
+ struct rte_flow_actions_template *at,
+ bool nt_mode)
{
memset(mhdr, 0, sizeof(*mhdr));
/* Modify header action without any commands is shared by default. */
- mhdr->shared = true;
+ if (!(nt_mode))
+ mhdr->shared = true;
mhdr->pos = at->mhdr_off;
}
@@ -2124,7 +2143,6 @@ mlx5_tbl_translate_modify_header(struct rte_eth_dev *dev,
} else {
typeof(mp_ctx->mh) *mh = &mp_ctx->mh;
uint32_t idx = mh->elements_num;
-
mh->pattern[mh->elements_num++] = pattern;
acts->mhdr->multi_pattern = 1;
acts->rule_acts[mhdr_ix].modify_header.pattern_idx = idx;
@@ -2270,7 +2288,7 @@ __flow_hw_translate_actions_template(struct rte_eth_dev *dev,
uint32_t target_grp = 0;
int table_type;
- flow_hw_modify_field_init(&mhdr, at);
+ flow_hw_modify_field_init(&mhdr, at, nt_mode);
if (attr->transfer)
type = MLX5DR_TABLE_TYPE_FDB;
else if (attr->egress)
@@ -3226,6 +3244,7 @@ flow_hw_actions_construct(struct rte_eth_dev *dev,
flow->res_idx - mp_segment->head_index;
rule_acts[pos].modify_header.data =
(uint8_t *)ap->mhdr_cmd;
+ MLX5_ASSERT(hw_acts->mhdr->mhdr_cmds_num <= MLX5_MHDR_MAX_CMD);
rte_memcpy(ap->mhdr_cmd, hw_acts->mhdr->mhdr_cmds,
sizeof(*ap->mhdr_cmd) * hw_acts->mhdr->mhdr_cmds_num);
}
@@ -12225,12 +12244,106 @@ static int flow_hw_prepare(struct rte_eth_dev *dev,
/*TODO: consider if other allocation is needed for actions translate. */
return 0;
}
+#define FLOW_HW_SET_DV_FIELDS(flow_attr, root, flags) \
+{ \
+ typeof(flow_attr) _flow_attr = (flow_attr); \
+ if (_flow_attr->transfer) \
+ dv_resource.ft_type = MLX5DV_FLOW_TABLE_TYPE_FDB; \
+ else \
+ dv_resource.ft_type = _flow_attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX : \
+ MLX5DV_FLOW_TABLE_TYPE_NIC_RX; \
+ root = _flow_attr->group ? 0 : 1; \
+ flags = mlx5_hw_act_flag[!!_flow_attr->group][get_mlx5dr_table_type(_flow_attr)]; \
+}
+
+static int
+flow_hw_modify_hdr_resource_register
+ (struct rte_eth_dev *dev,
+ struct rte_flow_template_table *table,
+ struct mlx5_hw_actions *hw_acts,
+ struct rte_flow_hw *dev_flow,
+ struct rte_flow_error *error)
+{
+ struct rte_flow_attr *attr = &table->cfg.attr.flow_attr;
+ struct mlx5_flow_dv_modify_hdr_resource *dv_resource_ptr = NULL;
+ struct mlx5_flow_dv_modify_hdr_resource dv_resource;
+ struct mlx5_tbl_multi_pattern_ctx *mpctx = &table->mpctx;
+ int ret;
+
+ if (hw_acts->mhdr) {
+ dv_resource.actions_num = hw_acts->mhdr->mhdr_cmds_num;
+ memcpy(dv_resource.actions, hw_acts->mhdr->mhdr_cmds,
+ sizeof(struct mlx5_modification_cmd) * dv_resource.actions_num);
+ } else {
+ return 0;
+ }
+ FLOW_HW_SET_DV_FIELDS(attr, dv_resource.root, dv_resource.flags);
+ /* Save a pointer to the pattern needed for DR layer created on actions translate. */
+ dv_resource.mh_dr_pattern = &table->mpctx.mh;
+ ret = __flow_modify_hdr_resource_register(dev, &dv_resource,
+ &dv_resource_ptr, error);
+ if (ret)
+ return ret;
+ MLX5_ASSERT(dv_resource_ptr);
+ dev_flow->nt2hws->modify_hdr = dv_resource_ptr;
+ /* keep action for the rule construction. */
+ mpctx->segments[0].mhdr_action = dv_resource_ptr->action;
+ /* Bulk size is 1, so index is 1. */
+ dev_flow->res_idx = 1;
+ return 0;
+}
+
+static int
+flow_hw_encap_decap_resource_register
+ (struct rte_eth_dev *dev,
+ struct rte_flow_template_table *table,
+ struct mlx5_hw_actions *hw_acts,
+ struct rte_flow_hw *dev_flow,
+ struct rte_flow_error *error)
+{
+ struct rte_flow_attr *attr = &table->cfg.attr.flow_attr;
+ struct mlx5_flow_dv_encap_decap_resource *dv_resource_ptr = NULL;
+ struct mlx5_flow_dv_encap_decap_resource dv_resource;
+ struct mlx5_tbl_multi_pattern_ctx *mpctx = &table->mpctx;
+ int ret;
+ bool is_root;
+ int ix;
+
+ if (hw_acts->encap_decap)
+ dv_resource.reformat_type = hw_acts->encap_decap->action_type;
+ else
+ return 0;
+ ix = mlx5_bwc_multi_pattern_reformat_to_index((enum mlx5dr_action_type)
+ dv_resource.reformat_type);
+ if (ix < 0)
+ return ix;
+ typeof(mpctx->reformat[0]) *reformat = mpctx->reformat + ix;
+ if (!reformat->elements_num)
+ return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
+ NULL, "No reformat action exist in the table.");
+ dv_resource.size = reformat->reformat_hdr->sz;
+ FLOW_HW_SET_DV_FIELDS(attr, is_root, dv_resource.flags);
+ MLX5_ASSERT(dv_resource.size <= MLX5_ENCAP_MAX_LEN);
+ memcpy(dv_resource.buf, reformat->reformat_hdr->data, dv_resource.size);
+ ret = __flow_encap_decap_resource_register(dev, &dv_resource, is_root,
+ &dv_resource_ptr, error);
+ if (ret)
+ return ret;
+ MLX5_ASSERT(dv_resource_ptr);
+ dev_flow->nt2hws->rix_encap_decap = dv_resource_ptr->idx;
+ /* keep action for the rule construction. */
+ mpctx->segments[0].reformat_action[ix] = dv_resource_ptr->action;
+ /* Bulk size is 1, so index is 1. */
+ dev_flow->res_idx = 1;
+ return 0;
+}
static int
flow_hw_translate_flow_actions(struct rte_eth_dev *dev,
const struct rte_flow_attr *attr,
const struct rte_flow_action actions[],
struct rte_flow_hw *flow,
+ struct mlx5_flow_hw_action_params *ap,
struct mlx5_hw_actions *hw_acts,
uint64_t item_flags,
bool external,
@@ -12249,12 +12362,28 @@ flow_hw_translate_flow_actions(struct rte_eth_dev *dev,
.transfer = attr->transfer,
};
struct rte_flow_action masks[MLX5_HW_MAX_ACTS];
- struct mlx5_flow_hw_action_params ap;
+ struct rte_flow_action_raw_encap encap_conf;
+ struct rte_flow_action_modify_field mh_conf[MLX5_HW_MAX_ACTS];
+
memset(&masks, 0, sizeof(masks));
int i = -1;
do {
i++;
masks[i].type = actions[i].type;
+ if (masks[i].type == RTE_FLOW_ACTION_TYPE_RAW_ENCAP) {
+ memset(&encap_conf, 0x00, sizeof(encap_conf));
+ encap_conf.size = ((const struct rte_flow_action_raw_encap *)
+ (actions[i].conf))->size;
+ masks[i].conf = &encap_conf;
+ }
+ if (masks[i].type == RTE_FLOW_ACTION_TYPE_MODIFY_FIELD) {
+ const struct rte_flow_action_modify_field *conf = actions[i].conf;
+ memset(&mh_conf, 0xff, sizeof(mh_conf[i]));
+ mh_conf[i].operation = conf->operation;
+ mh_conf[i].dst.field = conf->dst.field;
+ mh_conf[i].src.field = conf->src.field;
+ masks[i].conf = &mh_conf[i];
+ }
} while (masks[i].type != RTE_FLOW_ACTION_TYPE_END);
RTE_SET_USED(action_flags);
/* The group in the attribute translation was done in advance. */
@@ -12279,8 +12408,6 @@ flow_hw_translate_flow_actions(struct rte_eth_dev *dev,
ret = -rte_errno;
goto end;
}
- if (ret)
- goto clean_up;
grp.group_id = src_group;
table->grp = &grp;
table->type = table_type;
@@ -12290,19 +12417,25 @@ flow_hw_translate_flow_actions(struct rte_eth_dev *dev,
table->ats[0].action_template = at;
ret = __flow_hw_translate_actions_template(dev, &table->cfg, hw_acts, at,
&table->mpctx, true, error);
+ if (ret)
+ goto end;
+ /* handle bulk actions register. */
+ ret = flow_hw_encap_decap_resource_register(dev, table, hw_acts, flow, error);
+ if (ret)
+ goto clean_up;
+ ret = flow_hw_modify_hdr_resource_register(dev, table, hw_acts, flow, error);
if (ret)
goto clean_up;
table->ats[0].acts = *hw_acts;
- ret = flow_hw_actions_construct(dev, flow, &ap,
+ ret = flow_hw_actions_construct(dev, flow, ap,
&table->ats[0], item_flags, table,
actions, hw_acts->rule_acts, 0, error);
if (ret)
goto clean_up;
-
goto end;
clean_up:
/* Make sure that there is no garbage in the actions. */
- __flow_hw_actions_release(dev, hw_acts);
+ __flow_hw_action_template_destroy(dev, hw_acts);
end:
if (table)
mlx5_free(table);
@@ -12508,6 +12641,7 @@ static int flow_hw_create_flow(struct rte_eth_dev *dev,
{
int ret;
struct mlx5_hw_actions hw_act;
+ struct mlx5_flow_hw_action_params ap;
struct mlx5_flow_dv_matcher matcher = {
.mask = {
.size = sizeof(matcher.mask.buf),
@@ -12574,7 +12708,7 @@ static int flow_hw_create_flow(struct rte_eth_dev *dev,
goto error;
/* Note: the actions should be saved in the sub-flow rule itself for reference. */
- ret = flow_hw_translate_flow_actions(dev, attr, actions, *flow, &hw_act,
+ ret = flow_hw_translate_flow_actions(dev, attr, actions, *flow, &ap, &hw_act,
item_flags, external, error);
if (ret)
goto error;
@@ -12595,9 +12729,19 @@ static int flow_hw_create_flow(struct rte_eth_dev *dev,
if (ret)
goto error;
}
- return 0;
-
+ ret = 0;
error:
+ /*
+ * Release memory allocated.
+ * Cannot use __flow_hw_actions_release(dev, &hw_act);
+ * since it destroys the actions as well.
+ */
+ if (hw_act.encap_decap)
+ mlx5_free(hw_act.encap_decap);
+ if (hw_act.push_remove)
+ mlx5_free(hw_act.push_remove);
+ if (hw_act.mhdr)
+ mlx5_free(hw_act.mhdr);
return ret;
}
#endif
@@ -12606,6 +12750,7 @@ static void
flow_hw_destroy(struct rte_eth_dev *dev, struct rte_flow_hw *flow)
{
int ret;
+ struct mlx5_priv *priv = dev->data->dev_private;
if (!flow || !flow->nt2hws)
return;
@@ -12630,6 +12775,19 @@ flow_hw_destroy(struct rte_eth_dev *dev, struct rte_flow_hw *flow)
*/
if (flow->nt2hws->flow_aux)
mlx5_free(flow->nt2hws->flow_aux);
+
+ if (flow->nt2hws->rix_encap_decap) {
+ ret = flow_encap_decap_resource_release(dev, flow->nt2hws->rix_encap_decap);
+ if (ret)
+ DRV_LOG(ERR, "failed to release encap decap.");
+ }
+ if (flow->nt2hws->modify_hdr) {
+ MLX5_ASSERT(flow->nt2hws->modify_hdr->action);
+ ret = mlx5_hlist_unregister(priv->sh->modify_cmds,
+ &flow->nt2hws->modify_hdr->entry);
+ if (ret)
+ DRV_LOG(ERR, "failed to release modify action.");
+ }
}
#ifdef HAVE_MLX5_HWS_SUPPORT
--
2.25.1
From 5f3ca4ddaf1764ecd78f66629655508fd39b2aa7 Mon Sep 17 00:00:00 2001
From: Maayan Kashani <mkashani@nvidia.com>
Date: Sun, 10 Mar 2024 16:17:06 +0200
Subject: [PATCH 10/11] net/mlx5: use non const max number for ASO actions
For ASO max allocations in non-template mode,
Read FW capabilities instead of using consts.
Signed-off-by: Maayan Kashani <mkashani@nvidia.com>
---
drivers/net/mlx5/mlx5.h | 17 ++++++++++++-----
drivers/net/mlx5/mlx5_flow_hw.c | 13 +++++++++----
drivers/net/mlx5/mlx5_flow_meter.c | 25 +++++++++++++++++++++----
3 files changed, 42 insertions(+), 13 deletions(-)
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 67986a00b4..e635907c52 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -792,12 +792,18 @@ struct mlx5_dev_shared_port {
/* Only yellow color valid. */
#define MLX5_MTR_POLICY_MODE_OY 3
+/* Max number of meters. */
+#define MLX5_MTR_MAX(priv) (mlx5_flow_mtr_max_get(priv))
/* Max number of meters allocated in non template mode. */
-#define MLX5_MTR_NT_MAX (1 << 23)
-/* Max number of connection tracking allocated in non template mode */
-#define MLX5_CT_NT_MAX (1 << 23)
-/* Max number of counters allocated in non template mode */
-#define MLX5_CNT_MAX (1 << 23)
+#define MLX5_MTR_NT_MAX(priv) (MLX5_MTR_MAX(priv) >> 1)
+/* Max number of connection tracking. */
+#define MLX5_CT_MAX(priv) (1 << (priv)->sh->cdev->config.hca_attr.log_max_conn_track_offload)
+/* Max number of connection tracking allocated in non template mode. */
+#define MLX5_CT_NT_MAX(priv) (MLX5_CT_MAX(priv) >> 1)
+/* Max number of counters. */
+#define MLX5_CNT_MAX(priv) ((priv)->sh->hws_max_nb_counters)
+/* Max number of counters allocated in non template mode. */
+#define MLX5_CNT_NT_MAX(priv) (MLX5_CNT_MAX(priv) >> 1)
enum mlx5_meter_domain {
MLX5_MTR_DOMAIN_INGRESS,
@@ -2423,6 +2429,7 @@ mlx5_flow_meter_hierarchy_get_final_policy(struct rte_eth_dev *dev,
int mlx5_flow_meter_flush(struct rte_eth_dev *dev,
struct rte_mtr_error *error);
void mlx5_flow_meter_rxq_flush(struct rte_eth_dev *dev);
+uint32_t mlx5_flow_mtr_max_get(struct mlx5_priv *priv);
/* mlx5_os.c */
diff --git a/drivers/net/mlx5/mlx5_flow_hw.c b/drivers/net/mlx5/mlx5_flow_hw.c
index 41f20ed222..3022a86344 100644
--- a/drivers/net/mlx5/mlx5_flow_hw.c
+++ b/drivers/net/mlx5/mlx5_flow_hw.c
@@ -12522,6 +12522,7 @@ static int flow_hw_ensure_action_pools_allocated(struct rte_eth_dev *dev,
bool actions_end = false;
struct mlx5_priv *priv = dev->data->dev_private;
int ret;
+ uint obj_num;
for (; !actions_end; actions++) {
switch ((int)actions->type) {
@@ -12530,7 +12531,8 @@ static int flow_hw_ensure_action_pools_allocated(struct rte_eth_dev *dev,
if (!priv->hws_age_req) {
/* If no counters were previously allocated. */
if (!priv->hws_cpool) {
- ret = mlx5_hws_cnt_pool_create(dev, MLX5_CNT_MAX,
+ obj_num = MLX5_CNT_NT_MAX(priv);
+ ret = mlx5_hws_cnt_pool_create(dev, obj_num,
priv->nb_queue, NULL);
if (ret)
goto err;
@@ -12548,7 +12550,8 @@ static int flow_hw_ensure_action_pools_allocated(struct rte_eth_dev *dev,
case RTE_FLOW_ACTION_TYPE_COUNT:
/* If no counters were previously allocated. */
if (!priv->hws_cpool) {
- ret = mlx5_hws_cnt_pool_create(dev, MLX5_CNT_MAX,
+ obj_num = MLX5_CNT_NT_MAX(priv);
+ ret = mlx5_hws_cnt_pool_create(dev, obj_num,
priv->nb_queue, NULL);
if (ret)
goto err;
@@ -12557,7 +12560,8 @@ static int flow_hw_ensure_action_pools_allocated(struct rte_eth_dev *dev,
case RTE_FLOW_ACTION_TYPE_CONNTRACK:
/* If no CT were previously allocated. */
if (!priv->hws_ctpool) {
- ret = mlx5_flow_ct_init(dev, MLX5_CT_NT_MAX, priv->nb_queue);
+ obj_num = MLX5_CT_NT_MAX(priv);
+ ret = mlx5_flow_ct_init(dev, obj_num, priv->nb_queue);
if (ret)
goto err;
}
@@ -12565,7 +12569,8 @@ static int flow_hw_ensure_action_pools_allocated(struct rte_eth_dev *dev,
case RTE_FLOW_ACTION_TYPE_METER_MARK:
/* If no meters were previously allocated. */
if (!priv->hws_mpool) {
- ret = mlx5_flow_meter_init(dev, MLX5_MTR_NT_MAX, 0, 0,
+ obj_num = MLX5_MTR_NT_MAX(priv);
+ ret = mlx5_flow_meter_init(dev, obj_num, 0, 0,
priv->nb_queue);
if (ret)
goto err;
diff --git a/drivers/net/mlx5/mlx5_flow_meter.c b/drivers/net/mlx5/mlx5_flow_meter.c
index da3289b218..19d8607070 100644
--- a/drivers/net/mlx5/mlx5_flow_meter.c
+++ b/drivers/net/mlx5/mlx5_flow_meter.c
@@ -704,6 +704,26 @@ mlx5_flow_meter_param_fill(struct mlx5_flow_meter_profile *fmp,
return 0;
}
+/**
+ * Callback to get MTR maximum objects number.
+ *
+ * @param[in] priv
+ * Pointer to Ethernet device.
+ *
+ * @return
+ * Max number of meters.
+ */
+uint32_t
+mlx5_flow_mtr_max_get(struct mlx5_priv *priv)
+{
+ struct mlx5_hca_qos_attr *qattr = &priv->sh->cdev->config.hca_attr.qos;
+
+ /* Max number of meters. */
+ return ((priv->sh->meter_aso_en) ?
+ 1 << (qattr->log_max_num_meter_aso + 1) :
+ qattr->log_max_flow_meter);
+}
+
/**
* Callback to get MTR capabilities.
*
@@ -730,14 +750,11 @@ mlx5_flow_mtr_cap_get(struct rte_eth_dev *dev,
RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL,
"Meter is not supported");
memset(cap, 0, sizeof(*cap));
+ cap->n_max = mlx5_flow_mtr_max_get(priv);
if (priv->sh->meter_aso_en) {
- /* 2 meters per one ASO cache line. */
- cap->n_max = 1 << (qattr->log_max_num_meter_aso + 1);
cap->srtcm_rfc2697_packet_mode_supported = 1;
cap->trtcm_rfc2698_packet_mode_supported = 1;
cap->trtcm_rfc4115_packet_mode_supported = 1;
- } else {
- cap->n_max = 1 << qattr->log_max_flow_meter;
}
cap->srtcm_rfc2697_byte_mode_supported = 1;
cap->trtcm_rfc2698_byte_mode_supported = 1;
--
2.25.1
From 3833d67a02a9df5bda034c236923c6353ed363f6 Mon Sep 17 00:00:00 2001
From: Maayan Kashani <mkashani@nvidia.com>
Date: Sun, 10 Mar 2024 16:22:53 +0200
Subject: [PATCH 11/11] net/mlx5: initial design changes
Change flow_drv_list_create/destroy to mlx5_flow_list_create/destroy.
Remove resource release function inlining.
Check number of queues in template mode in hw configure function.
Use user priority to calculate matcher priority.
Signed-off-by: Maayan Kashani <mkashani@nvidia.com>
---
drivers/net/mlx5/mlx5_flow.c | 70 +++++++---------------
drivers/net/mlx5/mlx5_flow_hw.c | 100 +++++++++++++++-----------------
2 files changed, 68 insertions(+), 102 deletions(-)
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 68f20300a5..32d6defdb0 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -4928,19 +4928,6 @@ flow_check_hairpin_split(struct rte_eth_dev *dev,
return 0;
}
-/* Declare flow create/destroy prototype in advance. */
-
-static uintptr_t
-flow_drv_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
- const struct rte_flow_attr *attr,
- const struct rte_flow_item items[],
- const struct rte_flow_action actions[],
- bool external, struct rte_flow_error *error);
-
-static void
-flow_drv_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
- uintptr_t flow_idx);
-
int
flow_dv_mreg_match_cb(void *tool_ctx __rte_unused,
struct mlx5_list_entry *entry, void *cb_ctx)
@@ -5059,7 +5046,7 @@ flow_dv_mreg_create_cb(void *tool_ctx, void *cb_ctx)
* be applied, removed, deleted in arbitrary order
* by list traversing.
*/
- mcp_res->rix_flow = flow_drv_list_create(dev, MLX5_FLOW_TYPE_MCP,
+ mcp_res->rix_flow = mlx5_flow_list_create(dev, MLX5_FLOW_TYPE_MCP,
&attr, items, actions, false, error);
if (!mcp_res->rix_flow) {
mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], idx);
@@ -5153,7 +5140,7 @@ flow_dv_mreg_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
struct mlx5_priv *priv = dev->data->dev_private;
MLX5_ASSERT(mcp_res->rix_flow);
- flow_drv_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow);
+ mlx5_flow_list_destroy(dev, MLX5_FLOW_TYPE_MCP, mcp_res->rix_flow);
mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MCP], mcp_res->idx);
}
@@ -7540,7 +7527,7 @@ mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev)
};
struct rte_flow_error error;
- return (void *)(uintptr_t)flow_drv_list_create(dev, MLX5_FLOW_TYPE_CTL,
+ return (void *)(uintptr_t)mlx5_flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
&attr, &pattern,
actions, false, &error);
}
@@ -7608,14 +7595,14 @@ mlx5_flow_create_devx_sq_miss_flow(struct rte_eth_dev *dev, uint32_t sq_num)
* Creates group 0, highest priority jump flow.
* Matches txq to bypass kernel packets.
*/
- if (flow_drv_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, actions,
+ if (mlx5_flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern, actions,
false, &error) == 0)
return 0;
/* Create group 1, lowest priority redirect flow for txq. */
attr.group = 1;
actions[0].conf = &port;
actions[0].type = RTE_FLOW_ACTION_TYPE_PORT_ID;
- return flow_drv_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern,
+ return mlx5_flow_list_create(dev, MLX5_FLOW_TYPE_CTL, &attr, pattern,
actions, false, &error);
}
@@ -7771,7 +7758,7 @@ mlx5_flow_cache_flow_toggle(struct rte_eth_dev *dev, bool orig_prio)
flow_info->flow_idx_low_prio);
if (high && low) {
RTE_SWAP(*low, *high);
- flow_drv_list_destroy(dev, MLX5_FLOW_TYPE_GEN,
+ mlx5_flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN,
flow_info->flow_idx_low_prio);
flow_info->flow_idx_high_prio = 0;
}
@@ -7785,7 +7772,7 @@ mlx5_flow_cache_flow_toggle(struct rte_eth_dev *dev, bool orig_prio)
while (flow_info) {
if (flow_info->orig_prio != flow_info->attr.priority) {
if (flow_info->flow_idx_high_prio)
- flow_drv_list_destroy(dev, MLX5_FLOW_TYPE_GEN,
+ mlx5_flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN,
flow_info->flow_idx_high_prio);
else
break;
@@ -7940,12 +7927,13 @@ mlx5_flow_create(struct rte_eth_dev *dev,
RTE_PMD_MLX5_FLOW_ENGINE_FLAG_STANDBY_DUP_INGRESS)))
new_attr->priority += 1;
}
- flow_idx = flow_drv_list_create(dev, MLX5_FLOW_TYPE_GEN, attr, items, actions, true, error);
+ flow_idx = mlx5_flow_list_create(dev, MLX5_FLOW_TYPE_GEN, attr, items, actions,
+ true, error);
if (!flow_idx)
return NULL;
if (unlikely(mlx5_need_cache_flow(priv, attr))) {
if (mlx5_flow_cache_flow_info(dev, attr, prio, items, actions, flow_idx)) {
- flow_drv_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx);
+ mlx5_flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx);
flow_idx = 0;
}
}
@@ -7958,17 +7946,6 @@ mlx5_flow_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
const struct rte_flow_item items[],
const struct rte_flow_action actions[],
bool external, struct rte_flow_error *error)
-{
- return flow_drv_list_create(dev, type, attr, items, actions, external,
- error);
-}
-
-uintptr_t
-flow_drv_list_create(struct rte_eth_dev *dev, enum mlx5_flow_type type,
- const struct rte_flow_attr *attr,
- const struct rte_flow_item items[],
- const struct rte_flow_action actions[],
- bool external, struct rte_flow_error *error)
{
const struct mlx5_flow_driver_ops *fops;
enum mlx5_flow_drv_type drv_type = flow_get_drv_type(dev, attr);
@@ -8017,8 +7994,8 @@ flow_legacy_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
mlx5_ipool_free(priv->flows[type], flow_idx);
}
-static void
-flow_drv_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
+void
+mlx5_flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
uintptr_t flow_idx)
{
const struct mlx5_flow_driver_ops *fops;
@@ -8029,13 +8006,6 @@ flow_drv_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
fops->list_destroy(dev, type, flow_idx);
}
-void
-mlx5_flow_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
- uintptr_t flow_idx)
-{
- flow_drv_list_destroy(dev, type, flow_idx);
-}
-
/**
* Destroy all flows.
*
@@ -8064,9 +8034,9 @@ mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type,
#endif
MLX5_IPOOL_FOREACH(priv->flows[type], fidx, flow) {
if (priv->sh->config.dv_flow_en == 2) {
- flow_drv_list_destroy(dev, type, (uintptr_t)flow);
+ mlx5_flow_list_destroy(dev, type, (uintptr_t)flow);
} else {
- flow_drv_list_destroy(dev, type, fidx);
+ mlx5_flow_list_destroy(dev, type, fidx);
}
if (unlikely(mlx5_need_cache_flow(priv, NULL) && type == MLX5_FLOW_TYPE_GEN)) {
flow_info = LIST_FIRST(&mode_info->hot_upgrade);
@@ -8339,7 +8309,7 @@ mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev,
actions[0].type = RTE_FLOW_ACTION_TYPE_JUMP;
actions[0].conf = &jump;
actions[1].type = RTE_FLOW_ACTION_TYPE_END;
- flow_idx = flow_drv_list_create(dev, MLX5_FLOW_TYPE_CTL,
+ flow_idx = mlx5_flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
&attr, items, actions, false, &error);
if (!flow_idx) {
DRV_LOG(DEBUG,
@@ -8429,7 +8399,7 @@ mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
action_rss.types = 0;
for (i = 0; i != priv->reta_idx_n; ++i)
queue[i] = (*priv->reta_idx)[i];
- flow_idx = flow_drv_list_create(dev, MLX5_FLOW_TYPE_CTL,
+ flow_idx = mlx5_flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
&attr, items, actions, false, &error);
if (!flow_idx)
return -rte_errno;
@@ -8504,7 +8474,7 @@ mlx5_flow_lacp_miss(struct rte_eth_dev *dev)
},
};
struct rte_flow_error error;
- uint32_t flow_idx = flow_drv_list_create(dev, MLX5_FLOW_TYPE_CTL,
+ uint32_t flow_idx = mlx5_flow_list_create(dev, MLX5_FLOW_TYPE_CTL,
&attr, items, actions,
false, &error);
@@ -8528,7 +8498,7 @@ mlx5_flow_destroy(struct rte_eth_dev *dev,
struct rte_pmd_mlx5_flow_engine_mode_info *mode_info = &priv->mode_info;
struct mlx5_dv_flow_info *flow_info;
- flow_drv_list_destroy(dev, MLX5_FLOW_TYPE_GEN,
+ mlx5_flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN,
(uintptr_t)(void *)flow);
if (unlikely(mlx5_need_cache_flow(priv, NULL))) {
flow_info = LIST_FIRST(&mode_info->hot_upgrade);
@@ -9840,14 +9810,14 @@ mlx5_flow_discover_mreg_c(struct rte_eth_dev *dev)
if (!priv->sh->config.dv_flow_en)
break;
/* Create internal flow, validation skips copy action. */
- flow_idx = flow_drv_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr,
+ flow_idx = mlx5_flow_list_create(dev, MLX5_FLOW_TYPE_GEN, &attr,
items, actions, false, &error);
flow = mlx5_ipool_get(priv->flows[MLX5_FLOW_TYPE_GEN],
flow_idx);
if (!flow)
continue;
priv->sh->flow_mreg_c[n++] = idx;
- flow_drv_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx);
+ mlx5_flow_list_destroy(dev, MLX5_FLOW_TYPE_GEN, flow_idx);
}
for (; n < MLX5_MREG_C_NUM; ++n)
priv->sh->flow_mreg_c[n] = REG_NON;
diff --git a/drivers/net/mlx5/mlx5_flow_hw.c b/drivers/net/mlx5/mlx5_flow_hw.c
index 3022a86344..d938b5976a 100644
--- a/drivers/net/mlx5/mlx5_flow_hw.c
+++ b/drivers/net/mlx5/mlx5_flow_hw.c
@@ -2254,12 +2254,12 @@ mlx5_create_ipv6_ext_reformat(struct rte_eth_dev *dev,
*/
static int
__flow_hw_translate_actions_template(struct rte_eth_dev *dev,
- const struct mlx5_flow_template_table_cfg *cfg,
- struct mlx5_hw_actions *acts,
- struct rte_flow_actions_template *at,
- struct mlx5_tbl_multi_pattern_ctx *mp_ctx,
- bool nt_mode __rte_unused,
- struct rte_flow_error *error)
+ const struct mlx5_flow_template_table_cfg *cfg,
+ struct mlx5_hw_actions *acts,
+ struct rte_flow_actions_template *at,
+ struct mlx5_tbl_multi_pattern_ctx *mp_ctx,
+ bool nt_mode,
+ struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
const struct rte_flow_template_table_attr *table_attr = &cfg->attr;
@@ -10609,7 +10609,7 @@ static int
flow_hw_validate_attributes(const struct rte_flow_port_attr *port_attr,
uint16_t nb_queue,
const struct rte_flow_queue_attr *queue_attr[],
- struct rte_flow_error *error)
+ bool nt_mode, struct rte_flow_error *error)
{
uint32_t size;
unsigned int i;
@@ -10618,7 +10618,7 @@ flow_hw_validate_attributes(const struct rte_flow_port_attr *port_attr,
return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
"Port attributes must be non-NULL");
- if (nb_queue == 0)
+ if (nb_queue == 0 && !nt_mode)
return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
"At least one flow queue is required");
@@ -10695,7 +10695,7 @@ __flow_hw_configure(struct rte_eth_dev *dev,
rte_errno = EINVAL;
goto err;
}
- if (flow_hw_validate_attributes(port_attr, nb_queue, queue_attr, error))
+ if (flow_hw_validate_attributes(port_attr, nb_queue, queue_attr, nt_mode, error))
return -rte_errno;
/*
* Calling rte_flow_configure() again is allowed if
@@ -10713,7 +10713,7 @@ __flow_hw_configure(struct rte_eth_dev *dev,
}
}
/* If previous configuration was not default non template mode config. */
- if (!(priv->hw_attr->nt_mode)) {
+ if (!priv->hw_attr->nt_mode) {
if (flow_hw_compare_config(priv->hw_attr, port_attr, nb_queue, queue_attr))
return 0;
else
@@ -12176,6 +12176,7 @@ flow_hw_get_aged_flows(struct rte_eth_dev *dev, void **contexts,
/**
* Initialization function for non template API which calls
* flow_hw_configure with default values.
+ * Configure non queues cause 1 queue is configured by default for inner usage.
*
* @param[in] dev
* Pointer to the Ethernet device structure.
@@ -12185,8 +12186,6 @@ flow_hw_get_aged_flows(struct rte_eth_dev *dev, void **contexts,
* @return
* 0 on success, a negative errno value otherwise and rte_errno is set.
*/
- /* Configure non queues cause 1 queue is configured by default for inner usage. */
-
int
flow_hw_init(struct rte_eth_dev *dev,
struct rte_flow_error *error)
@@ -12214,10 +12213,10 @@ flow_hw_init(struct rte_eth_dev *dev,
}
static int flow_hw_prepare(struct rte_eth_dev *dev,
- const struct rte_flow_action actions[] __rte_unused,
- enum mlx5_flow_type type,
- struct rte_flow_hw **flow,
- struct rte_flow_error *error)
+ const struct rte_flow_action actions[] __rte_unused,
+ enum mlx5_flow_type type,
+ struct rte_flow_hw **flow,
+ struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
uint32_t idx = 0;
@@ -12340,14 +12339,14 @@ flow_hw_encap_decap_resource_register
static int
flow_hw_translate_flow_actions(struct rte_eth_dev *dev,
- const struct rte_flow_attr *attr,
- const struct rte_flow_action actions[],
- struct rte_flow_hw *flow,
- struct mlx5_flow_hw_action_params *ap,
- struct mlx5_hw_actions *hw_acts,
- uint64_t item_flags,
- bool external,
- struct rte_flow_error *error)
+ const struct rte_flow_attr *attr,
+ const struct rte_flow_action actions[],
+ struct rte_flow_hw *flow,
+ struct mlx5_flow_hw_action_params *ap,
+ struct mlx5_hw_actions *hw_acts,
+ uint64_t item_flags,
+ bool external,
+ struct rte_flow_error *error)
{
int ret = 0;
uint32_t src_group = 0;
@@ -12445,12 +12444,12 @@ flow_hw_translate_flow_actions(struct rte_eth_dev *dev,
}
static int flow_hw_register_matcher(struct rte_eth_dev *dev,
- const struct rte_flow_attr *attr,
- const struct rte_flow_item items[],
- bool external,
- struct rte_flow_hw *flow,
- struct mlx5_flow_dv_matcher *matcher,
- struct rte_flow_error *error)
+ const struct rte_flow_attr *attr,
+ const struct rte_flow_item items[],
+ bool external,
+ struct rte_flow_hw *flow,
+ struct mlx5_flow_dv_matcher *matcher,
+ struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct rte_flow_error sub_error = {
@@ -12481,10 +12480,7 @@ static int flow_hw_register_matcher(struct rte_eth_dev *dev,
matcher->crc = rte_raw_cksum((const void *)matcher->mask.buf,
matcher->mask.size);
- matcher->priority = mlx5_get_matcher_priority(dev, attr,
- matcher->priority,
- external);
-
+ matcher->priority = attr->priority;
ret = __translate_group(dev, attr, external, attr->group, &group, error);
if (ret)
return ret;
@@ -12592,10 +12588,10 @@ static int flow_hw_ensure_action_pools_allocated(struct rte_eth_dev *dev,
/* TODO: remove dev if not used */
static int flow_hw_apply(struct rte_eth_dev *dev __rte_unused,
- const struct rte_flow_item items[],
- struct mlx5dr_rule_action rule_actions[],
- struct rte_flow_hw *flow,
- struct rte_flow_error *error)
+ const struct rte_flow_item items[],
+ struct mlx5dr_rule_action rule_actions[],
+ struct rte_flow_hw *flow,
+ struct rte_flow_error *error)
{
struct mlx5dr_bwc_rule *rule = NULL;
@@ -12636,13 +12632,13 @@ static int flow_hw_apply(struct rte_eth_dev *dev __rte_unused,
* 0 on success, negative errno value otherwise and rte_errno set.
*/
static int flow_hw_create_flow(struct rte_eth_dev *dev,
- enum mlx5_flow_type type,
- const struct rte_flow_attr *attr,
- const struct rte_flow_item items[],
- const struct rte_flow_action actions[],
- bool external,
- struct rte_flow_hw **flow,
- struct rte_flow_error *error)
+ enum mlx5_flow_type type,
+ const struct rte_flow_attr *attr,
+ const struct rte_flow_item items[],
+ const struct rte_flow_action actions[],
+ bool external,
+ struct rte_flow_hw **flow,
+ struct rte_flow_error *error)
{
int ret;
struct mlx5_hw_actions hw_act;
@@ -12807,7 +12803,7 @@ flow_hw_destroy(struct rte_eth_dev *dev, struct rte_flow_hw *flow)
* Address of flow to destroy.
*/
static void flow_hw_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type type,
- uintptr_t flow_addr)
+ uintptr_t flow_addr)
{
struct mlx5_priv *priv = dev->data->dev_private;
/* Get flow via idx */
@@ -12843,12 +12839,12 @@ static void flow_hw_list_destroy(struct rte_eth_dev *dev, enum mlx5_flow_type ty
* A flow addr on success, 0 otherwise and rte_errno is set.
*/
static uintptr_t flow_hw_list_create(struct rte_eth_dev *dev,
- enum mlx5_flow_type type,
- const struct rte_flow_attr *attr,
- const struct rte_flow_item items[],
- const struct rte_flow_action actions[],
- bool external,
- struct rte_flow_error *error)
+ enum mlx5_flow_type type,
+ const struct rte_flow_attr *attr,
+ const struct rte_flow_item items[],
+ const struct rte_flow_action actions[],
+ bool external,
+ struct rte_flow_error *error)
{
int ret;
struct rte_flow_hw *flow = NULL;
--
2.25.1
^ permalink raw reply [flat|nested] 7+ messages in thread
* [PATCH v2 10/34] net/mlx5: fix segfault on counter pool destroy
[not found] ` <20240603080505.2641-1-mkashani@nvidia.com>
@ 2024-06-03 8:04 ` Maayan Kashani
0 siblings, 0 replies; 7+ messages in thread
From: Maayan Kashani @ 2024-06-03 8:04 UTC (permalink / raw)
To: dev
Cc: mkashani, dsosnowski, rasland, stable, Viacheslav Ovsiienko,
Ori Kam, Suanming Mou, Matan Azrad
If the counter pool was not added to list,
and an error state was reached,
on attempt to destroy the counter pool,
segmentation fault was received during list remove action.
Added a check to verify the list is not empty before trying to
remove the cpool from the list.
Invalid state, leading to segfault,
can also be reached in the following scenario:
1. mlx5_hws_cnt_pool_init() does a zmalloc and initializes most
of the fields of cpool, but does not initialize the next field.
2. mlx5_hws_cnt_pool_dcs_alloc() attempts to bulk allocate flow counters.
If this fails, we skip straight to 4.
In HW, this can fail simply if FW doesn't support bulk flow
counter allocation.
3. Right before the goto error, we insert the cpool to the hws_cpool_list.
This is where the next field is initialized.
4. mlx5_hws_cnt_pool_destroy() assumes the cpool's next field
is initialized and SEGVs if not.
So, added a guard against cases where the entry was uninitialized
(checking le_prev field is not NULL).
Fixes: 6ac2104ac125 ("net/mlx5: fix counter query during port close")
Cc: stable@dpdk.org
Signed-off-by: Maayan Kashani <mkashani@nvidia.com>
---
drivers/net/mlx5/mlx5_hws_cnt.c | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/drivers/net/mlx5/mlx5_hws_cnt.c b/drivers/net/mlx5/mlx5_hws_cnt.c
index 36d422bdfa..a46a4bd94e 100644
--- a/drivers/net/mlx5/mlx5_hws_cnt.c
+++ b/drivers/net/mlx5/mlx5_hws_cnt.c
@@ -718,7 +718,9 @@ mlx5_hws_cnt_pool_destroy(struct mlx5_dev_ctx_shared *sh,
* Maybe blocked for at most 200ms here.
*/
rte_spinlock_lock(&sh->cpool_lock);
- LIST_REMOVE(cpool, next);
+ /* Try to remove cpool before it was added to list caused segfault. */
+ if (!LIST_EMPTY(&sh->hws_cpool_list) && cpool->next.le_prev)
+ LIST_REMOVE(cpool, next);
rte_spinlock_unlock(&sh->cpool_lock);
if (cpool->cfg.host_cpool == NULL) {
if (--sh->cnt_svc->refcnt == 0)
--
2.25.1
^ permalink raw reply [flat|nested] 7+ messages in thread
* [PATCH v3 06/11] net/mlx5: fix segfault on counter pool destroy
[not found] ` <20240603104850.9935-1-mkashani@nvidia.com>
@ 2024-06-03 10:48 ` Maayan Kashani
[not found] ` <20240606102317.172553-1-mkashani@nvidia.com>
1 sibling, 0 replies; 7+ messages in thread
From: Maayan Kashani @ 2024-06-03 10:48 UTC (permalink / raw)
To: dev
Cc: mkashani, dsosnowski, rasland, stable, Viacheslav Ovsiienko,
Ori Kam, Suanming Mou, Matan Azrad
If the counter pool was not added to list,
and an error state was reached,
on attempt to destroy the counter pool,
segmentation fault was received during list remove action.
Added a check to verify the list is not empty before trying to
remove the cpool from the list.
Invalid state, leading to segfault,
can also be reached in the following scenario:
1. mlx5_hws_cnt_pool_init() does a zmalloc and initializes most
of the fields of cpool, but does not initialize the next field.
2. mlx5_hws_cnt_pool_dcs_alloc() attempts to bulk allocate flow counters.
If this fails, we skip straight to 4.
In HW, this can fail simply if FW doesn't support bulk flow
counter allocation.
3. Right before the goto error, we insert the cpool to the hws_cpool_list.
This is where the next field is initialized.
4. mlx5_hws_cnt_pool_destroy() assumes the cpool's next field
is initialized and SEGVs if not.
So, added a guard against cases where the entry was uninitialized
(checking le_prev field is not NULL).
Fixes: 6ac2104ac125 ("net/mlx5: fix counter query during port close")
Cc: stable@dpdk.org
Signed-off-by: Maayan Kashani <mkashani@nvidia.com>
---
drivers/net/mlx5/mlx5_hws_cnt.c | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/drivers/net/mlx5/mlx5_hws_cnt.c b/drivers/net/mlx5/mlx5_hws_cnt.c
index 36d422bdfa..a46a4bd94e 100644
--- a/drivers/net/mlx5/mlx5_hws_cnt.c
+++ b/drivers/net/mlx5/mlx5_hws_cnt.c
@@ -718,7 +718,9 @@ mlx5_hws_cnt_pool_destroy(struct mlx5_dev_ctx_shared *sh,
* Maybe blocked for at most 200ms here.
*/
rte_spinlock_lock(&sh->cpool_lock);
- LIST_REMOVE(cpool, next);
+ /* Try to remove cpool before it was added to list caused segfault. */
+ if (!LIST_EMPTY(&sh->hws_cpool_list) && cpool->next.le_prev)
+ LIST_REMOVE(cpool, next);
rte_spinlock_unlock(&sh->cpool_lock);
if (cpool->cfg.host_cpool == NULL) {
if (--sh->cnt_svc->refcnt == 0)
--
2.25.1
^ permalink raw reply [flat|nested] 7+ messages in thread
* [PATCH v4 06/11] net/mlx5: fix segfault on counter pool destroy
[not found] ` <20240606102317.172553-1-mkashani@nvidia.com>
@ 2024-06-06 10:23 ` Maayan Kashani
[not found] ` <20240606123256.177947-1-mkashani@nvidia.com>
1 sibling, 0 replies; 7+ messages in thread
From: Maayan Kashani @ 2024-06-06 10:23 UTC (permalink / raw)
To: dev
Cc: mkashani, dsosnowski, rasland, stable, Viacheslav Ovsiienko,
Ori Kam, Suanming Mou, Matan Azrad
If the counter pool was not added to list,
and an error state was reached,
on attempt to destroy the counter pool,
segmentation fault was received during list remove action.
Added a check to verify the list is not empty before trying to
remove the cpool from the list.
Invalid state, leading to segfault,
can also be reached in the following scenario:
1. mlx5_hws_cnt_pool_init() does a zmalloc and initializes most
of the fields of cpool, but does not initialize the next field.
2. mlx5_hws_cnt_pool_dcs_alloc() attempts to bulk allocate flow counters.
If this fails, we skip straight to 4.
In HW, this can fail simply if FW doesn't support bulk flow
counter allocation.
3. Right before the goto error, we insert the cpool to the hws_cpool_list.
This is where the next field is initialized.
4. mlx5_hws_cnt_pool_destroy() assumes the cpool's next field
is initialized and SEGVs if not.
So, added a guard against cases where the entry was uninitialized
(checking le_prev field is not NULL).
Fixes: 6ac2104ac125 ("net/mlx5: fix counter query during port close")
Cc: stable@dpdk.org
Signed-off-by: Maayan Kashani <mkashani@nvidia.com>
Acked-by: Dariusz Sosnowski <dsosnowski@nvidia.com>
---
drivers/net/mlx5/mlx5_hws_cnt.c | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/drivers/net/mlx5/mlx5_hws_cnt.c b/drivers/net/mlx5/mlx5_hws_cnt.c
index 36d422bdfa..a46a4bd94e 100644
--- a/drivers/net/mlx5/mlx5_hws_cnt.c
+++ b/drivers/net/mlx5/mlx5_hws_cnt.c
@@ -718,7 +718,9 @@ mlx5_hws_cnt_pool_destroy(struct mlx5_dev_ctx_shared *sh,
* Maybe blocked for at most 200ms here.
*/
rte_spinlock_lock(&sh->cpool_lock);
- LIST_REMOVE(cpool, next);
+ /* Try to remove cpool before it was added to list caused segfault. */
+ if (!LIST_EMPTY(&sh->hws_cpool_list) && cpool->next.le_prev)
+ LIST_REMOVE(cpool, next);
rte_spinlock_unlock(&sh->cpool_lock);
if (cpool->cfg.host_cpool == NULL) {
if (--sh->cnt_svc->refcnt == 0)
--
2.21.0
^ permalink raw reply [flat|nested] 7+ messages in thread
* [PATCH v5 06/11] net/mlx5: fix segfault on counter pool destroy
[not found] ` <20240606123256.177947-1-mkashani@nvidia.com>
@ 2024-06-06 12:32 ` Maayan Kashani
[not found] ` <20240609085600.87274-1-mkashani@nvidia.com>
1 sibling, 0 replies; 7+ messages in thread
From: Maayan Kashani @ 2024-06-06 12:32 UTC (permalink / raw)
To: dev
Cc: mkashani, dsosnowski, rasland, stable, Viacheslav Ovsiienko,
Ori Kam, Suanming Mou, Matan Azrad
If the counter pool was not added to list,
and an error state was reached,
on attempt to destroy the counter pool,
segmentation fault was received during list remove action.
Added a check to verify the list is not empty before trying to
remove the cpool from the list.
Invalid state, leading to segfault,
can also be reached in the following scenario:
1. mlx5_hws_cnt_pool_init() does a zmalloc and initializes most
of the fields of cpool, but does not initialize the next field.
2. mlx5_hws_cnt_pool_dcs_alloc() attempts to bulk allocate flow counters.
If this fails, we skip straight to 4.
In HW, this can fail simply if FW doesn't support bulk flow
counter allocation.
3. Right before the goto error, we insert the cpool to the hws_cpool_list.
This is where the next field is initialized.
4. mlx5_hws_cnt_pool_destroy() assumes the cpool's next field
is initialized and SEGVs if not.
So, added a guard against cases where the entry was uninitialized
(checking le_prev field is not NULL).
Fixes: 6ac2104ac125 ("net/mlx5: fix counter query during port close")
Cc: stable@dpdk.org
Signed-off-by: Maayan Kashani <mkashani@nvidia.com>
Acked-by: Dariusz Sosnowski <dsosnowski@nvidia.com>
---
drivers/net/mlx5/mlx5_hws_cnt.c | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/drivers/net/mlx5/mlx5_hws_cnt.c b/drivers/net/mlx5/mlx5_hws_cnt.c
index 36d422bdfa..a46a4bd94e 100644
--- a/drivers/net/mlx5/mlx5_hws_cnt.c
+++ b/drivers/net/mlx5/mlx5_hws_cnt.c
@@ -718,7 +718,9 @@ mlx5_hws_cnt_pool_destroy(struct mlx5_dev_ctx_shared *sh,
* Maybe blocked for at most 200ms here.
*/
rte_spinlock_lock(&sh->cpool_lock);
- LIST_REMOVE(cpool, next);
+ /* Try to remove cpool before it was added to list caused segfault. */
+ if (!LIST_EMPTY(&sh->hws_cpool_list) && cpool->next.le_prev)
+ LIST_REMOVE(cpool, next);
rte_spinlock_unlock(&sh->cpool_lock);
if (cpool->cfg.host_cpool == NULL) {
if (--sh->cnt_svc->refcnt == 0)
--
2.21.0
^ permalink raw reply [flat|nested] 7+ messages in thread
* [PATCH v6 06/11] net/mlx5: fix segfault on counter pool destroy
[not found] ` <20240609085600.87274-1-mkashani@nvidia.com>
@ 2024-06-09 8:55 ` Maayan Kashani
[not found] ` <20240609110107.92009-1-mkashani@nvidia.com>
1 sibling, 0 replies; 7+ messages in thread
From: Maayan Kashani @ 2024-06-09 8:55 UTC (permalink / raw)
To: dev
Cc: mkashani, dsosnowski, rasland, stable, Viacheslav Ovsiienko,
Ori Kam, Suanming Mou, Matan Azrad
If the counter pool was not added to list,
and an error state was reached,
on attempt to destroy the counter pool,
segmentation fault was received during list remove action.
Added a check to verify the list is not empty before trying to
remove the cpool from the list.
Invalid state, leading to segfault,
can also be reached in the following scenario:
1. mlx5_hws_cnt_pool_init() does a zmalloc and initializes most
of the fields of cpool, but does not initialize the next field.
2. mlx5_hws_cnt_pool_dcs_alloc() attempts to bulk allocate flow counters.
If this fails, we skip straight to 4.
In HW, this can fail simply if FW doesn't support bulk flow
counter allocation.
3. Right before the goto error, we insert the cpool to the hws_cpool_list.
This is where the next field is initialized.
4. mlx5_hws_cnt_pool_destroy() assumes the cpool's next field
is initialized and SEGVs if not.
So, added a guard against cases where the entry was uninitialized
(checking le_prev field is not NULL).
Fixes: 6ac2104ac125 ("net/mlx5: fix counter query during port close")
Cc: stable@dpdk.org
Signed-off-by: Maayan Kashani <mkashani@nvidia.com>
Acked-by: Dariusz Sosnowski <dsosnowski@nvidia.com>
---
drivers/net/mlx5/mlx5_hws_cnt.c | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/drivers/net/mlx5/mlx5_hws_cnt.c b/drivers/net/mlx5/mlx5_hws_cnt.c
index 36d422bdfa5..a46a4bd94e8 100644
--- a/drivers/net/mlx5/mlx5_hws_cnt.c
+++ b/drivers/net/mlx5/mlx5_hws_cnt.c
@@ -718,7 +718,9 @@ mlx5_hws_cnt_pool_destroy(struct mlx5_dev_ctx_shared *sh,
* Maybe blocked for at most 200ms here.
*/
rte_spinlock_lock(&sh->cpool_lock);
- LIST_REMOVE(cpool, next);
+ /* Try to remove cpool before it was added to list caused segfault. */
+ if (!LIST_EMPTY(&sh->hws_cpool_list) && cpool->next.le_prev)
+ LIST_REMOVE(cpool, next);
rte_spinlock_unlock(&sh->cpool_lock);
if (cpool->cfg.host_cpool == NULL) {
if (--sh->cnt_svc->refcnt == 0)
--
2.21.0
^ permalink raw reply [flat|nested] 7+ messages in thread
* [PATCH v7 06/11] net/mlx5: fix segfault on counter pool destroy
[not found] ` <20240609110107.92009-1-mkashani@nvidia.com>
@ 2024-06-09 11:01 ` Maayan Kashani
0 siblings, 0 replies; 7+ messages in thread
From: Maayan Kashani @ 2024-06-09 11:01 UTC (permalink / raw)
To: dev
Cc: mkashani, dsosnowski, rasland, stable, Viacheslav Ovsiienko,
Ori Kam, Suanming Mou, Matan Azrad
If the counter pool was not added to list,
and an error state was reached,
on attempt to destroy the counter pool,
segmentation fault was received during list remove action.
Added a check to verify the list is not empty before trying to
remove the cpool from the list.
Invalid state, leading to segfault,
can also be reached in the following scenario:
1. mlx5_hws_cnt_pool_init() does a zmalloc and initializes most
of the fields of cpool, but does not initialize the next field.
2. mlx5_hws_cnt_pool_dcs_alloc() attempts to bulk allocate flow counters.
If this fails, we skip straight to 4.
In HW, this can fail simply if FW doesn't support bulk flow
counter allocation.
3. Right before the goto error, we insert the cpool to the hws_cpool_list.
This is where the next field is initialized.
4. mlx5_hws_cnt_pool_destroy() assumes the cpool's next field
is initialized and SEGVs if not.
So, added a guard against cases where the entry was uninitialized
(checking le_prev field is not NULL).
Fixes: 6ac2104ac125 ("net/mlx5: fix counter query during port close")
Cc: stable@dpdk.org
Signed-off-by: Maayan Kashani <mkashani@nvidia.com>
Acked-by: Dariusz Sosnowski <dsosnowski@nvidia.com>
---
drivers/net/mlx5/mlx5_hws_cnt.c | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/drivers/net/mlx5/mlx5_hws_cnt.c b/drivers/net/mlx5/mlx5_hws_cnt.c
index 36d422bdfa5..a46a4bd94e8 100644
--- a/drivers/net/mlx5/mlx5_hws_cnt.c
+++ b/drivers/net/mlx5/mlx5_hws_cnt.c
@@ -718,7 +718,9 @@ mlx5_hws_cnt_pool_destroy(struct mlx5_dev_ctx_shared *sh,
* Maybe blocked for at most 200ms here.
*/
rte_spinlock_lock(&sh->cpool_lock);
- LIST_REMOVE(cpool, next);
+ /* Try to remove cpool before it was added to list caused segfault. */
+ if (!LIST_EMPTY(&sh->hws_cpool_list) && cpool->next.le_prev)
+ LIST_REMOVE(cpool, next);
rte_spinlock_unlock(&sh->cpool_lock);
if (cpool->cfg.host_cpool == NULL) {
if (--sh->cnt_svc->refcnt == 0)
--
2.21.0
^ permalink raw reply [flat|nested] 7+ messages in thread
end of thread, other threads:[~2024-06-09 11:01 UTC | newest]
Thread overview: 7+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2024-06-02 10:28 [PATCH 01/11] net/mlx5: initial design of non template to hws Maayan Kashani
[not found] ` <20240603080505.2641-1-mkashani@nvidia.com>
2024-06-03 8:04 ` [PATCH v2 10/34] net/mlx5: fix segfault on counter pool destroy Maayan Kashani
[not found] ` <20240603104850.9935-1-mkashani@nvidia.com>
2024-06-03 10:48 ` [PATCH v3 06/11] " Maayan Kashani
[not found] ` <20240606102317.172553-1-mkashani@nvidia.com>
2024-06-06 10:23 ` [PATCH v4 " Maayan Kashani
[not found] ` <20240606123256.177947-1-mkashani@nvidia.com>
2024-06-06 12:32 ` [PATCH v5 " Maayan Kashani
[not found] ` <20240609085600.87274-1-mkashani@nvidia.com>
2024-06-09 8:55 ` [PATCH v6 " Maayan Kashani
[not found] ` <20240609110107.92009-1-mkashani@nvidia.com>
2024-06-09 11:01 ` [PATCH v7 " Maayan Kashani
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).