* [dpdk-stable] [PATCH 02/13] net/mlx5: fix meter statistics [not found] <20210331073632.1443011-1-lizh@nvidia.com> @ 2021-03-31 7:36 ` Li Zhang [not found] ` <20210402151430.1531529-1-lizh@nvidia.com> ` (6 subsequent siblings) 7 siblings, 0 replies; 8+ messages in thread From: Li Zhang @ 2021-03-31 7:36 UTC (permalink / raw) To: dekelp, orika, viacheslavo, matan, shahafs, Suanming Mou Cc: dev, thomas, rasland, roniba, Shun Hao, stable From: Shun Hao <shunh@nvidia.com> This fixes the meter statistics issue that when using multiple meters, only one meter has stats value. To match the correct meter in policer table, now the meter_id is also used in its match criteria, so only one color and one drop matcher are needed. And both meter_id and flow_id will be written to related registers in meter prefix flow. Fixes: 46a5e6bc6a ("net/mlx5: prepare meter flow tables") Cc: stable@dpdk.org Signed-off-by: Shun Hao <shunh@nvidia.com> --- drivers/net/mlx5/linux/mlx5_os.c | 9 +- drivers/net/mlx5/mlx5.c | 7 +- drivers/net/mlx5/mlx5.h | 25 +- drivers/net/mlx5/mlx5_flow.c | 182 +++++++---- drivers/net/mlx5/mlx5_flow.h | 40 +-- drivers/net/mlx5/mlx5_flow_dv.c | 489 ++++++++++++++++++----------- drivers/net/mlx5/mlx5_flow_meter.c | 53 +++- 7 files changed, 512 insertions(+), 293 deletions(-) diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c index 2d5bcab4cf..b7eca9d153 100644 --- a/drivers/net/mlx5/linux/mlx5_os.c +++ b/drivers/net/mlx5/linux/mlx5_os.c @@ -316,7 +316,13 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv) } sh->tx_domain = domain; #ifdef HAVE_MLX5DV_DR_ESWITCH + sh->esw_drop_action = mlx5_glue->dr_create_flow_action_drop(); if (priv->config.dv_esw_en) { + if (!sh->esw_drop_action) { + DRV_LOG(ERR, "Failed to create eswitch drop action"); + err = errno; + goto error; + } domain = mlx5_glue->dr_create_domain (sh->ctx, MLX5DV_DR_DOMAIN_TYPE_FDB); if (!domain) { @@ -325,7 +331,6 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv) goto error; } sh->fdb_domain = domain; - sh->esw_drop_action = mlx5_glue->dr_create_flow_action_drop(); } #endif if (!sh->tunnel_hub) @@ -1273,7 +1278,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev, - 1 + REG_C_0; priv->mtr_en = 1; priv->mtr_reg_share = - config->hca_attr.qos.flow_meter; + config->hca_attr.qos.flow_meter; DRV_LOG(DEBUG, "The REG_C meter uses is %d", priv->mtr_color_reg); } diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index 3538cc8c20..d41c098f65 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -275,10 +275,13 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = { }, #endif [MLX5_IPOOL_MTR] = { + /** + * The ipool index should grow continually from small to big, + * for meter idx, so not set grow_trunk to avoid meter index + * not jump continually. + */ .size = sizeof(struct mlx5_flow_meter), .trunk_size = 64, - .grow_trunk = 3, - .grow_shift = 2, .need_lock = 1, .release_mem_en = 1, .malloc = mlx5_malloc, diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index 6faba4fbb1..95469c605a 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -188,6 +188,16 @@ struct mlx5_stats_ctrl { /* Maximal number of segments to split. */ #define MLX5_MAX_RXQ_NSEG (1u << MLX5_MAX_LOG_RQ_SEGS) +/* The bit size of one register. */ +#define MLX5_REG_BITS 32 + +/* Idle bits for non-color usage in color register. */ +#define MLX5_MTR_IDLE_BITS_IN_COLOR_REG (MLX5_REG_BITS - MLX5_MTR_COLOR_BITS) + +#define UINT32_T(x) ((uint32_t)(x)) + +#define LS32_MASK(bits) ((UINT32_T(1) << (bits)) - 1) + /* LRO configurations structure. */ struct mlx5_lro_config { uint32_t supported:1; /* Whether LRO is supported. */ @@ -944,9 +954,9 @@ struct mlx5_priv { unsigned int representor:1; /* Device is a port representor. */ unsigned int master:1; /* Device is a E-Switch master. */ unsigned int txpp_en:1; /* Tx packet pacing enabled. */ + unsigned int sampler_en:1; /* Whether support sampler. */ unsigned int mtr_en:1; /* Whether support meter. */ unsigned int mtr_reg_share:1; /* Whether support meter REG_C share. */ - unsigned int sampler_en:1; /* Whether support sampler. */ uint16_t domain_id; /* Switch domain identifier. */ uint16_t vport_id; /* Associated VF vport index (if any). */ uint32_t vport_meta_tag; /* Used for vport index match ove VF LAG. */ @@ -1003,6 +1013,10 @@ struct mlx5_priv { uint32_t rss_shared_actions; /* RSS shared actions. */ struct mlx5_devx_obj *q_counters; /* DevX queue counter object. */ uint32_t counter_set_id; /* Queue counter ID to set in DevX objects. */ + uint8_t max_mtr_bits; + /* Indicate how many bits are used by meter id at the most. */ + uint8_t max_mtr_flow_bits; + /* Indicate how many bits are used by meter flow id at the most. */ }; #define PORT_ID(priv) ((priv)->dev_data->port_id) @@ -1271,11 +1285,10 @@ int mlx5_pmd_socket_init(void); int mlx5_flow_meter_ops_get(struct rte_eth_dev *dev, void *arg); struct mlx5_flow_meter *mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id); -struct mlx5_flow_meter *mlx5_flow_meter_attach - (struct mlx5_priv *priv, - uint32_t meter_id, - const struct rte_flow_attr *attr, - struct rte_flow_error *error); +int mlx5_flow_meter_attach(struct mlx5_priv *priv, + struct mlx5_flow_meter *fm, + const struct rte_flow_attr *attr, + struct rte_flow_error *error); void mlx5_flow_meter_detach(struct mlx5_flow_meter *fm); /* mlx5_os.c */ diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c index c347f8130e..cffd6129e8 100644 --- a/drivers/net/mlx5/mlx5_flow.c +++ b/drivers/net/mlx5/mlx5_flow.c @@ -764,9 +764,9 @@ mlx5_flow_get_reg_id(struct rte_eth_dev *dev, return REG_C_0; } break; - case MLX5_MTR_SFX: + case MLX5_MTR_ID: /* - * If meter color and flow match share one register, flow match + * If meter color and meter id share one register, flow match * should use the meter color register for match. */ if (priv->mtr_reg_share) @@ -3051,7 +3051,8 @@ flow_mreg_split_qrss_release(struct rte_eth_dev *dev, SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, handle_idx, dev_handle, next) - if (dev_handle->split_flow_id) + if (dev_handle->split_flow_id && + !dev_handle->is_meter_flow_id) mlx5_ipool_free(priv->sh->ipool [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], dev_handle->split_flow_id); @@ -3690,23 +3691,30 @@ flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], * * @param[in] actions * Pointer to the list of actions. - * @param[out] mtr + * @param[out] has_mtr * Pointer to the meter exist flag. + * @param[out] meter_id + * Pointer to the meter id. * * @return * Total number of actions. */ static int -flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr) +flow_check_meter_action(const struct rte_flow_action actions[], + bool *has_mtr, + uint32_t *meter_id) { + const struct rte_flow_action_meter *mtr = NULL; int actions_n = 0; - MLX5_ASSERT(mtr); - *mtr = 0; + MLX5_ASSERT(has_mtr); + *has_mtr = false; for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { switch (actions->type) { case RTE_FLOW_ACTION_TYPE_METER: - *mtr = 1; + mtr = actions->conf; + *meter_id = mtr->mtr_id; + *has_mtr = true; break; default: break; @@ -4363,8 +4371,10 @@ flow_create_split_inner(struct rte_eth_dev *dev, * header will be in the prefix sub flow, as not to take the * L3 tunnel header into account. * - * @param dev + * @param[in] dev * Pointer to Ethernet device. + * @param[in] fm + * Pointer to flow meter structure. * @param[in] items * Pattern specification (list terminated by the END pattern item). * @param[out] sfx_items @@ -4379,29 +4389,38 @@ flow_create_split_inner(struct rte_eth_dev *dev, * The pattern items for the suffix flow. * @param[out] tag_sfx * Pointer to suffix flow tag. + * @param[out] error + * Perform verbose error reporting if not NULL. * * @return - * 0 on success. + * The flow id, 0 otherwise and rte_errno is set. */ -static int +static uint32_t flow_meter_split_prep(struct rte_eth_dev *dev, - const struct rte_flow_item items[], - struct rte_flow_item sfx_items[], - const struct rte_flow_action actions[], - struct rte_flow_action actions_sfx[], - struct rte_flow_action actions_pre[]) + struct mlx5_flow_meter *fm, + const struct rte_flow_item items[], + struct rte_flow_item sfx_items[], + const struct rte_flow_action actions[], + struct rte_flow_action actions_sfx[], + struct rte_flow_action actions_pre[], + struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; struct rte_flow_action *tag_action = NULL; struct rte_flow_item *tag_item; struct mlx5_rte_flow_action_set_tag *set_tag; - struct rte_flow_error error; const struct rte_flow_action_raw_encap *raw_encap; const struct rte_flow_action_raw_decap *raw_decap; - struct mlx5_rte_flow_item_tag *tag_spec; - struct mlx5_rte_flow_item_tag *tag_mask; + struct mlx5_rte_flow_item_tag *tag_item_spec; + struct mlx5_rte_flow_item_tag *tag_item_mask; uint32_t tag_id = 0; bool copy_vlan = false; + uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; + uint8_t mtr_reg_bits = priv->mtr_reg_share ? + MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; + uint8_t flow_id_bits = 0; + int shift; + uint32_t flow_id_val = 0; /* Prepare the actions for prefix and suffix flow. */ for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { @@ -4410,10 +4429,7 @@ flow_meter_split_prep(struct rte_eth_dev *dev, switch (actions->type) { case RTE_FLOW_ACTION_TYPE_METER: /* Add the extra tag action first. */ - tag_action = actions_pre; - tag_action->type = (enum rte_flow_action_type) - MLX5_RTE_FLOW_ACTION_TYPE_TAG; - actions_pre++; + tag_action = actions_pre++; action_cur = &actions_pre; break; case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: @@ -4446,23 +4462,22 @@ flow_meter_split_prep(struct rte_eth_dev *dev, actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; actions_pre->type = RTE_FLOW_ACTION_TYPE_END; actions_pre++; - /* Set the tag. */ - set_tag = (void *)actions_pre; - set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); - mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], - &tag_id); - if (tag_id >= (1 << (sizeof(tag_id) * 8 - MLX5_MTR_COLOR_BITS))) { - DRV_LOG(ERR, "Port %u meter flow id exceed max limit.", - dev->data->port_id); - mlx5_ipool_free(priv->sh->ipool - [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], tag_id); - return 0; - } else if (!tag_id) { - return 0; + /* Generate meter flow_id only if support multiple flows per meter. */ + mlx5_ipool_malloc(fm->flow_ipool, &tag_id); + if (!tag_id) + return rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Failed to allocate meter flow id."); + flow_id_bits = MLX5_REG_BITS - __builtin_clz(tag_id - 1); + flow_id_bits = flow_id_bits ? flow_id_bits : 1; + if ((flow_id_bits + priv->max_mtr_bits) > mtr_reg_bits) { + mlx5_ipool_free(fm->flow_ipool, tag_id); + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Meter flow id exceeds max limit."); } - set_tag->data = tag_id << MLX5_MTR_COLOR_BITS; - assert(tag_action); - tag_action->conf = set_tag; + if (flow_id_bits > priv->max_mtr_flow_bits) + priv->max_mtr_flow_bits = flow_id_bits; /* Prepare the suffix subflow items. */ tag_item = sfx_items++; for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { @@ -4491,16 +4506,34 @@ flow_meter_split_prep(struct rte_eth_dev *dev, } sfx_items->type = RTE_FLOW_ITEM_TYPE_END; sfx_items++; - tag_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; - tag_spec->data = tag_id << MLX5_MTR_COLOR_BITS; - tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); - tag_mask = tag_spec + 1; - tag_mask->data = 0xffffff00; + /* Build tag actions and items for meter_id/meter flow_id. */ + assert(tag_action); + set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; + tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; + tag_item_mask = tag_item_spec + 1; + /* + * The color Reg bits used by flow_id are growing from + * msb to lsb, so must do bit reverse for flow_id val in RegC. + */ + for (shift = 0; shift < flow_id_bits; shift++) + flow_id_val = (flow_id_val << 1) | + (((tag_id - 1) >> shift) & 0x1); + /* Both flow_id and meter_id share the same register. */ + set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 0, error); + set_tag->data = + (fm->idx | (flow_id_val << (mtr_reg_bits - flow_id_bits))) + << mtr_id_offset; + tag_item_spec->id = set_tag->id; + tag_item_spec->data = set_tag->data; + tag_item_mask->data = UINT32_MAX << mtr_id_offset; + tag_action->type = (enum rte_flow_action_type) + MLX5_RTE_FLOW_ACTION_TYPE_TAG; + tag_action->conf = set_tag; tag_item->type = (enum rte_flow_item_type) - MLX5_RTE_FLOW_ITEM_TYPE_TAG; - tag_item->spec = tag_spec; + MLX5_RTE_FLOW_ITEM_TYPE_TAG; + tag_item->spec = tag_item_spec; tag_item->last = NULL; - tag_item->mask = tag_mask; + tag_item->mask = tag_item_mask; return tag_id; } @@ -5200,25 +5233,49 @@ flow_create_split_meter(struct rte_eth_dev *dev, struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); struct rte_flow_action *sfx_actions = NULL; struct rte_flow_action *pre_actions = NULL; struct rte_flow_item *sfx_items = NULL; struct mlx5_flow *dev_flow = NULL; struct rte_flow_attr sfx_attr = *attr; - uint32_t mtr = 0; + struct mlx5_flow_meter *fm = NULL; + bool has_mtr = false; + uint32_t meter_id; uint32_t mtr_tag_id = 0; size_t act_size; size_t item_size; int actions_n = 0; - int ret; + int ret = 0; if (priv->mtr_en) - actions_n = flow_check_meter_action(actions, &mtr); - if (mtr) { - /* The five prefix actions: meter, decap, encap, tag, end. */ + actions_n = flow_check_meter_action(actions, &has_mtr, + &meter_id); + if (has_mtr) { + if (flow->meter) { + fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], + flow->meter); + if (!fm) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "Meter not found."); + } else { + fm = mlx5_flow_meter_find(priv, meter_id); + if (!fm) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "Meter not found."); + ret = mlx5_flow_meter_attach(priv, fm, + &sfx_attr, error); + if (ret) + return -rte_errno; + flow->meter = fm->idx; + } + wks->fm = fm; + /* The prefix actions: meter, decap, encap, tag, end. */ act_size = sizeof(struct rte_flow_action) * (actions_n + 5) + sizeof(struct mlx5_rte_flow_action_set_tag); - /* tag, vlan, port id, end. */ + /* The suffix items: tag, vlan, port id, end. */ #define METER_SUFFIX_ITEM 4 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + sizeof(struct mlx5_rte_flow_item_tag) * 2; @@ -5232,9 +5289,9 @@ flow_create_split_meter(struct rte_eth_dev *dev, sfx_items = (struct rte_flow_item *)((char *)sfx_actions + act_size); pre_actions = sfx_actions + actions_n; - mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items, + mtr_tag_id = flow_meter_split_prep(dev, fm, items, sfx_items, actions, sfx_actions, - pre_actions); + pre_actions, error); if (!mtr_tag_id) { ret = -rte_errno; goto exit; @@ -5245,10 +5302,12 @@ flow_create_split_meter(struct rte_eth_dev *dev, attr, items, pre_actions, flow_split_info, error); if (ret) { + mlx5_ipool_free(fm->flow_ipool, mtr_tag_id); ret = -rte_errno; goto exit; } dev_flow->handle->split_flow_id = mtr_tag_id; + dev_flow->handle->is_meter_flow_id = 1; /* Setting the sfx group atrr. */ sfx_attr.group = sfx_attr.transfer ? (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : @@ -6520,20 +6579,17 @@ mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, * * @param[in] dev * Pointer to Ethernet device. - * @param[in] fm - * Pointer to the flow meter. * * @return * Pointer to table set on success, NULL otherwise. */ struct mlx5_meter_domains_infos * -mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm) +mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev) { const struct mlx5_flow_driver_ops *fops; fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); - return fops->create_mtr_tbls(dev, fm); + return fops->create_mtr_tbls(dev); } /** @@ -6558,7 +6614,7 @@ mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, } /** - * Create policer rules. + * Prepare policer rules. * * @param[in] dev * Pointer to Ethernet device. @@ -6571,14 +6627,14 @@ mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, * 0 on success, -1 otherwise. */ int -mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, +mlx5_flow_prepare_policer_rules(struct rte_eth_dev *dev, struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr) { const struct mlx5_flow_driver_ops *fops; fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); - return fops->create_policer_rules(dev, fm, attr); + return fops->prepare_policer_rules(dev, fm, attr); } /** diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index 8324e188e1..c92e746a04 100644 --- a/drivers/net/mlx5/mlx5_flow.h +++ b/drivers/net/mlx5/mlx5_flow.h @@ -79,7 +79,7 @@ enum mlx5_feature_name { MLX5_APP_TAG, MLX5_COPY_MARK, MLX5_MTR_COLOR, - MLX5_MTR_SFX, + MLX5_MTR_ID, MLX5_ASO_FLOW_HIT, }; @@ -655,7 +655,8 @@ struct mlx5_flow_handle { uint64_t layers; /**< Bit-fields of present layers, see MLX5_FLOW_LAYER_*. */ void *drv_flow; /**< pointer to driver flow object. */ - uint32_t split_flow_id:28; /**< Sub flow unique match flow id. */ + uint32_t split_flow_id:27; /**< Sub flow unique match flow id. */ + uint32_t is_meter_flow_id:1; /**< Indate if flow_id is for meter. */ uint32_t mark:1; /**< Metadate rxq mark flag. */ uint32_t fate_action:3; /**< Fate action type. */ union { @@ -842,14 +843,16 @@ struct mlx5_meter_domain_info { /**< Meter table. */ struct mlx5_flow_tbl_resource *sfx_tbl; /**< Meter suffix table. */ - void *any_matcher; - /**< Meter color not match default criteria. */ - void *color_matcher; - /**< Meter color match criteria. */ + struct mlx5_flow_dv_matcher *drop_matcher; + /**< Matcher for Drop. */ + struct mlx5_flow_dv_matcher *color_matcher; + /**< Matcher for Color. */ void *jump_actn; /**< Meter match action. */ - void *policer_rules[RTE_MTR_DROPPED + 1]; - /**< Meter policer for the match. */ + void *green_rule; + /**< Meter green rule. */ + void *drop_rule; + /**< Meter drop rule. */ }; /* Meter table set for TX RX FDB. */ @@ -862,10 +865,10 @@ struct mlx5_meter_domains_infos { /**< RX meter table. */ struct mlx5_meter_domain_info transfer; /**< FDB meter table. */ - void *drop_actn; - /**< Drop action as not matched. */ - void *count_actns[RTE_MTR_DROPPED + 1]; - /**< Counters for match and unmatched statistics. */ + void *green_count; + /**< Counters for green rule. */ + void *drop_count; + /**< Counters for green rule. */ uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)]; /**< Flow meter parameter. */ size_t fmp_size; @@ -928,6 +931,8 @@ struct mlx5_flow_meter { /**< Meter state. */ uint32_t shared:1; /**< Meter shared or not. */ + struct mlx5_indexed_pool *flow_ipool; + /**< Index pool for flow id. */ }; /* RFC2697 parameter structure. */ @@ -1148,6 +1153,7 @@ struct mlx5_flow_workspace { struct mlx5_flow_rss_desc rss_desc; uint32_t rssq_num; /* Allocated queue num in rss_desc. */ uint32_t flow_idx; /* Intermediate device flow index. */ + struct mlx5_flow_meter *fm; /* Pointer to the meter in flow. */ }; struct mlx5_flow_split_info { @@ -1188,8 +1194,7 @@ typedef int (*mlx5_flow_query_t)(struct rte_eth_dev *dev, void *data, struct rte_flow_error *error); typedef struct mlx5_meter_domains_infos *(*mlx5_flow_create_mtr_tbls_t) - (struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm); + (struct rte_eth_dev *dev); typedef int (*mlx5_flow_destroy_mtr_tbls_t)(struct rte_eth_dev *dev, struct mlx5_meter_domains_infos *tbls); typedef int (*mlx5_flow_create_policer_rules_t) @@ -1252,7 +1257,7 @@ struct mlx5_flow_driver_ops { mlx5_flow_query_t query; mlx5_flow_create_mtr_tbls_t create_mtr_tbls; mlx5_flow_destroy_mtr_tbls_t destroy_mtr_tbls; - mlx5_flow_create_policer_rules_t create_policer_rules; + mlx5_flow_create_policer_rules_t prepare_policer_rules; mlx5_flow_destroy_policer_rules_t destroy_policer_rules; mlx5_flow_counter_alloc_t counter_alloc; mlx5_flow_counter_free_t counter_free; @@ -1452,11 +1457,10 @@ int mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, const struct rte_flow_item_ecpri *acc_mask, struct rte_flow_error *error); struct mlx5_meter_domains_infos *mlx5_flow_create_mtr_tbls - (struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm); + (struct rte_eth_dev *dev); int mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, struct mlx5_meter_domains_infos *tbl); -int mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, +int mlx5_flow_prepare_policer_rules(struct rte_eth_dev *dev, struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr); int mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 23e5849783..ce9857d3f7 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -10982,14 +10982,12 @@ flow_dv_translate(struct rte_eth_dev *dev, const struct rte_flow_action_rss *rss; const struct rte_flow_action *action = actions; const uint8_t *rss_key; - const struct rte_flow_action_meter *mtr; struct mlx5_flow_tbl_resource *tbl; struct mlx5_aso_age_action *age_act; uint32_t port_id = 0; struct mlx5_flow_dv_port_id_action_resource port_id_resource; int action_type = actions->type; const struct rte_flow_action *found_action = NULL; - struct mlx5_flow_meter *fm = NULL; uint32_t jump_group = 0; if (!mlx5_flow_os_action_supported(action_type)) @@ -11414,33 +11412,13 @@ flow_dv_translate(struct rte_eth_dev *dev, MLX5_FLOW_FATE_DEFAULT_MISS; break; case RTE_FLOW_ACTION_TYPE_METER: - mtr = actions->conf; - if (!flow->meter) { - fm = mlx5_flow_meter_attach(priv, mtr->mtr_id, - attr, error); - if (!fm) - return rte_flow_error_set(error, - rte_errno, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "meter not found " - "or invalid parameters"); - flow->meter = fm->idx; - } + if (!wks->fm) + return rte_flow_error_set(error, rte_errno, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "Failed to get meter in flow."); /* Set the meter action. */ - if (!fm) { - fm = mlx5_ipool_get(priv->sh->ipool - [MLX5_IPOOL_MTR], flow->meter); - if (!fm) - return rte_flow_error_set(error, - rte_errno, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "meter not found " - "or invalid parameters"); - } dev_flow->dv.actions[actions_n++] = - fm->mfts->meter_action; + wks->fm->mfts->meter_action; action_flags |= MLX5_FLOW_ACTION_METER; break; case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: @@ -12536,6 +12514,7 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) { struct mlx5_flow_handle *dev_handle; struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_meter *fm = NULL; uint32_t srss = 0; if (!flow) @@ -12546,8 +12525,6 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) flow->counter = 0; } if (flow->meter) { - struct mlx5_flow_meter *fm; - fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], flow->meter); if (fm) @@ -12589,6 +12566,10 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) flow_dv_fate_resource_release(dev, dev_handle); else if (!srss) srss = dev_handle->rix_srss; + if (fm && dev_handle->is_meter_flow_id && + dev_handle->split_flow_id) + mlx5_ipool_free(fm->flow_ipool, + dev_handle->split_flow_id); mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], tmp_idx); } @@ -13274,49 +13255,20 @@ flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev, if (!mtd || !priv->config.dv_flow_en) return 0; - if (mtd->ingress.policer_rules[RTE_MTR_DROPPED]) - claim_zero(mlx5_flow_os_destroy_flow - (mtd->ingress.policer_rules[RTE_MTR_DROPPED])); - if (mtd->egress.policer_rules[RTE_MTR_DROPPED]) - claim_zero(mlx5_flow_os_destroy_flow - (mtd->egress.policer_rules[RTE_MTR_DROPPED])); - if (mtd->transfer.policer_rules[RTE_MTR_DROPPED]) - claim_zero(mlx5_flow_os_destroy_flow - (mtd->transfer.policer_rules[RTE_MTR_DROPPED])); - if (mtd->egress.color_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->egress.color_matcher)); - if (mtd->egress.any_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->egress.any_matcher)); if (mtd->egress.tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->egress.tbl); if (mtd->egress.sfx_tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->egress.sfx_tbl); - if (mtd->ingress.color_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->ingress.color_matcher)); - if (mtd->ingress.any_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->ingress.any_matcher)); if (mtd->ingress.tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->ingress.tbl); if (mtd->ingress.sfx_tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->ingress.sfx_tbl); - if (mtd->transfer.color_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->transfer.color_matcher)); - if (mtd->transfer.any_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->transfer.any_matcher)); if (mtd->transfer.tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->transfer.tbl); if (mtd->transfer.sfx_tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->transfer.sfx_tbl); - if (mtd->drop_actn) - claim_zero(mlx5_flow_os_destroy_flow_action(mtd->drop_actn)); mlx5_free(mtd); return 0; } @@ -13335,37 +13287,17 @@ flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev, * Table attribute. * @param[in] transfer * Table attribute. - * @param[in] color_reg_c_idx - * Reg C index for color match. * * @return - * 0 on success, -1 otherwise and rte_errno is set. + * 0 on success, -1 otherwise. */ static int flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, struct mlx5_meter_domains_infos *mtb, - uint8_t egress, uint8_t transfer, - uint32_t color_reg_c_idx) + uint8_t egress, uint8_t transfer) { - struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_dev_ctx_shared *sh = priv->sh; - struct mlx5_flow_dv_match_params mask = { - .size = sizeof(mask.buf), - }; - struct mlx5_flow_dv_match_params value = { - .size = sizeof(value.buf), - }; - struct mlx5dv_flow_matcher_attr dv_attr = { - .type = IBV_FLOW_ATTR_NORMAL, - .priority = 0, - .match_criteria_enable = 0, - .match_mask = (void *)&mask, - }; - void *actions[METER_ACTIONS]; - struct mlx5_meter_domain_info *dtb; struct rte_flow_error error; - int i = 0; - int ret; + struct mlx5_meter_domain_info *dtb; if (transfer) dtb = &mtb->transfer; @@ -13390,41 +13322,7 @@ flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, DRV_LOG(ERR, "Failed to create meter suffix table."); return -1; } - /* Create matchers, Any and Color. */ - dv_attr.priority = 3; - dv_attr.match_criteria_enable = 0; - ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj, - &dtb->any_matcher); - if (ret) { - DRV_LOG(ERR, "Failed to create meter" - " policer default matcher."); - goto error_exit; - } - dv_attr.priority = 0; - dv_attr.match_criteria_enable = - 1 << MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT; - flow_dv_match_meta_reg(mask.buf, value.buf, color_reg_c_idx, - rte_col_2_mlx5_col(RTE_COLORS), UINT8_MAX); - ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj, - &dtb->color_matcher); - if (ret) { - DRV_LOG(ERR, "Failed to create meter policer color matcher."); - goto error_exit; - } - if (mtb->count_actns[RTE_MTR_DROPPED]) - actions[i++] = mtb->count_actns[RTE_MTR_DROPPED]; - actions[i++] = mtb->drop_actn; - /* Default rule: lowest priority, match any, actions: drop. */ - ret = mlx5_flow_os_create_flow(dtb->any_matcher, (void *)&value, i, - actions, - &dtb->policer_rules[RTE_MTR_DROPPED]); - if (ret) { - DRV_LOG(ERR, "Failed to create meter policer drop rule."); - goto error_exit; - } return 0; -error_exit: - return -1; } /** @@ -13433,20 +13331,16 @@ flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, * * @param[in] dev * Pointer to Ethernet device. - * @param[in] fm - * Pointer to the flow meter. * * @return * Pointer to table set on success, NULL otherwise and rte_errno is set. */ static struct mlx5_meter_domains_infos * -flow_dv_create_mtr_tbl(struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm) +flow_dv_create_mtr_tbl(struct rte_eth_dev *dev) { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_meter_domains_infos *mtb; int ret; - int i; if (!priv->mtr_en) { rte_errno = ENOTSUP; @@ -13457,37 +13351,21 @@ flow_dv_create_mtr_tbl(struct rte_eth_dev *dev, DRV_LOG(ERR, "Failed to allocate memory for meter."); return NULL; } - /* Create meter count actions */ - for (i = 0; i <= RTE_MTR_DROPPED; i++) { - struct mlx5_flow_counter *cnt; - if (!fm->policer_stats.cnt[i]) - continue; - cnt = flow_dv_counter_get_by_idx(dev, - fm->policer_stats.cnt[i], NULL); - mtb->count_actns[i] = cnt->action; - } - /* Create drop action. */ - ret = mlx5_flow_os_create_flow_action_drop(&mtb->drop_actn); - if (ret) { - DRV_LOG(ERR, "Failed to create drop action."); - goto error_exit; - } /* Egress meter table. */ - ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0, priv->mtr_color_reg); + ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0); if (ret) { DRV_LOG(ERR, "Failed to prepare egress meter table."); goto error_exit; } /* Ingress meter table. */ - ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0, priv->mtr_color_reg); + ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0); if (ret) { DRV_LOG(ERR, "Failed to prepare ingress meter table."); goto error_exit; } /* FDB meter table. */ if (priv->config.dv_esw_en) { - ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1, - priv->mtr_color_reg); + ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1); if (ret) { DRV_LOG(ERR, "Failed to prepare fdb meter table."); goto error_exit; @@ -13499,24 +13377,153 @@ flow_dv_create_mtr_tbl(struct rte_eth_dev *dev, return NULL; } +/** + * Destroy the meter table matchers. + * Lock free, (mutex should be acquired by caller). + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in,out] dtb + * Pointer to DV meter table. + * + * @return + * Always 0. + */ +static int +flow_dv_destroy_mtr_matchers(struct rte_eth_dev *dev, + struct mlx5_meter_domain_info *dtb) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_tbl_data_entry *tbl; + + if (!priv->config.dv_flow_en) + return 0; + if (dtb->drop_matcher) { + tbl = container_of(dtb->drop_matcher->tbl, typeof(*tbl), tbl); + mlx5_cache_unregister(&tbl->matchers, + &dtb->drop_matcher->entry); + dtb->drop_matcher = NULL; + } + if (dtb->color_matcher) { + tbl = container_of(dtb->color_matcher->tbl, typeof(*tbl), tbl); + mlx5_cache_unregister(&tbl->matchers, + &dtb->color_matcher->entry); + dtb->color_matcher = NULL; + } + return 0; +} + +/** + * Create the matchers for meter table. + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in] color_reg_c_idx + * Reg C index for color match. + * @param[in] mtr_id_reg_c_idx + * Reg C index for meter_id match. + * @param[in] mtr_id_mask + * Mask for meter_id match criteria. + * @param[in,out] dtb + * Pointer to DV meter table. + * @param[out] error + * Perform verbose error reporting if not NULL. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_prepare_mtr_matchers(struct rte_eth_dev *dev, + uint32_t color_reg_c_idx, + uint32_t mtr_id_reg_c_idx, + uint32_t mtr_id_mask, + struct mlx5_meter_domain_info *dtb, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_tbl_data_entry *tbl_data; + struct mlx5_cache_entry *entry; + struct mlx5_flow_dv_matcher matcher = { + .mask = { + .size = sizeof(matcher.mask.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), + }, + .tbl = dtb->tbl, + }; + struct mlx5_flow_dv_match_params value = { + .size = sizeof(value.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), + }; + struct mlx5_flow_cb_ctx ctx = { + .error = error, + .data = &matcher, + }; + + tbl_data = container_of(dtb->tbl, struct mlx5_flow_tbl_data_entry, tbl); + if (!dtb->drop_matcher) { + /* Create matchers for Drop. */ + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + mtr_id_reg_c_idx, 0, mtr_id_mask); + matcher.priority = MLX5_REG_BITS * 2 - priv->max_mtr_bits; + matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf, + matcher.mask.size); + entry = mlx5_cache_register(&tbl_data->matchers, &ctx); + if (!entry) { + DRV_LOG(ERR, "Failed to register meter drop matcher."); + return -1; + } + dtb->drop_matcher = + container_of(entry, struct mlx5_flow_dv_matcher, entry); + } + if (!dtb->color_matcher) { + /* Create matchers for Color + meter_id. */ + if (priv->mtr_reg_share) { + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + color_reg_c_idx, 0, + (mtr_id_mask | + LS32_MASK(MLX5_MTR_COLOR_BITS))); + } else { + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + color_reg_c_idx, 0, + LS32_MASK(MLX5_MTR_COLOR_BITS)); + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + mtr_id_reg_c_idx, 0, mtr_id_mask); + } + matcher.priority = MLX5_REG_BITS - priv->max_mtr_bits; + matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf, + matcher.mask.size); + entry = mlx5_cache_register(&tbl_data->matchers, &ctx); + if (!entry) { + DRV_LOG(ERR, "Failed to register meter color matcher."); + return -1; + } + dtb->color_matcher = + container_of(entry, struct mlx5_flow_dv_matcher, entry); + } + return 0; +} + /** * Destroy domain policer rule. * + * @param[in] dev + * Pointer to Ethernet device. * @param[in] dt * Pointer to domain table. */ static void -flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt) +flow_dv_destroy_domain_policer_rule(struct rte_eth_dev *dev, + struct mlx5_meter_domain_info *dt) { - int i; - - for (i = 0; i < RTE_MTR_DROPPED; i++) { - if (dt->policer_rules[i]) { - claim_zero(mlx5_flow_os_destroy_flow - (dt->policer_rules[i])); - dt->policer_rules[i] = NULL; - } + if (dt->drop_rule) { + claim_zero(mlx5_flow_os_destroy_flow(dt->drop_rule)); + dt->drop_rule = NULL; + } + if (dt->green_rule) { + claim_zero(mlx5_flow_os_destroy_flow(dt->green_rule)); + dt->green_rule = NULL; } + flow_dv_destroy_mtr_matchers(dev, dt); if (dt->jump_actn) { claim_zero(mlx5_flow_os_destroy_flow_action(dt->jump_actn)); dt->jump_actn = NULL; @@ -13537,7 +13544,7 @@ flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt) * Always 0. */ static int -flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused, +flow_dv_destroy_policer_rules(struct rte_eth_dev *dev, const struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr) { @@ -13546,39 +13553,55 @@ flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused, if (!mtb) return 0; if (attr->egress) - flow_dv_destroy_domain_policer_rule(&mtb->egress); + flow_dv_destroy_domain_policer_rule(dev, &mtb->egress); if (attr->ingress) - flow_dv_destroy_domain_policer_rule(&mtb->ingress); + flow_dv_destroy_domain_policer_rule(dev, &mtb->ingress); if (attr->transfer) - flow_dv_destroy_domain_policer_rule(&mtb->transfer); + flow_dv_destroy_domain_policer_rule(dev, &mtb->transfer); return 0; } /** * Create specify domain meter policer rule. * + * @param[in] dev + * Pointer to Ethernet device. * @param[in] fm * Pointer to flow meter structure. * @param[in] mtb * Pointer to DV meter table set. - * @param[in] mtr_reg_c - * Color match REG_C. + * @param[out] drop_rule + * The address of pointer saving drop rule. + * @param[out] color_rule + * The address of pointer saving green rule. * * @return * 0 on success, -1 otherwise. */ static int -flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, +flow_dv_create_policer_forward_rule(struct rte_eth_dev *dev, + struct mlx5_flow_meter *fm, struct mlx5_meter_domain_info *dtb, - uint8_t mtr_reg_c) + void **drop_rule, + void **green_rule) { + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flow_dv_match_params matcher = { - .size = sizeof(matcher.buf), + .size = sizeof(matcher.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), }; struct mlx5_flow_dv_match_params value = { - .size = sizeof(value.buf), + .size = sizeof(value.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), }; struct mlx5_meter_domains_infos *mtb = fm->mfts; + struct rte_flow_error error; + uint32_t color_reg_c = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, + 0, &error); + uint32_t mtr_id_reg_c = mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, + 0, &error); + uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; + uint32_t mtr_id_mask = LS32_MASK(priv->max_mtr_bits) << mtr_id_offset; void *actions[METER_ACTIONS]; int i; int ret = 0; @@ -13591,25 +13614,52 @@ flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, DRV_LOG(ERR, "Failed to create policer jump action."); goto error; } - for (i = 0; i < RTE_MTR_DROPPED; i++) { - int j = 0; - - flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_reg_c, - rte_col_2_mlx5_col(i), UINT8_MAX); - if (mtb->count_actns[i]) - actions[j++] = mtb->count_actns[i]; - if (fm->action[i] == MTR_POLICER_ACTION_DROP) - actions[j++] = mtb->drop_actn; - else - actions[j++] = dtb->jump_actn; - ret = mlx5_flow_os_create_flow(dtb->color_matcher, - (void *)&value, j, actions, - &dtb->policer_rules[i]); + /* Prepare matchers. */ + if (!dtb->drop_matcher || !dtb->color_matcher) { + ret = flow_dv_prepare_mtr_matchers(dev, color_reg_c, + mtr_id_reg_c, mtr_id_mask, + dtb, &error); if (ret) { - DRV_LOG(ERR, "Failed to create policer rule."); + DRV_LOG(ERR, "Failed to setup matchers for mtr table."); goto error; } } + /* Create Drop flow, matching meter_id only. */ + i = 0; + flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_id_reg_c, + (fm->idx << mtr_id_offset), UINT32_MAX); + if (mtb->drop_count) + actions[i++] = mtb->drop_count; + actions[i++] = priv->sh->esw_drop_action; + ret = mlx5_flow_os_create_flow(dtb->drop_matcher->matcher_object, + (void *)&value, i, actions, drop_rule); + if (ret) { + DRV_LOG(ERR, "Failed to create meter policer drop rule."); + goto error; + } + /* Create flow matching Green color + meter_id. */ + i = 0; + if (priv->mtr_reg_share) { + flow_dv_match_meta_reg(matcher.buf, value.buf, color_reg_c, + ((fm->idx << mtr_id_offset) | + rte_col_2_mlx5_col(RTE_COLOR_GREEN)), + UINT32_MAX); + } else { + flow_dv_match_meta_reg(matcher.buf, value.buf, color_reg_c, + rte_col_2_mlx5_col(RTE_COLOR_GREEN), + UINT32_MAX); + flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_id_reg_c, + fm->idx, UINT32_MAX); + } + if (mtb->green_count) + actions[i++] = mtb->green_count; + actions[i++] = dtb->jump_actn; + ret = mlx5_flow_os_create_flow(dtb->color_matcher->matcher_object, + (void *)&value, i, actions, green_rule); + if (ret) { + DRV_LOG(ERR, "Failed to create meter policer color rule."); + goto error; + } return 0; error: rte_errno = errno; @@ -13617,7 +13667,8 @@ flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, } /** - * Create policer rules. + * Prepare policer rules for all domains. + * If meter already initialized, this will replace all old rules with new ones. * * @param[in] dev * Pointer to Ethernet device. @@ -13630,41 +13681,107 @@ flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, * 0 on success, -1 otherwise. */ static int -flow_dv_create_policer_rules(struct rte_eth_dev *dev, - struct mlx5_flow_meter *fm, - const struct rte_flow_attr *attr) +flow_dv_prepare_policer_rules(struct rte_eth_dev *dev, + struct mlx5_flow_meter *fm, + const struct rte_flow_attr *attr) { - struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_meter_domains_infos *mtb = fm->mfts; + bool initialized = false; + struct mlx5_flow_counter *cnt; + void *egress_drop_rule = NULL; + void *egress_green_rule = NULL; + void *ingress_drop_rule = NULL; + void *ingress_green_rule = NULL; + void *transfer_drop_rule = NULL; + void *transfer_green_rule = NULL; int ret; + /* Get the statistics counters for green/drop. */ + if (fm->policer_stats.cnt[RTE_COLOR_GREEN]) { + cnt = flow_dv_counter_get_by_idx(dev, + fm->policer_stats.cnt[RTE_COLOR_GREEN], + NULL); + mtb->green_count = cnt->action; + } else { + mtb->green_count = NULL; + } + if (fm->policer_stats.cnt[RTE_MTR_DROPPED]) { + cnt = flow_dv_counter_get_by_idx(dev, + fm->policer_stats.cnt[RTE_MTR_DROPPED], + NULL); + mtb->drop_count = cnt->action; + } else { + mtb->drop_count = NULL; + } + /** + * If flow meter has been initilized, all policer rules + * are created. So can get if meter initialized by checking + * any policer rule. + */ + if (mtb->egress.drop_rule) + initialized = true; if (attr->egress) { - ret = flow_dv_create_policer_forward_rule(fm, &mtb->egress, - priv->mtr_color_reg); + ret = flow_dv_create_policer_forward_rule(dev, + fm, &mtb->egress, + &egress_drop_rule, &egress_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create egress policer."); goto error; } } if (attr->ingress) { - ret = flow_dv_create_policer_forward_rule(fm, &mtb->ingress, - priv->mtr_color_reg); + ret = flow_dv_create_policer_forward_rule(dev, + fm, &mtb->ingress, + &ingress_drop_rule, &ingress_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create ingress policer."); goto error; } } if (attr->transfer) { - ret = flow_dv_create_policer_forward_rule(fm, &mtb->transfer, - priv->mtr_color_reg); + ret = flow_dv_create_policer_forward_rule(dev, + fm, &mtb->transfer, + &transfer_drop_rule, &transfer_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create transfer policer."); goto error; } } + /* Replace old flows if existing. */ + if (mtb->egress.drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->egress.drop_rule)); + if (mtb->egress.green_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->egress.green_rule)); + if (mtb->ingress.drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->ingress.drop_rule)); + if (mtb->ingress.green_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->ingress.green_rule)); + if (mtb->transfer.drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->transfer.drop_rule)); + if (mtb->transfer.green_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->transfer.green_rule)); + mtb->egress.drop_rule = egress_drop_rule; + mtb->egress.green_rule = egress_green_rule; + mtb->ingress.drop_rule = ingress_drop_rule; + mtb->ingress.green_rule = ingress_green_rule; + mtb->transfer.drop_rule = transfer_drop_rule; + mtb->transfer.green_rule = transfer_green_rule; return 0; error: - flow_dv_destroy_policer_rules(dev, fm, attr); + if (egress_drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(egress_drop_rule)); + if (egress_green_rule) + claim_zero(mlx5_flow_os_destroy_flow(egress_green_rule)); + if (ingress_drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(ingress_drop_rule)); + if (ingress_green_rule) + claim_zero(mlx5_flow_os_destroy_flow(ingress_green_rule)); + if (transfer_drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(transfer_drop_rule)); + if (transfer_green_rule) + claim_zero(mlx5_flow_os_destroy_flow(transfer_green_rule)); + if (!initialized) + flow_dv_destroy_policer_rules(dev, fm, attr); return -1; } @@ -13960,7 +14077,7 @@ const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = { .query = flow_dv_query, .create_mtr_tbls = flow_dv_create_mtr_tbl, .destroy_mtr_tbls = flow_dv_destroy_mtr_tbl, - .create_policer_rules = flow_dv_create_policer_rules, + .prepare_policer_rules = flow_dv_prepare_policer_rules, .destroy_policer_rules = flow_dv_destroy_policer_rules, .counter_alloc = flow_dv_counter_allocate, .counter_free = flow_dv_counter_free, diff --git a/drivers/net/mlx5/mlx5_flow_meter.c b/drivers/net/mlx5/mlx5_flow_meter.c index dbc574b508..68f5c344cf 100644 --- a/drivers/net/mlx5/mlx5_flow_meter.c +++ b/drivers/net/mlx5/mlx5_flow_meter.c @@ -474,6 +474,12 @@ mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id, MTR_POLICER_ACTION_COLOR_RED }; int i; + /* Meter must use global drop action. */ + if (!priv->sh->esw_drop_action) + return -rte_mtr_error_set(error, ENOTSUP, + RTE_MTR_ERROR_TYPE_MTR_PARAMS, + NULL, + "No drop action ready for meter."); /* Meter params must not be NULL. */ if (params == NULL) return -rte_mtr_error_set(error, EINVAL, @@ -630,9 +636,18 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, .egress = 1, .transfer = priv->config.dv_esw_en ? 1 : 0, }; + struct mlx5_indexed_pool_config flow_ipool_cfg = { + .size = 0, + .trunk_size = 64, + .need_lock = 1, + .type = "mlx5_flow_mtr_flow_id_pool", + }; int ret; unsigned int i; uint32_t idx = 0; + uint8_t mtr_id_bits; + uint8_t mtr_reg_bits = priv->mtr_reg_share ? + MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; if (!priv->mtr_en) return -rte_mtr_error_set(error, ENOTSUP, @@ -654,6 +669,13 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, return -rte_mtr_error_set(error, ENOMEM, RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, "Memory alloc failed for meter."); + mtr_id_bits = MLX5_REG_BITS - __builtin_clz(idx); + if ((mtr_id_bits + priv->max_mtr_flow_bits) > mtr_reg_bits) { + DRV_LOG(ERR, "Meter number exceeds max limit."); + goto error; + } + if (mtr_id_bits > priv->max_mtr_bits) + priv->max_mtr_bits = mtr_id_bits; fm->idx = idx; /* Fill the flow meter parameters. */ fm->meter_id = meter_id; @@ -667,10 +689,10 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, if (!fm->policer_stats.cnt[i]) goto error; } - fm->mfts = mlx5_flow_create_mtr_tbls(dev, fm); + fm->mfts = mlx5_flow_create_mtr_tbls(dev); if (!fm->mfts) goto error; - ret = mlx5_flow_create_policer_rules(dev, fm, &attr); + ret = mlx5_flow_prepare_policer_rules(dev, fm, &attr); if (ret) goto error; /* Add to the flow meter list. */ @@ -679,6 +701,9 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, fm->shared = !!shared; fm->policer_stats.stats_mask = params->stats_mask; fm->profile->ref_cnt++; + fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg); + if (!fm->flow_ipool) + goto error; rte_spinlock_init(&fm->sl); return 0; error: @@ -749,6 +774,8 @@ mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id, if (fm->policer_stats.cnt[i]) mlx5_counter_free(dev, fm->policer_stats.cnt[i]); /* Free meter flow table */ + if (fm->flow_ipool) + mlx5_ipool_destroy(fm->flow_ipool); mlx5_flow_destroy_policer_rules(dev, fm, &attr); mlx5_flow_destroy_mtr_tbls(dev, fm->mfts); mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], fm->idx); @@ -1153,30 +1180,24 @@ mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id) * * @param [in] priv * Pointer to mlx5 private data. - * @param [in] meter_id - * Flow meter id. + * @param[in] fm + * Pointer to flow meter. * @param [in] attr * Pointer to flow attributes. * @param [out] error * Pointer to error structure. * - * @return the flow meter pointer, NULL otherwise. + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. */ -struct mlx5_flow_meter * -mlx5_flow_meter_attach(struct mlx5_priv *priv, uint32_t meter_id, +int +mlx5_flow_meter_attach(struct mlx5_priv *priv, + struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr, struct rte_flow_error *error) { - struct mlx5_flow_meter *fm; int ret = 0; - fm = mlx5_flow_meter_find(priv, meter_id); - if (fm == NULL) { - rte_flow_error_set(error, ENOENT, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "Meter object id not valid"); - return fm; - } rte_spinlock_lock(&fm->sl); if (fm->mfts->meter_action) { if (fm->shared && @@ -1210,7 +1231,7 @@ mlx5_flow_meter_attach(struct mlx5_priv *priv, uint32_t meter_id, fm->mfts->meter_action ? "Meter attr not match" : "Meter action create failed"); - return ret ? NULL : fm; + return ret ? -rte_errno : 0; } /** -- 2.27.0 ^ permalink raw reply [flat|nested] 8+ messages in thread
[parent not found: <20210402151430.1531529-1-lizh@nvidia.com>]
* [dpdk-stable] [PATCH v2 02/13] net/mlx5: fix meter statistics [not found] ` <20210402151430.1531529-1-lizh@nvidia.com> @ 2021-04-02 15:14 ` Li Zhang 0 siblings, 0 replies; 8+ messages in thread From: Li Zhang @ 2021-04-02 15:14 UTC (permalink / raw) To: lizh; +Cc: Shun Hao, stable From: Shun Hao <shunh@nvidia.com> This fixes the meter statistics issue that when using multiple meters, only one meter has stats value. To match the correct meter in policer table, now the meter_id is also used in its match criteria, so only one color and one drop matcher are needed. And both meter_id and flow_id will be written to related registers in meter prefix flow. Fixes: 46a5e6bc6a ("net/mlx5: prepare meter flow tables") Cc: stable@dpdk.org Signed-off-by: Shun Hao <shunh@nvidia.com> --- drivers/net/mlx5/linux/mlx5_os.c | 9 +- drivers/net/mlx5/mlx5.c | 7 +- drivers/net/mlx5/mlx5.h | 25 +- drivers/net/mlx5/mlx5_flow.c | 182 +++++++---- drivers/net/mlx5/mlx5_flow.h | 40 +-- drivers/net/mlx5/mlx5_flow_dv.c | 489 ++++++++++++++++++----------- drivers/net/mlx5/mlx5_flow_meter.c | 53 +++- 7 files changed, 512 insertions(+), 293 deletions(-) diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c index 5e3ae9f10e..c3ae1972de 100644 --- a/drivers/net/mlx5/linux/mlx5_os.c +++ b/drivers/net/mlx5/linux/mlx5_os.c @@ -316,7 +316,13 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv) } sh->tx_domain = domain; #ifdef HAVE_MLX5DV_DR_ESWITCH + sh->esw_drop_action = mlx5_glue->dr_create_flow_action_drop(); if (priv->config.dv_esw_en) { + if (!sh->esw_drop_action) { + DRV_LOG(ERR, "Failed to create eswitch drop action"); + err = errno; + goto error; + } domain = mlx5_glue->dr_create_domain (sh->ctx, MLX5DV_DR_DOMAIN_TYPE_FDB); if (!domain) { @@ -325,7 +331,6 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv) goto error; } sh->fdb_domain = domain; - sh->esw_drop_action = mlx5_glue->dr_create_flow_action_drop(); } #endif if (!sh->tunnel_hub) @@ -1225,7 +1230,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev, - 1 + REG_C_0; priv->mtr_en = 1; priv->mtr_reg_share = - config->hca_attr.qos.flow_meter; + config->hca_attr.qos.flow_meter; DRV_LOG(DEBUG, "The REG_C meter uses is %d", priv->mtr_color_reg); } diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index d9372f0a00..a1744b1017 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -275,10 +275,13 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = { }, #endif [MLX5_IPOOL_MTR] = { + /** + * The ipool index should grow continually from small to big, + * for meter idx, so not set grow_trunk to avoid meter index + * not jump continually. + */ .size = sizeof(struct mlx5_flow_meter), .trunk_size = 64, - .grow_trunk = 3, - .grow_shift = 2, .need_lock = 1, .release_mem_en = 1, .malloc = mlx5_malloc, diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index e4963bd107..05b6a082f5 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -187,6 +187,16 @@ struct mlx5_stats_ctrl { /* Maximal number of segments to split. */ #define MLX5_MAX_RXQ_NSEG (1u << MLX5_MAX_LOG_RQ_SEGS) +/* The bit size of one register. */ +#define MLX5_REG_BITS 32 + +/* Idle bits for non-color usage in color register. */ +#define MLX5_MTR_IDLE_BITS_IN_COLOR_REG (MLX5_REG_BITS - MLX5_MTR_COLOR_BITS) + +#define UINT32_T(x) ((uint32_t)(x)) + +#define LS32_MASK(bits) ((UINT32_T(1) << (bits)) - 1) + /* LRO configurations structure. */ struct mlx5_lro_config { uint32_t supported:1; /* Whether LRO is supported. */ @@ -928,9 +938,9 @@ struct mlx5_priv { unsigned int representor:1; /* Device is a port representor. */ unsigned int master:1; /* Device is a E-Switch master. */ unsigned int txpp_en:1; /* Tx packet pacing enabled. */ + unsigned int sampler_en:1; /* Whether support sampler. */ unsigned int mtr_en:1; /* Whether support meter. */ unsigned int mtr_reg_share:1; /* Whether support meter REG_C share. */ - unsigned int sampler_en:1; /* Whether support sampler. */ uint16_t domain_id; /* Switch domain identifier. */ uint16_t vport_id; /* Associated VF vport index (if any). */ uint32_t vport_meta_tag; /* Used for vport index match ove VF LAG. */ @@ -989,6 +999,10 @@ struct mlx5_priv { uint32_t rss_shared_actions; /* RSS shared actions. */ struct mlx5_devx_obj *q_counters; /* DevX queue counter object. */ uint32_t counter_set_id; /* Queue counter ID to set in DevX objects. */ + uint8_t max_mtr_bits; + /* Indicate how many bits are used by meter id at the most. */ + uint8_t max_mtr_flow_bits; + /* Indicate how many bits are used by meter flow id at the most. */ }; #define PORT_ID(priv) ((priv)->dev_data->port_id) @@ -1246,11 +1260,10 @@ int mlx5_pmd_socket_init(void); int mlx5_flow_meter_ops_get(struct rte_eth_dev *dev, void *arg); struct mlx5_flow_meter *mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id); -struct mlx5_flow_meter *mlx5_flow_meter_attach - (struct mlx5_priv *priv, - uint32_t meter_id, - const struct rte_flow_attr *attr, - struct rte_flow_error *error); +int mlx5_flow_meter_attach(struct mlx5_priv *priv, + struct mlx5_flow_meter *fm, + const struct rte_flow_attr *attr, + struct rte_flow_error *error); void mlx5_flow_meter_detach(struct mlx5_flow_meter *fm); /* mlx5_os.c */ diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c index c347f8130e..cffd6129e8 100644 --- a/drivers/net/mlx5/mlx5_flow.c +++ b/drivers/net/mlx5/mlx5_flow.c @@ -764,9 +764,9 @@ mlx5_flow_get_reg_id(struct rte_eth_dev *dev, return REG_C_0; } break; - case MLX5_MTR_SFX: + case MLX5_MTR_ID: /* - * If meter color and flow match share one register, flow match + * If meter color and meter id share one register, flow match * should use the meter color register for match. */ if (priv->mtr_reg_share) @@ -3051,7 +3051,8 @@ flow_mreg_split_qrss_release(struct rte_eth_dev *dev, SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, handle_idx, dev_handle, next) - if (dev_handle->split_flow_id) + if (dev_handle->split_flow_id && + !dev_handle->is_meter_flow_id) mlx5_ipool_free(priv->sh->ipool [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], dev_handle->split_flow_id); @@ -3690,23 +3691,30 @@ flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], * * @param[in] actions * Pointer to the list of actions. - * @param[out] mtr + * @param[out] has_mtr * Pointer to the meter exist flag. + * @param[out] meter_id + * Pointer to the meter id. * * @return * Total number of actions. */ static int -flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr) +flow_check_meter_action(const struct rte_flow_action actions[], + bool *has_mtr, + uint32_t *meter_id) { + const struct rte_flow_action_meter *mtr = NULL; int actions_n = 0; - MLX5_ASSERT(mtr); - *mtr = 0; + MLX5_ASSERT(has_mtr); + *has_mtr = false; for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { switch (actions->type) { case RTE_FLOW_ACTION_TYPE_METER: - *mtr = 1; + mtr = actions->conf; + *meter_id = mtr->mtr_id; + *has_mtr = true; break; default: break; @@ -4363,8 +4371,10 @@ flow_create_split_inner(struct rte_eth_dev *dev, * header will be in the prefix sub flow, as not to take the * L3 tunnel header into account. * - * @param dev + * @param[in] dev * Pointer to Ethernet device. + * @param[in] fm + * Pointer to flow meter structure. * @param[in] items * Pattern specification (list terminated by the END pattern item). * @param[out] sfx_items @@ -4379,29 +4389,38 @@ flow_create_split_inner(struct rte_eth_dev *dev, * The pattern items for the suffix flow. * @param[out] tag_sfx * Pointer to suffix flow tag. + * @param[out] error + * Perform verbose error reporting if not NULL. * * @return - * 0 on success. + * The flow id, 0 otherwise and rte_errno is set. */ -static int +static uint32_t flow_meter_split_prep(struct rte_eth_dev *dev, - const struct rte_flow_item items[], - struct rte_flow_item sfx_items[], - const struct rte_flow_action actions[], - struct rte_flow_action actions_sfx[], - struct rte_flow_action actions_pre[]) + struct mlx5_flow_meter *fm, + const struct rte_flow_item items[], + struct rte_flow_item sfx_items[], + const struct rte_flow_action actions[], + struct rte_flow_action actions_sfx[], + struct rte_flow_action actions_pre[], + struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; struct rte_flow_action *tag_action = NULL; struct rte_flow_item *tag_item; struct mlx5_rte_flow_action_set_tag *set_tag; - struct rte_flow_error error; const struct rte_flow_action_raw_encap *raw_encap; const struct rte_flow_action_raw_decap *raw_decap; - struct mlx5_rte_flow_item_tag *tag_spec; - struct mlx5_rte_flow_item_tag *tag_mask; + struct mlx5_rte_flow_item_tag *tag_item_spec; + struct mlx5_rte_flow_item_tag *tag_item_mask; uint32_t tag_id = 0; bool copy_vlan = false; + uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; + uint8_t mtr_reg_bits = priv->mtr_reg_share ? + MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; + uint8_t flow_id_bits = 0; + int shift; + uint32_t flow_id_val = 0; /* Prepare the actions for prefix and suffix flow. */ for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { @@ -4410,10 +4429,7 @@ flow_meter_split_prep(struct rte_eth_dev *dev, switch (actions->type) { case RTE_FLOW_ACTION_TYPE_METER: /* Add the extra tag action first. */ - tag_action = actions_pre; - tag_action->type = (enum rte_flow_action_type) - MLX5_RTE_FLOW_ACTION_TYPE_TAG; - actions_pre++; + tag_action = actions_pre++; action_cur = &actions_pre; break; case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: @@ -4446,23 +4462,22 @@ flow_meter_split_prep(struct rte_eth_dev *dev, actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; actions_pre->type = RTE_FLOW_ACTION_TYPE_END; actions_pre++; - /* Set the tag. */ - set_tag = (void *)actions_pre; - set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); - mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], - &tag_id); - if (tag_id >= (1 << (sizeof(tag_id) * 8 - MLX5_MTR_COLOR_BITS))) { - DRV_LOG(ERR, "Port %u meter flow id exceed max limit.", - dev->data->port_id); - mlx5_ipool_free(priv->sh->ipool - [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], tag_id); - return 0; - } else if (!tag_id) { - return 0; + /* Generate meter flow_id only if support multiple flows per meter. */ + mlx5_ipool_malloc(fm->flow_ipool, &tag_id); + if (!tag_id) + return rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Failed to allocate meter flow id."); + flow_id_bits = MLX5_REG_BITS - __builtin_clz(tag_id - 1); + flow_id_bits = flow_id_bits ? flow_id_bits : 1; + if ((flow_id_bits + priv->max_mtr_bits) > mtr_reg_bits) { + mlx5_ipool_free(fm->flow_ipool, tag_id); + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Meter flow id exceeds max limit."); } - set_tag->data = tag_id << MLX5_MTR_COLOR_BITS; - assert(tag_action); - tag_action->conf = set_tag; + if (flow_id_bits > priv->max_mtr_flow_bits) + priv->max_mtr_flow_bits = flow_id_bits; /* Prepare the suffix subflow items. */ tag_item = sfx_items++; for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { @@ -4491,16 +4506,34 @@ flow_meter_split_prep(struct rte_eth_dev *dev, } sfx_items->type = RTE_FLOW_ITEM_TYPE_END; sfx_items++; - tag_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; - tag_spec->data = tag_id << MLX5_MTR_COLOR_BITS; - tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); - tag_mask = tag_spec + 1; - tag_mask->data = 0xffffff00; + /* Build tag actions and items for meter_id/meter flow_id. */ + assert(tag_action); + set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; + tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; + tag_item_mask = tag_item_spec + 1; + /* + * The color Reg bits used by flow_id are growing from + * msb to lsb, so must do bit reverse for flow_id val in RegC. + */ + for (shift = 0; shift < flow_id_bits; shift++) + flow_id_val = (flow_id_val << 1) | + (((tag_id - 1) >> shift) & 0x1); + /* Both flow_id and meter_id share the same register. */ + set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 0, error); + set_tag->data = + (fm->idx | (flow_id_val << (mtr_reg_bits - flow_id_bits))) + << mtr_id_offset; + tag_item_spec->id = set_tag->id; + tag_item_spec->data = set_tag->data; + tag_item_mask->data = UINT32_MAX << mtr_id_offset; + tag_action->type = (enum rte_flow_action_type) + MLX5_RTE_FLOW_ACTION_TYPE_TAG; + tag_action->conf = set_tag; tag_item->type = (enum rte_flow_item_type) - MLX5_RTE_FLOW_ITEM_TYPE_TAG; - tag_item->spec = tag_spec; + MLX5_RTE_FLOW_ITEM_TYPE_TAG; + tag_item->spec = tag_item_spec; tag_item->last = NULL; - tag_item->mask = tag_mask; + tag_item->mask = tag_item_mask; return tag_id; } @@ -5200,25 +5233,49 @@ flow_create_split_meter(struct rte_eth_dev *dev, struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); struct rte_flow_action *sfx_actions = NULL; struct rte_flow_action *pre_actions = NULL; struct rte_flow_item *sfx_items = NULL; struct mlx5_flow *dev_flow = NULL; struct rte_flow_attr sfx_attr = *attr; - uint32_t mtr = 0; + struct mlx5_flow_meter *fm = NULL; + bool has_mtr = false; + uint32_t meter_id; uint32_t mtr_tag_id = 0; size_t act_size; size_t item_size; int actions_n = 0; - int ret; + int ret = 0; if (priv->mtr_en) - actions_n = flow_check_meter_action(actions, &mtr); - if (mtr) { - /* The five prefix actions: meter, decap, encap, tag, end. */ + actions_n = flow_check_meter_action(actions, &has_mtr, + &meter_id); + if (has_mtr) { + if (flow->meter) { + fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], + flow->meter); + if (!fm) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "Meter not found."); + } else { + fm = mlx5_flow_meter_find(priv, meter_id); + if (!fm) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "Meter not found."); + ret = mlx5_flow_meter_attach(priv, fm, + &sfx_attr, error); + if (ret) + return -rte_errno; + flow->meter = fm->idx; + } + wks->fm = fm; + /* The prefix actions: meter, decap, encap, tag, end. */ act_size = sizeof(struct rte_flow_action) * (actions_n + 5) + sizeof(struct mlx5_rte_flow_action_set_tag); - /* tag, vlan, port id, end. */ + /* The suffix items: tag, vlan, port id, end. */ #define METER_SUFFIX_ITEM 4 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + sizeof(struct mlx5_rte_flow_item_tag) * 2; @@ -5232,9 +5289,9 @@ flow_create_split_meter(struct rte_eth_dev *dev, sfx_items = (struct rte_flow_item *)((char *)sfx_actions + act_size); pre_actions = sfx_actions + actions_n; - mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items, + mtr_tag_id = flow_meter_split_prep(dev, fm, items, sfx_items, actions, sfx_actions, - pre_actions); + pre_actions, error); if (!mtr_tag_id) { ret = -rte_errno; goto exit; @@ -5245,10 +5302,12 @@ flow_create_split_meter(struct rte_eth_dev *dev, attr, items, pre_actions, flow_split_info, error); if (ret) { + mlx5_ipool_free(fm->flow_ipool, mtr_tag_id); ret = -rte_errno; goto exit; } dev_flow->handle->split_flow_id = mtr_tag_id; + dev_flow->handle->is_meter_flow_id = 1; /* Setting the sfx group atrr. */ sfx_attr.group = sfx_attr.transfer ? (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : @@ -6520,20 +6579,17 @@ mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, * * @param[in] dev * Pointer to Ethernet device. - * @param[in] fm - * Pointer to the flow meter. * * @return * Pointer to table set on success, NULL otherwise. */ struct mlx5_meter_domains_infos * -mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm) +mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev) { const struct mlx5_flow_driver_ops *fops; fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); - return fops->create_mtr_tbls(dev, fm); + return fops->create_mtr_tbls(dev); } /** @@ -6558,7 +6614,7 @@ mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, } /** - * Create policer rules. + * Prepare policer rules. * * @param[in] dev * Pointer to Ethernet device. @@ -6571,14 +6627,14 @@ mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, * 0 on success, -1 otherwise. */ int -mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, +mlx5_flow_prepare_policer_rules(struct rte_eth_dev *dev, struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr) { const struct mlx5_flow_driver_ops *fops; fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); - return fops->create_policer_rules(dev, fm, attr); + return fops->prepare_policer_rules(dev, fm, attr); } /** diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index 8324e188e1..c92e746a04 100644 --- a/drivers/net/mlx5/mlx5_flow.h +++ b/drivers/net/mlx5/mlx5_flow.h @@ -79,7 +79,7 @@ enum mlx5_feature_name { MLX5_APP_TAG, MLX5_COPY_MARK, MLX5_MTR_COLOR, - MLX5_MTR_SFX, + MLX5_MTR_ID, MLX5_ASO_FLOW_HIT, }; @@ -655,7 +655,8 @@ struct mlx5_flow_handle { uint64_t layers; /**< Bit-fields of present layers, see MLX5_FLOW_LAYER_*. */ void *drv_flow; /**< pointer to driver flow object. */ - uint32_t split_flow_id:28; /**< Sub flow unique match flow id. */ + uint32_t split_flow_id:27; /**< Sub flow unique match flow id. */ + uint32_t is_meter_flow_id:1; /**< Indate if flow_id is for meter. */ uint32_t mark:1; /**< Metadate rxq mark flag. */ uint32_t fate_action:3; /**< Fate action type. */ union { @@ -842,14 +843,16 @@ struct mlx5_meter_domain_info { /**< Meter table. */ struct mlx5_flow_tbl_resource *sfx_tbl; /**< Meter suffix table. */ - void *any_matcher; - /**< Meter color not match default criteria. */ - void *color_matcher; - /**< Meter color match criteria. */ + struct mlx5_flow_dv_matcher *drop_matcher; + /**< Matcher for Drop. */ + struct mlx5_flow_dv_matcher *color_matcher; + /**< Matcher for Color. */ void *jump_actn; /**< Meter match action. */ - void *policer_rules[RTE_MTR_DROPPED + 1]; - /**< Meter policer for the match. */ + void *green_rule; + /**< Meter green rule. */ + void *drop_rule; + /**< Meter drop rule. */ }; /* Meter table set for TX RX FDB. */ @@ -862,10 +865,10 @@ struct mlx5_meter_domains_infos { /**< RX meter table. */ struct mlx5_meter_domain_info transfer; /**< FDB meter table. */ - void *drop_actn; - /**< Drop action as not matched. */ - void *count_actns[RTE_MTR_DROPPED + 1]; - /**< Counters for match and unmatched statistics. */ + void *green_count; + /**< Counters for green rule. */ + void *drop_count; + /**< Counters for green rule. */ uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)]; /**< Flow meter parameter. */ size_t fmp_size; @@ -928,6 +931,8 @@ struct mlx5_flow_meter { /**< Meter state. */ uint32_t shared:1; /**< Meter shared or not. */ + struct mlx5_indexed_pool *flow_ipool; + /**< Index pool for flow id. */ }; /* RFC2697 parameter structure. */ @@ -1148,6 +1153,7 @@ struct mlx5_flow_workspace { struct mlx5_flow_rss_desc rss_desc; uint32_t rssq_num; /* Allocated queue num in rss_desc. */ uint32_t flow_idx; /* Intermediate device flow index. */ + struct mlx5_flow_meter *fm; /* Pointer to the meter in flow. */ }; struct mlx5_flow_split_info { @@ -1188,8 +1194,7 @@ typedef int (*mlx5_flow_query_t)(struct rte_eth_dev *dev, void *data, struct rte_flow_error *error); typedef struct mlx5_meter_domains_infos *(*mlx5_flow_create_mtr_tbls_t) - (struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm); + (struct rte_eth_dev *dev); typedef int (*mlx5_flow_destroy_mtr_tbls_t)(struct rte_eth_dev *dev, struct mlx5_meter_domains_infos *tbls); typedef int (*mlx5_flow_create_policer_rules_t) @@ -1252,7 +1257,7 @@ struct mlx5_flow_driver_ops { mlx5_flow_query_t query; mlx5_flow_create_mtr_tbls_t create_mtr_tbls; mlx5_flow_destroy_mtr_tbls_t destroy_mtr_tbls; - mlx5_flow_create_policer_rules_t create_policer_rules; + mlx5_flow_create_policer_rules_t prepare_policer_rules; mlx5_flow_destroy_policer_rules_t destroy_policer_rules; mlx5_flow_counter_alloc_t counter_alloc; mlx5_flow_counter_free_t counter_free; @@ -1452,11 +1457,10 @@ int mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, const struct rte_flow_item_ecpri *acc_mask, struct rte_flow_error *error); struct mlx5_meter_domains_infos *mlx5_flow_create_mtr_tbls - (struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm); + (struct rte_eth_dev *dev); int mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, struct mlx5_meter_domains_infos *tbl); -int mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, +int mlx5_flow_prepare_policer_rules(struct rte_eth_dev *dev, struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr); int mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 23e5849783..ce9857d3f7 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -10982,14 +10982,12 @@ flow_dv_translate(struct rte_eth_dev *dev, const struct rte_flow_action_rss *rss; const struct rte_flow_action *action = actions; const uint8_t *rss_key; - const struct rte_flow_action_meter *mtr; struct mlx5_flow_tbl_resource *tbl; struct mlx5_aso_age_action *age_act; uint32_t port_id = 0; struct mlx5_flow_dv_port_id_action_resource port_id_resource; int action_type = actions->type; const struct rte_flow_action *found_action = NULL; - struct mlx5_flow_meter *fm = NULL; uint32_t jump_group = 0; if (!mlx5_flow_os_action_supported(action_type)) @@ -11414,33 +11412,13 @@ flow_dv_translate(struct rte_eth_dev *dev, MLX5_FLOW_FATE_DEFAULT_MISS; break; case RTE_FLOW_ACTION_TYPE_METER: - mtr = actions->conf; - if (!flow->meter) { - fm = mlx5_flow_meter_attach(priv, mtr->mtr_id, - attr, error); - if (!fm) - return rte_flow_error_set(error, - rte_errno, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "meter not found " - "or invalid parameters"); - flow->meter = fm->idx; - } + if (!wks->fm) + return rte_flow_error_set(error, rte_errno, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "Failed to get meter in flow."); /* Set the meter action. */ - if (!fm) { - fm = mlx5_ipool_get(priv->sh->ipool - [MLX5_IPOOL_MTR], flow->meter); - if (!fm) - return rte_flow_error_set(error, - rte_errno, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "meter not found " - "or invalid parameters"); - } dev_flow->dv.actions[actions_n++] = - fm->mfts->meter_action; + wks->fm->mfts->meter_action; action_flags |= MLX5_FLOW_ACTION_METER; break; case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: @@ -12536,6 +12514,7 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) { struct mlx5_flow_handle *dev_handle; struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_meter *fm = NULL; uint32_t srss = 0; if (!flow) @@ -12546,8 +12525,6 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) flow->counter = 0; } if (flow->meter) { - struct mlx5_flow_meter *fm; - fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], flow->meter); if (fm) @@ -12589,6 +12566,10 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) flow_dv_fate_resource_release(dev, dev_handle); else if (!srss) srss = dev_handle->rix_srss; + if (fm && dev_handle->is_meter_flow_id && + dev_handle->split_flow_id) + mlx5_ipool_free(fm->flow_ipool, + dev_handle->split_flow_id); mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], tmp_idx); } @@ -13274,49 +13255,20 @@ flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev, if (!mtd || !priv->config.dv_flow_en) return 0; - if (mtd->ingress.policer_rules[RTE_MTR_DROPPED]) - claim_zero(mlx5_flow_os_destroy_flow - (mtd->ingress.policer_rules[RTE_MTR_DROPPED])); - if (mtd->egress.policer_rules[RTE_MTR_DROPPED]) - claim_zero(mlx5_flow_os_destroy_flow - (mtd->egress.policer_rules[RTE_MTR_DROPPED])); - if (mtd->transfer.policer_rules[RTE_MTR_DROPPED]) - claim_zero(mlx5_flow_os_destroy_flow - (mtd->transfer.policer_rules[RTE_MTR_DROPPED])); - if (mtd->egress.color_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->egress.color_matcher)); - if (mtd->egress.any_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->egress.any_matcher)); if (mtd->egress.tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->egress.tbl); if (mtd->egress.sfx_tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->egress.sfx_tbl); - if (mtd->ingress.color_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->ingress.color_matcher)); - if (mtd->ingress.any_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->ingress.any_matcher)); if (mtd->ingress.tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->ingress.tbl); if (mtd->ingress.sfx_tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->ingress.sfx_tbl); - if (mtd->transfer.color_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->transfer.color_matcher)); - if (mtd->transfer.any_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->transfer.any_matcher)); if (mtd->transfer.tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->transfer.tbl); if (mtd->transfer.sfx_tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->transfer.sfx_tbl); - if (mtd->drop_actn) - claim_zero(mlx5_flow_os_destroy_flow_action(mtd->drop_actn)); mlx5_free(mtd); return 0; } @@ -13335,37 +13287,17 @@ flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev, * Table attribute. * @param[in] transfer * Table attribute. - * @param[in] color_reg_c_idx - * Reg C index for color match. * * @return - * 0 on success, -1 otherwise and rte_errno is set. + * 0 on success, -1 otherwise. */ static int flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, struct mlx5_meter_domains_infos *mtb, - uint8_t egress, uint8_t transfer, - uint32_t color_reg_c_idx) + uint8_t egress, uint8_t transfer) { - struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_dev_ctx_shared *sh = priv->sh; - struct mlx5_flow_dv_match_params mask = { - .size = sizeof(mask.buf), - }; - struct mlx5_flow_dv_match_params value = { - .size = sizeof(value.buf), - }; - struct mlx5dv_flow_matcher_attr dv_attr = { - .type = IBV_FLOW_ATTR_NORMAL, - .priority = 0, - .match_criteria_enable = 0, - .match_mask = (void *)&mask, - }; - void *actions[METER_ACTIONS]; - struct mlx5_meter_domain_info *dtb; struct rte_flow_error error; - int i = 0; - int ret; + struct mlx5_meter_domain_info *dtb; if (transfer) dtb = &mtb->transfer; @@ -13390,41 +13322,7 @@ flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, DRV_LOG(ERR, "Failed to create meter suffix table."); return -1; } - /* Create matchers, Any and Color. */ - dv_attr.priority = 3; - dv_attr.match_criteria_enable = 0; - ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj, - &dtb->any_matcher); - if (ret) { - DRV_LOG(ERR, "Failed to create meter" - " policer default matcher."); - goto error_exit; - } - dv_attr.priority = 0; - dv_attr.match_criteria_enable = - 1 << MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT; - flow_dv_match_meta_reg(mask.buf, value.buf, color_reg_c_idx, - rte_col_2_mlx5_col(RTE_COLORS), UINT8_MAX); - ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj, - &dtb->color_matcher); - if (ret) { - DRV_LOG(ERR, "Failed to create meter policer color matcher."); - goto error_exit; - } - if (mtb->count_actns[RTE_MTR_DROPPED]) - actions[i++] = mtb->count_actns[RTE_MTR_DROPPED]; - actions[i++] = mtb->drop_actn; - /* Default rule: lowest priority, match any, actions: drop. */ - ret = mlx5_flow_os_create_flow(dtb->any_matcher, (void *)&value, i, - actions, - &dtb->policer_rules[RTE_MTR_DROPPED]); - if (ret) { - DRV_LOG(ERR, "Failed to create meter policer drop rule."); - goto error_exit; - } return 0; -error_exit: - return -1; } /** @@ -13433,20 +13331,16 @@ flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, * * @param[in] dev * Pointer to Ethernet device. - * @param[in] fm - * Pointer to the flow meter. * * @return * Pointer to table set on success, NULL otherwise and rte_errno is set. */ static struct mlx5_meter_domains_infos * -flow_dv_create_mtr_tbl(struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm) +flow_dv_create_mtr_tbl(struct rte_eth_dev *dev) { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_meter_domains_infos *mtb; int ret; - int i; if (!priv->mtr_en) { rte_errno = ENOTSUP; @@ -13457,37 +13351,21 @@ flow_dv_create_mtr_tbl(struct rte_eth_dev *dev, DRV_LOG(ERR, "Failed to allocate memory for meter."); return NULL; } - /* Create meter count actions */ - for (i = 0; i <= RTE_MTR_DROPPED; i++) { - struct mlx5_flow_counter *cnt; - if (!fm->policer_stats.cnt[i]) - continue; - cnt = flow_dv_counter_get_by_idx(dev, - fm->policer_stats.cnt[i], NULL); - mtb->count_actns[i] = cnt->action; - } - /* Create drop action. */ - ret = mlx5_flow_os_create_flow_action_drop(&mtb->drop_actn); - if (ret) { - DRV_LOG(ERR, "Failed to create drop action."); - goto error_exit; - } /* Egress meter table. */ - ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0, priv->mtr_color_reg); + ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0); if (ret) { DRV_LOG(ERR, "Failed to prepare egress meter table."); goto error_exit; } /* Ingress meter table. */ - ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0, priv->mtr_color_reg); + ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0); if (ret) { DRV_LOG(ERR, "Failed to prepare ingress meter table."); goto error_exit; } /* FDB meter table. */ if (priv->config.dv_esw_en) { - ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1, - priv->mtr_color_reg); + ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1); if (ret) { DRV_LOG(ERR, "Failed to prepare fdb meter table."); goto error_exit; @@ -13499,24 +13377,153 @@ flow_dv_create_mtr_tbl(struct rte_eth_dev *dev, return NULL; } +/** + * Destroy the meter table matchers. + * Lock free, (mutex should be acquired by caller). + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in,out] dtb + * Pointer to DV meter table. + * + * @return + * Always 0. + */ +static int +flow_dv_destroy_mtr_matchers(struct rte_eth_dev *dev, + struct mlx5_meter_domain_info *dtb) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_tbl_data_entry *tbl; + + if (!priv->config.dv_flow_en) + return 0; + if (dtb->drop_matcher) { + tbl = container_of(dtb->drop_matcher->tbl, typeof(*tbl), tbl); + mlx5_cache_unregister(&tbl->matchers, + &dtb->drop_matcher->entry); + dtb->drop_matcher = NULL; + } + if (dtb->color_matcher) { + tbl = container_of(dtb->color_matcher->tbl, typeof(*tbl), tbl); + mlx5_cache_unregister(&tbl->matchers, + &dtb->color_matcher->entry); + dtb->color_matcher = NULL; + } + return 0; +} + +/** + * Create the matchers for meter table. + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in] color_reg_c_idx + * Reg C index for color match. + * @param[in] mtr_id_reg_c_idx + * Reg C index for meter_id match. + * @param[in] mtr_id_mask + * Mask for meter_id match criteria. + * @param[in,out] dtb + * Pointer to DV meter table. + * @param[out] error + * Perform verbose error reporting if not NULL. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_prepare_mtr_matchers(struct rte_eth_dev *dev, + uint32_t color_reg_c_idx, + uint32_t mtr_id_reg_c_idx, + uint32_t mtr_id_mask, + struct mlx5_meter_domain_info *dtb, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_tbl_data_entry *tbl_data; + struct mlx5_cache_entry *entry; + struct mlx5_flow_dv_matcher matcher = { + .mask = { + .size = sizeof(matcher.mask.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), + }, + .tbl = dtb->tbl, + }; + struct mlx5_flow_dv_match_params value = { + .size = sizeof(value.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), + }; + struct mlx5_flow_cb_ctx ctx = { + .error = error, + .data = &matcher, + }; + + tbl_data = container_of(dtb->tbl, struct mlx5_flow_tbl_data_entry, tbl); + if (!dtb->drop_matcher) { + /* Create matchers for Drop. */ + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + mtr_id_reg_c_idx, 0, mtr_id_mask); + matcher.priority = MLX5_REG_BITS * 2 - priv->max_mtr_bits; + matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf, + matcher.mask.size); + entry = mlx5_cache_register(&tbl_data->matchers, &ctx); + if (!entry) { + DRV_LOG(ERR, "Failed to register meter drop matcher."); + return -1; + } + dtb->drop_matcher = + container_of(entry, struct mlx5_flow_dv_matcher, entry); + } + if (!dtb->color_matcher) { + /* Create matchers for Color + meter_id. */ + if (priv->mtr_reg_share) { + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + color_reg_c_idx, 0, + (mtr_id_mask | + LS32_MASK(MLX5_MTR_COLOR_BITS))); + } else { + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + color_reg_c_idx, 0, + LS32_MASK(MLX5_MTR_COLOR_BITS)); + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + mtr_id_reg_c_idx, 0, mtr_id_mask); + } + matcher.priority = MLX5_REG_BITS - priv->max_mtr_bits; + matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf, + matcher.mask.size); + entry = mlx5_cache_register(&tbl_data->matchers, &ctx); + if (!entry) { + DRV_LOG(ERR, "Failed to register meter color matcher."); + return -1; + } + dtb->color_matcher = + container_of(entry, struct mlx5_flow_dv_matcher, entry); + } + return 0; +} + /** * Destroy domain policer rule. * + * @param[in] dev + * Pointer to Ethernet device. * @param[in] dt * Pointer to domain table. */ static void -flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt) +flow_dv_destroy_domain_policer_rule(struct rte_eth_dev *dev, + struct mlx5_meter_domain_info *dt) { - int i; - - for (i = 0; i < RTE_MTR_DROPPED; i++) { - if (dt->policer_rules[i]) { - claim_zero(mlx5_flow_os_destroy_flow - (dt->policer_rules[i])); - dt->policer_rules[i] = NULL; - } + if (dt->drop_rule) { + claim_zero(mlx5_flow_os_destroy_flow(dt->drop_rule)); + dt->drop_rule = NULL; + } + if (dt->green_rule) { + claim_zero(mlx5_flow_os_destroy_flow(dt->green_rule)); + dt->green_rule = NULL; } + flow_dv_destroy_mtr_matchers(dev, dt); if (dt->jump_actn) { claim_zero(mlx5_flow_os_destroy_flow_action(dt->jump_actn)); dt->jump_actn = NULL; @@ -13537,7 +13544,7 @@ flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt) * Always 0. */ static int -flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused, +flow_dv_destroy_policer_rules(struct rte_eth_dev *dev, const struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr) { @@ -13546,39 +13553,55 @@ flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused, if (!mtb) return 0; if (attr->egress) - flow_dv_destroy_domain_policer_rule(&mtb->egress); + flow_dv_destroy_domain_policer_rule(dev, &mtb->egress); if (attr->ingress) - flow_dv_destroy_domain_policer_rule(&mtb->ingress); + flow_dv_destroy_domain_policer_rule(dev, &mtb->ingress); if (attr->transfer) - flow_dv_destroy_domain_policer_rule(&mtb->transfer); + flow_dv_destroy_domain_policer_rule(dev, &mtb->transfer); return 0; } /** * Create specify domain meter policer rule. * + * @param[in] dev + * Pointer to Ethernet device. * @param[in] fm * Pointer to flow meter structure. * @param[in] mtb * Pointer to DV meter table set. - * @param[in] mtr_reg_c - * Color match REG_C. + * @param[out] drop_rule + * The address of pointer saving drop rule. + * @param[out] color_rule + * The address of pointer saving green rule. * * @return * 0 on success, -1 otherwise. */ static int -flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, +flow_dv_create_policer_forward_rule(struct rte_eth_dev *dev, + struct mlx5_flow_meter *fm, struct mlx5_meter_domain_info *dtb, - uint8_t mtr_reg_c) + void **drop_rule, + void **green_rule) { + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flow_dv_match_params matcher = { - .size = sizeof(matcher.buf), + .size = sizeof(matcher.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), }; struct mlx5_flow_dv_match_params value = { - .size = sizeof(value.buf), + .size = sizeof(value.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), }; struct mlx5_meter_domains_infos *mtb = fm->mfts; + struct rte_flow_error error; + uint32_t color_reg_c = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, + 0, &error); + uint32_t mtr_id_reg_c = mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, + 0, &error); + uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; + uint32_t mtr_id_mask = LS32_MASK(priv->max_mtr_bits) << mtr_id_offset; void *actions[METER_ACTIONS]; int i; int ret = 0; @@ -13591,25 +13614,52 @@ flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, DRV_LOG(ERR, "Failed to create policer jump action."); goto error; } - for (i = 0; i < RTE_MTR_DROPPED; i++) { - int j = 0; - - flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_reg_c, - rte_col_2_mlx5_col(i), UINT8_MAX); - if (mtb->count_actns[i]) - actions[j++] = mtb->count_actns[i]; - if (fm->action[i] == MTR_POLICER_ACTION_DROP) - actions[j++] = mtb->drop_actn; - else - actions[j++] = dtb->jump_actn; - ret = mlx5_flow_os_create_flow(dtb->color_matcher, - (void *)&value, j, actions, - &dtb->policer_rules[i]); + /* Prepare matchers. */ + if (!dtb->drop_matcher || !dtb->color_matcher) { + ret = flow_dv_prepare_mtr_matchers(dev, color_reg_c, + mtr_id_reg_c, mtr_id_mask, + dtb, &error); if (ret) { - DRV_LOG(ERR, "Failed to create policer rule."); + DRV_LOG(ERR, "Failed to setup matchers for mtr table."); goto error; } } + /* Create Drop flow, matching meter_id only. */ + i = 0; + flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_id_reg_c, + (fm->idx << mtr_id_offset), UINT32_MAX); + if (mtb->drop_count) + actions[i++] = mtb->drop_count; + actions[i++] = priv->sh->esw_drop_action; + ret = mlx5_flow_os_create_flow(dtb->drop_matcher->matcher_object, + (void *)&value, i, actions, drop_rule); + if (ret) { + DRV_LOG(ERR, "Failed to create meter policer drop rule."); + goto error; + } + /* Create flow matching Green color + meter_id. */ + i = 0; + if (priv->mtr_reg_share) { + flow_dv_match_meta_reg(matcher.buf, value.buf, color_reg_c, + ((fm->idx << mtr_id_offset) | + rte_col_2_mlx5_col(RTE_COLOR_GREEN)), + UINT32_MAX); + } else { + flow_dv_match_meta_reg(matcher.buf, value.buf, color_reg_c, + rte_col_2_mlx5_col(RTE_COLOR_GREEN), + UINT32_MAX); + flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_id_reg_c, + fm->idx, UINT32_MAX); + } + if (mtb->green_count) + actions[i++] = mtb->green_count; + actions[i++] = dtb->jump_actn; + ret = mlx5_flow_os_create_flow(dtb->color_matcher->matcher_object, + (void *)&value, i, actions, green_rule); + if (ret) { + DRV_LOG(ERR, "Failed to create meter policer color rule."); + goto error; + } return 0; error: rte_errno = errno; @@ -13617,7 +13667,8 @@ flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, } /** - * Create policer rules. + * Prepare policer rules for all domains. + * If meter already initialized, this will replace all old rules with new ones. * * @param[in] dev * Pointer to Ethernet device. @@ -13630,41 +13681,107 @@ flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, * 0 on success, -1 otherwise. */ static int -flow_dv_create_policer_rules(struct rte_eth_dev *dev, - struct mlx5_flow_meter *fm, - const struct rte_flow_attr *attr) +flow_dv_prepare_policer_rules(struct rte_eth_dev *dev, + struct mlx5_flow_meter *fm, + const struct rte_flow_attr *attr) { - struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_meter_domains_infos *mtb = fm->mfts; + bool initialized = false; + struct mlx5_flow_counter *cnt; + void *egress_drop_rule = NULL; + void *egress_green_rule = NULL; + void *ingress_drop_rule = NULL; + void *ingress_green_rule = NULL; + void *transfer_drop_rule = NULL; + void *transfer_green_rule = NULL; int ret; + /* Get the statistics counters for green/drop. */ + if (fm->policer_stats.cnt[RTE_COLOR_GREEN]) { + cnt = flow_dv_counter_get_by_idx(dev, + fm->policer_stats.cnt[RTE_COLOR_GREEN], + NULL); + mtb->green_count = cnt->action; + } else { + mtb->green_count = NULL; + } + if (fm->policer_stats.cnt[RTE_MTR_DROPPED]) { + cnt = flow_dv_counter_get_by_idx(dev, + fm->policer_stats.cnt[RTE_MTR_DROPPED], + NULL); + mtb->drop_count = cnt->action; + } else { + mtb->drop_count = NULL; + } + /** + * If flow meter has been initilized, all policer rules + * are created. So can get if meter initialized by checking + * any policer rule. + */ + if (mtb->egress.drop_rule) + initialized = true; if (attr->egress) { - ret = flow_dv_create_policer_forward_rule(fm, &mtb->egress, - priv->mtr_color_reg); + ret = flow_dv_create_policer_forward_rule(dev, + fm, &mtb->egress, + &egress_drop_rule, &egress_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create egress policer."); goto error; } } if (attr->ingress) { - ret = flow_dv_create_policer_forward_rule(fm, &mtb->ingress, - priv->mtr_color_reg); + ret = flow_dv_create_policer_forward_rule(dev, + fm, &mtb->ingress, + &ingress_drop_rule, &ingress_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create ingress policer."); goto error; } } if (attr->transfer) { - ret = flow_dv_create_policer_forward_rule(fm, &mtb->transfer, - priv->mtr_color_reg); + ret = flow_dv_create_policer_forward_rule(dev, + fm, &mtb->transfer, + &transfer_drop_rule, &transfer_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create transfer policer."); goto error; } } + /* Replace old flows if existing. */ + if (mtb->egress.drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->egress.drop_rule)); + if (mtb->egress.green_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->egress.green_rule)); + if (mtb->ingress.drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->ingress.drop_rule)); + if (mtb->ingress.green_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->ingress.green_rule)); + if (mtb->transfer.drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->transfer.drop_rule)); + if (mtb->transfer.green_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->transfer.green_rule)); + mtb->egress.drop_rule = egress_drop_rule; + mtb->egress.green_rule = egress_green_rule; + mtb->ingress.drop_rule = ingress_drop_rule; + mtb->ingress.green_rule = ingress_green_rule; + mtb->transfer.drop_rule = transfer_drop_rule; + mtb->transfer.green_rule = transfer_green_rule; return 0; error: - flow_dv_destroy_policer_rules(dev, fm, attr); + if (egress_drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(egress_drop_rule)); + if (egress_green_rule) + claim_zero(mlx5_flow_os_destroy_flow(egress_green_rule)); + if (ingress_drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(ingress_drop_rule)); + if (ingress_green_rule) + claim_zero(mlx5_flow_os_destroy_flow(ingress_green_rule)); + if (transfer_drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(transfer_drop_rule)); + if (transfer_green_rule) + claim_zero(mlx5_flow_os_destroy_flow(transfer_green_rule)); + if (!initialized) + flow_dv_destroy_policer_rules(dev, fm, attr); return -1; } @@ -13960,7 +14077,7 @@ const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = { .query = flow_dv_query, .create_mtr_tbls = flow_dv_create_mtr_tbl, .destroy_mtr_tbls = flow_dv_destroy_mtr_tbl, - .create_policer_rules = flow_dv_create_policer_rules, + .prepare_policer_rules = flow_dv_prepare_policer_rules, .destroy_policer_rules = flow_dv_destroy_policer_rules, .counter_alloc = flow_dv_counter_allocate, .counter_free = flow_dv_counter_free, diff --git a/drivers/net/mlx5/mlx5_flow_meter.c b/drivers/net/mlx5/mlx5_flow_meter.c index dbc574b508..68f5c344cf 100644 --- a/drivers/net/mlx5/mlx5_flow_meter.c +++ b/drivers/net/mlx5/mlx5_flow_meter.c @@ -474,6 +474,12 @@ mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id, MTR_POLICER_ACTION_COLOR_RED }; int i; + /* Meter must use global drop action. */ + if (!priv->sh->esw_drop_action) + return -rte_mtr_error_set(error, ENOTSUP, + RTE_MTR_ERROR_TYPE_MTR_PARAMS, + NULL, + "No drop action ready for meter."); /* Meter params must not be NULL. */ if (params == NULL) return -rte_mtr_error_set(error, EINVAL, @@ -630,9 +636,18 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, .egress = 1, .transfer = priv->config.dv_esw_en ? 1 : 0, }; + struct mlx5_indexed_pool_config flow_ipool_cfg = { + .size = 0, + .trunk_size = 64, + .need_lock = 1, + .type = "mlx5_flow_mtr_flow_id_pool", + }; int ret; unsigned int i; uint32_t idx = 0; + uint8_t mtr_id_bits; + uint8_t mtr_reg_bits = priv->mtr_reg_share ? + MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; if (!priv->mtr_en) return -rte_mtr_error_set(error, ENOTSUP, @@ -654,6 +669,13 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, return -rte_mtr_error_set(error, ENOMEM, RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, "Memory alloc failed for meter."); + mtr_id_bits = MLX5_REG_BITS - __builtin_clz(idx); + if ((mtr_id_bits + priv->max_mtr_flow_bits) > mtr_reg_bits) { + DRV_LOG(ERR, "Meter number exceeds max limit."); + goto error; + } + if (mtr_id_bits > priv->max_mtr_bits) + priv->max_mtr_bits = mtr_id_bits; fm->idx = idx; /* Fill the flow meter parameters. */ fm->meter_id = meter_id; @@ -667,10 +689,10 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, if (!fm->policer_stats.cnt[i]) goto error; } - fm->mfts = mlx5_flow_create_mtr_tbls(dev, fm); + fm->mfts = mlx5_flow_create_mtr_tbls(dev); if (!fm->mfts) goto error; - ret = mlx5_flow_create_policer_rules(dev, fm, &attr); + ret = mlx5_flow_prepare_policer_rules(dev, fm, &attr); if (ret) goto error; /* Add to the flow meter list. */ @@ -679,6 +701,9 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, fm->shared = !!shared; fm->policer_stats.stats_mask = params->stats_mask; fm->profile->ref_cnt++; + fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg); + if (!fm->flow_ipool) + goto error; rte_spinlock_init(&fm->sl); return 0; error: @@ -749,6 +774,8 @@ mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id, if (fm->policer_stats.cnt[i]) mlx5_counter_free(dev, fm->policer_stats.cnt[i]); /* Free meter flow table */ + if (fm->flow_ipool) + mlx5_ipool_destroy(fm->flow_ipool); mlx5_flow_destroy_policer_rules(dev, fm, &attr); mlx5_flow_destroy_mtr_tbls(dev, fm->mfts); mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], fm->idx); @@ -1153,30 +1180,24 @@ mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id) * * @param [in] priv * Pointer to mlx5 private data. - * @param [in] meter_id - * Flow meter id. + * @param[in] fm + * Pointer to flow meter. * @param [in] attr * Pointer to flow attributes. * @param [out] error * Pointer to error structure. * - * @return the flow meter pointer, NULL otherwise. + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. */ -struct mlx5_flow_meter * -mlx5_flow_meter_attach(struct mlx5_priv *priv, uint32_t meter_id, +int +mlx5_flow_meter_attach(struct mlx5_priv *priv, + struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr, struct rte_flow_error *error) { - struct mlx5_flow_meter *fm; int ret = 0; - fm = mlx5_flow_meter_find(priv, meter_id); - if (fm == NULL) { - rte_flow_error_set(error, ENOENT, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "Meter object id not valid"); - return fm; - } rte_spinlock_lock(&fm->sl); if (fm->mfts->meter_action) { if (fm->shared && @@ -1210,7 +1231,7 @@ mlx5_flow_meter_attach(struct mlx5_priv *priv, uint32_t meter_id, fm->mfts->meter_action ? "Meter attr not match" : "Meter action create failed"); - return ret ? NULL : fm; + return ret ? -rte_errno : 0; } /** -- 2.27.0 ^ permalink raw reply [flat|nested] 8+ messages in thread
[parent not found: <20210402151627.1531623-1-lizh@nvidia.com>]
* [dpdk-stable] [PATCH v2 02/13] net/mlx5: fix meter statistics [not found] ` <20210402151627.1531623-1-lizh@nvidia.com> @ 2021-04-02 15:16 ` Li Zhang 0 siblings, 0 replies; 8+ messages in thread From: Li Zhang @ 2021-04-02 15:16 UTC (permalink / raw) To: dekelp, orika, viacheslavo, matan, shahafs, Suanming Mou Cc: dev, thomas, rasland, roniba, Shun Hao, stable From: Shun Hao <shunh@nvidia.com> This fixes the meter statistics issue that when using multiple meters, only one meter has stats value. To match the correct meter in policer table, now the meter_id is also used in its match criteria, so only one color and one drop matcher are needed. And both meter_id and flow_id will be written to related registers in meter prefix flow. Fixes: 46a5e6bc6a ("net/mlx5: prepare meter flow tables") Cc: stable@dpdk.org Signed-off-by: Shun Hao <shunh@nvidia.com> --- drivers/net/mlx5/linux/mlx5_os.c | 9 +- drivers/net/mlx5/mlx5.c | 7 +- drivers/net/mlx5/mlx5.h | 25 +- drivers/net/mlx5/mlx5_flow.c | 182 +++++++---- drivers/net/mlx5/mlx5_flow.h | 40 +-- drivers/net/mlx5/mlx5_flow_dv.c | 489 ++++++++++++++++++----------- drivers/net/mlx5/mlx5_flow_meter.c | 53 +++- 7 files changed, 512 insertions(+), 293 deletions(-) diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c index 5e3ae9f10e..c3ae1972de 100644 --- a/drivers/net/mlx5/linux/mlx5_os.c +++ b/drivers/net/mlx5/linux/mlx5_os.c @@ -316,7 +316,13 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv) } sh->tx_domain = domain; #ifdef HAVE_MLX5DV_DR_ESWITCH + sh->esw_drop_action = mlx5_glue->dr_create_flow_action_drop(); if (priv->config.dv_esw_en) { + if (!sh->esw_drop_action) { + DRV_LOG(ERR, "Failed to create eswitch drop action"); + err = errno; + goto error; + } domain = mlx5_glue->dr_create_domain (sh->ctx, MLX5DV_DR_DOMAIN_TYPE_FDB); if (!domain) { @@ -325,7 +331,6 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv) goto error; } sh->fdb_domain = domain; - sh->esw_drop_action = mlx5_glue->dr_create_flow_action_drop(); } #endif if (!sh->tunnel_hub) @@ -1225,7 +1230,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev, - 1 + REG_C_0; priv->mtr_en = 1; priv->mtr_reg_share = - config->hca_attr.qos.flow_meter; + config->hca_attr.qos.flow_meter; DRV_LOG(DEBUG, "The REG_C meter uses is %d", priv->mtr_color_reg); } diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index d9372f0a00..a1744b1017 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -275,10 +275,13 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = { }, #endif [MLX5_IPOOL_MTR] = { + /** + * The ipool index should grow continually from small to big, + * for meter idx, so not set grow_trunk to avoid meter index + * not jump continually. + */ .size = sizeof(struct mlx5_flow_meter), .trunk_size = 64, - .grow_trunk = 3, - .grow_shift = 2, .need_lock = 1, .release_mem_en = 1, .malloc = mlx5_malloc, diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index e4963bd107..05b6a082f5 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -187,6 +187,16 @@ struct mlx5_stats_ctrl { /* Maximal number of segments to split. */ #define MLX5_MAX_RXQ_NSEG (1u << MLX5_MAX_LOG_RQ_SEGS) +/* The bit size of one register. */ +#define MLX5_REG_BITS 32 + +/* Idle bits for non-color usage in color register. */ +#define MLX5_MTR_IDLE_BITS_IN_COLOR_REG (MLX5_REG_BITS - MLX5_MTR_COLOR_BITS) + +#define UINT32_T(x) ((uint32_t)(x)) + +#define LS32_MASK(bits) ((UINT32_T(1) << (bits)) - 1) + /* LRO configurations structure. */ struct mlx5_lro_config { uint32_t supported:1; /* Whether LRO is supported. */ @@ -928,9 +938,9 @@ struct mlx5_priv { unsigned int representor:1; /* Device is a port representor. */ unsigned int master:1; /* Device is a E-Switch master. */ unsigned int txpp_en:1; /* Tx packet pacing enabled. */ + unsigned int sampler_en:1; /* Whether support sampler. */ unsigned int mtr_en:1; /* Whether support meter. */ unsigned int mtr_reg_share:1; /* Whether support meter REG_C share. */ - unsigned int sampler_en:1; /* Whether support sampler. */ uint16_t domain_id; /* Switch domain identifier. */ uint16_t vport_id; /* Associated VF vport index (if any). */ uint32_t vport_meta_tag; /* Used for vport index match ove VF LAG. */ @@ -989,6 +999,10 @@ struct mlx5_priv { uint32_t rss_shared_actions; /* RSS shared actions. */ struct mlx5_devx_obj *q_counters; /* DevX queue counter object. */ uint32_t counter_set_id; /* Queue counter ID to set in DevX objects. */ + uint8_t max_mtr_bits; + /* Indicate how many bits are used by meter id at the most. */ + uint8_t max_mtr_flow_bits; + /* Indicate how many bits are used by meter flow id at the most. */ }; #define PORT_ID(priv) ((priv)->dev_data->port_id) @@ -1246,11 +1260,10 @@ int mlx5_pmd_socket_init(void); int mlx5_flow_meter_ops_get(struct rte_eth_dev *dev, void *arg); struct mlx5_flow_meter *mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id); -struct mlx5_flow_meter *mlx5_flow_meter_attach - (struct mlx5_priv *priv, - uint32_t meter_id, - const struct rte_flow_attr *attr, - struct rte_flow_error *error); +int mlx5_flow_meter_attach(struct mlx5_priv *priv, + struct mlx5_flow_meter *fm, + const struct rte_flow_attr *attr, + struct rte_flow_error *error); void mlx5_flow_meter_detach(struct mlx5_flow_meter *fm); /* mlx5_os.c */ diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c index c347f8130e..cffd6129e8 100644 --- a/drivers/net/mlx5/mlx5_flow.c +++ b/drivers/net/mlx5/mlx5_flow.c @@ -764,9 +764,9 @@ mlx5_flow_get_reg_id(struct rte_eth_dev *dev, return REG_C_0; } break; - case MLX5_MTR_SFX: + case MLX5_MTR_ID: /* - * If meter color and flow match share one register, flow match + * If meter color and meter id share one register, flow match * should use the meter color register for match. */ if (priv->mtr_reg_share) @@ -3051,7 +3051,8 @@ flow_mreg_split_qrss_release(struct rte_eth_dev *dev, SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, handle_idx, dev_handle, next) - if (dev_handle->split_flow_id) + if (dev_handle->split_flow_id && + !dev_handle->is_meter_flow_id) mlx5_ipool_free(priv->sh->ipool [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], dev_handle->split_flow_id); @@ -3690,23 +3691,30 @@ flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], * * @param[in] actions * Pointer to the list of actions. - * @param[out] mtr + * @param[out] has_mtr * Pointer to the meter exist flag. + * @param[out] meter_id + * Pointer to the meter id. * * @return * Total number of actions. */ static int -flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr) +flow_check_meter_action(const struct rte_flow_action actions[], + bool *has_mtr, + uint32_t *meter_id) { + const struct rte_flow_action_meter *mtr = NULL; int actions_n = 0; - MLX5_ASSERT(mtr); - *mtr = 0; + MLX5_ASSERT(has_mtr); + *has_mtr = false; for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { switch (actions->type) { case RTE_FLOW_ACTION_TYPE_METER: - *mtr = 1; + mtr = actions->conf; + *meter_id = mtr->mtr_id; + *has_mtr = true; break; default: break; @@ -4363,8 +4371,10 @@ flow_create_split_inner(struct rte_eth_dev *dev, * header will be in the prefix sub flow, as not to take the * L3 tunnel header into account. * - * @param dev + * @param[in] dev * Pointer to Ethernet device. + * @param[in] fm + * Pointer to flow meter structure. * @param[in] items * Pattern specification (list terminated by the END pattern item). * @param[out] sfx_items @@ -4379,29 +4389,38 @@ flow_create_split_inner(struct rte_eth_dev *dev, * The pattern items for the suffix flow. * @param[out] tag_sfx * Pointer to suffix flow tag. + * @param[out] error + * Perform verbose error reporting if not NULL. * * @return - * 0 on success. + * The flow id, 0 otherwise and rte_errno is set. */ -static int +static uint32_t flow_meter_split_prep(struct rte_eth_dev *dev, - const struct rte_flow_item items[], - struct rte_flow_item sfx_items[], - const struct rte_flow_action actions[], - struct rte_flow_action actions_sfx[], - struct rte_flow_action actions_pre[]) + struct mlx5_flow_meter *fm, + const struct rte_flow_item items[], + struct rte_flow_item sfx_items[], + const struct rte_flow_action actions[], + struct rte_flow_action actions_sfx[], + struct rte_flow_action actions_pre[], + struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; struct rte_flow_action *tag_action = NULL; struct rte_flow_item *tag_item; struct mlx5_rte_flow_action_set_tag *set_tag; - struct rte_flow_error error; const struct rte_flow_action_raw_encap *raw_encap; const struct rte_flow_action_raw_decap *raw_decap; - struct mlx5_rte_flow_item_tag *tag_spec; - struct mlx5_rte_flow_item_tag *tag_mask; + struct mlx5_rte_flow_item_tag *tag_item_spec; + struct mlx5_rte_flow_item_tag *tag_item_mask; uint32_t tag_id = 0; bool copy_vlan = false; + uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; + uint8_t mtr_reg_bits = priv->mtr_reg_share ? + MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; + uint8_t flow_id_bits = 0; + int shift; + uint32_t flow_id_val = 0; /* Prepare the actions for prefix and suffix flow. */ for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { @@ -4410,10 +4429,7 @@ flow_meter_split_prep(struct rte_eth_dev *dev, switch (actions->type) { case RTE_FLOW_ACTION_TYPE_METER: /* Add the extra tag action first. */ - tag_action = actions_pre; - tag_action->type = (enum rte_flow_action_type) - MLX5_RTE_FLOW_ACTION_TYPE_TAG; - actions_pre++; + tag_action = actions_pre++; action_cur = &actions_pre; break; case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: @@ -4446,23 +4462,22 @@ flow_meter_split_prep(struct rte_eth_dev *dev, actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; actions_pre->type = RTE_FLOW_ACTION_TYPE_END; actions_pre++; - /* Set the tag. */ - set_tag = (void *)actions_pre; - set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); - mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], - &tag_id); - if (tag_id >= (1 << (sizeof(tag_id) * 8 - MLX5_MTR_COLOR_BITS))) { - DRV_LOG(ERR, "Port %u meter flow id exceed max limit.", - dev->data->port_id); - mlx5_ipool_free(priv->sh->ipool - [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], tag_id); - return 0; - } else if (!tag_id) { - return 0; + /* Generate meter flow_id only if support multiple flows per meter. */ + mlx5_ipool_malloc(fm->flow_ipool, &tag_id); + if (!tag_id) + return rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Failed to allocate meter flow id."); + flow_id_bits = MLX5_REG_BITS - __builtin_clz(tag_id - 1); + flow_id_bits = flow_id_bits ? flow_id_bits : 1; + if ((flow_id_bits + priv->max_mtr_bits) > mtr_reg_bits) { + mlx5_ipool_free(fm->flow_ipool, tag_id); + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Meter flow id exceeds max limit."); } - set_tag->data = tag_id << MLX5_MTR_COLOR_BITS; - assert(tag_action); - tag_action->conf = set_tag; + if (flow_id_bits > priv->max_mtr_flow_bits) + priv->max_mtr_flow_bits = flow_id_bits; /* Prepare the suffix subflow items. */ tag_item = sfx_items++; for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { @@ -4491,16 +4506,34 @@ flow_meter_split_prep(struct rte_eth_dev *dev, } sfx_items->type = RTE_FLOW_ITEM_TYPE_END; sfx_items++; - tag_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; - tag_spec->data = tag_id << MLX5_MTR_COLOR_BITS; - tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); - tag_mask = tag_spec + 1; - tag_mask->data = 0xffffff00; + /* Build tag actions and items for meter_id/meter flow_id. */ + assert(tag_action); + set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; + tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; + tag_item_mask = tag_item_spec + 1; + /* + * The color Reg bits used by flow_id are growing from + * msb to lsb, so must do bit reverse for flow_id val in RegC. + */ + for (shift = 0; shift < flow_id_bits; shift++) + flow_id_val = (flow_id_val << 1) | + (((tag_id - 1) >> shift) & 0x1); + /* Both flow_id and meter_id share the same register. */ + set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 0, error); + set_tag->data = + (fm->idx | (flow_id_val << (mtr_reg_bits - flow_id_bits))) + << mtr_id_offset; + tag_item_spec->id = set_tag->id; + tag_item_spec->data = set_tag->data; + tag_item_mask->data = UINT32_MAX << mtr_id_offset; + tag_action->type = (enum rte_flow_action_type) + MLX5_RTE_FLOW_ACTION_TYPE_TAG; + tag_action->conf = set_tag; tag_item->type = (enum rte_flow_item_type) - MLX5_RTE_FLOW_ITEM_TYPE_TAG; - tag_item->spec = tag_spec; + MLX5_RTE_FLOW_ITEM_TYPE_TAG; + tag_item->spec = tag_item_spec; tag_item->last = NULL; - tag_item->mask = tag_mask; + tag_item->mask = tag_item_mask; return tag_id; } @@ -5200,25 +5233,49 @@ flow_create_split_meter(struct rte_eth_dev *dev, struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); struct rte_flow_action *sfx_actions = NULL; struct rte_flow_action *pre_actions = NULL; struct rte_flow_item *sfx_items = NULL; struct mlx5_flow *dev_flow = NULL; struct rte_flow_attr sfx_attr = *attr; - uint32_t mtr = 0; + struct mlx5_flow_meter *fm = NULL; + bool has_mtr = false; + uint32_t meter_id; uint32_t mtr_tag_id = 0; size_t act_size; size_t item_size; int actions_n = 0; - int ret; + int ret = 0; if (priv->mtr_en) - actions_n = flow_check_meter_action(actions, &mtr); - if (mtr) { - /* The five prefix actions: meter, decap, encap, tag, end. */ + actions_n = flow_check_meter_action(actions, &has_mtr, + &meter_id); + if (has_mtr) { + if (flow->meter) { + fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], + flow->meter); + if (!fm) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "Meter not found."); + } else { + fm = mlx5_flow_meter_find(priv, meter_id); + if (!fm) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "Meter not found."); + ret = mlx5_flow_meter_attach(priv, fm, + &sfx_attr, error); + if (ret) + return -rte_errno; + flow->meter = fm->idx; + } + wks->fm = fm; + /* The prefix actions: meter, decap, encap, tag, end. */ act_size = sizeof(struct rte_flow_action) * (actions_n + 5) + sizeof(struct mlx5_rte_flow_action_set_tag); - /* tag, vlan, port id, end. */ + /* The suffix items: tag, vlan, port id, end. */ #define METER_SUFFIX_ITEM 4 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + sizeof(struct mlx5_rte_flow_item_tag) * 2; @@ -5232,9 +5289,9 @@ flow_create_split_meter(struct rte_eth_dev *dev, sfx_items = (struct rte_flow_item *)((char *)sfx_actions + act_size); pre_actions = sfx_actions + actions_n; - mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items, + mtr_tag_id = flow_meter_split_prep(dev, fm, items, sfx_items, actions, sfx_actions, - pre_actions); + pre_actions, error); if (!mtr_tag_id) { ret = -rte_errno; goto exit; @@ -5245,10 +5302,12 @@ flow_create_split_meter(struct rte_eth_dev *dev, attr, items, pre_actions, flow_split_info, error); if (ret) { + mlx5_ipool_free(fm->flow_ipool, mtr_tag_id); ret = -rte_errno; goto exit; } dev_flow->handle->split_flow_id = mtr_tag_id; + dev_flow->handle->is_meter_flow_id = 1; /* Setting the sfx group atrr. */ sfx_attr.group = sfx_attr.transfer ? (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : @@ -6520,20 +6579,17 @@ mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, * * @param[in] dev * Pointer to Ethernet device. - * @param[in] fm - * Pointer to the flow meter. * * @return * Pointer to table set on success, NULL otherwise. */ struct mlx5_meter_domains_infos * -mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm) +mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev) { const struct mlx5_flow_driver_ops *fops; fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); - return fops->create_mtr_tbls(dev, fm); + return fops->create_mtr_tbls(dev); } /** @@ -6558,7 +6614,7 @@ mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, } /** - * Create policer rules. + * Prepare policer rules. * * @param[in] dev * Pointer to Ethernet device. @@ -6571,14 +6627,14 @@ mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, * 0 on success, -1 otherwise. */ int -mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, +mlx5_flow_prepare_policer_rules(struct rte_eth_dev *dev, struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr) { const struct mlx5_flow_driver_ops *fops; fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); - return fops->create_policer_rules(dev, fm, attr); + return fops->prepare_policer_rules(dev, fm, attr); } /** diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index 8324e188e1..c92e746a04 100644 --- a/drivers/net/mlx5/mlx5_flow.h +++ b/drivers/net/mlx5/mlx5_flow.h @@ -79,7 +79,7 @@ enum mlx5_feature_name { MLX5_APP_TAG, MLX5_COPY_MARK, MLX5_MTR_COLOR, - MLX5_MTR_SFX, + MLX5_MTR_ID, MLX5_ASO_FLOW_HIT, }; @@ -655,7 +655,8 @@ struct mlx5_flow_handle { uint64_t layers; /**< Bit-fields of present layers, see MLX5_FLOW_LAYER_*. */ void *drv_flow; /**< pointer to driver flow object. */ - uint32_t split_flow_id:28; /**< Sub flow unique match flow id. */ + uint32_t split_flow_id:27; /**< Sub flow unique match flow id. */ + uint32_t is_meter_flow_id:1; /**< Indate if flow_id is for meter. */ uint32_t mark:1; /**< Metadate rxq mark flag. */ uint32_t fate_action:3; /**< Fate action type. */ union { @@ -842,14 +843,16 @@ struct mlx5_meter_domain_info { /**< Meter table. */ struct mlx5_flow_tbl_resource *sfx_tbl; /**< Meter suffix table. */ - void *any_matcher; - /**< Meter color not match default criteria. */ - void *color_matcher; - /**< Meter color match criteria. */ + struct mlx5_flow_dv_matcher *drop_matcher; + /**< Matcher for Drop. */ + struct mlx5_flow_dv_matcher *color_matcher; + /**< Matcher for Color. */ void *jump_actn; /**< Meter match action. */ - void *policer_rules[RTE_MTR_DROPPED + 1]; - /**< Meter policer for the match. */ + void *green_rule; + /**< Meter green rule. */ + void *drop_rule; + /**< Meter drop rule. */ }; /* Meter table set for TX RX FDB. */ @@ -862,10 +865,10 @@ struct mlx5_meter_domains_infos { /**< RX meter table. */ struct mlx5_meter_domain_info transfer; /**< FDB meter table. */ - void *drop_actn; - /**< Drop action as not matched. */ - void *count_actns[RTE_MTR_DROPPED + 1]; - /**< Counters for match and unmatched statistics. */ + void *green_count; + /**< Counters for green rule. */ + void *drop_count; + /**< Counters for green rule. */ uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)]; /**< Flow meter parameter. */ size_t fmp_size; @@ -928,6 +931,8 @@ struct mlx5_flow_meter { /**< Meter state. */ uint32_t shared:1; /**< Meter shared or not. */ + struct mlx5_indexed_pool *flow_ipool; + /**< Index pool for flow id. */ }; /* RFC2697 parameter structure. */ @@ -1148,6 +1153,7 @@ struct mlx5_flow_workspace { struct mlx5_flow_rss_desc rss_desc; uint32_t rssq_num; /* Allocated queue num in rss_desc. */ uint32_t flow_idx; /* Intermediate device flow index. */ + struct mlx5_flow_meter *fm; /* Pointer to the meter in flow. */ }; struct mlx5_flow_split_info { @@ -1188,8 +1194,7 @@ typedef int (*mlx5_flow_query_t)(struct rte_eth_dev *dev, void *data, struct rte_flow_error *error); typedef struct mlx5_meter_domains_infos *(*mlx5_flow_create_mtr_tbls_t) - (struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm); + (struct rte_eth_dev *dev); typedef int (*mlx5_flow_destroy_mtr_tbls_t)(struct rte_eth_dev *dev, struct mlx5_meter_domains_infos *tbls); typedef int (*mlx5_flow_create_policer_rules_t) @@ -1252,7 +1257,7 @@ struct mlx5_flow_driver_ops { mlx5_flow_query_t query; mlx5_flow_create_mtr_tbls_t create_mtr_tbls; mlx5_flow_destroy_mtr_tbls_t destroy_mtr_tbls; - mlx5_flow_create_policer_rules_t create_policer_rules; + mlx5_flow_create_policer_rules_t prepare_policer_rules; mlx5_flow_destroy_policer_rules_t destroy_policer_rules; mlx5_flow_counter_alloc_t counter_alloc; mlx5_flow_counter_free_t counter_free; @@ -1452,11 +1457,10 @@ int mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, const struct rte_flow_item_ecpri *acc_mask, struct rte_flow_error *error); struct mlx5_meter_domains_infos *mlx5_flow_create_mtr_tbls - (struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm); + (struct rte_eth_dev *dev); int mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, struct mlx5_meter_domains_infos *tbl); -int mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, +int mlx5_flow_prepare_policer_rules(struct rte_eth_dev *dev, struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr); int mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 23e5849783..ce9857d3f7 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -10982,14 +10982,12 @@ flow_dv_translate(struct rte_eth_dev *dev, const struct rte_flow_action_rss *rss; const struct rte_flow_action *action = actions; const uint8_t *rss_key; - const struct rte_flow_action_meter *mtr; struct mlx5_flow_tbl_resource *tbl; struct mlx5_aso_age_action *age_act; uint32_t port_id = 0; struct mlx5_flow_dv_port_id_action_resource port_id_resource; int action_type = actions->type; const struct rte_flow_action *found_action = NULL; - struct mlx5_flow_meter *fm = NULL; uint32_t jump_group = 0; if (!mlx5_flow_os_action_supported(action_type)) @@ -11414,33 +11412,13 @@ flow_dv_translate(struct rte_eth_dev *dev, MLX5_FLOW_FATE_DEFAULT_MISS; break; case RTE_FLOW_ACTION_TYPE_METER: - mtr = actions->conf; - if (!flow->meter) { - fm = mlx5_flow_meter_attach(priv, mtr->mtr_id, - attr, error); - if (!fm) - return rte_flow_error_set(error, - rte_errno, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "meter not found " - "or invalid parameters"); - flow->meter = fm->idx; - } + if (!wks->fm) + return rte_flow_error_set(error, rte_errno, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "Failed to get meter in flow."); /* Set the meter action. */ - if (!fm) { - fm = mlx5_ipool_get(priv->sh->ipool - [MLX5_IPOOL_MTR], flow->meter); - if (!fm) - return rte_flow_error_set(error, - rte_errno, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "meter not found " - "or invalid parameters"); - } dev_flow->dv.actions[actions_n++] = - fm->mfts->meter_action; + wks->fm->mfts->meter_action; action_flags |= MLX5_FLOW_ACTION_METER; break; case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: @@ -12536,6 +12514,7 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) { struct mlx5_flow_handle *dev_handle; struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_meter *fm = NULL; uint32_t srss = 0; if (!flow) @@ -12546,8 +12525,6 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) flow->counter = 0; } if (flow->meter) { - struct mlx5_flow_meter *fm; - fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], flow->meter); if (fm) @@ -12589,6 +12566,10 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) flow_dv_fate_resource_release(dev, dev_handle); else if (!srss) srss = dev_handle->rix_srss; + if (fm && dev_handle->is_meter_flow_id && + dev_handle->split_flow_id) + mlx5_ipool_free(fm->flow_ipool, + dev_handle->split_flow_id); mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], tmp_idx); } @@ -13274,49 +13255,20 @@ flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev, if (!mtd || !priv->config.dv_flow_en) return 0; - if (mtd->ingress.policer_rules[RTE_MTR_DROPPED]) - claim_zero(mlx5_flow_os_destroy_flow - (mtd->ingress.policer_rules[RTE_MTR_DROPPED])); - if (mtd->egress.policer_rules[RTE_MTR_DROPPED]) - claim_zero(mlx5_flow_os_destroy_flow - (mtd->egress.policer_rules[RTE_MTR_DROPPED])); - if (mtd->transfer.policer_rules[RTE_MTR_DROPPED]) - claim_zero(mlx5_flow_os_destroy_flow - (mtd->transfer.policer_rules[RTE_MTR_DROPPED])); - if (mtd->egress.color_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->egress.color_matcher)); - if (mtd->egress.any_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->egress.any_matcher)); if (mtd->egress.tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->egress.tbl); if (mtd->egress.sfx_tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->egress.sfx_tbl); - if (mtd->ingress.color_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->ingress.color_matcher)); - if (mtd->ingress.any_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->ingress.any_matcher)); if (mtd->ingress.tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->ingress.tbl); if (mtd->ingress.sfx_tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->ingress.sfx_tbl); - if (mtd->transfer.color_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->transfer.color_matcher)); - if (mtd->transfer.any_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->transfer.any_matcher)); if (mtd->transfer.tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->transfer.tbl); if (mtd->transfer.sfx_tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->transfer.sfx_tbl); - if (mtd->drop_actn) - claim_zero(mlx5_flow_os_destroy_flow_action(mtd->drop_actn)); mlx5_free(mtd); return 0; } @@ -13335,37 +13287,17 @@ flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev, * Table attribute. * @param[in] transfer * Table attribute. - * @param[in] color_reg_c_idx - * Reg C index for color match. * * @return - * 0 on success, -1 otherwise and rte_errno is set. + * 0 on success, -1 otherwise. */ static int flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, struct mlx5_meter_domains_infos *mtb, - uint8_t egress, uint8_t transfer, - uint32_t color_reg_c_idx) + uint8_t egress, uint8_t transfer) { - struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_dev_ctx_shared *sh = priv->sh; - struct mlx5_flow_dv_match_params mask = { - .size = sizeof(mask.buf), - }; - struct mlx5_flow_dv_match_params value = { - .size = sizeof(value.buf), - }; - struct mlx5dv_flow_matcher_attr dv_attr = { - .type = IBV_FLOW_ATTR_NORMAL, - .priority = 0, - .match_criteria_enable = 0, - .match_mask = (void *)&mask, - }; - void *actions[METER_ACTIONS]; - struct mlx5_meter_domain_info *dtb; struct rte_flow_error error; - int i = 0; - int ret; + struct mlx5_meter_domain_info *dtb; if (transfer) dtb = &mtb->transfer; @@ -13390,41 +13322,7 @@ flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, DRV_LOG(ERR, "Failed to create meter suffix table."); return -1; } - /* Create matchers, Any and Color. */ - dv_attr.priority = 3; - dv_attr.match_criteria_enable = 0; - ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj, - &dtb->any_matcher); - if (ret) { - DRV_LOG(ERR, "Failed to create meter" - " policer default matcher."); - goto error_exit; - } - dv_attr.priority = 0; - dv_attr.match_criteria_enable = - 1 << MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT; - flow_dv_match_meta_reg(mask.buf, value.buf, color_reg_c_idx, - rte_col_2_mlx5_col(RTE_COLORS), UINT8_MAX); - ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj, - &dtb->color_matcher); - if (ret) { - DRV_LOG(ERR, "Failed to create meter policer color matcher."); - goto error_exit; - } - if (mtb->count_actns[RTE_MTR_DROPPED]) - actions[i++] = mtb->count_actns[RTE_MTR_DROPPED]; - actions[i++] = mtb->drop_actn; - /* Default rule: lowest priority, match any, actions: drop. */ - ret = mlx5_flow_os_create_flow(dtb->any_matcher, (void *)&value, i, - actions, - &dtb->policer_rules[RTE_MTR_DROPPED]); - if (ret) { - DRV_LOG(ERR, "Failed to create meter policer drop rule."); - goto error_exit; - } return 0; -error_exit: - return -1; } /** @@ -13433,20 +13331,16 @@ flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, * * @param[in] dev * Pointer to Ethernet device. - * @param[in] fm - * Pointer to the flow meter. * * @return * Pointer to table set on success, NULL otherwise and rte_errno is set. */ static struct mlx5_meter_domains_infos * -flow_dv_create_mtr_tbl(struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm) +flow_dv_create_mtr_tbl(struct rte_eth_dev *dev) { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_meter_domains_infos *mtb; int ret; - int i; if (!priv->mtr_en) { rte_errno = ENOTSUP; @@ -13457,37 +13351,21 @@ flow_dv_create_mtr_tbl(struct rte_eth_dev *dev, DRV_LOG(ERR, "Failed to allocate memory for meter."); return NULL; } - /* Create meter count actions */ - for (i = 0; i <= RTE_MTR_DROPPED; i++) { - struct mlx5_flow_counter *cnt; - if (!fm->policer_stats.cnt[i]) - continue; - cnt = flow_dv_counter_get_by_idx(dev, - fm->policer_stats.cnt[i], NULL); - mtb->count_actns[i] = cnt->action; - } - /* Create drop action. */ - ret = mlx5_flow_os_create_flow_action_drop(&mtb->drop_actn); - if (ret) { - DRV_LOG(ERR, "Failed to create drop action."); - goto error_exit; - } /* Egress meter table. */ - ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0, priv->mtr_color_reg); + ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0); if (ret) { DRV_LOG(ERR, "Failed to prepare egress meter table."); goto error_exit; } /* Ingress meter table. */ - ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0, priv->mtr_color_reg); + ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0); if (ret) { DRV_LOG(ERR, "Failed to prepare ingress meter table."); goto error_exit; } /* FDB meter table. */ if (priv->config.dv_esw_en) { - ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1, - priv->mtr_color_reg); + ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1); if (ret) { DRV_LOG(ERR, "Failed to prepare fdb meter table."); goto error_exit; @@ -13499,24 +13377,153 @@ flow_dv_create_mtr_tbl(struct rte_eth_dev *dev, return NULL; } +/** + * Destroy the meter table matchers. + * Lock free, (mutex should be acquired by caller). + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in,out] dtb + * Pointer to DV meter table. + * + * @return + * Always 0. + */ +static int +flow_dv_destroy_mtr_matchers(struct rte_eth_dev *dev, + struct mlx5_meter_domain_info *dtb) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_tbl_data_entry *tbl; + + if (!priv->config.dv_flow_en) + return 0; + if (dtb->drop_matcher) { + tbl = container_of(dtb->drop_matcher->tbl, typeof(*tbl), tbl); + mlx5_cache_unregister(&tbl->matchers, + &dtb->drop_matcher->entry); + dtb->drop_matcher = NULL; + } + if (dtb->color_matcher) { + tbl = container_of(dtb->color_matcher->tbl, typeof(*tbl), tbl); + mlx5_cache_unregister(&tbl->matchers, + &dtb->color_matcher->entry); + dtb->color_matcher = NULL; + } + return 0; +} + +/** + * Create the matchers for meter table. + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in] color_reg_c_idx + * Reg C index for color match. + * @param[in] mtr_id_reg_c_idx + * Reg C index for meter_id match. + * @param[in] mtr_id_mask + * Mask for meter_id match criteria. + * @param[in,out] dtb + * Pointer to DV meter table. + * @param[out] error + * Perform verbose error reporting if not NULL. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_prepare_mtr_matchers(struct rte_eth_dev *dev, + uint32_t color_reg_c_idx, + uint32_t mtr_id_reg_c_idx, + uint32_t mtr_id_mask, + struct mlx5_meter_domain_info *dtb, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_tbl_data_entry *tbl_data; + struct mlx5_cache_entry *entry; + struct mlx5_flow_dv_matcher matcher = { + .mask = { + .size = sizeof(matcher.mask.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), + }, + .tbl = dtb->tbl, + }; + struct mlx5_flow_dv_match_params value = { + .size = sizeof(value.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), + }; + struct mlx5_flow_cb_ctx ctx = { + .error = error, + .data = &matcher, + }; + + tbl_data = container_of(dtb->tbl, struct mlx5_flow_tbl_data_entry, tbl); + if (!dtb->drop_matcher) { + /* Create matchers for Drop. */ + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + mtr_id_reg_c_idx, 0, mtr_id_mask); + matcher.priority = MLX5_REG_BITS * 2 - priv->max_mtr_bits; + matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf, + matcher.mask.size); + entry = mlx5_cache_register(&tbl_data->matchers, &ctx); + if (!entry) { + DRV_LOG(ERR, "Failed to register meter drop matcher."); + return -1; + } + dtb->drop_matcher = + container_of(entry, struct mlx5_flow_dv_matcher, entry); + } + if (!dtb->color_matcher) { + /* Create matchers for Color + meter_id. */ + if (priv->mtr_reg_share) { + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + color_reg_c_idx, 0, + (mtr_id_mask | + LS32_MASK(MLX5_MTR_COLOR_BITS))); + } else { + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + color_reg_c_idx, 0, + LS32_MASK(MLX5_MTR_COLOR_BITS)); + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + mtr_id_reg_c_idx, 0, mtr_id_mask); + } + matcher.priority = MLX5_REG_BITS - priv->max_mtr_bits; + matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf, + matcher.mask.size); + entry = mlx5_cache_register(&tbl_data->matchers, &ctx); + if (!entry) { + DRV_LOG(ERR, "Failed to register meter color matcher."); + return -1; + } + dtb->color_matcher = + container_of(entry, struct mlx5_flow_dv_matcher, entry); + } + return 0; +} + /** * Destroy domain policer rule. * + * @param[in] dev + * Pointer to Ethernet device. * @param[in] dt * Pointer to domain table. */ static void -flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt) +flow_dv_destroy_domain_policer_rule(struct rte_eth_dev *dev, + struct mlx5_meter_domain_info *dt) { - int i; - - for (i = 0; i < RTE_MTR_DROPPED; i++) { - if (dt->policer_rules[i]) { - claim_zero(mlx5_flow_os_destroy_flow - (dt->policer_rules[i])); - dt->policer_rules[i] = NULL; - } + if (dt->drop_rule) { + claim_zero(mlx5_flow_os_destroy_flow(dt->drop_rule)); + dt->drop_rule = NULL; + } + if (dt->green_rule) { + claim_zero(mlx5_flow_os_destroy_flow(dt->green_rule)); + dt->green_rule = NULL; } + flow_dv_destroy_mtr_matchers(dev, dt); if (dt->jump_actn) { claim_zero(mlx5_flow_os_destroy_flow_action(dt->jump_actn)); dt->jump_actn = NULL; @@ -13537,7 +13544,7 @@ flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt) * Always 0. */ static int -flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused, +flow_dv_destroy_policer_rules(struct rte_eth_dev *dev, const struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr) { @@ -13546,39 +13553,55 @@ flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused, if (!mtb) return 0; if (attr->egress) - flow_dv_destroy_domain_policer_rule(&mtb->egress); + flow_dv_destroy_domain_policer_rule(dev, &mtb->egress); if (attr->ingress) - flow_dv_destroy_domain_policer_rule(&mtb->ingress); + flow_dv_destroy_domain_policer_rule(dev, &mtb->ingress); if (attr->transfer) - flow_dv_destroy_domain_policer_rule(&mtb->transfer); + flow_dv_destroy_domain_policer_rule(dev, &mtb->transfer); return 0; } /** * Create specify domain meter policer rule. * + * @param[in] dev + * Pointer to Ethernet device. * @param[in] fm * Pointer to flow meter structure. * @param[in] mtb * Pointer to DV meter table set. - * @param[in] mtr_reg_c - * Color match REG_C. + * @param[out] drop_rule + * The address of pointer saving drop rule. + * @param[out] color_rule + * The address of pointer saving green rule. * * @return * 0 on success, -1 otherwise. */ static int -flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, +flow_dv_create_policer_forward_rule(struct rte_eth_dev *dev, + struct mlx5_flow_meter *fm, struct mlx5_meter_domain_info *dtb, - uint8_t mtr_reg_c) + void **drop_rule, + void **green_rule) { + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flow_dv_match_params matcher = { - .size = sizeof(matcher.buf), + .size = sizeof(matcher.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), }; struct mlx5_flow_dv_match_params value = { - .size = sizeof(value.buf), + .size = sizeof(value.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), }; struct mlx5_meter_domains_infos *mtb = fm->mfts; + struct rte_flow_error error; + uint32_t color_reg_c = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, + 0, &error); + uint32_t mtr_id_reg_c = mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, + 0, &error); + uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; + uint32_t mtr_id_mask = LS32_MASK(priv->max_mtr_bits) << mtr_id_offset; void *actions[METER_ACTIONS]; int i; int ret = 0; @@ -13591,25 +13614,52 @@ flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, DRV_LOG(ERR, "Failed to create policer jump action."); goto error; } - for (i = 0; i < RTE_MTR_DROPPED; i++) { - int j = 0; - - flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_reg_c, - rte_col_2_mlx5_col(i), UINT8_MAX); - if (mtb->count_actns[i]) - actions[j++] = mtb->count_actns[i]; - if (fm->action[i] == MTR_POLICER_ACTION_DROP) - actions[j++] = mtb->drop_actn; - else - actions[j++] = dtb->jump_actn; - ret = mlx5_flow_os_create_flow(dtb->color_matcher, - (void *)&value, j, actions, - &dtb->policer_rules[i]); + /* Prepare matchers. */ + if (!dtb->drop_matcher || !dtb->color_matcher) { + ret = flow_dv_prepare_mtr_matchers(dev, color_reg_c, + mtr_id_reg_c, mtr_id_mask, + dtb, &error); if (ret) { - DRV_LOG(ERR, "Failed to create policer rule."); + DRV_LOG(ERR, "Failed to setup matchers for mtr table."); goto error; } } + /* Create Drop flow, matching meter_id only. */ + i = 0; + flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_id_reg_c, + (fm->idx << mtr_id_offset), UINT32_MAX); + if (mtb->drop_count) + actions[i++] = mtb->drop_count; + actions[i++] = priv->sh->esw_drop_action; + ret = mlx5_flow_os_create_flow(dtb->drop_matcher->matcher_object, + (void *)&value, i, actions, drop_rule); + if (ret) { + DRV_LOG(ERR, "Failed to create meter policer drop rule."); + goto error; + } + /* Create flow matching Green color + meter_id. */ + i = 0; + if (priv->mtr_reg_share) { + flow_dv_match_meta_reg(matcher.buf, value.buf, color_reg_c, + ((fm->idx << mtr_id_offset) | + rte_col_2_mlx5_col(RTE_COLOR_GREEN)), + UINT32_MAX); + } else { + flow_dv_match_meta_reg(matcher.buf, value.buf, color_reg_c, + rte_col_2_mlx5_col(RTE_COLOR_GREEN), + UINT32_MAX); + flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_id_reg_c, + fm->idx, UINT32_MAX); + } + if (mtb->green_count) + actions[i++] = mtb->green_count; + actions[i++] = dtb->jump_actn; + ret = mlx5_flow_os_create_flow(dtb->color_matcher->matcher_object, + (void *)&value, i, actions, green_rule); + if (ret) { + DRV_LOG(ERR, "Failed to create meter policer color rule."); + goto error; + } return 0; error: rte_errno = errno; @@ -13617,7 +13667,8 @@ flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, } /** - * Create policer rules. + * Prepare policer rules for all domains. + * If meter already initialized, this will replace all old rules with new ones. * * @param[in] dev * Pointer to Ethernet device. @@ -13630,41 +13681,107 @@ flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, * 0 on success, -1 otherwise. */ static int -flow_dv_create_policer_rules(struct rte_eth_dev *dev, - struct mlx5_flow_meter *fm, - const struct rte_flow_attr *attr) +flow_dv_prepare_policer_rules(struct rte_eth_dev *dev, + struct mlx5_flow_meter *fm, + const struct rte_flow_attr *attr) { - struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_meter_domains_infos *mtb = fm->mfts; + bool initialized = false; + struct mlx5_flow_counter *cnt; + void *egress_drop_rule = NULL; + void *egress_green_rule = NULL; + void *ingress_drop_rule = NULL; + void *ingress_green_rule = NULL; + void *transfer_drop_rule = NULL; + void *transfer_green_rule = NULL; int ret; + /* Get the statistics counters for green/drop. */ + if (fm->policer_stats.cnt[RTE_COLOR_GREEN]) { + cnt = flow_dv_counter_get_by_idx(dev, + fm->policer_stats.cnt[RTE_COLOR_GREEN], + NULL); + mtb->green_count = cnt->action; + } else { + mtb->green_count = NULL; + } + if (fm->policer_stats.cnt[RTE_MTR_DROPPED]) { + cnt = flow_dv_counter_get_by_idx(dev, + fm->policer_stats.cnt[RTE_MTR_DROPPED], + NULL); + mtb->drop_count = cnt->action; + } else { + mtb->drop_count = NULL; + } + /** + * If flow meter has been initilized, all policer rules + * are created. So can get if meter initialized by checking + * any policer rule. + */ + if (mtb->egress.drop_rule) + initialized = true; if (attr->egress) { - ret = flow_dv_create_policer_forward_rule(fm, &mtb->egress, - priv->mtr_color_reg); + ret = flow_dv_create_policer_forward_rule(dev, + fm, &mtb->egress, + &egress_drop_rule, &egress_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create egress policer."); goto error; } } if (attr->ingress) { - ret = flow_dv_create_policer_forward_rule(fm, &mtb->ingress, - priv->mtr_color_reg); + ret = flow_dv_create_policer_forward_rule(dev, + fm, &mtb->ingress, + &ingress_drop_rule, &ingress_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create ingress policer."); goto error; } } if (attr->transfer) { - ret = flow_dv_create_policer_forward_rule(fm, &mtb->transfer, - priv->mtr_color_reg); + ret = flow_dv_create_policer_forward_rule(dev, + fm, &mtb->transfer, + &transfer_drop_rule, &transfer_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create transfer policer."); goto error; } } + /* Replace old flows if existing. */ + if (mtb->egress.drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->egress.drop_rule)); + if (mtb->egress.green_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->egress.green_rule)); + if (mtb->ingress.drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->ingress.drop_rule)); + if (mtb->ingress.green_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->ingress.green_rule)); + if (mtb->transfer.drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->transfer.drop_rule)); + if (mtb->transfer.green_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->transfer.green_rule)); + mtb->egress.drop_rule = egress_drop_rule; + mtb->egress.green_rule = egress_green_rule; + mtb->ingress.drop_rule = ingress_drop_rule; + mtb->ingress.green_rule = ingress_green_rule; + mtb->transfer.drop_rule = transfer_drop_rule; + mtb->transfer.green_rule = transfer_green_rule; return 0; error: - flow_dv_destroy_policer_rules(dev, fm, attr); + if (egress_drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(egress_drop_rule)); + if (egress_green_rule) + claim_zero(mlx5_flow_os_destroy_flow(egress_green_rule)); + if (ingress_drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(ingress_drop_rule)); + if (ingress_green_rule) + claim_zero(mlx5_flow_os_destroy_flow(ingress_green_rule)); + if (transfer_drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(transfer_drop_rule)); + if (transfer_green_rule) + claim_zero(mlx5_flow_os_destroy_flow(transfer_green_rule)); + if (!initialized) + flow_dv_destroy_policer_rules(dev, fm, attr); return -1; } @@ -13960,7 +14077,7 @@ const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = { .query = flow_dv_query, .create_mtr_tbls = flow_dv_create_mtr_tbl, .destroy_mtr_tbls = flow_dv_destroy_mtr_tbl, - .create_policer_rules = flow_dv_create_policer_rules, + .prepare_policer_rules = flow_dv_prepare_policer_rules, .destroy_policer_rules = flow_dv_destroy_policer_rules, .counter_alloc = flow_dv_counter_allocate, .counter_free = flow_dv_counter_free, diff --git a/drivers/net/mlx5/mlx5_flow_meter.c b/drivers/net/mlx5/mlx5_flow_meter.c index dbc574b508..68f5c344cf 100644 --- a/drivers/net/mlx5/mlx5_flow_meter.c +++ b/drivers/net/mlx5/mlx5_flow_meter.c @@ -474,6 +474,12 @@ mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id, MTR_POLICER_ACTION_COLOR_RED }; int i; + /* Meter must use global drop action. */ + if (!priv->sh->esw_drop_action) + return -rte_mtr_error_set(error, ENOTSUP, + RTE_MTR_ERROR_TYPE_MTR_PARAMS, + NULL, + "No drop action ready for meter."); /* Meter params must not be NULL. */ if (params == NULL) return -rte_mtr_error_set(error, EINVAL, @@ -630,9 +636,18 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, .egress = 1, .transfer = priv->config.dv_esw_en ? 1 : 0, }; + struct mlx5_indexed_pool_config flow_ipool_cfg = { + .size = 0, + .trunk_size = 64, + .need_lock = 1, + .type = "mlx5_flow_mtr_flow_id_pool", + }; int ret; unsigned int i; uint32_t idx = 0; + uint8_t mtr_id_bits; + uint8_t mtr_reg_bits = priv->mtr_reg_share ? + MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; if (!priv->mtr_en) return -rte_mtr_error_set(error, ENOTSUP, @@ -654,6 +669,13 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, return -rte_mtr_error_set(error, ENOMEM, RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, "Memory alloc failed for meter."); + mtr_id_bits = MLX5_REG_BITS - __builtin_clz(idx); + if ((mtr_id_bits + priv->max_mtr_flow_bits) > mtr_reg_bits) { + DRV_LOG(ERR, "Meter number exceeds max limit."); + goto error; + } + if (mtr_id_bits > priv->max_mtr_bits) + priv->max_mtr_bits = mtr_id_bits; fm->idx = idx; /* Fill the flow meter parameters. */ fm->meter_id = meter_id; @@ -667,10 +689,10 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, if (!fm->policer_stats.cnt[i]) goto error; } - fm->mfts = mlx5_flow_create_mtr_tbls(dev, fm); + fm->mfts = mlx5_flow_create_mtr_tbls(dev); if (!fm->mfts) goto error; - ret = mlx5_flow_create_policer_rules(dev, fm, &attr); + ret = mlx5_flow_prepare_policer_rules(dev, fm, &attr); if (ret) goto error; /* Add to the flow meter list. */ @@ -679,6 +701,9 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, fm->shared = !!shared; fm->policer_stats.stats_mask = params->stats_mask; fm->profile->ref_cnt++; + fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg); + if (!fm->flow_ipool) + goto error; rte_spinlock_init(&fm->sl); return 0; error: @@ -749,6 +774,8 @@ mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id, if (fm->policer_stats.cnt[i]) mlx5_counter_free(dev, fm->policer_stats.cnt[i]); /* Free meter flow table */ + if (fm->flow_ipool) + mlx5_ipool_destroy(fm->flow_ipool); mlx5_flow_destroy_policer_rules(dev, fm, &attr); mlx5_flow_destroy_mtr_tbls(dev, fm->mfts); mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], fm->idx); @@ -1153,30 +1180,24 @@ mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id) * * @param [in] priv * Pointer to mlx5 private data. - * @param [in] meter_id - * Flow meter id. + * @param[in] fm + * Pointer to flow meter. * @param [in] attr * Pointer to flow attributes. * @param [out] error * Pointer to error structure. * - * @return the flow meter pointer, NULL otherwise. + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. */ -struct mlx5_flow_meter * -mlx5_flow_meter_attach(struct mlx5_priv *priv, uint32_t meter_id, +int +mlx5_flow_meter_attach(struct mlx5_priv *priv, + struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr, struct rte_flow_error *error) { - struct mlx5_flow_meter *fm; int ret = 0; - fm = mlx5_flow_meter_find(priv, meter_id); - if (fm == NULL) { - rte_flow_error_set(error, ENOENT, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "Meter object id not valid"); - return fm; - } rte_spinlock_lock(&fm->sl); if (fm->mfts->meter_action) { if (fm->shared && @@ -1210,7 +1231,7 @@ mlx5_flow_meter_attach(struct mlx5_priv *priv, uint32_t meter_id, fm->mfts->meter_action ? "Meter attr not match" : "Meter action create failed"); - return ret ? NULL : fm; + return ret ? -rte_errno : 0; } /** -- 2.27.0 ^ permalink raw reply [flat|nested] 8+ messages in thread
[parent not found: <20210412160817.1986716-1-lizh@nvidia.com>]
* [dpdk-stable] [PATCH v3 03/14] net/mlx5: fix meter statistics [not found] ` <20210412160817.1986716-1-lizh@nvidia.com> @ 2021-04-12 16:08 ` Li Zhang 0 siblings, 0 replies; 8+ messages in thread From: Li Zhang @ 2021-04-12 16:08 UTC (permalink / raw) To: lizh; +Cc: Shun Hao, stable, Matan Azrad From: Shun Hao <shunh@nvidia.com> Currently, packets after meter will be steered to a global policer table, which includes green/red color rules for every meter, so as to have counter statistics of each color in every meter. There's a bug that all the rules in global policer table are matching only color criteria, so all packets will be counted to one meter only, and other meter statistics are always zero. This patch does these: 1. The rules in policer table matches both meter index and color, so packet after meter could be counted to the correct meter counter. 2. The meter index and flow index are now sharing the available register bits dynamically. Meter index starts from lsb, and flow index starts from msb. Fixes: 46a5e6bc6a ("net/mlx5: prepare meter flow tables") Cc: stable@dpdk.org Signed-off-by: Shun Hao <shunh@nvidia.com> Acked-by: Matan Azrad <matan@nvidia.com> --- drivers/net/mlx5/mlx5.c | 7 +- drivers/net/mlx5/mlx5.h | 15 +- drivers/net/mlx5/mlx5_flow.c | 183 +++++++---- drivers/net/mlx5/mlx5_flow.h | 40 +-- drivers/net/mlx5/mlx5_flow_dv.c | 489 ++++++++++++++++++----------- drivers/net/mlx5/mlx5_flow_meter.c | 53 +++- 6 files changed, 496 insertions(+), 291 deletions(-) diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index 9557d06afa..734dee9f19 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -275,10 +275,13 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = { }, #endif [MLX5_IPOOL_MTR] = { + /** + * The ipool index should grow continually from small to big, + * for meter idx, so not set grow_trunk to avoid meter index + * not jump continually. + */ .size = sizeof(struct mlx5_flow_meter), .trunk_size = 64, - .grow_trunk = 3, - .grow_shift = 2, .need_lock = 1, .release_mem_en = 1, .malloc = mlx5_malloc, diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index 0f69f9d125..b50acaca41 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -944,9 +944,9 @@ struct mlx5_priv { unsigned int representor:1; /* Device is a port representor. */ unsigned int master:1; /* Device is a E-Switch master. */ unsigned int txpp_en:1; /* Tx packet pacing enabled. */ + unsigned int sampler_en:1; /* Whether support sampler. */ unsigned int mtr_en:1; /* Whether support meter. */ unsigned int mtr_reg_share:1; /* Whether support meter REG_C share. */ - unsigned int sampler_en:1; /* Whether support sampler. */ uint16_t domain_id; /* Switch domain identifier. */ uint16_t vport_id; /* Associated VF vport index (if any). */ uint32_t vport_meta_tag; /* Used for vport index match ove VF LAG. */ @@ -1003,6 +1003,10 @@ struct mlx5_priv { uint32_t rss_shared_actions; /* RSS shared actions. */ struct mlx5_devx_obj *q_counters; /* DevX queue counter object. */ uint32_t counter_set_id; /* Queue counter ID to set in DevX objects. */ + uint8_t max_mtr_bits; + /* Indicate how many bits are used by meter id at the most. */ + uint8_t max_mtr_flow_bits; + /* Indicate how many bits are used by meter flow id at the most. */ }; #define PORT_ID(priv) ((priv)->dev_data->port_id) @@ -1271,11 +1275,10 @@ int mlx5_pmd_socket_init(void); int mlx5_flow_meter_ops_get(struct rte_eth_dev *dev, void *arg); struct mlx5_flow_meter *mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id); -struct mlx5_flow_meter *mlx5_flow_meter_attach - (struct mlx5_priv *priv, - uint32_t meter_id, - const struct rte_flow_attr *attr, - struct rte_flow_error *error); +int mlx5_flow_meter_attach(struct mlx5_priv *priv, + struct mlx5_flow_meter *fm, + const struct rte_flow_attr *attr, + struct rte_flow_error *error); void mlx5_flow_meter_detach(struct mlx5_flow_meter *fm); /* mlx5_os.c */ diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c index 668c32cf51..63ff6acbbf 100644 --- a/drivers/net/mlx5/mlx5_flow.c +++ b/drivers/net/mlx5/mlx5_flow.c @@ -748,9 +748,9 @@ mlx5_flow_get_reg_id(struct rte_eth_dev *dev, return REG_C_0; } break; - case MLX5_MTR_SFX: + case MLX5_MTR_ID: /* - * If meter color and flow match share one register, flow match + * If meter color and meter id share one register, flow match * should use the meter color register for match. */ if (priv->mtr_reg_share) @@ -3035,7 +3035,8 @@ flow_mreg_split_qrss_release(struct rte_eth_dev *dev, SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, handle_idx, dev_handle, next) - if (dev_handle->split_flow_id) + if (dev_handle->split_flow_id && + !dev_handle->is_meter_flow_id) mlx5_ipool_free(priv->sh->ipool [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], dev_handle->split_flow_id); @@ -3674,23 +3675,30 @@ flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], * * @param[in] actions * Pointer to the list of actions. - * @param[out] mtr + * @param[out] has_mtr * Pointer to the meter exist flag. + * @param[out] meter_id + * Pointer to the meter id. * * @return * Total number of actions. */ static int -flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr) +flow_check_meter_action(const struct rte_flow_action actions[], + bool *has_mtr, + uint32_t *meter_id) { + const struct rte_flow_action_meter *mtr = NULL; int actions_n = 0; - MLX5_ASSERT(mtr); - *mtr = 0; + MLX5_ASSERT(has_mtr); + *has_mtr = false; for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { switch (actions->type) { case RTE_FLOW_ACTION_TYPE_METER: - *mtr = 1; + mtr = actions->conf; + *meter_id = mtr->mtr_id; + *has_mtr = true; break; default: break; @@ -4347,8 +4355,10 @@ flow_create_split_inner(struct rte_eth_dev *dev, * header will be in the prefix sub flow, as not to take the * L3 tunnel header into account. * - * @param dev + * @param[in] dev * Pointer to Ethernet device. + * @param[in] fm + * Pointer to flow meter structure. * @param[in] items * Pattern specification (list terminated by the END pattern item). * @param[out] sfx_items @@ -4363,29 +4373,39 @@ flow_create_split_inner(struct rte_eth_dev *dev, * The pattern items for the suffix flow. * @param[out] tag_sfx * Pointer to suffix flow tag. + * @param[out] error + * Perform verbose error reporting if not NULL. * * @return - * 0 on success. + * The flow id, 0 otherwise and rte_errno is set. */ -static int +static uint32_t flow_meter_split_prep(struct rte_eth_dev *dev, - const struct rte_flow_item items[], - struct rte_flow_item sfx_items[], - const struct rte_flow_action actions[], - struct rte_flow_action actions_sfx[], - struct rte_flow_action actions_pre[]) + struct mlx5_flow_meter *fm, + const struct rte_flow_item items[], + struct rte_flow_item sfx_items[], + const struct rte_flow_action actions[], + struct rte_flow_action actions_sfx[], + struct rte_flow_action actions_pre[], + struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; struct rte_flow_action *tag_action = NULL; struct rte_flow_item *tag_item; struct mlx5_rte_flow_action_set_tag *set_tag; - struct rte_flow_error error; const struct rte_flow_action_raw_encap *raw_encap; const struct rte_flow_action_raw_decap *raw_decap; - struct mlx5_rte_flow_item_tag *tag_spec; - struct mlx5_rte_flow_item_tag *tag_mask; + struct mlx5_rte_flow_item_tag *tag_item_spec; + struct mlx5_rte_flow_item_tag *tag_item_mask; uint32_t tag_id = 0; bool copy_vlan = false; + uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; + uint8_t mtr_reg_bits = priv->mtr_reg_share ? + MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; + uint32_t flow_id = 0; + uint32_t flow_id_reversed = 0; + uint8_t flow_id_bits = 0; + int shift; /* Prepare the actions for prefix and suffix flow. */ for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { @@ -4394,10 +4414,7 @@ flow_meter_split_prep(struct rte_eth_dev *dev, switch (actions->type) { case RTE_FLOW_ACTION_TYPE_METER: /* Add the extra tag action first. */ - tag_action = actions_pre; - tag_action->type = (enum rte_flow_action_type) - MLX5_RTE_FLOW_ACTION_TYPE_TAG; - actions_pre++; + tag_action = actions_pre++; action_cur = &actions_pre; break; case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: @@ -4430,23 +4447,22 @@ flow_meter_split_prep(struct rte_eth_dev *dev, actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; actions_pre->type = RTE_FLOW_ACTION_TYPE_END; actions_pre++; - /* Set the tag. */ - set_tag = (void *)actions_pre; - set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); - mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], - &tag_id); - if (tag_id >= (1 << (sizeof(tag_id) * 8 - MLX5_MTR_COLOR_BITS))) { - DRV_LOG(ERR, "Port %u meter flow id exceed max limit.", - dev->data->port_id); - mlx5_ipool_free(priv->sh->ipool - [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], tag_id); - return 0; - } else if (!tag_id) { - return 0; + mlx5_ipool_malloc(fm->flow_ipool, &tag_id); + if (!tag_id) + return rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Failed to allocate meter flow id."); + flow_id = tag_id - 1; + flow_id_bits = MLX5_REG_BITS - __builtin_clz(flow_id); + flow_id_bits = flow_id_bits ? flow_id_bits : 1; + if ((flow_id_bits + priv->max_mtr_bits) > mtr_reg_bits) { + mlx5_ipool_free(fm->flow_ipool, tag_id); + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Meter flow id exceeds max limit."); } - set_tag->data = tag_id << MLX5_MTR_COLOR_BITS; - assert(tag_action); - tag_action->conf = set_tag; + if (flow_id_bits > priv->max_mtr_flow_bits) + priv->max_mtr_flow_bits = flow_id_bits; /* Prepare the suffix subflow items. */ tag_item = sfx_items++; for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { @@ -4475,16 +4491,34 @@ flow_meter_split_prep(struct rte_eth_dev *dev, } sfx_items->type = RTE_FLOW_ITEM_TYPE_END; sfx_items++; - tag_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; - tag_spec->data = tag_id << MLX5_MTR_COLOR_BITS; - tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); - tag_mask = tag_spec + 1; - tag_mask->data = 0xffffff00; + /* Build tag actions and items for meter_id/meter flow_id. */ + assert(tag_action); + set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; + tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; + tag_item_mask = tag_item_spec + 1; + /* + * The color Reg bits used by flow_id are growing from + * msb to lsb, so must do bit reverse for flow_id val in RegC. + */ + for (shift = 0; shift < flow_id_bits; shift++) + flow_id_reversed = (flow_id_reversed << 1) | + ((flow_id >> shift) & 0x1); + /* Both flow_id and meter_id share the same register. */ + set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 0, error); + set_tag->data = + (fm->idx | (flow_id_reversed << (mtr_reg_bits - flow_id_bits))) + << mtr_id_offset; + tag_item_spec->id = set_tag->id; + tag_item_spec->data = set_tag->data; + tag_item_mask->data = UINT32_MAX << mtr_id_offset; + tag_action->type = (enum rte_flow_action_type) + MLX5_RTE_FLOW_ACTION_TYPE_TAG; + tag_action->conf = set_tag; tag_item->type = (enum rte_flow_item_type) - MLX5_RTE_FLOW_ITEM_TYPE_TAG; - tag_item->spec = tag_spec; + MLX5_RTE_FLOW_ITEM_TYPE_TAG; + tag_item->spec = tag_item_spec; tag_item->last = NULL; - tag_item->mask = tag_mask; + tag_item->mask = tag_item_mask; return tag_id; } @@ -5184,25 +5218,49 @@ flow_create_split_meter(struct rte_eth_dev *dev, struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); struct rte_flow_action *sfx_actions = NULL; struct rte_flow_action *pre_actions = NULL; struct rte_flow_item *sfx_items = NULL; struct mlx5_flow *dev_flow = NULL; struct rte_flow_attr sfx_attr = *attr; - uint32_t mtr = 0; + struct mlx5_flow_meter *fm = NULL; + bool has_mtr = false; + uint32_t meter_id; uint32_t mtr_tag_id = 0; size_t act_size; size_t item_size; int actions_n = 0; - int ret; + int ret = 0; if (priv->mtr_en) - actions_n = flow_check_meter_action(actions, &mtr); - if (mtr) { - /* The five prefix actions: meter, decap, encap, tag, end. */ + actions_n = flow_check_meter_action(actions, &has_mtr, + &meter_id); + if (has_mtr) { + if (flow->meter) { + fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], + flow->meter); + if (!fm) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "Meter not found."); + } else { + fm = mlx5_flow_meter_find(priv, meter_id); + if (!fm) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "Meter not found."); + ret = mlx5_flow_meter_attach(priv, fm, + &sfx_attr, error); + if (ret) + return -rte_errno; + flow->meter = fm->idx; + } + wks->fm = fm; + /* The prefix actions: meter, decap, encap, tag, end. */ act_size = sizeof(struct rte_flow_action) * (actions_n + 5) + sizeof(struct mlx5_rte_flow_action_set_tag); - /* tag, vlan, port id, end. */ + /* The suffix items: tag, vlan, port id, end. */ #define METER_SUFFIX_ITEM 4 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + sizeof(struct mlx5_rte_flow_item_tag) * 2; @@ -5216,9 +5274,9 @@ flow_create_split_meter(struct rte_eth_dev *dev, sfx_items = (struct rte_flow_item *)((char *)sfx_actions + act_size); pre_actions = sfx_actions + actions_n; - mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items, + mtr_tag_id = flow_meter_split_prep(dev, fm, items, sfx_items, actions, sfx_actions, - pre_actions); + pre_actions, error); if (!mtr_tag_id) { ret = -rte_errno; goto exit; @@ -5229,10 +5287,12 @@ flow_create_split_meter(struct rte_eth_dev *dev, attr, items, pre_actions, flow_split_info, error); if (ret) { + mlx5_ipool_free(fm->flow_ipool, mtr_tag_id); ret = -rte_errno; goto exit; } dev_flow->handle->split_flow_id = mtr_tag_id; + dev_flow->handle->is_meter_flow_id = 1; /* Setting the sfx group atrr. */ sfx_attr.group = sfx_attr.transfer ? (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : @@ -6504,20 +6564,17 @@ mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, * * @param[in] dev * Pointer to Ethernet device. - * @param[in] fm - * Pointer to the flow meter. * * @return * Pointer to table set on success, NULL otherwise. */ struct mlx5_meter_domains_infos * -mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm) +mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev) { const struct mlx5_flow_driver_ops *fops; fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); - return fops->create_mtr_tbls(dev, fm); + return fops->create_mtr_tbls(dev); } /** @@ -6542,7 +6599,7 @@ mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, } /** - * Create policer rules. + * Prepare policer rules. * * @param[in] dev * Pointer to Ethernet device. @@ -6555,14 +6612,14 @@ mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, * 0 on success, -1 otherwise. */ int -mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, +mlx5_flow_prepare_policer_rules(struct rte_eth_dev *dev, struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr) { const struct mlx5_flow_driver_ops *fops; fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); - return fops->create_policer_rules(dev, fm, attr); + return fops->prepare_policer_rules(dev, fm, attr); } /** diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index ec673c29ab..4482a456f0 100644 --- a/drivers/net/mlx5/mlx5_flow.h +++ b/drivers/net/mlx5/mlx5_flow.h @@ -79,7 +79,7 @@ enum mlx5_feature_name { MLX5_APP_TAG, MLX5_COPY_MARK, MLX5_MTR_COLOR, - MLX5_MTR_SFX, + MLX5_MTR_ID, MLX5_ASO_FLOW_HIT, }; @@ -655,7 +655,8 @@ struct mlx5_flow_handle { uint64_t layers; /**< Bit-fields of present layers, see MLX5_FLOW_LAYER_*. */ void *drv_flow; /**< pointer to driver flow object. */ - uint32_t split_flow_id:28; /**< Sub flow unique match flow id. */ + uint32_t split_flow_id:27; /**< Sub flow unique match flow id. */ + uint32_t is_meter_flow_id:1; /**< Indate if flow_id is for meter. */ uint32_t mark:1; /**< Metadate rxq mark flag. */ uint32_t fate_action:3; /**< Fate action type. */ union { @@ -842,14 +843,16 @@ struct mlx5_meter_domain_info { /**< Meter table. */ struct mlx5_flow_tbl_resource *sfx_tbl; /**< Meter suffix table. */ - void *any_matcher; - /**< Meter color not match default criteria. */ - void *color_matcher; - /**< Meter color match criteria. */ + struct mlx5_flow_dv_matcher *drop_matcher; + /**< Matcher for Drop. */ + struct mlx5_flow_dv_matcher *color_matcher; + /**< Matcher for Color. */ void *jump_actn; /**< Meter match action. */ - void *policer_rules[RTE_MTR_DROPPED + 1]; - /**< Meter policer for the match. */ + void *green_rule; + /**< Meter green rule. */ + void *drop_rule; + /**< Meter drop rule. */ }; /* Meter table set for TX RX FDB. */ @@ -862,10 +865,10 @@ struct mlx5_meter_domains_infos { /**< RX meter table. */ struct mlx5_meter_domain_info transfer; /**< FDB meter table. */ - void *drop_actn; - /**< Drop action as not matched. */ - void *count_actns[RTE_MTR_DROPPED + 1]; - /**< Counters for match and unmatched statistics. */ + void *green_count; + /**< Counters for green rule. */ + void *drop_count; + /**< Counters for green rule. */ uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)]; /**< Flow meter parameter. */ size_t fmp_size; @@ -928,6 +931,8 @@ struct mlx5_flow_meter { /**< Meter state. */ uint32_t shared:1; /**< Meter shared or not. */ + struct mlx5_indexed_pool *flow_ipool; + /**< Index pool for flow id. */ }; /* RFC2697 parameter structure. */ @@ -1166,6 +1171,7 @@ struct mlx5_flow_workspace { struct mlx5_flow_rss_desc rss_desc; uint32_t rssq_num; /* Allocated queue num in rss_desc. */ uint32_t flow_idx; /* Intermediate device flow index. */ + struct mlx5_flow_meter *fm; /* Pointer to the meter in flow. */ }; struct mlx5_flow_split_info { @@ -1206,8 +1212,7 @@ typedef int (*mlx5_flow_query_t)(struct rte_eth_dev *dev, void *data, struct rte_flow_error *error); typedef struct mlx5_meter_domains_infos *(*mlx5_flow_create_mtr_tbls_t) - (struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm); + (struct rte_eth_dev *dev); typedef int (*mlx5_flow_destroy_mtr_tbls_t)(struct rte_eth_dev *dev, struct mlx5_meter_domains_infos *tbls); typedef int (*mlx5_flow_create_policer_rules_t) @@ -1270,7 +1275,7 @@ struct mlx5_flow_driver_ops { mlx5_flow_query_t query; mlx5_flow_create_mtr_tbls_t create_mtr_tbls; mlx5_flow_destroy_mtr_tbls_t destroy_mtr_tbls; - mlx5_flow_create_policer_rules_t create_policer_rules; + mlx5_flow_create_policer_rules_t prepare_policer_rules; mlx5_flow_destroy_policer_rules_t destroy_policer_rules; mlx5_flow_counter_alloc_t counter_alloc; mlx5_flow_counter_free_t counter_free; @@ -1470,11 +1475,10 @@ int mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, const struct rte_flow_item_ecpri *acc_mask, struct rte_flow_error *error); struct mlx5_meter_domains_infos *mlx5_flow_create_mtr_tbls - (struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm); + (struct rte_eth_dev *dev); int mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, struct mlx5_meter_domains_infos *tbl); -int mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, +int mlx5_flow_prepare_policer_rules(struct rte_eth_dev *dev, struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr); int mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 07a0ee5abb..14ed219fd5 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -10981,14 +10981,12 @@ flow_dv_translate(struct rte_eth_dev *dev, const struct rte_flow_action_rss *rss; const struct rte_flow_action *action = actions; const uint8_t *rss_key; - const struct rte_flow_action_meter *mtr; struct mlx5_flow_tbl_resource *tbl; struct mlx5_aso_age_action *age_act; uint32_t port_id = 0; struct mlx5_flow_dv_port_id_action_resource port_id_resource; int action_type = actions->type; const struct rte_flow_action *found_action = NULL; - struct mlx5_flow_meter *fm = NULL; uint32_t jump_group = 0; if (!mlx5_flow_os_action_supported(action_type)) @@ -11413,33 +11411,13 @@ flow_dv_translate(struct rte_eth_dev *dev, MLX5_FLOW_FATE_DEFAULT_MISS; break; case RTE_FLOW_ACTION_TYPE_METER: - mtr = actions->conf; - if (!flow->meter) { - fm = mlx5_flow_meter_attach(priv, mtr->mtr_id, - attr, error); - if (!fm) - return rte_flow_error_set(error, - rte_errno, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "meter not found " - "or invalid parameters"); - flow->meter = fm->idx; - } + if (!wks->fm) + return rte_flow_error_set(error, rte_errno, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "Failed to get meter in flow."); /* Set the meter action. */ - if (!fm) { - fm = mlx5_ipool_get(priv->sh->ipool - [MLX5_IPOOL_MTR], flow->meter); - if (!fm) - return rte_flow_error_set(error, - rte_errno, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "meter not found " - "or invalid parameters"); - } dev_flow->dv.actions[actions_n++] = - fm->mfts->meter_action; + wks->fm->mfts->meter_action; action_flags |= MLX5_FLOW_ACTION_METER; break; case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: @@ -12588,6 +12566,7 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) { struct mlx5_flow_handle *dev_handle; struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_meter *fm = NULL; uint32_t srss = 0; if (!flow) @@ -12598,8 +12577,6 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) flow->counter = 0; } if (flow->meter) { - struct mlx5_flow_meter *fm; - fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], flow->meter); if (fm) @@ -12641,6 +12618,10 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) flow_dv_fate_resource_release(dev, dev_handle); else if (!srss) srss = dev_handle->rix_srss; + if (fm && dev_handle->is_meter_flow_id && + dev_handle->split_flow_id) + mlx5_ipool_free(fm->flow_ipool, + dev_handle->split_flow_id); mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], tmp_idx); } @@ -13406,49 +13387,20 @@ flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev, if (!mtd || !priv->config.dv_flow_en) return 0; - if (mtd->ingress.policer_rules[RTE_MTR_DROPPED]) - claim_zero(mlx5_flow_os_destroy_flow - (mtd->ingress.policer_rules[RTE_MTR_DROPPED])); - if (mtd->egress.policer_rules[RTE_MTR_DROPPED]) - claim_zero(mlx5_flow_os_destroy_flow - (mtd->egress.policer_rules[RTE_MTR_DROPPED])); - if (mtd->transfer.policer_rules[RTE_MTR_DROPPED]) - claim_zero(mlx5_flow_os_destroy_flow - (mtd->transfer.policer_rules[RTE_MTR_DROPPED])); - if (mtd->egress.color_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->egress.color_matcher)); - if (mtd->egress.any_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->egress.any_matcher)); if (mtd->egress.tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->egress.tbl); if (mtd->egress.sfx_tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->egress.sfx_tbl); - if (mtd->ingress.color_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->ingress.color_matcher)); - if (mtd->ingress.any_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->ingress.any_matcher)); if (mtd->ingress.tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->ingress.tbl); if (mtd->ingress.sfx_tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->ingress.sfx_tbl); - if (mtd->transfer.color_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->transfer.color_matcher)); - if (mtd->transfer.any_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->transfer.any_matcher)); if (mtd->transfer.tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->transfer.tbl); if (mtd->transfer.sfx_tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->transfer.sfx_tbl); - if (mtd->drop_actn) - claim_zero(mlx5_flow_os_destroy_flow_action(mtd->drop_actn)); mlx5_free(mtd); return 0; } @@ -13467,37 +13419,17 @@ flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev, * Table attribute. * @param[in] transfer * Table attribute. - * @param[in] color_reg_c_idx - * Reg C index for color match. * * @return - * 0 on success, -1 otherwise and rte_errno is set. + * 0 on success, -1 otherwise. */ static int flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, struct mlx5_meter_domains_infos *mtb, - uint8_t egress, uint8_t transfer, - uint32_t color_reg_c_idx) + uint8_t egress, uint8_t transfer) { - struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_dev_ctx_shared *sh = priv->sh; - struct mlx5_flow_dv_match_params mask = { - .size = sizeof(mask.buf), - }; - struct mlx5_flow_dv_match_params value = { - .size = sizeof(value.buf), - }; - struct mlx5dv_flow_matcher_attr dv_attr = { - .type = IBV_FLOW_ATTR_NORMAL, - .priority = 0, - .match_criteria_enable = 0, - .match_mask = (void *)&mask, - }; - void *actions[METER_ACTIONS]; - struct mlx5_meter_domain_info *dtb; struct rte_flow_error error; - int i = 0; - int ret; + struct mlx5_meter_domain_info *dtb; if (transfer) dtb = &mtb->transfer; @@ -13522,41 +13454,7 @@ flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, DRV_LOG(ERR, "Failed to create meter suffix table."); return -1; } - /* Create matchers, Any and Color. */ - dv_attr.priority = 3; - dv_attr.match_criteria_enable = 0; - ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj, - &dtb->any_matcher); - if (ret) { - DRV_LOG(ERR, "Failed to create meter" - " policer default matcher."); - goto error_exit; - } - dv_attr.priority = 0; - dv_attr.match_criteria_enable = - 1 << MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT; - flow_dv_match_meta_reg(mask.buf, value.buf, color_reg_c_idx, - rte_col_2_mlx5_col(RTE_COLORS), UINT8_MAX); - ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj, - &dtb->color_matcher); - if (ret) { - DRV_LOG(ERR, "Failed to create meter policer color matcher."); - goto error_exit; - } - if (mtb->count_actns[RTE_MTR_DROPPED]) - actions[i++] = mtb->count_actns[RTE_MTR_DROPPED]; - actions[i++] = mtb->drop_actn; - /* Default rule: lowest priority, match any, actions: drop. */ - ret = mlx5_flow_os_create_flow(dtb->any_matcher, (void *)&value, i, - actions, - &dtb->policer_rules[RTE_MTR_DROPPED]); - if (ret) { - DRV_LOG(ERR, "Failed to create meter policer drop rule."); - goto error_exit; - } return 0; -error_exit: - return -1; } /** @@ -13565,20 +13463,16 @@ flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, * * @param[in] dev * Pointer to Ethernet device. - * @param[in] fm - * Pointer to the flow meter. * * @return * Pointer to table set on success, NULL otherwise and rte_errno is set. */ static struct mlx5_meter_domains_infos * -flow_dv_create_mtr_tbl(struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm) +flow_dv_create_mtr_tbl(struct rte_eth_dev *dev) { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_meter_domains_infos *mtb; int ret; - int i; if (!priv->mtr_en) { rte_errno = ENOTSUP; @@ -13589,37 +13483,21 @@ flow_dv_create_mtr_tbl(struct rte_eth_dev *dev, DRV_LOG(ERR, "Failed to allocate memory for meter."); return NULL; } - /* Create meter count actions */ - for (i = 0; i <= RTE_MTR_DROPPED; i++) { - struct mlx5_flow_counter *cnt; - if (!fm->policer_stats.cnt[i]) - continue; - cnt = flow_dv_counter_get_by_idx(dev, - fm->policer_stats.cnt[i], NULL); - mtb->count_actns[i] = cnt->action; - } - /* Create drop action. */ - ret = mlx5_flow_os_create_flow_action_drop(&mtb->drop_actn); - if (ret) { - DRV_LOG(ERR, "Failed to create drop action."); - goto error_exit; - } /* Egress meter table. */ - ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0, priv->mtr_color_reg); + ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0); if (ret) { DRV_LOG(ERR, "Failed to prepare egress meter table."); goto error_exit; } /* Ingress meter table. */ - ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0, priv->mtr_color_reg); + ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0); if (ret) { DRV_LOG(ERR, "Failed to prepare ingress meter table."); goto error_exit; } /* FDB meter table. */ if (priv->config.dv_esw_en) { - ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1, - priv->mtr_color_reg); + ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1); if (ret) { DRV_LOG(ERR, "Failed to prepare fdb meter table."); goto error_exit; @@ -13631,24 +13509,152 @@ flow_dv_create_mtr_tbl(struct rte_eth_dev *dev, return NULL; } +/** + * Destroy the meter table matchers. + * Lock free, (mutex should be acquired by caller). + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in,out] dtb + * Pointer to DV meter table. + * + * @return + * Always 0. + */ +static int +flow_dv_destroy_mtr_matchers(struct rte_eth_dev *dev, + struct mlx5_meter_domain_info *dtb) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_tbl_data_entry *tbl; + + if (!priv->config.dv_flow_en) + return 0; + if (dtb->drop_matcher) { + tbl = container_of(dtb->drop_matcher->tbl, typeof(*tbl), tbl); + mlx5_cache_unregister(&tbl->matchers, + &dtb->drop_matcher->entry); + dtb->drop_matcher = NULL; + } + if (dtb->color_matcher) { + tbl = container_of(dtb->color_matcher->tbl, typeof(*tbl), tbl); + mlx5_cache_unregister(&tbl->matchers, + &dtb->color_matcher->entry); + dtb->color_matcher = NULL; + } + return 0; +} + +/** + * Create the matchers for meter table. + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in] color_reg_c_idx + * Reg C index for color match. + * @param[in] mtr_id_reg_c_idx + * Reg C index for meter_id match. + * @param[in] mtr_id_mask + * Mask for meter_id match criteria. + * @param[in,out] dtb + * Pointer to DV meter table. + * @param[out] error + * Perform verbose error reporting if not NULL. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_prepare_mtr_matchers(struct rte_eth_dev *dev, + uint32_t color_reg_c_idx, + uint32_t mtr_id_reg_c_idx, + uint32_t mtr_id_mask, + struct mlx5_meter_domain_info *dtb, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_tbl_data_entry *tbl_data; + struct mlx5_cache_entry *entry; + struct mlx5_flow_dv_matcher matcher = { + .mask = { + .size = sizeof(matcher.mask.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), + }, + .tbl = dtb->tbl, + }; + struct mlx5_flow_dv_match_params value = { + .size = sizeof(value.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), + }; + struct mlx5_flow_cb_ctx ctx = { + .error = error, + .data = &matcher, + }; + uint32_t color_mask = (UINT32_C(1) << MLX5_MTR_COLOR_BITS) - 1; + + tbl_data = container_of(dtb->tbl, struct mlx5_flow_tbl_data_entry, tbl); + if (!dtb->drop_matcher) { + /* Create matchers for Drop. */ + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + mtr_id_reg_c_idx, 0, mtr_id_mask); + matcher.priority = MLX5_REG_BITS * 2 - priv->max_mtr_bits; + matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf, + matcher.mask.size); + entry = mlx5_cache_register(&tbl_data->matchers, &ctx); + if (!entry) { + DRV_LOG(ERR, "Failed to register meter drop matcher."); + return -1; + } + dtb->drop_matcher = + container_of(entry, struct mlx5_flow_dv_matcher, entry); + } + if (!dtb->color_matcher) { + /* Create matchers for Color + meter_id. */ + if (priv->mtr_reg_share) { + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + color_reg_c_idx, 0, + (mtr_id_mask | color_mask)); + } else { + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + color_reg_c_idx, 0, color_mask); + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + mtr_id_reg_c_idx, 0, mtr_id_mask); + } + matcher.priority = MLX5_REG_BITS - priv->max_mtr_bits; + matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf, + matcher.mask.size); + entry = mlx5_cache_register(&tbl_data->matchers, &ctx); + if (!entry) { + DRV_LOG(ERR, "Failed to register meter color matcher."); + return -1; + } + dtb->color_matcher = + container_of(entry, struct mlx5_flow_dv_matcher, entry); + } + return 0; +} + /** * Destroy domain policer rule. * + * @param[in] dev + * Pointer to Ethernet device. * @param[in] dt * Pointer to domain table. */ static void -flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt) +flow_dv_destroy_domain_policer_rule(struct rte_eth_dev *dev, + struct mlx5_meter_domain_info *dt) { - int i; - - for (i = 0; i < RTE_MTR_DROPPED; i++) { - if (dt->policer_rules[i]) { - claim_zero(mlx5_flow_os_destroy_flow - (dt->policer_rules[i])); - dt->policer_rules[i] = NULL; - } + if (dt->drop_rule) { + claim_zero(mlx5_flow_os_destroy_flow(dt->drop_rule)); + dt->drop_rule = NULL; } + if (dt->green_rule) { + claim_zero(mlx5_flow_os_destroy_flow(dt->green_rule)); + dt->green_rule = NULL; + } + flow_dv_destroy_mtr_matchers(dev, dt); if (dt->jump_actn) { claim_zero(mlx5_flow_os_destroy_flow_action(dt->jump_actn)); dt->jump_actn = NULL; @@ -13669,7 +13675,7 @@ flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt) * Always 0. */ static int -flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused, +flow_dv_destroy_policer_rules(struct rte_eth_dev *dev, const struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr) { @@ -13678,39 +13684,56 @@ flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused, if (!mtb) return 0; if (attr->egress) - flow_dv_destroy_domain_policer_rule(&mtb->egress); + flow_dv_destroy_domain_policer_rule(dev, &mtb->egress); if (attr->ingress) - flow_dv_destroy_domain_policer_rule(&mtb->ingress); + flow_dv_destroy_domain_policer_rule(dev, &mtb->ingress); if (attr->transfer) - flow_dv_destroy_domain_policer_rule(&mtb->transfer); + flow_dv_destroy_domain_policer_rule(dev, &mtb->transfer); return 0; } /** * Create specify domain meter policer rule. * + * @param[in] dev + * Pointer to Ethernet device. * @param[in] fm * Pointer to flow meter structure. * @param[in] mtb * Pointer to DV meter table set. - * @param[in] mtr_reg_c - * Color match REG_C. + * @param[out] drop_rule + * The address of pointer saving drop rule. + * @param[out] color_rule + * The address of pointer saving green rule. * * @return * 0 on success, -1 otherwise. */ static int -flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, +flow_dv_create_policer_forward_rule(struct rte_eth_dev *dev, + struct mlx5_flow_meter *fm, struct mlx5_meter_domain_info *dtb, - uint8_t mtr_reg_c) + void **drop_rule, + void **green_rule) { + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flow_dv_match_params matcher = { - .size = sizeof(matcher.buf), + .size = sizeof(matcher.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), }; struct mlx5_flow_dv_match_params value = { - .size = sizeof(value.buf), + .size = sizeof(value.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), }; struct mlx5_meter_domains_infos *mtb = fm->mfts; + struct rte_flow_error error; + uint32_t color_reg_c = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, + 0, &error); + uint32_t mtr_id_reg_c = mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, + 0, &error); + uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; + uint32_t mtr_id_mask = + ((UINT32_C(1) << priv->max_mtr_bits) - 1) << mtr_id_offset; void *actions[METER_ACTIONS]; int i; int ret = 0; @@ -13723,25 +13746,52 @@ flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, DRV_LOG(ERR, "Failed to create policer jump action."); goto error; } - for (i = 0; i < RTE_MTR_DROPPED; i++) { - int j = 0; - - flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_reg_c, - rte_col_2_mlx5_col(i), UINT8_MAX); - if (mtb->count_actns[i]) - actions[j++] = mtb->count_actns[i]; - if (fm->action[i] == MTR_POLICER_ACTION_DROP) - actions[j++] = mtb->drop_actn; - else - actions[j++] = dtb->jump_actn; - ret = mlx5_flow_os_create_flow(dtb->color_matcher, - (void *)&value, j, actions, - &dtb->policer_rules[i]); + /* Prepare matchers. */ + if (!dtb->drop_matcher || !dtb->color_matcher) { + ret = flow_dv_prepare_mtr_matchers(dev, color_reg_c, + mtr_id_reg_c, mtr_id_mask, + dtb, &error); if (ret) { - DRV_LOG(ERR, "Failed to create policer rule."); + DRV_LOG(ERR, "Failed to setup matchers for mtr table."); goto error; } } + /* Create Drop flow, matching meter_id only. */ + i = 0; + flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_id_reg_c, + (fm->idx << mtr_id_offset), UINT32_MAX); + if (mtb->drop_count) + actions[i++] = mtb->drop_count; + actions[i++] = priv->sh->dr_drop_action; + ret = mlx5_flow_os_create_flow(dtb->drop_matcher->matcher_object, + (void *)&value, i, actions, drop_rule); + if (ret) { + DRV_LOG(ERR, "Failed to create meter policer drop rule."); + goto error; + } + /* Create flow matching Green color + meter_id. */ + i = 0; + if (priv->mtr_reg_share) { + flow_dv_match_meta_reg(matcher.buf, value.buf, color_reg_c, + ((fm->idx << mtr_id_offset) | + rte_col_2_mlx5_col(RTE_COLOR_GREEN)), + UINT32_MAX); + } else { + flow_dv_match_meta_reg(matcher.buf, value.buf, color_reg_c, + rte_col_2_mlx5_col(RTE_COLOR_GREEN), + UINT32_MAX); + flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_id_reg_c, + fm->idx, UINT32_MAX); + } + if (mtb->green_count) + actions[i++] = mtb->green_count; + actions[i++] = dtb->jump_actn; + ret = mlx5_flow_os_create_flow(dtb->color_matcher->matcher_object, + (void *)&value, i, actions, green_rule); + if (ret) { + DRV_LOG(ERR, "Failed to create meter policer color rule."); + goto error; + } return 0; error: rte_errno = errno; @@ -13749,7 +13799,8 @@ flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, } /** - * Create policer rules. + * Prepare policer rules for all domains. + * If meter already initialized, this will replace all old rules with new ones. * * @param[in] dev * Pointer to Ethernet device. @@ -13762,41 +13813,107 @@ flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, * 0 on success, -1 otherwise. */ static int -flow_dv_create_policer_rules(struct rte_eth_dev *dev, - struct mlx5_flow_meter *fm, - const struct rte_flow_attr *attr) +flow_dv_prepare_policer_rules(struct rte_eth_dev *dev, + struct mlx5_flow_meter *fm, + const struct rte_flow_attr *attr) { - struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_meter_domains_infos *mtb = fm->mfts; + bool initialized = false; + struct mlx5_flow_counter *cnt; + void *egress_drop_rule = NULL; + void *egress_green_rule = NULL; + void *ingress_drop_rule = NULL; + void *ingress_green_rule = NULL; + void *transfer_drop_rule = NULL; + void *transfer_green_rule = NULL; int ret; + /* Get the statistics counters for green/drop. */ + if (fm->policer_stats.cnt[RTE_COLOR_GREEN]) { + cnt = flow_dv_counter_get_by_idx(dev, + fm->policer_stats.cnt[RTE_COLOR_GREEN], + NULL); + mtb->green_count = cnt->action; + } else { + mtb->green_count = NULL; + } + if (fm->policer_stats.cnt[RTE_MTR_DROPPED]) { + cnt = flow_dv_counter_get_by_idx(dev, + fm->policer_stats.cnt[RTE_MTR_DROPPED], + NULL); + mtb->drop_count = cnt->action; + } else { + mtb->drop_count = NULL; + } + /** + * If flow meter has been initialized, all policer rules + * are created. So can get if meter initialized by checking + * any policer rule. + */ + if (mtb->egress.drop_rule) + initialized = true; if (attr->egress) { - ret = flow_dv_create_policer_forward_rule(fm, &mtb->egress, - priv->mtr_color_reg); + ret = flow_dv_create_policer_forward_rule(dev, + fm, &mtb->egress, + &egress_drop_rule, &egress_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create egress policer."); goto error; } } if (attr->ingress) { - ret = flow_dv_create_policer_forward_rule(fm, &mtb->ingress, - priv->mtr_color_reg); + ret = flow_dv_create_policer_forward_rule(dev, + fm, &mtb->ingress, + &ingress_drop_rule, &ingress_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create ingress policer."); goto error; } } if (attr->transfer) { - ret = flow_dv_create_policer_forward_rule(fm, &mtb->transfer, - priv->mtr_color_reg); + ret = flow_dv_create_policer_forward_rule(dev, + fm, &mtb->transfer, + &transfer_drop_rule, &transfer_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create transfer policer."); goto error; } } + /* Replace old flows if existing. */ + if (mtb->egress.drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->egress.drop_rule)); + if (mtb->egress.green_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->egress.green_rule)); + if (mtb->ingress.drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->ingress.drop_rule)); + if (mtb->ingress.green_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->ingress.green_rule)); + if (mtb->transfer.drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->transfer.drop_rule)); + if (mtb->transfer.green_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->transfer.green_rule)); + mtb->egress.drop_rule = egress_drop_rule; + mtb->egress.green_rule = egress_green_rule; + mtb->ingress.drop_rule = ingress_drop_rule; + mtb->ingress.green_rule = ingress_green_rule; + mtb->transfer.drop_rule = transfer_drop_rule; + mtb->transfer.green_rule = transfer_green_rule; return 0; error: - flow_dv_destroy_policer_rules(dev, fm, attr); + if (egress_drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(egress_drop_rule)); + if (egress_green_rule) + claim_zero(mlx5_flow_os_destroy_flow(egress_green_rule)); + if (ingress_drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(ingress_drop_rule)); + if (ingress_green_rule) + claim_zero(mlx5_flow_os_destroy_flow(ingress_green_rule)); + if (transfer_drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(transfer_drop_rule)); + if (transfer_green_rule) + claim_zero(mlx5_flow_os_destroy_flow(transfer_green_rule)); + if (!initialized) + flow_dv_destroy_policer_rules(dev, fm, attr); return -1; } @@ -14093,7 +14210,7 @@ const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = { .query = flow_dv_query, .create_mtr_tbls = flow_dv_create_mtr_tbl, .destroy_mtr_tbls = flow_dv_destroy_mtr_tbl, - .create_policer_rules = flow_dv_create_policer_rules, + .prepare_policer_rules = flow_dv_prepare_policer_rules, .destroy_policer_rules = flow_dv_destroy_policer_rules, .counter_alloc = flow_dv_counter_allocate, .counter_free = flow_dv_counter_free, diff --git a/drivers/net/mlx5/mlx5_flow_meter.c b/drivers/net/mlx5/mlx5_flow_meter.c index dbc574b508..7f7693b698 100644 --- a/drivers/net/mlx5/mlx5_flow_meter.c +++ b/drivers/net/mlx5/mlx5_flow_meter.c @@ -474,6 +474,12 @@ mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id, MTR_POLICER_ACTION_COLOR_RED }; int i; + /* Meter must use global drop action. */ + if (!priv->sh->dr_drop_action) + return -rte_mtr_error_set(error, ENOTSUP, + RTE_MTR_ERROR_TYPE_MTR_PARAMS, + NULL, + "No drop action ready for meter."); /* Meter params must not be NULL. */ if (params == NULL) return -rte_mtr_error_set(error, EINVAL, @@ -630,9 +636,18 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, .egress = 1, .transfer = priv->config.dv_esw_en ? 1 : 0, }; + struct mlx5_indexed_pool_config flow_ipool_cfg = { + .size = 0, + .trunk_size = 64, + .need_lock = 1, + .type = "mlx5_flow_mtr_flow_id_pool", + }; int ret; unsigned int i; uint32_t idx = 0; + uint8_t mtr_id_bits; + uint8_t mtr_reg_bits = priv->mtr_reg_share ? + MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; if (!priv->mtr_en) return -rte_mtr_error_set(error, ENOTSUP, @@ -654,6 +669,13 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, return -rte_mtr_error_set(error, ENOMEM, RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, "Memory alloc failed for meter."); + mtr_id_bits = MLX5_REG_BITS - __builtin_clz(idx); + if ((mtr_id_bits + priv->max_mtr_flow_bits) > mtr_reg_bits) { + DRV_LOG(ERR, "Meter number exceeds max limit."); + goto error; + } + if (mtr_id_bits > priv->max_mtr_bits) + priv->max_mtr_bits = mtr_id_bits; fm->idx = idx; /* Fill the flow meter parameters. */ fm->meter_id = meter_id; @@ -667,10 +689,10 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, if (!fm->policer_stats.cnt[i]) goto error; } - fm->mfts = mlx5_flow_create_mtr_tbls(dev, fm); + fm->mfts = mlx5_flow_create_mtr_tbls(dev); if (!fm->mfts) goto error; - ret = mlx5_flow_create_policer_rules(dev, fm, &attr); + ret = mlx5_flow_prepare_policer_rules(dev, fm, &attr); if (ret) goto error; /* Add to the flow meter list. */ @@ -679,6 +701,9 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, fm->shared = !!shared; fm->policer_stats.stats_mask = params->stats_mask; fm->profile->ref_cnt++; + fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg); + if (!fm->flow_ipool) + goto error; rte_spinlock_init(&fm->sl); return 0; error: @@ -749,6 +774,8 @@ mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id, if (fm->policer_stats.cnt[i]) mlx5_counter_free(dev, fm->policer_stats.cnt[i]); /* Free meter flow table */ + if (fm->flow_ipool) + mlx5_ipool_destroy(fm->flow_ipool); mlx5_flow_destroy_policer_rules(dev, fm, &attr); mlx5_flow_destroy_mtr_tbls(dev, fm->mfts); mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], fm->idx); @@ -1153,30 +1180,24 @@ mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id) * * @param [in] priv * Pointer to mlx5 private data. - * @param [in] meter_id - * Flow meter id. + * @param[in] fm + * Pointer to flow meter. * @param [in] attr * Pointer to flow attributes. * @param [out] error * Pointer to error structure. * - * @return the flow meter pointer, NULL otherwise. + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. */ -struct mlx5_flow_meter * -mlx5_flow_meter_attach(struct mlx5_priv *priv, uint32_t meter_id, +int +mlx5_flow_meter_attach(struct mlx5_priv *priv, + struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr, struct rte_flow_error *error) { - struct mlx5_flow_meter *fm; int ret = 0; - fm = mlx5_flow_meter_find(priv, meter_id); - if (fm == NULL) { - rte_flow_error_set(error, ENOENT, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "Meter object id not valid"); - return fm; - } rte_spinlock_lock(&fm->sl); if (fm->mfts->meter_action) { if (fm->shared && @@ -1210,7 +1231,7 @@ mlx5_flow_meter_attach(struct mlx5_priv *priv, uint32_t meter_id, fm->mfts->meter_action ? "Meter attr not match" : "Meter action create failed"); - return ret ? NULL : fm; + return ret ? -rte_errno : 0; } /** -- 2.27.0 ^ permalink raw reply [flat|nested] 8+ messages in thread
[parent not found: <20210413001033.1999355-1-lizh@nvidia.com>]
* [dpdk-stable] [PATCH v3 03/14] net/mlx5: fix meter statistics [not found] ` <20210413001033.1999355-1-lizh@nvidia.com> @ 2021-04-13 0:10 ` Li Zhang 0 siblings, 0 replies; 8+ messages in thread From: Li Zhang @ 2021-04-13 0:10 UTC (permalink / raw) To: dekelp, orika, viacheslavo, matan, shahafs, Suanming Mou Cc: dev, thomas, rasland, roniba, Shun Hao, stable From: Shun Hao <shunh@nvidia.com> Currently, packets after meter will be steered to a global policer table, which includes green/red color rules for every meter, so as to have counter statistics of each color in every meter. There's a bug that all the rules in global policer table are matching only color criteria, so all packets will be counted to one meter only, and other meter statistics are always zero. This patch does these: 1. The rules in policer table matches both meter index and color, so packet after meter could be counted to the correct meter counter. 2. The meter index and flow index are now sharing the available register bits dynamically. Meter index starts from lsb, and flow index starts from msb. Fixes: 46a5e6bc6a ("net/mlx5: prepare meter flow tables") Cc: stable@dpdk.org Signed-off-by: Shun Hao <shunh@nvidia.com> Acked-by: Matan Azrad <matan@nvidia.com> --- drivers/net/mlx5/mlx5.c | 7 +- drivers/net/mlx5/mlx5.h | 15 +- drivers/net/mlx5/mlx5_flow.c | 183 +++++++---- drivers/net/mlx5/mlx5_flow.h | 40 +-- drivers/net/mlx5/mlx5_flow_dv.c | 489 ++++++++++++++++++----------- drivers/net/mlx5/mlx5_flow_meter.c | 53 +++- 6 files changed, 496 insertions(+), 291 deletions(-) diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index 9557d06afa..734dee9f19 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -275,10 +275,13 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = { }, #endif [MLX5_IPOOL_MTR] = { + /** + * The ipool index should grow continually from small to big, + * for meter idx, so not set grow_trunk to avoid meter index + * not jump continually. + */ .size = sizeof(struct mlx5_flow_meter), .trunk_size = 64, - .grow_trunk = 3, - .grow_shift = 2, .need_lock = 1, .release_mem_en = 1, .malloc = mlx5_malloc, diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index 0f69f9d125..b50acaca41 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -944,9 +944,9 @@ struct mlx5_priv { unsigned int representor:1; /* Device is a port representor. */ unsigned int master:1; /* Device is a E-Switch master. */ unsigned int txpp_en:1; /* Tx packet pacing enabled. */ + unsigned int sampler_en:1; /* Whether support sampler. */ unsigned int mtr_en:1; /* Whether support meter. */ unsigned int mtr_reg_share:1; /* Whether support meter REG_C share. */ - unsigned int sampler_en:1; /* Whether support sampler. */ uint16_t domain_id; /* Switch domain identifier. */ uint16_t vport_id; /* Associated VF vport index (if any). */ uint32_t vport_meta_tag; /* Used for vport index match ove VF LAG. */ @@ -1003,6 +1003,10 @@ struct mlx5_priv { uint32_t rss_shared_actions; /* RSS shared actions. */ struct mlx5_devx_obj *q_counters; /* DevX queue counter object. */ uint32_t counter_set_id; /* Queue counter ID to set in DevX objects. */ + uint8_t max_mtr_bits; + /* Indicate how many bits are used by meter id at the most. */ + uint8_t max_mtr_flow_bits; + /* Indicate how many bits are used by meter flow id at the most. */ }; #define PORT_ID(priv) ((priv)->dev_data->port_id) @@ -1271,11 +1275,10 @@ int mlx5_pmd_socket_init(void); int mlx5_flow_meter_ops_get(struct rte_eth_dev *dev, void *arg); struct mlx5_flow_meter *mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id); -struct mlx5_flow_meter *mlx5_flow_meter_attach - (struct mlx5_priv *priv, - uint32_t meter_id, - const struct rte_flow_attr *attr, - struct rte_flow_error *error); +int mlx5_flow_meter_attach(struct mlx5_priv *priv, + struct mlx5_flow_meter *fm, + const struct rte_flow_attr *attr, + struct rte_flow_error *error); void mlx5_flow_meter_detach(struct mlx5_flow_meter *fm); /* mlx5_os.c */ diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c index 668c32cf51..63ff6acbbf 100644 --- a/drivers/net/mlx5/mlx5_flow.c +++ b/drivers/net/mlx5/mlx5_flow.c @@ -748,9 +748,9 @@ mlx5_flow_get_reg_id(struct rte_eth_dev *dev, return REG_C_0; } break; - case MLX5_MTR_SFX: + case MLX5_MTR_ID: /* - * If meter color and flow match share one register, flow match + * If meter color and meter id share one register, flow match * should use the meter color register for match. */ if (priv->mtr_reg_share) @@ -3035,7 +3035,8 @@ flow_mreg_split_qrss_release(struct rte_eth_dev *dev, SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, handle_idx, dev_handle, next) - if (dev_handle->split_flow_id) + if (dev_handle->split_flow_id && + !dev_handle->is_meter_flow_id) mlx5_ipool_free(priv->sh->ipool [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], dev_handle->split_flow_id); @@ -3674,23 +3675,30 @@ flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], * * @param[in] actions * Pointer to the list of actions. - * @param[out] mtr + * @param[out] has_mtr * Pointer to the meter exist flag. + * @param[out] meter_id + * Pointer to the meter id. * * @return * Total number of actions. */ static int -flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr) +flow_check_meter_action(const struct rte_flow_action actions[], + bool *has_mtr, + uint32_t *meter_id) { + const struct rte_flow_action_meter *mtr = NULL; int actions_n = 0; - MLX5_ASSERT(mtr); - *mtr = 0; + MLX5_ASSERT(has_mtr); + *has_mtr = false; for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { switch (actions->type) { case RTE_FLOW_ACTION_TYPE_METER: - *mtr = 1; + mtr = actions->conf; + *meter_id = mtr->mtr_id; + *has_mtr = true; break; default: break; @@ -4347,8 +4355,10 @@ flow_create_split_inner(struct rte_eth_dev *dev, * header will be in the prefix sub flow, as not to take the * L3 tunnel header into account. * - * @param dev + * @param[in] dev * Pointer to Ethernet device. + * @param[in] fm + * Pointer to flow meter structure. * @param[in] items * Pattern specification (list terminated by the END pattern item). * @param[out] sfx_items @@ -4363,29 +4373,39 @@ flow_create_split_inner(struct rte_eth_dev *dev, * The pattern items for the suffix flow. * @param[out] tag_sfx * Pointer to suffix flow tag. + * @param[out] error + * Perform verbose error reporting if not NULL. * * @return - * 0 on success. + * The flow id, 0 otherwise and rte_errno is set. */ -static int +static uint32_t flow_meter_split_prep(struct rte_eth_dev *dev, - const struct rte_flow_item items[], - struct rte_flow_item sfx_items[], - const struct rte_flow_action actions[], - struct rte_flow_action actions_sfx[], - struct rte_flow_action actions_pre[]) + struct mlx5_flow_meter *fm, + const struct rte_flow_item items[], + struct rte_flow_item sfx_items[], + const struct rte_flow_action actions[], + struct rte_flow_action actions_sfx[], + struct rte_flow_action actions_pre[], + struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; struct rte_flow_action *tag_action = NULL; struct rte_flow_item *tag_item; struct mlx5_rte_flow_action_set_tag *set_tag; - struct rte_flow_error error; const struct rte_flow_action_raw_encap *raw_encap; const struct rte_flow_action_raw_decap *raw_decap; - struct mlx5_rte_flow_item_tag *tag_spec; - struct mlx5_rte_flow_item_tag *tag_mask; + struct mlx5_rte_flow_item_tag *tag_item_spec; + struct mlx5_rte_flow_item_tag *tag_item_mask; uint32_t tag_id = 0; bool copy_vlan = false; + uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; + uint8_t mtr_reg_bits = priv->mtr_reg_share ? + MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; + uint32_t flow_id = 0; + uint32_t flow_id_reversed = 0; + uint8_t flow_id_bits = 0; + int shift; /* Prepare the actions for prefix and suffix flow. */ for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { @@ -4394,10 +4414,7 @@ flow_meter_split_prep(struct rte_eth_dev *dev, switch (actions->type) { case RTE_FLOW_ACTION_TYPE_METER: /* Add the extra tag action first. */ - tag_action = actions_pre; - tag_action->type = (enum rte_flow_action_type) - MLX5_RTE_FLOW_ACTION_TYPE_TAG; - actions_pre++; + tag_action = actions_pre++; action_cur = &actions_pre; break; case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: @@ -4430,23 +4447,22 @@ flow_meter_split_prep(struct rte_eth_dev *dev, actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; actions_pre->type = RTE_FLOW_ACTION_TYPE_END; actions_pre++; - /* Set the tag. */ - set_tag = (void *)actions_pre; - set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); - mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], - &tag_id); - if (tag_id >= (1 << (sizeof(tag_id) * 8 - MLX5_MTR_COLOR_BITS))) { - DRV_LOG(ERR, "Port %u meter flow id exceed max limit.", - dev->data->port_id); - mlx5_ipool_free(priv->sh->ipool - [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], tag_id); - return 0; - } else if (!tag_id) { - return 0; + mlx5_ipool_malloc(fm->flow_ipool, &tag_id); + if (!tag_id) + return rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Failed to allocate meter flow id."); + flow_id = tag_id - 1; + flow_id_bits = MLX5_REG_BITS - __builtin_clz(flow_id); + flow_id_bits = flow_id_bits ? flow_id_bits : 1; + if ((flow_id_bits + priv->max_mtr_bits) > mtr_reg_bits) { + mlx5_ipool_free(fm->flow_ipool, tag_id); + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Meter flow id exceeds max limit."); } - set_tag->data = tag_id << MLX5_MTR_COLOR_BITS; - assert(tag_action); - tag_action->conf = set_tag; + if (flow_id_bits > priv->max_mtr_flow_bits) + priv->max_mtr_flow_bits = flow_id_bits; /* Prepare the suffix subflow items. */ tag_item = sfx_items++; for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { @@ -4475,16 +4491,34 @@ flow_meter_split_prep(struct rte_eth_dev *dev, } sfx_items->type = RTE_FLOW_ITEM_TYPE_END; sfx_items++; - tag_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; - tag_spec->data = tag_id << MLX5_MTR_COLOR_BITS; - tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); - tag_mask = tag_spec + 1; - tag_mask->data = 0xffffff00; + /* Build tag actions and items for meter_id/meter flow_id. */ + assert(tag_action); + set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; + tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; + tag_item_mask = tag_item_spec + 1; + /* + * The color Reg bits used by flow_id are growing from + * msb to lsb, so must do bit reverse for flow_id val in RegC. + */ + for (shift = 0; shift < flow_id_bits; shift++) + flow_id_reversed = (flow_id_reversed << 1) | + ((flow_id >> shift) & 0x1); + /* Both flow_id and meter_id share the same register. */ + set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 0, error); + set_tag->data = + (fm->idx | (flow_id_reversed << (mtr_reg_bits - flow_id_bits))) + << mtr_id_offset; + tag_item_spec->id = set_tag->id; + tag_item_spec->data = set_tag->data; + tag_item_mask->data = UINT32_MAX << mtr_id_offset; + tag_action->type = (enum rte_flow_action_type) + MLX5_RTE_FLOW_ACTION_TYPE_TAG; + tag_action->conf = set_tag; tag_item->type = (enum rte_flow_item_type) - MLX5_RTE_FLOW_ITEM_TYPE_TAG; - tag_item->spec = tag_spec; + MLX5_RTE_FLOW_ITEM_TYPE_TAG; + tag_item->spec = tag_item_spec; tag_item->last = NULL; - tag_item->mask = tag_mask; + tag_item->mask = tag_item_mask; return tag_id; } @@ -5184,25 +5218,49 @@ flow_create_split_meter(struct rte_eth_dev *dev, struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); struct rte_flow_action *sfx_actions = NULL; struct rte_flow_action *pre_actions = NULL; struct rte_flow_item *sfx_items = NULL; struct mlx5_flow *dev_flow = NULL; struct rte_flow_attr sfx_attr = *attr; - uint32_t mtr = 0; + struct mlx5_flow_meter *fm = NULL; + bool has_mtr = false; + uint32_t meter_id; uint32_t mtr_tag_id = 0; size_t act_size; size_t item_size; int actions_n = 0; - int ret; + int ret = 0; if (priv->mtr_en) - actions_n = flow_check_meter_action(actions, &mtr); - if (mtr) { - /* The five prefix actions: meter, decap, encap, tag, end. */ + actions_n = flow_check_meter_action(actions, &has_mtr, + &meter_id); + if (has_mtr) { + if (flow->meter) { + fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], + flow->meter); + if (!fm) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "Meter not found."); + } else { + fm = mlx5_flow_meter_find(priv, meter_id); + if (!fm) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "Meter not found."); + ret = mlx5_flow_meter_attach(priv, fm, + &sfx_attr, error); + if (ret) + return -rte_errno; + flow->meter = fm->idx; + } + wks->fm = fm; + /* The prefix actions: meter, decap, encap, tag, end. */ act_size = sizeof(struct rte_flow_action) * (actions_n + 5) + sizeof(struct mlx5_rte_flow_action_set_tag); - /* tag, vlan, port id, end. */ + /* The suffix items: tag, vlan, port id, end. */ #define METER_SUFFIX_ITEM 4 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + sizeof(struct mlx5_rte_flow_item_tag) * 2; @@ -5216,9 +5274,9 @@ flow_create_split_meter(struct rte_eth_dev *dev, sfx_items = (struct rte_flow_item *)((char *)sfx_actions + act_size); pre_actions = sfx_actions + actions_n; - mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items, + mtr_tag_id = flow_meter_split_prep(dev, fm, items, sfx_items, actions, sfx_actions, - pre_actions); + pre_actions, error); if (!mtr_tag_id) { ret = -rte_errno; goto exit; @@ -5229,10 +5287,12 @@ flow_create_split_meter(struct rte_eth_dev *dev, attr, items, pre_actions, flow_split_info, error); if (ret) { + mlx5_ipool_free(fm->flow_ipool, mtr_tag_id); ret = -rte_errno; goto exit; } dev_flow->handle->split_flow_id = mtr_tag_id; + dev_flow->handle->is_meter_flow_id = 1; /* Setting the sfx group atrr. */ sfx_attr.group = sfx_attr.transfer ? (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : @@ -6504,20 +6564,17 @@ mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, * * @param[in] dev * Pointer to Ethernet device. - * @param[in] fm - * Pointer to the flow meter. * * @return * Pointer to table set on success, NULL otherwise. */ struct mlx5_meter_domains_infos * -mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm) +mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev) { const struct mlx5_flow_driver_ops *fops; fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); - return fops->create_mtr_tbls(dev, fm); + return fops->create_mtr_tbls(dev); } /** @@ -6542,7 +6599,7 @@ mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, } /** - * Create policer rules. + * Prepare policer rules. * * @param[in] dev * Pointer to Ethernet device. @@ -6555,14 +6612,14 @@ mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, * 0 on success, -1 otherwise. */ int -mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, +mlx5_flow_prepare_policer_rules(struct rte_eth_dev *dev, struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr) { const struct mlx5_flow_driver_ops *fops; fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); - return fops->create_policer_rules(dev, fm, attr); + return fops->prepare_policer_rules(dev, fm, attr); } /** diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index ec673c29ab..4482a456f0 100644 --- a/drivers/net/mlx5/mlx5_flow.h +++ b/drivers/net/mlx5/mlx5_flow.h @@ -79,7 +79,7 @@ enum mlx5_feature_name { MLX5_APP_TAG, MLX5_COPY_MARK, MLX5_MTR_COLOR, - MLX5_MTR_SFX, + MLX5_MTR_ID, MLX5_ASO_FLOW_HIT, }; @@ -655,7 +655,8 @@ struct mlx5_flow_handle { uint64_t layers; /**< Bit-fields of present layers, see MLX5_FLOW_LAYER_*. */ void *drv_flow; /**< pointer to driver flow object. */ - uint32_t split_flow_id:28; /**< Sub flow unique match flow id. */ + uint32_t split_flow_id:27; /**< Sub flow unique match flow id. */ + uint32_t is_meter_flow_id:1; /**< Indate if flow_id is for meter. */ uint32_t mark:1; /**< Metadate rxq mark flag. */ uint32_t fate_action:3; /**< Fate action type. */ union { @@ -842,14 +843,16 @@ struct mlx5_meter_domain_info { /**< Meter table. */ struct mlx5_flow_tbl_resource *sfx_tbl; /**< Meter suffix table. */ - void *any_matcher; - /**< Meter color not match default criteria. */ - void *color_matcher; - /**< Meter color match criteria. */ + struct mlx5_flow_dv_matcher *drop_matcher; + /**< Matcher for Drop. */ + struct mlx5_flow_dv_matcher *color_matcher; + /**< Matcher for Color. */ void *jump_actn; /**< Meter match action. */ - void *policer_rules[RTE_MTR_DROPPED + 1]; - /**< Meter policer for the match. */ + void *green_rule; + /**< Meter green rule. */ + void *drop_rule; + /**< Meter drop rule. */ }; /* Meter table set for TX RX FDB. */ @@ -862,10 +865,10 @@ struct mlx5_meter_domains_infos { /**< RX meter table. */ struct mlx5_meter_domain_info transfer; /**< FDB meter table. */ - void *drop_actn; - /**< Drop action as not matched. */ - void *count_actns[RTE_MTR_DROPPED + 1]; - /**< Counters for match and unmatched statistics. */ + void *green_count; + /**< Counters for green rule. */ + void *drop_count; + /**< Counters for green rule. */ uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)]; /**< Flow meter parameter. */ size_t fmp_size; @@ -928,6 +931,8 @@ struct mlx5_flow_meter { /**< Meter state. */ uint32_t shared:1; /**< Meter shared or not. */ + struct mlx5_indexed_pool *flow_ipool; + /**< Index pool for flow id. */ }; /* RFC2697 parameter structure. */ @@ -1166,6 +1171,7 @@ struct mlx5_flow_workspace { struct mlx5_flow_rss_desc rss_desc; uint32_t rssq_num; /* Allocated queue num in rss_desc. */ uint32_t flow_idx; /* Intermediate device flow index. */ + struct mlx5_flow_meter *fm; /* Pointer to the meter in flow. */ }; struct mlx5_flow_split_info { @@ -1206,8 +1212,7 @@ typedef int (*mlx5_flow_query_t)(struct rte_eth_dev *dev, void *data, struct rte_flow_error *error); typedef struct mlx5_meter_domains_infos *(*mlx5_flow_create_mtr_tbls_t) - (struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm); + (struct rte_eth_dev *dev); typedef int (*mlx5_flow_destroy_mtr_tbls_t)(struct rte_eth_dev *dev, struct mlx5_meter_domains_infos *tbls); typedef int (*mlx5_flow_create_policer_rules_t) @@ -1270,7 +1275,7 @@ struct mlx5_flow_driver_ops { mlx5_flow_query_t query; mlx5_flow_create_mtr_tbls_t create_mtr_tbls; mlx5_flow_destroy_mtr_tbls_t destroy_mtr_tbls; - mlx5_flow_create_policer_rules_t create_policer_rules; + mlx5_flow_create_policer_rules_t prepare_policer_rules; mlx5_flow_destroy_policer_rules_t destroy_policer_rules; mlx5_flow_counter_alloc_t counter_alloc; mlx5_flow_counter_free_t counter_free; @@ -1470,11 +1475,10 @@ int mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, const struct rte_flow_item_ecpri *acc_mask, struct rte_flow_error *error); struct mlx5_meter_domains_infos *mlx5_flow_create_mtr_tbls - (struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm); + (struct rte_eth_dev *dev); int mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, struct mlx5_meter_domains_infos *tbl); -int mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, +int mlx5_flow_prepare_policer_rules(struct rte_eth_dev *dev, struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr); int mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 07a0ee5abb..14ed219fd5 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -10981,14 +10981,12 @@ flow_dv_translate(struct rte_eth_dev *dev, const struct rte_flow_action_rss *rss; const struct rte_flow_action *action = actions; const uint8_t *rss_key; - const struct rte_flow_action_meter *mtr; struct mlx5_flow_tbl_resource *tbl; struct mlx5_aso_age_action *age_act; uint32_t port_id = 0; struct mlx5_flow_dv_port_id_action_resource port_id_resource; int action_type = actions->type; const struct rte_flow_action *found_action = NULL; - struct mlx5_flow_meter *fm = NULL; uint32_t jump_group = 0; if (!mlx5_flow_os_action_supported(action_type)) @@ -11413,33 +11411,13 @@ flow_dv_translate(struct rte_eth_dev *dev, MLX5_FLOW_FATE_DEFAULT_MISS; break; case RTE_FLOW_ACTION_TYPE_METER: - mtr = actions->conf; - if (!flow->meter) { - fm = mlx5_flow_meter_attach(priv, mtr->mtr_id, - attr, error); - if (!fm) - return rte_flow_error_set(error, - rte_errno, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "meter not found " - "or invalid parameters"); - flow->meter = fm->idx; - } + if (!wks->fm) + return rte_flow_error_set(error, rte_errno, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "Failed to get meter in flow."); /* Set the meter action. */ - if (!fm) { - fm = mlx5_ipool_get(priv->sh->ipool - [MLX5_IPOOL_MTR], flow->meter); - if (!fm) - return rte_flow_error_set(error, - rte_errno, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "meter not found " - "or invalid parameters"); - } dev_flow->dv.actions[actions_n++] = - fm->mfts->meter_action; + wks->fm->mfts->meter_action; action_flags |= MLX5_FLOW_ACTION_METER; break; case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: @@ -12588,6 +12566,7 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) { struct mlx5_flow_handle *dev_handle; struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_meter *fm = NULL; uint32_t srss = 0; if (!flow) @@ -12598,8 +12577,6 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) flow->counter = 0; } if (flow->meter) { - struct mlx5_flow_meter *fm; - fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], flow->meter); if (fm) @@ -12641,6 +12618,10 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) flow_dv_fate_resource_release(dev, dev_handle); else if (!srss) srss = dev_handle->rix_srss; + if (fm && dev_handle->is_meter_flow_id && + dev_handle->split_flow_id) + mlx5_ipool_free(fm->flow_ipool, + dev_handle->split_flow_id); mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], tmp_idx); } @@ -13406,49 +13387,20 @@ flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev, if (!mtd || !priv->config.dv_flow_en) return 0; - if (mtd->ingress.policer_rules[RTE_MTR_DROPPED]) - claim_zero(mlx5_flow_os_destroy_flow - (mtd->ingress.policer_rules[RTE_MTR_DROPPED])); - if (mtd->egress.policer_rules[RTE_MTR_DROPPED]) - claim_zero(mlx5_flow_os_destroy_flow - (mtd->egress.policer_rules[RTE_MTR_DROPPED])); - if (mtd->transfer.policer_rules[RTE_MTR_DROPPED]) - claim_zero(mlx5_flow_os_destroy_flow - (mtd->transfer.policer_rules[RTE_MTR_DROPPED])); - if (mtd->egress.color_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->egress.color_matcher)); - if (mtd->egress.any_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->egress.any_matcher)); if (mtd->egress.tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->egress.tbl); if (mtd->egress.sfx_tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->egress.sfx_tbl); - if (mtd->ingress.color_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->ingress.color_matcher)); - if (mtd->ingress.any_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->ingress.any_matcher)); if (mtd->ingress.tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->ingress.tbl); if (mtd->ingress.sfx_tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->ingress.sfx_tbl); - if (mtd->transfer.color_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->transfer.color_matcher)); - if (mtd->transfer.any_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->transfer.any_matcher)); if (mtd->transfer.tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->transfer.tbl); if (mtd->transfer.sfx_tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->transfer.sfx_tbl); - if (mtd->drop_actn) - claim_zero(mlx5_flow_os_destroy_flow_action(mtd->drop_actn)); mlx5_free(mtd); return 0; } @@ -13467,37 +13419,17 @@ flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev, * Table attribute. * @param[in] transfer * Table attribute. - * @param[in] color_reg_c_idx - * Reg C index for color match. * * @return - * 0 on success, -1 otherwise and rte_errno is set. + * 0 on success, -1 otherwise. */ static int flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, struct mlx5_meter_domains_infos *mtb, - uint8_t egress, uint8_t transfer, - uint32_t color_reg_c_idx) + uint8_t egress, uint8_t transfer) { - struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_dev_ctx_shared *sh = priv->sh; - struct mlx5_flow_dv_match_params mask = { - .size = sizeof(mask.buf), - }; - struct mlx5_flow_dv_match_params value = { - .size = sizeof(value.buf), - }; - struct mlx5dv_flow_matcher_attr dv_attr = { - .type = IBV_FLOW_ATTR_NORMAL, - .priority = 0, - .match_criteria_enable = 0, - .match_mask = (void *)&mask, - }; - void *actions[METER_ACTIONS]; - struct mlx5_meter_domain_info *dtb; struct rte_flow_error error; - int i = 0; - int ret; + struct mlx5_meter_domain_info *dtb; if (transfer) dtb = &mtb->transfer; @@ -13522,41 +13454,7 @@ flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, DRV_LOG(ERR, "Failed to create meter suffix table."); return -1; } - /* Create matchers, Any and Color. */ - dv_attr.priority = 3; - dv_attr.match_criteria_enable = 0; - ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj, - &dtb->any_matcher); - if (ret) { - DRV_LOG(ERR, "Failed to create meter" - " policer default matcher."); - goto error_exit; - } - dv_attr.priority = 0; - dv_attr.match_criteria_enable = - 1 << MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT; - flow_dv_match_meta_reg(mask.buf, value.buf, color_reg_c_idx, - rte_col_2_mlx5_col(RTE_COLORS), UINT8_MAX); - ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj, - &dtb->color_matcher); - if (ret) { - DRV_LOG(ERR, "Failed to create meter policer color matcher."); - goto error_exit; - } - if (mtb->count_actns[RTE_MTR_DROPPED]) - actions[i++] = mtb->count_actns[RTE_MTR_DROPPED]; - actions[i++] = mtb->drop_actn; - /* Default rule: lowest priority, match any, actions: drop. */ - ret = mlx5_flow_os_create_flow(dtb->any_matcher, (void *)&value, i, - actions, - &dtb->policer_rules[RTE_MTR_DROPPED]); - if (ret) { - DRV_LOG(ERR, "Failed to create meter policer drop rule."); - goto error_exit; - } return 0; -error_exit: - return -1; } /** @@ -13565,20 +13463,16 @@ flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, * * @param[in] dev * Pointer to Ethernet device. - * @param[in] fm - * Pointer to the flow meter. * * @return * Pointer to table set on success, NULL otherwise and rte_errno is set. */ static struct mlx5_meter_domains_infos * -flow_dv_create_mtr_tbl(struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm) +flow_dv_create_mtr_tbl(struct rte_eth_dev *dev) { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_meter_domains_infos *mtb; int ret; - int i; if (!priv->mtr_en) { rte_errno = ENOTSUP; @@ -13589,37 +13483,21 @@ flow_dv_create_mtr_tbl(struct rte_eth_dev *dev, DRV_LOG(ERR, "Failed to allocate memory for meter."); return NULL; } - /* Create meter count actions */ - for (i = 0; i <= RTE_MTR_DROPPED; i++) { - struct mlx5_flow_counter *cnt; - if (!fm->policer_stats.cnt[i]) - continue; - cnt = flow_dv_counter_get_by_idx(dev, - fm->policer_stats.cnt[i], NULL); - mtb->count_actns[i] = cnt->action; - } - /* Create drop action. */ - ret = mlx5_flow_os_create_flow_action_drop(&mtb->drop_actn); - if (ret) { - DRV_LOG(ERR, "Failed to create drop action."); - goto error_exit; - } /* Egress meter table. */ - ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0, priv->mtr_color_reg); + ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0); if (ret) { DRV_LOG(ERR, "Failed to prepare egress meter table."); goto error_exit; } /* Ingress meter table. */ - ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0, priv->mtr_color_reg); + ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0); if (ret) { DRV_LOG(ERR, "Failed to prepare ingress meter table."); goto error_exit; } /* FDB meter table. */ if (priv->config.dv_esw_en) { - ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1, - priv->mtr_color_reg); + ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1); if (ret) { DRV_LOG(ERR, "Failed to prepare fdb meter table."); goto error_exit; @@ -13631,24 +13509,152 @@ flow_dv_create_mtr_tbl(struct rte_eth_dev *dev, return NULL; } +/** + * Destroy the meter table matchers. + * Lock free, (mutex should be acquired by caller). + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in,out] dtb + * Pointer to DV meter table. + * + * @return + * Always 0. + */ +static int +flow_dv_destroy_mtr_matchers(struct rte_eth_dev *dev, + struct mlx5_meter_domain_info *dtb) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_tbl_data_entry *tbl; + + if (!priv->config.dv_flow_en) + return 0; + if (dtb->drop_matcher) { + tbl = container_of(dtb->drop_matcher->tbl, typeof(*tbl), tbl); + mlx5_cache_unregister(&tbl->matchers, + &dtb->drop_matcher->entry); + dtb->drop_matcher = NULL; + } + if (dtb->color_matcher) { + tbl = container_of(dtb->color_matcher->tbl, typeof(*tbl), tbl); + mlx5_cache_unregister(&tbl->matchers, + &dtb->color_matcher->entry); + dtb->color_matcher = NULL; + } + return 0; +} + +/** + * Create the matchers for meter table. + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in] color_reg_c_idx + * Reg C index for color match. + * @param[in] mtr_id_reg_c_idx + * Reg C index for meter_id match. + * @param[in] mtr_id_mask + * Mask for meter_id match criteria. + * @param[in,out] dtb + * Pointer to DV meter table. + * @param[out] error + * Perform verbose error reporting if not NULL. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_prepare_mtr_matchers(struct rte_eth_dev *dev, + uint32_t color_reg_c_idx, + uint32_t mtr_id_reg_c_idx, + uint32_t mtr_id_mask, + struct mlx5_meter_domain_info *dtb, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_tbl_data_entry *tbl_data; + struct mlx5_cache_entry *entry; + struct mlx5_flow_dv_matcher matcher = { + .mask = { + .size = sizeof(matcher.mask.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), + }, + .tbl = dtb->tbl, + }; + struct mlx5_flow_dv_match_params value = { + .size = sizeof(value.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), + }; + struct mlx5_flow_cb_ctx ctx = { + .error = error, + .data = &matcher, + }; + uint32_t color_mask = (UINT32_C(1) << MLX5_MTR_COLOR_BITS) - 1; + + tbl_data = container_of(dtb->tbl, struct mlx5_flow_tbl_data_entry, tbl); + if (!dtb->drop_matcher) { + /* Create matchers for Drop. */ + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + mtr_id_reg_c_idx, 0, mtr_id_mask); + matcher.priority = MLX5_REG_BITS * 2 - priv->max_mtr_bits; + matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf, + matcher.mask.size); + entry = mlx5_cache_register(&tbl_data->matchers, &ctx); + if (!entry) { + DRV_LOG(ERR, "Failed to register meter drop matcher."); + return -1; + } + dtb->drop_matcher = + container_of(entry, struct mlx5_flow_dv_matcher, entry); + } + if (!dtb->color_matcher) { + /* Create matchers for Color + meter_id. */ + if (priv->mtr_reg_share) { + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + color_reg_c_idx, 0, + (mtr_id_mask | color_mask)); + } else { + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + color_reg_c_idx, 0, color_mask); + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + mtr_id_reg_c_idx, 0, mtr_id_mask); + } + matcher.priority = MLX5_REG_BITS - priv->max_mtr_bits; + matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf, + matcher.mask.size); + entry = mlx5_cache_register(&tbl_data->matchers, &ctx); + if (!entry) { + DRV_LOG(ERR, "Failed to register meter color matcher."); + return -1; + } + dtb->color_matcher = + container_of(entry, struct mlx5_flow_dv_matcher, entry); + } + return 0; +} + /** * Destroy domain policer rule. * + * @param[in] dev + * Pointer to Ethernet device. * @param[in] dt * Pointer to domain table. */ static void -flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt) +flow_dv_destroy_domain_policer_rule(struct rte_eth_dev *dev, + struct mlx5_meter_domain_info *dt) { - int i; - - for (i = 0; i < RTE_MTR_DROPPED; i++) { - if (dt->policer_rules[i]) { - claim_zero(mlx5_flow_os_destroy_flow - (dt->policer_rules[i])); - dt->policer_rules[i] = NULL; - } + if (dt->drop_rule) { + claim_zero(mlx5_flow_os_destroy_flow(dt->drop_rule)); + dt->drop_rule = NULL; } + if (dt->green_rule) { + claim_zero(mlx5_flow_os_destroy_flow(dt->green_rule)); + dt->green_rule = NULL; + } + flow_dv_destroy_mtr_matchers(dev, dt); if (dt->jump_actn) { claim_zero(mlx5_flow_os_destroy_flow_action(dt->jump_actn)); dt->jump_actn = NULL; @@ -13669,7 +13675,7 @@ flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt) * Always 0. */ static int -flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused, +flow_dv_destroy_policer_rules(struct rte_eth_dev *dev, const struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr) { @@ -13678,39 +13684,56 @@ flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused, if (!mtb) return 0; if (attr->egress) - flow_dv_destroy_domain_policer_rule(&mtb->egress); + flow_dv_destroy_domain_policer_rule(dev, &mtb->egress); if (attr->ingress) - flow_dv_destroy_domain_policer_rule(&mtb->ingress); + flow_dv_destroy_domain_policer_rule(dev, &mtb->ingress); if (attr->transfer) - flow_dv_destroy_domain_policer_rule(&mtb->transfer); + flow_dv_destroy_domain_policer_rule(dev, &mtb->transfer); return 0; } /** * Create specify domain meter policer rule. * + * @param[in] dev + * Pointer to Ethernet device. * @param[in] fm * Pointer to flow meter structure. * @param[in] mtb * Pointer to DV meter table set. - * @param[in] mtr_reg_c - * Color match REG_C. + * @param[out] drop_rule + * The address of pointer saving drop rule. + * @param[out] color_rule + * The address of pointer saving green rule. * * @return * 0 on success, -1 otherwise. */ static int -flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, +flow_dv_create_policer_forward_rule(struct rte_eth_dev *dev, + struct mlx5_flow_meter *fm, struct mlx5_meter_domain_info *dtb, - uint8_t mtr_reg_c) + void **drop_rule, + void **green_rule) { + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flow_dv_match_params matcher = { - .size = sizeof(matcher.buf), + .size = sizeof(matcher.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), }; struct mlx5_flow_dv_match_params value = { - .size = sizeof(value.buf), + .size = sizeof(value.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), }; struct mlx5_meter_domains_infos *mtb = fm->mfts; + struct rte_flow_error error; + uint32_t color_reg_c = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, + 0, &error); + uint32_t mtr_id_reg_c = mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, + 0, &error); + uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; + uint32_t mtr_id_mask = + ((UINT32_C(1) << priv->max_mtr_bits) - 1) << mtr_id_offset; void *actions[METER_ACTIONS]; int i; int ret = 0; @@ -13723,25 +13746,52 @@ flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, DRV_LOG(ERR, "Failed to create policer jump action."); goto error; } - for (i = 0; i < RTE_MTR_DROPPED; i++) { - int j = 0; - - flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_reg_c, - rte_col_2_mlx5_col(i), UINT8_MAX); - if (mtb->count_actns[i]) - actions[j++] = mtb->count_actns[i]; - if (fm->action[i] == MTR_POLICER_ACTION_DROP) - actions[j++] = mtb->drop_actn; - else - actions[j++] = dtb->jump_actn; - ret = mlx5_flow_os_create_flow(dtb->color_matcher, - (void *)&value, j, actions, - &dtb->policer_rules[i]); + /* Prepare matchers. */ + if (!dtb->drop_matcher || !dtb->color_matcher) { + ret = flow_dv_prepare_mtr_matchers(dev, color_reg_c, + mtr_id_reg_c, mtr_id_mask, + dtb, &error); if (ret) { - DRV_LOG(ERR, "Failed to create policer rule."); + DRV_LOG(ERR, "Failed to setup matchers for mtr table."); goto error; } } + /* Create Drop flow, matching meter_id only. */ + i = 0; + flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_id_reg_c, + (fm->idx << mtr_id_offset), UINT32_MAX); + if (mtb->drop_count) + actions[i++] = mtb->drop_count; + actions[i++] = priv->sh->dr_drop_action; + ret = mlx5_flow_os_create_flow(dtb->drop_matcher->matcher_object, + (void *)&value, i, actions, drop_rule); + if (ret) { + DRV_LOG(ERR, "Failed to create meter policer drop rule."); + goto error; + } + /* Create flow matching Green color + meter_id. */ + i = 0; + if (priv->mtr_reg_share) { + flow_dv_match_meta_reg(matcher.buf, value.buf, color_reg_c, + ((fm->idx << mtr_id_offset) | + rte_col_2_mlx5_col(RTE_COLOR_GREEN)), + UINT32_MAX); + } else { + flow_dv_match_meta_reg(matcher.buf, value.buf, color_reg_c, + rte_col_2_mlx5_col(RTE_COLOR_GREEN), + UINT32_MAX); + flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_id_reg_c, + fm->idx, UINT32_MAX); + } + if (mtb->green_count) + actions[i++] = mtb->green_count; + actions[i++] = dtb->jump_actn; + ret = mlx5_flow_os_create_flow(dtb->color_matcher->matcher_object, + (void *)&value, i, actions, green_rule); + if (ret) { + DRV_LOG(ERR, "Failed to create meter policer color rule."); + goto error; + } return 0; error: rte_errno = errno; @@ -13749,7 +13799,8 @@ flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, } /** - * Create policer rules. + * Prepare policer rules for all domains. + * If meter already initialized, this will replace all old rules with new ones. * * @param[in] dev * Pointer to Ethernet device. @@ -13762,41 +13813,107 @@ flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, * 0 on success, -1 otherwise. */ static int -flow_dv_create_policer_rules(struct rte_eth_dev *dev, - struct mlx5_flow_meter *fm, - const struct rte_flow_attr *attr) +flow_dv_prepare_policer_rules(struct rte_eth_dev *dev, + struct mlx5_flow_meter *fm, + const struct rte_flow_attr *attr) { - struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_meter_domains_infos *mtb = fm->mfts; + bool initialized = false; + struct mlx5_flow_counter *cnt; + void *egress_drop_rule = NULL; + void *egress_green_rule = NULL; + void *ingress_drop_rule = NULL; + void *ingress_green_rule = NULL; + void *transfer_drop_rule = NULL; + void *transfer_green_rule = NULL; int ret; + /* Get the statistics counters for green/drop. */ + if (fm->policer_stats.cnt[RTE_COLOR_GREEN]) { + cnt = flow_dv_counter_get_by_idx(dev, + fm->policer_stats.cnt[RTE_COLOR_GREEN], + NULL); + mtb->green_count = cnt->action; + } else { + mtb->green_count = NULL; + } + if (fm->policer_stats.cnt[RTE_MTR_DROPPED]) { + cnt = flow_dv_counter_get_by_idx(dev, + fm->policer_stats.cnt[RTE_MTR_DROPPED], + NULL); + mtb->drop_count = cnt->action; + } else { + mtb->drop_count = NULL; + } + /** + * If flow meter has been initialized, all policer rules + * are created. So can get if meter initialized by checking + * any policer rule. + */ + if (mtb->egress.drop_rule) + initialized = true; if (attr->egress) { - ret = flow_dv_create_policer_forward_rule(fm, &mtb->egress, - priv->mtr_color_reg); + ret = flow_dv_create_policer_forward_rule(dev, + fm, &mtb->egress, + &egress_drop_rule, &egress_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create egress policer."); goto error; } } if (attr->ingress) { - ret = flow_dv_create_policer_forward_rule(fm, &mtb->ingress, - priv->mtr_color_reg); + ret = flow_dv_create_policer_forward_rule(dev, + fm, &mtb->ingress, + &ingress_drop_rule, &ingress_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create ingress policer."); goto error; } } if (attr->transfer) { - ret = flow_dv_create_policer_forward_rule(fm, &mtb->transfer, - priv->mtr_color_reg); + ret = flow_dv_create_policer_forward_rule(dev, + fm, &mtb->transfer, + &transfer_drop_rule, &transfer_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create transfer policer."); goto error; } } + /* Replace old flows if existing. */ + if (mtb->egress.drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->egress.drop_rule)); + if (mtb->egress.green_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->egress.green_rule)); + if (mtb->ingress.drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->ingress.drop_rule)); + if (mtb->ingress.green_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->ingress.green_rule)); + if (mtb->transfer.drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->transfer.drop_rule)); + if (mtb->transfer.green_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->transfer.green_rule)); + mtb->egress.drop_rule = egress_drop_rule; + mtb->egress.green_rule = egress_green_rule; + mtb->ingress.drop_rule = ingress_drop_rule; + mtb->ingress.green_rule = ingress_green_rule; + mtb->transfer.drop_rule = transfer_drop_rule; + mtb->transfer.green_rule = transfer_green_rule; return 0; error: - flow_dv_destroy_policer_rules(dev, fm, attr); + if (egress_drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(egress_drop_rule)); + if (egress_green_rule) + claim_zero(mlx5_flow_os_destroy_flow(egress_green_rule)); + if (ingress_drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(ingress_drop_rule)); + if (ingress_green_rule) + claim_zero(mlx5_flow_os_destroy_flow(ingress_green_rule)); + if (transfer_drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(transfer_drop_rule)); + if (transfer_green_rule) + claim_zero(mlx5_flow_os_destroy_flow(transfer_green_rule)); + if (!initialized) + flow_dv_destroy_policer_rules(dev, fm, attr); return -1; } @@ -14093,7 +14210,7 @@ const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = { .query = flow_dv_query, .create_mtr_tbls = flow_dv_create_mtr_tbl, .destroy_mtr_tbls = flow_dv_destroy_mtr_tbl, - .create_policer_rules = flow_dv_create_policer_rules, + .prepare_policer_rules = flow_dv_prepare_policer_rules, .destroy_policer_rules = flow_dv_destroy_policer_rules, .counter_alloc = flow_dv_counter_allocate, .counter_free = flow_dv_counter_free, diff --git a/drivers/net/mlx5/mlx5_flow_meter.c b/drivers/net/mlx5/mlx5_flow_meter.c index dbc574b508..7f7693b698 100644 --- a/drivers/net/mlx5/mlx5_flow_meter.c +++ b/drivers/net/mlx5/mlx5_flow_meter.c @@ -474,6 +474,12 @@ mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id, MTR_POLICER_ACTION_COLOR_RED }; int i; + /* Meter must use global drop action. */ + if (!priv->sh->dr_drop_action) + return -rte_mtr_error_set(error, ENOTSUP, + RTE_MTR_ERROR_TYPE_MTR_PARAMS, + NULL, + "No drop action ready for meter."); /* Meter params must not be NULL. */ if (params == NULL) return -rte_mtr_error_set(error, EINVAL, @@ -630,9 +636,18 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, .egress = 1, .transfer = priv->config.dv_esw_en ? 1 : 0, }; + struct mlx5_indexed_pool_config flow_ipool_cfg = { + .size = 0, + .trunk_size = 64, + .need_lock = 1, + .type = "mlx5_flow_mtr_flow_id_pool", + }; int ret; unsigned int i; uint32_t idx = 0; + uint8_t mtr_id_bits; + uint8_t mtr_reg_bits = priv->mtr_reg_share ? + MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; if (!priv->mtr_en) return -rte_mtr_error_set(error, ENOTSUP, @@ -654,6 +669,13 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, return -rte_mtr_error_set(error, ENOMEM, RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, "Memory alloc failed for meter."); + mtr_id_bits = MLX5_REG_BITS - __builtin_clz(idx); + if ((mtr_id_bits + priv->max_mtr_flow_bits) > mtr_reg_bits) { + DRV_LOG(ERR, "Meter number exceeds max limit."); + goto error; + } + if (mtr_id_bits > priv->max_mtr_bits) + priv->max_mtr_bits = mtr_id_bits; fm->idx = idx; /* Fill the flow meter parameters. */ fm->meter_id = meter_id; @@ -667,10 +689,10 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, if (!fm->policer_stats.cnt[i]) goto error; } - fm->mfts = mlx5_flow_create_mtr_tbls(dev, fm); + fm->mfts = mlx5_flow_create_mtr_tbls(dev); if (!fm->mfts) goto error; - ret = mlx5_flow_create_policer_rules(dev, fm, &attr); + ret = mlx5_flow_prepare_policer_rules(dev, fm, &attr); if (ret) goto error; /* Add to the flow meter list. */ @@ -679,6 +701,9 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, fm->shared = !!shared; fm->policer_stats.stats_mask = params->stats_mask; fm->profile->ref_cnt++; + fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg); + if (!fm->flow_ipool) + goto error; rte_spinlock_init(&fm->sl); return 0; error: @@ -749,6 +774,8 @@ mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id, if (fm->policer_stats.cnt[i]) mlx5_counter_free(dev, fm->policer_stats.cnt[i]); /* Free meter flow table */ + if (fm->flow_ipool) + mlx5_ipool_destroy(fm->flow_ipool); mlx5_flow_destroy_policer_rules(dev, fm, &attr); mlx5_flow_destroy_mtr_tbls(dev, fm->mfts); mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], fm->idx); @@ -1153,30 +1180,24 @@ mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id) * * @param [in] priv * Pointer to mlx5 private data. - * @param [in] meter_id - * Flow meter id. + * @param[in] fm + * Pointer to flow meter. * @param [in] attr * Pointer to flow attributes. * @param [out] error * Pointer to error structure. * - * @return the flow meter pointer, NULL otherwise. + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. */ -struct mlx5_flow_meter * -mlx5_flow_meter_attach(struct mlx5_priv *priv, uint32_t meter_id, +int +mlx5_flow_meter_attach(struct mlx5_priv *priv, + struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr, struct rte_flow_error *error) { - struct mlx5_flow_meter *fm; int ret = 0; - fm = mlx5_flow_meter_find(priv, meter_id); - if (fm == NULL) { - rte_flow_error_set(error, ENOENT, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "Meter object id not valid"); - return fm; - } rte_spinlock_lock(&fm->sl); if (fm->mfts->meter_action) { if (fm->shared && @@ -1210,7 +1231,7 @@ mlx5_flow_meter_attach(struct mlx5_priv *priv, uint32_t meter_id, fm->mfts->meter_action ? "Meter attr not match" : "Meter action create failed"); - return ret ? NULL : fm; + return ret ? -rte_errno : 0; } /** -- 2.27.0 ^ permalink raw reply [flat|nested] 8+ messages in thread
[parent not found: <20210414025736.31142-1-lizh@nvidia.com>]
* [dpdk-stable] [PATCH v4 03/14] net/mlx5: fix meter statistics [not found] ` <20210414025736.31142-1-lizh@nvidia.com> @ 2021-04-14 2:57 ` Li Zhang 0 siblings, 0 replies; 8+ messages in thread From: Li Zhang @ 2021-04-14 2:57 UTC (permalink / raw) To: dekelp, orika, viacheslavo, matan, shahafs, Suanming Mou Cc: dev, thomas, rasland, roniba, Shun Hao, stable From: Shun Hao <shunh@nvidia.com> Currently, packets after meter will be steered to a global policer table, which includes green/red color rules for every meter, so as to have counter statistics of each color in every meter. There's a bug that all the rules in global policer table are matching only color criteria, so all packets will be counted to one meter only, and other meter statistics are always zero. This patch does these: 1. The rules in policer table matches both meter index and color, so packet after meter could be counted to the correct meter counter. 2. The meter index and flow index are now sharing the available register bits dynamically. Meter index starts from lsb, and flow index starts from msb. Fixes: 46a5e6bc6a ("net/mlx5: prepare meter flow tables") Cc: stable@dpdk.org Signed-off-by: Shun Hao <shunh@nvidia.com> Acked-by: Matan Azrad <matan@nvidia.com> --- drivers/net/mlx5/mlx5.c | 7 +- drivers/net/mlx5/mlx5.h | 15 +- drivers/net/mlx5/mlx5_flow.c | 183 +++++++---- drivers/net/mlx5/mlx5_flow.h | 40 +-- drivers/net/mlx5/mlx5_flow_dv.c | 489 ++++++++++++++++++----------- drivers/net/mlx5/mlx5_flow_meter.c | 53 +++- 6 files changed, 496 insertions(+), 291 deletions(-) diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index 9557d06afa..734dee9f19 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -275,10 +275,13 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = { }, #endif [MLX5_IPOOL_MTR] = { + /** + * The ipool index should grow continually from small to big, + * for meter idx, so not set grow_trunk to avoid meter index + * not jump continually. + */ .size = sizeof(struct mlx5_flow_meter), .trunk_size = 64, - .grow_trunk = 3, - .grow_shift = 2, .need_lock = 1, .release_mem_en = 1, .malloc = mlx5_malloc, diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index 0f69f9d125..b50acaca41 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -944,9 +944,9 @@ struct mlx5_priv { unsigned int representor:1; /* Device is a port representor. */ unsigned int master:1; /* Device is a E-Switch master. */ unsigned int txpp_en:1; /* Tx packet pacing enabled. */ + unsigned int sampler_en:1; /* Whether support sampler. */ unsigned int mtr_en:1; /* Whether support meter. */ unsigned int mtr_reg_share:1; /* Whether support meter REG_C share. */ - unsigned int sampler_en:1; /* Whether support sampler. */ uint16_t domain_id; /* Switch domain identifier. */ uint16_t vport_id; /* Associated VF vport index (if any). */ uint32_t vport_meta_tag; /* Used for vport index match ove VF LAG. */ @@ -1003,6 +1003,10 @@ struct mlx5_priv { uint32_t rss_shared_actions; /* RSS shared actions. */ struct mlx5_devx_obj *q_counters; /* DevX queue counter object. */ uint32_t counter_set_id; /* Queue counter ID to set in DevX objects. */ + uint8_t max_mtr_bits; + /* Indicate how many bits are used by meter id at the most. */ + uint8_t max_mtr_flow_bits; + /* Indicate how many bits are used by meter flow id at the most. */ }; #define PORT_ID(priv) ((priv)->dev_data->port_id) @@ -1271,11 +1275,10 @@ int mlx5_pmd_socket_init(void); int mlx5_flow_meter_ops_get(struct rte_eth_dev *dev, void *arg); struct mlx5_flow_meter *mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id); -struct mlx5_flow_meter *mlx5_flow_meter_attach - (struct mlx5_priv *priv, - uint32_t meter_id, - const struct rte_flow_attr *attr, - struct rte_flow_error *error); +int mlx5_flow_meter_attach(struct mlx5_priv *priv, + struct mlx5_flow_meter *fm, + const struct rte_flow_attr *attr, + struct rte_flow_error *error); void mlx5_flow_meter_detach(struct mlx5_flow_meter *fm); /* mlx5_os.c */ diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c index 668c32cf51..63ff6acbbf 100644 --- a/drivers/net/mlx5/mlx5_flow.c +++ b/drivers/net/mlx5/mlx5_flow.c @@ -748,9 +748,9 @@ mlx5_flow_get_reg_id(struct rte_eth_dev *dev, return REG_C_0; } break; - case MLX5_MTR_SFX: + case MLX5_MTR_ID: /* - * If meter color and flow match share one register, flow match + * If meter color and meter id share one register, flow match * should use the meter color register for match. */ if (priv->mtr_reg_share) @@ -3035,7 +3035,8 @@ flow_mreg_split_qrss_release(struct rte_eth_dev *dev, SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, handle_idx, dev_handle, next) - if (dev_handle->split_flow_id) + if (dev_handle->split_flow_id && + !dev_handle->is_meter_flow_id) mlx5_ipool_free(priv->sh->ipool [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], dev_handle->split_flow_id); @@ -3674,23 +3675,30 @@ flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], * * @param[in] actions * Pointer to the list of actions. - * @param[out] mtr + * @param[out] has_mtr * Pointer to the meter exist flag. + * @param[out] meter_id + * Pointer to the meter id. * * @return * Total number of actions. */ static int -flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr) +flow_check_meter_action(const struct rte_flow_action actions[], + bool *has_mtr, + uint32_t *meter_id) { + const struct rte_flow_action_meter *mtr = NULL; int actions_n = 0; - MLX5_ASSERT(mtr); - *mtr = 0; + MLX5_ASSERT(has_mtr); + *has_mtr = false; for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { switch (actions->type) { case RTE_FLOW_ACTION_TYPE_METER: - *mtr = 1; + mtr = actions->conf; + *meter_id = mtr->mtr_id; + *has_mtr = true; break; default: break; @@ -4347,8 +4355,10 @@ flow_create_split_inner(struct rte_eth_dev *dev, * header will be in the prefix sub flow, as not to take the * L3 tunnel header into account. * - * @param dev + * @param[in] dev * Pointer to Ethernet device. + * @param[in] fm + * Pointer to flow meter structure. * @param[in] items * Pattern specification (list terminated by the END pattern item). * @param[out] sfx_items @@ -4363,29 +4373,39 @@ flow_create_split_inner(struct rte_eth_dev *dev, * The pattern items for the suffix flow. * @param[out] tag_sfx * Pointer to suffix flow tag. + * @param[out] error + * Perform verbose error reporting if not NULL. * * @return - * 0 on success. + * The flow id, 0 otherwise and rte_errno is set. */ -static int +static uint32_t flow_meter_split_prep(struct rte_eth_dev *dev, - const struct rte_flow_item items[], - struct rte_flow_item sfx_items[], - const struct rte_flow_action actions[], - struct rte_flow_action actions_sfx[], - struct rte_flow_action actions_pre[]) + struct mlx5_flow_meter *fm, + const struct rte_flow_item items[], + struct rte_flow_item sfx_items[], + const struct rte_flow_action actions[], + struct rte_flow_action actions_sfx[], + struct rte_flow_action actions_pre[], + struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; struct rte_flow_action *tag_action = NULL; struct rte_flow_item *tag_item; struct mlx5_rte_flow_action_set_tag *set_tag; - struct rte_flow_error error; const struct rte_flow_action_raw_encap *raw_encap; const struct rte_flow_action_raw_decap *raw_decap; - struct mlx5_rte_flow_item_tag *tag_spec; - struct mlx5_rte_flow_item_tag *tag_mask; + struct mlx5_rte_flow_item_tag *tag_item_spec; + struct mlx5_rte_flow_item_tag *tag_item_mask; uint32_t tag_id = 0; bool copy_vlan = false; + uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; + uint8_t mtr_reg_bits = priv->mtr_reg_share ? + MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; + uint32_t flow_id = 0; + uint32_t flow_id_reversed = 0; + uint8_t flow_id_bits = 0; + int shift; /* Prepare the actions for prefix and suffix flow. */ for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { @@ -4394,10 +4414,7 @@ flow_meter_split_prep(struct rte_eth_dev *dev, switch (actions->type) { case RTE_FLOW_ACTION_TYPE_METER: /* Add the extra tag action first. */ - tag_action = actions_pre; - tag_action->type = (enum rte_flow_action_type) - MLX5_RTE_FLOW_ACTION_TYPE_TAG; - actions_pre++; + tag_action = actions_pre++; action_cur = &actions_pre; break; case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: @@ -4430,23 +4447,22 @@ flow_meter_split_prep(struct rte_eth_dev *dev, actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; actions_pre->type = RTE_FLOW_ACTION_TYPE_END; actions_pre++; - /* Set the tag. */ - set_tag = (void *)actions_pre; - set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); - mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], - &tag_id); - if (tag_id >= (1 << (sizeof(tag_id) * 8 - MLX5_MTR_COLOR_BITS))) { - DRV_LOG(ERR, "Port %u meter flow id exceed max limit.", - dev->data->port_id); - mlx5_ipool_free(priv->sh->ipool - [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], tag_id); - return 0; - } else if (!tag_id) { - return 0; + mlx5_ipool_malloc(fm->flow_ipool, &tag_id); + if (!tag_id) + return rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Failed to allocate meter flow id."); + flow_id = tag_id - 1; + flow_id_bits = MLX5_REG_BITS - __builtin_clz(flow_id); + flow_id_bits = flow_id_bits ? flow_id_bits : 1; + if ((flow_id_bits + priv->max_mtr_bits) > mtr_reg_bits) { + mlx5_ipool_free(fm->flow_ipool, tag_id); + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Meter flow id exceeds max limit."); } - set_tag->data = tag_id << MLX5_MTR_COLOR_BITS; - assert(tag_action); - tag_action->conf = set_tag; + if (flow_id_bits > priv->max_mtr_flow_bits) + priv->max_mtr_flow_bits = flow_id_bits; /* Prepare the suffix subflow items. */ tag_item = sfx_items++; for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { @@ -4475,16 +4491,34 @@ flow_meter_split_prep(struct rte_eth_dev *dev, } sfx_items->type = RTE_FLOW_ITEM_TYPE_END; sfx_items++; - tag_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; - tag_spec->data = tag_id << MLX5_MTR_COLOR_BITS; - tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); - tag_mask = tag_spec + 1; - tag_mask->data = 0xffffff00; + /* Build tag actions and items for meter_id/meter flow_id. */ + assert(tag_action); + set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; + tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; + tag_item_mask = tag_item_spec + 1; + /* + * The color Reg bits used by flow_id are growing from + * msb to lsb, so must do bit reverse for flow_id val in RegC. + */ + for (shift = 0; shift < flow_id_bits; shift++) + flow_id_reversed = (flow_id_reversed << 1) | + ((flow_id >> shift) & 0x1); + /* Both flow_id and meter_id share the same register. */ + set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 0, error); + set_tag->data = + (fm->idx | (flow_id_reversed << (mtr_reg_bits - flow_id_bits))) + << mtr_id_offset; + tag_item_spec->id = set_tag->id; + tag_item_spec->data = set_tag->data; + tag_item_mask->data = UINT32_MAX << mtr_id_offset; + tag_action->type = (enum rte_flow_action_type) + MLX5_RTE_FLOW_ACTION_TYPE_TAG; + tag_action->conf = set_tag; tag_item->type = (enum rte_flow_item_type) - MLX5_RTE_FLOW_ITEM_TYPE_TAG; - tag_item->spec = tag_spec; + MLX5_RTE_FLOW_ITEM_TYPE_TAG; + tag_item->spec = tag_item_spec; tag_item->last = NULL; - tag_item->mask = tag_mask; + tag_item->mask = tag_item_mask; return tag_id; } @@ -5184,25 +5218,49 @@ flow_create_split_meter(struct rte_eth_dev *dev, struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); struct rte_flow_action *sfx_actions = NULL; struct rte_flow_action *pre_actions = NULL; struct rte_flow_item *sfx_items = NULL; struct mlx5_flow *dev_flow = NULL; struct rte_flow_attr sfx_attr = *attr; - uint32_t mtr = 0; + struct mlx5_flow_meter *fm = NULL; + bool has_mtr = false; + uint32_t meter_id; uint32_t mtr_tag_id = 0; size_t act_size; size_t item_size; int actions_n = 0; - int ret; + int ret = 0; if (priv->mtr_en) - actions_n = flow_check_meter_action(actions, &mtr); - if (mtr) { - /* The five prefix actions: meter, decap, encap, tag, end. */ + actions_n = flow_check_meter_action(actions, &has_mtr, + &meter_id); + if (has_mtr) { + if (flow->meter) { + fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], + flow->meter); + if (!fm) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "Meter not found."); + } else { + fm = mlx5_flow_meter_find(priv, meter_id); + if (!fm) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "Meter not found."); + ret = mlx5_flow_meter_attach(priv, fm, + &sfx_attr, error); + if (ret) + return -rte_errno; + flow->meter = fm->idx; + } + wks->fm = fm; + /* The prefix actions: meter, decap, encap, tag, end. */ act_size = sizeof(struct rte_flow_action) * (actions_n + 5) + sizeof(struct mlx5_rte_flow_action_set_tag); - /* tag, vlan, port id, end. */ + /* The suffix items: tag, vlan, port id, end. */ #define METER_SUFFIX_ITEM 4 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + sizeof(struct mlx5_rte_flow_item_tag) * 2; @@ -5216,9 +5274,9 @@ flow_create_split_meter(struct rte_eth_dev *dev, sfx_items = (struct rte_flow_item *)((char *)sfx_actions + act_size); pre_actions = sfx_actions + actions_n; - mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items, + mtr_tag_id = flow_meter_split_prep(dev, fm, items, sfx_items, actions, sfx_actions, - pre_actions); + pre_actions, error); if (!mtr_tag_id) { ret = -rte_errno; goto exit; @@ -5229,10 +5287,12 @@ flow_create_split_meter(struct rte_eth_dev *dev, attr, items, pre_actions, flow_split_info, error); if (ret) { + mlx5_ipool_free(fm->flow_ipool, mtr_tag_id); ret = -rte_errno; goto exit; } dev_flow->handle->split_flow_id = mtr_tag_id; + dev_flow->handle->is_meter_flow_id = 1; /* Setting the sfx group atrr. */ sfx_attr.group = sfx_attr.transfer ? (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : @@ -6504,20 +6564,17 @@ mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, * * @param[in] dev * Pointer to Ethernet device. - * @param[in] fm - * Pointer to the flow meter. * * @return * Pointer to table set on success, NULL otherwise. */ struct mlx5_meter_domains_infos * -mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm) +mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev) { const struct mlx5_flow_driver_ops *fops; fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); - return fops->create_mtr_tbls(dev, fm); + return fops->create_mtr_tbls(dev); } /** @@ -6542,7 +6599,7 @@ mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, } /** - * Create policer rules. + * Prepare policer rules. * * @param[in] dev * Pointer to Ethernet device. @@ -6555,14 +6612,14 @@ mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, * 0 on success, -1 otherwise. */ int -mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, +mlx5_flow_prepare_policer_rules(struct rte_eth_dev *dev, struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr) { const struct mlx5_flow_driver_ops *fops; fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); - return fops->create_policer_rules(dev, fm, attr); + return fops->prepare_policer_rules(dev, fm, attr); } /** diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index ec673c29ab..4482a456f0 100644 --- a/drivers/net/mlx5/mlx5_flow.h +++ b/drivers/net/mlx5/mlx5_flow.h @@ -79,7 +79,7 @@ enum mlx5_feature_name { MLX5_APP_TAG, MLX5_COPY_MARK, MLX5_MTR_COLOR, - MLX5_MTR_SFX, + MLX5_MTR_ID, MLX5_ASO_FLOW_HIT, }; @@ -655,7 +655,8 @@ struct mlx5_flow_handle { uint64_t layers; /**< Bit-fields of present layers, see MLX5_FLOW_LAYER_*. */ void *drv_flow; /**< pointer to driver flow object. */ - uint32_t split_flow_id:28; /**< Sub flow unique match flow id. */ + uint32_t split_flow_id:27; /**< Sub flow unique match flow id. */ + uint32_t is_meter_flow_id:1; /**< Indate if flow_id is for meter. */ uint32_t mark:1; /**< Metadate rxq mark flag. */ uint32_t fate_action:3; /**< Fate action type. */ union { @@ -842,14 +843,16 @@ struct mlx5_meter_domain_info { /**< Meter table. */ struct mlx5_flow_tbl_resource *sfx_tbl; /**< Meter suffix table. */ - void *any_matcher; - /**< Meter color not match default criteria. */ - void *color_matcher; - /**< Meter color match criteria. */ + struct mlx5_flow_dv_matcher *drop_matcher; + /**< Matcher for Drop. */ + struct mlx5_flow_dv_matcher *color_matcher; + /**< Matcher for Color. */ void *jump_actn; /**< Meter match action. */ - void *policer_rules[RTE_MTR_DROPPED + 1]; - /**< Meter policer for the match. */ + void *green_rule; + /**< Meter green rule. */ + void *drop_rule; + /**< Meter drop rule. */ }; /* Meter table set for TX RX FDB. */ @@ -862,10 +865,10 @@ struct mlx5_meter_domains_infos { /**< RX meter table. */ struct mlx5_meter_domain_info transfer; /**< FDB meter table. */ - void *drop_actn; - /**< Drop action as not matched. */ - void *count_actns[RTE_MTR_DROPPED + 1]; - /**< Counters for match and unmatched statistics. */ + void *green_count; + /**< Counters for green rule. */ + void *drop_count; + /**< Counters for green rule. */ uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)]; /**< Flow meter parameter. */ size_t fmp_size; @@ -928,6 +931,8 @@ struct mlx5_flow_meter { /**< Meter state. */ uint32_t shared:1; /**< Meter shared or not. */ + struct mlx5_indexed_pool *flow_ipool; + /**< Index pool for flow id. */ }; /* RFC2697 parameter structure. */ @@ -1166,6 +1171,7 @@ struct mlx5_flow_workspace { struct mlx5_flow_rss_desc rss_desc; uint32_t rssq_num; /* Allocated queue num in rss_desc. */ uint32_t flow_idx; /* Intermediate device flow index. */ + struct mlx5_flow_meter *fm; /* Pointer to the meter in flow. */ }; struct mlx5_flow_split_info { @@ -1206,8 +1212,7 @@ typedef int (*mlx5_flow_query_t)(struct rte_eth_dev *dev, void *data, struct rte_flow_error *error); typedef struct mlx5_meter_domains_infos *(*mlx5_flow_create_mtr_tbls_t) - (struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm); + (struct rte_eth_dev *dev); typedef int (*mlx5_flow_destroy_mtr_tbls_t)(struct rte_eth_dev *dev, struct mlx5_meter_domains_infos *tbls); typedef int (*mlx5_flow_create_policer_rules_t) @@ -1270,7 +1275,7 @@ struct mlx5_flow_driver_ops { mlx5_flow_query_t query; mlx5_flow_create_mtr_tbls_t create_mtr_tbls; mlx5_flow_destroy_mtr_tbls_t destroy_mtr_tbls; - mlx5_flow_create_policer_rules_t create_policer_rules; + mlx5_flow_create_policer_rules_t prepare_policer_rules; mlx5_flow_destroy_policer_rules_t destroy_policer_rules; mlx5_flow_counter_alloc_t counter_alloc; mlx5_flow_counter_free_t counter_free; @@ -1470,11 +1475,10 @@ int mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, const struct rte_flow_item_ecpri *acc_mask, struct rte_flow_error *error); struct mlx5_meter_domains_infos *mlx5_flow_create_mtr_tbls - (struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm); + (struct rte_eth_dev *dev); int mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, struct mlx5_meter_domains_infos *tbl); -int mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, +int mlx5_flow_prepare_policer_rules(struct rte_eth_dev *dev, struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr); int mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 07a0ee5abb..14ed219fd5 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -10981,14 +10981,12 @@ flow_dv_translate(struct rte_eth_dev *dev, const struct rte_flow_action_rss *rss; const struct rte_flow_action *action = actions; const uint8_t *rss_key; - const struct rte_flow_action_meter *mtr; struct mlx5_flow_tbl_resource *tbl; struct mlx5_aso_age_action *age_act; uint32_t port_id = 0; struct mlx5_flow_dv_port_id_action_resource port_id_resource; int action_type = actions->type; const struct rte_flow_action *found_action = NULL; - struct mlx5_flow_meter *fm = NULL; uint32_t jump_group = 0; if (!mlx5_flow_os_action_supported(action_type)) @@ -11413,33 +11411,13 @@ flow_dv_translate(struct rte_eth_dev *dev, MLX5_FLOW_FATE_DEFAULT_MISS; break; case RTE_FLOW_ACTION_TYPE_METER: - mtr = actions->conf; - if (!flow->meter) { - fm = mlx5_flow_meter_attach(priv, mtr->mtr_id, - attr, error); - if (!fm) - return rte_flow_error_set(error, - rte_errno, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "meter not found " - "or invalid parameters"); - flow->meter = fm->idx; - } + if (!wks->fm) + return rte_flow_error_set(error, rte_errno, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "Failed to get meter in flow."); /* Set the meter action. */ - if (!fm) { - fm = mlx5_ipool_get(priv->sh->ipool - [MLX5_IPOOL_MTR], flow->meter); - if (!fm) - return rte_flow_error_set(error, - rte_errno, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "meter not found " - "or invalid parameters"); - } dev_flow->dv.actions[actions_n++] = - fm->mfts->meter_action; + wks->fm->mfts->meter_action; action_flags |= MLX5_FLOW_ACTION_METER; break; case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: @@ -12588,6 +12566,7 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) { struct mlx5_flow_handle *dev_handle; struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_meter *fm = NULL; uint32_t srss = 0; if (!flow) @@ -12598,8 +12577,6 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) flow->counter = 0; } if (flow->meter) { - struct mlx5_flow_meter *fm; - fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], flow->meter); if (fm) @@ -12641,6 +12618,10 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) flow_dv_fate_resource_release(dev, dev_handle); else if (!srss) srss = dev_handle->rix_srss; + if (fm && dev_handle->is_meter_flow_id && + dev_handle->split_flow_id) + mlx5_ipool_free(fm->flow_ipool, + dev_handle->split_flow_id); mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], tmp_idx); } @@ -13406,49 +13387,20 @@ flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev, if (!mtd || !priv->config.dv_flow_en) return 0; - if (mtd->ingress.policer_rules[RTE_MTR_DROPPED]) - claim_zero(mlx5_flow_os_destroy_flow - (mtd->ingress.policer_rules[RTE_MTR_DROPPED])); - if (mtd->egress.policer_rules[RTE_MTR_DROPPED]) - claim_zero(mlx5_flow_os_destroy_flow - (mtd->egress.policer_rules[RTE_MTR_DROPPED])); - if (mtd->transfer.policer_rules[RTE_MTR_DROPPED]) - claim_zero(mlx5_flow_os_destroy_flow - (mtd->transfer.policer_rules[RTE_MTR_DROPPED])); - if (mtd->egress.color_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->egress.color_matcher)); - if (mtd->egress.any_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->egress.any_matcher)); if (mtd->egress.tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->egress.tbl); if (mtd->egress.sfx_tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->egress.sfx_tbl); - if (mtd->ingress.color_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->ingress.color_matcher)); - if (mtd->ingress.any_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->ingress.any_matcher)); if (mtd->ingress.tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->ingress.tbl); if (mtd->ingress.sfx_tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->ingress.sfx_tbl); - if (mtd->transfer.color_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->transfer.color_matcher)); - if (mtd->transfer.any_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->transfer.any_matcher)); if (mtd->transfer.tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->transfer.tbl); if (mtd->transfer.sfx_tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->transfer.sfx_tbl); - if (mtd->drop_actn) - claim_zero(mlx5_flow_os_destroy_flow_action(mtd->drop_actn)); mlx5_free(mtd); return 0; } @@ -13467,37 +13419,17 @@ flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev, * Table attribute. * @param[in] transfer * Table attribute. - * @param[in] color_reg_c_idx - * Reg C index for color match. * * @return - * 0 on success, -1 otherwise and rte_errno is set. + * 0 on success, -1 otherwise. */ static int flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, struct mlx5_meter_domains_infos *mtb, - uint8_t egress, uint8_t transfer, - uint32_t color_reg_c_idx) + uint8_t egress, uint8_t transfer) { - struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_dev_ctx_shared *sh = priv->sh; - struct mlx5_flow_dv_match_params mask = { - .size = sizeof(mask.buf), - }; - struct mlx5_flow_dv_match_params value = { - .size = sizeof(value.buf), - }; - struct mlx5dv_flow_matcher_attr dv_attr = { - .type = IBV_FLOW_ATTR_NORMAL, - .priority = 0, - .match_criteria_enable = 0, - .match_mask = (void *)&mask, - }; - void *actions[METER_ACTIONS]; - struct mlx5_meter_domain_info *dtb; struct rte_flow_error error; - int i = 0; - int ret; + struct mlx5_meter_domain_info *dtb; if (transfer) dtb = &mtb->transfer; @@ -13522,41 +13454,7 @@ flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, DRV_LOG(ERR, "Failed to create meter suffix table."); return -1; } - /* Create matchers, Any and Color. */ - dv_attr.priority = 3; - dv_attr.match_criteria_enable = 0; - ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj, - &dtb->any_matcher); - if (ret) { - DRV_LOG(ERR, "Failed to create meter" - " policer default matcher."); - goto error_exit; - } - dv_attr.priority = 0; - dv_attr.match_criteria_enable = - 1 << MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT; - flow_dv_match_meta_reg(mask.buf, value.buf, color_reg_c_idx, - rte_col_2_mlx5_col(RTE_COLORS), UINT8_MAX); - ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj, - &dtb->color_matcher); - if (ret) { - DRV_LOG(ERR, "Failed to create meter policer color matcher."); - goto error_exit; - } - if (mtb->count_actns[RTE_MTR_DROPPED]) - actions[i++] = mtb->count_actns[RTE_MTR_DROPPED]; - actions[i++] = mtb->drop_actn; - /* Default rule: lowest priority, match any, actions: drop. */ - ret = mlx5_flow_os_create_flow(dtb->any_matcher, (void *)&value, i, - actions, - &dtb->policer_rules[RTE_MTR_DROPPED]); - if (ret) { - DRV_LOG(ERR, "Failed to create meter policer drop rule."); - goto error_exit; - } return 0; -error_exit: - return -1; } /** @@ -13565,20 +13463,16 @@ flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, * * @param[in] dev * Pointer to Ethernet device. - * @param[in] fm - * Pointer to the flow meter. * * @return * Pointer to table set on success, NULL otherwise and rte_errno is set. */ static struct mlx5_meter_domains_infos * -flow_dv_create_mtr_tbl(struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm) +flow_dv_create_mtr_tbl(struct rte_eth_dev *dev) { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_meter_domains_infos *mtb; int ret; - int i; if (!priv->mtr_en) { rte_errno = ENOTSUP; @@ -13589,37 +13483,21 @@ flow_dv_create_mtr_tbl(struct rte_eth_dev *dev, DRV_LOG(ERR, "Failed to allocate memory for meter."); return NULL; } - /* Create meter count actions */ - for (i = 0; i <= RTE_MTR_DROPPED; i++) { - struct mlx5_flow_counter *cnt; - if (!fm->policer_stats.cnt[i]) - continue; - cnt = flow_dv_counter_get_by_idx(dev, - fm->policer_stats.cnt[i], NULL); - mtb->count_actns[i] = cnt->action; - } - /* Create drop action. */ - ret = mlx5_flow_os_create_flow_action_drop(&mtb->drop_actn); - if (ret) { - DRV_LOG(ERR, "Failed to create drop action."); - goto error_exit; - } /* Egress meter table. */ - ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0, priv->mtr_color_reg); + ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0); if (ret) { DRV_LOG(ERR, "Failed to prepare egress meter table."); goto error_exit; } /* Ingress meter table. */ - ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0, priv->mtr_color_reg); + ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0); if (ret) { DRV_LOG(ERR, "Failed to prepare ingress meter table."); goto error_exit; } /* FDB meter table. */ if (priv->config.dv_esw_en) { - ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1, - priv->mtr_color_reg); + ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1); if (ret) { DRV_LOG(ERR, "Failed to prepare fdb meter table."); goto error_exit; @@ -13631,24 +13509,152 @@ flow_dv_create_mtr_tbl(struct rte_eth_dev *dev, return NULL; } +/** + * Destroy the meter table matchers. + * Lock free, (mutex should be acquired by caller). + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in,out] dtb + * Pointer to DV meter table. + * + * @return + * Always 0. + */ +static int +flow_dv_destroy_mtr_matchers(struct rte_eth_dev *dev, + struct mlx5_meter_domain_info *dtb) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_tbl_data_entry *tbl; + + if (!priv->config.dv_flow_en) + return 0; + if (dtb->drop_matcher) { + tbl = container_of(dtb->drop_matcher->tbl, typeof(*tbl), tbl); + mlx5_cache_unregister(&tbl->matchers, + &dtb->drop_matcher->entry); + dtb->drop_matcher = NULL; + } + if (dtb->color_matcher) { + tbl = container_of(dtb->color_matcher->tbl, typeof(*tbl), tbl); + mlx5_cache_unregister(&tbl->matchers, + &dtb->color_matcher->entry); + dtb->color_matcher = NULL; + } + return 0; +} + +/** + * Create the matchers for meter table. + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in] color_reg_c_idx + * Reg C index for color match. + * @param[in] mtr_id_reg_c_idx + * Reg C index for meter_id match. + * @param[in] mtr_id_mask + * Mask for meter_id match criteria. + * @param[in,out] dtb + * Pointer to DV meter table. + * @param[out] error + * Perform verbose error reporting if not NULL. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_prepare_mtr_matchers(struct rte_eth_dev *dev, + uint32_t color_reg_c_idx, + uint32_t mtr_id_reg_c_idx, + uint32_t mtr_id_mask, + struct mlx5_meter_domain_info *dtb, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_tbl_data_entry *tbl_data; + struct mlx5_cache_entry *entry; + struct mlx5_flow_dv_matcher matcher = { + .mask = { + .size = sizeof(matcher.mask.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), + }, + .tbl = dtb->tbl, + }; + struct mlx5_flow_dv_match_params value = { + .size = sizeof(value.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), + }; + struct mlx5_flow_cb_ctx ctx = { + .error = error, + .data = &matcher, + }; + uint32_t color_mask = (UINT32_C(1) << MLX5_MTR_COLOR_BITS) - 1; + + tbl_data = container_of(dtb->tbl, struct mlx5_flow_tbl_data_entry, tbl); + if (!dtb->drop_matcher) { + /* Create matchers for Drop. */ + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + mtr_id_reg_c_idx, 0, mtr_id_mask); + matcher.priority = MLX5_REG_BITS * 2 - priv->max_mtr_bits; + matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf, + matcher.mask.size); + entry = mlx5_cache_register(&tbl_data->matchers, &ctx); + if (!entry) { + DRV_LOG(ERR, "Failed to register meter drop matcher."); + return -1; + } + dtb->drop_matcher = + container_of(entry, struct mlx5_flow_dv_matcher, entry); + } + if (!dtb->color_matcher) { + /* Create matchers for Color + meter_id. */ + if (priv->mtr_reg_share) { + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + color_reg_c_idx, 0, + (mtr_id_mask | color_mask)); + } else { + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + color_reg_c_idx, 0, color_mask); + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + mtr_id_reg_c_idx, 0, mtr_id_mask); + } + matcher.priority = MLX5_REG_BITS - priv->max_mtr_bits; + matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf, + matcher.mask.size); + entry = mlx5_cache_register(&tbl_data->matchers, &ctx); + if (!entry) { + DRV_LOG(ERR, "Failed to register meter color matcher."); + return -1; + } + dtb->color_matcher = + container_of(entry, struct mlx5_flow_dv_matcher, entry); + } + return 0; +} + /** * Destroy domain policer rule. * + * @param[in] dev + * Pointer to Ethernet device. * @param[in] dt * Pointer to domain table. */ static void -flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt) +flow_dv_destroy_domain_policer_rule(struct rte_eth_dev *dev, + struct mlx5_meter_domain_info *dt) { - int i; - - for (i = 0; i < RTE_MTR_DROPPED; i++) { - if (dt->policer_rules[i]) { - claim_zero(mlx5_flow_os_destroy_flow - (dt->policer_rules[i])); - dt->policer_rules[i] = NULL; - } + if (dt->drop_rule) { + claim_zero(mlx5_flow_os_destroy_flow(dt->drop_rule)); + dt->drop_rule = NULL; } + if (dt->green_rule) { + claim_zero(mlx5_flow_os_destroy_flow(dt->green_rule)); + dt->green_rule = NULL; + } + flow_dv_destroy_mtr_matchers(dev, dt); if (dt->jump_actn) { claim_zero(mlx5_flow_os_destroy_flow_action(dt->jump_actn)); dt->jump_actn = NULL; @@ -13669,7 +13675,7 @@ flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt) * Always 0. */ static int -flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused, +flow_dv_destroy_policer_rules(struct rte_eth_dev *dev, const struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr) { @@ -13678,39 +13684,56 @@ flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused, if (!mtb) return 0; if (attr->egress) - flow_dv_destroy_domain_policer_rule(&mtb->egress); + flow_dv_destroy_domain_policer_rule(dev, &mtb->egress); if (attr->ingress) - flow_dv_destroy_domain_policer_rule(&mtb->ingress); + flow_dv_destroy_domain_policer_rule(dev, &mtb->ingress); if (attr->transfer) - flow_dv_destroy_domain_policer_rule(&mtb->transfer); + flow_dv_destroy_domain_policer_rule(dev, &mtb->transfer); return 0; } /** * Create specify domain meter policer rule. * + * @param[in] dev + * Pointer to Ethernet device. * @param[in] fm * Pointer to flow meter structure. * @param[in] mtb * Pointer to DV meter table set. - * @param[in] mtr_reg_c - * Color match REG_C. + * @param[out] drop_rule + * The address of pointer saving drop rule. + * @param[out] color_rule + * The address of pointer saving green rule. * * @return * 0 on success, -1 otherwise. */ static int -flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, +flow_dv_create_policer_forward_rule(struct rte_eth_dev *dev, + struct mlx5_flow_meter *fm, struct mlx5_meter_domain_info *dtb, - uint8_t mtr_reg_c) + void **drop_rule, + void **green_rule) { + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flow_dv_match_params matcher = { - .size = sizeof(matcher.buf), + .size = sizeof(matcher.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), }; struct mlx5_flow_dv_match_params value = { - .size = sizeof(value.buf), + .size = sizeof(value.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), }; struct mlx5_meter_domains_infos *mtb = fm->mfts; + struct rte_flow_error error; + uint32_t color_reg_c = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, + 0, &error); + uint32_t mtr_id_reg_c = mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, + 0, &error); + uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; + uint32_t mtr_id_mask = + ((UINT32_C(1) << priv->max_mtr_bits) - 1) << mtr_id_offset; void *actions[METER_ACTIONS]; int i; int ret = 0; @@ -13723,25 +13746,52 @@ flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, DRV_LOG(ERR, "Failed to create policer jump action."); goto error; } - for (i = 0; i < RTE_MTR_DROPPED; i++) { - int j = 0; - - flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_reg_c, - rte_col_2_mlx5_col(i), UINT8_MAX); - if (mtb->count_actns[i]) - actions[j++] = mtb->count_actns[i]; - if (fm->action[i] == MTR_POLICER_ACTION_DROP) - actions[j++] = mtb->drop_actn; - else - actions[j++] = dtb->jump_actn; - ret = mlx5_flow_os_create_flow(dtb->color_matcher, - (void *)&value, j, actions, - &dtb->policer_rules[i]); + /* Prepare matchers. */ + if (!dtb->drop_matcher || !dtb->color_matcher) { + ret = flow_dv_prepare_mtr_matchers(dev, color_reg_c, + mtr_id_reg_c, mtr_id_mask, + dtb, &error); if (ret) { - DRV_LOG(ERR, "Failed to create policer rule."); + DRV_LOG(ERR, "Failed to setup matchers for mtr table."); goto error; } } + /* Create Drop flow, matching meter_id only. */ + i = 0; + flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_id_reg_c, + (fm->idx << mtr_id_offset), UINT32_MAX); + if (mtb->drop_count) + actions[i++] = mtb->drop_count; + actions[i++] = priv->sh->dr_drop_action; + ret = mlx5_flow_os_create_flow(dtb->drop_matcher->matcher_object, + (void *)&value, i, actions, drop_rule); + if (ret) { + DRV_LOG(ERR, "Failed to create meter policer drop rule."); + goto error; + } + /* Create flow matching Green color + meter_id. */ + i = 0; + if (priv->mtr_reg_share) { + flow_dv_match_meta_reg(matcher.buf, value.buf, color_reg_c, + ((fm->idx << mtr_id_offset) | + rte_col_2_mlx5_col(RTE_COLOR_GREEN)), + UINT32_MAX); + } else { + flow_dv_match_meta_reg(matcher.buf, value.buf, color_reg_c, + rte_col_2_mlx5_col(RTE_COLOR_GREEN), + UINT32_MAX); + flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_id_reg_c, + fm->idx, UINT32_MAX); + } + if (mtb->green_count) + actions[i++] = mtb->green_count; + actions[i++] = dtb->jump_actn; + ret = mlx5_flow_os_create_flow(dtb->color_matcher->matcher_object, + (void *)&value, i, actions, green_rule); + if (ret) { + DRV_LOG(ERR, "Failed to create meter policer color rule."); + goto error; + } return 0; error: rte_errno = errno; @@ -13749,7 +13799,8 @@ flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, } /** - * Create policer rules. + * Prepare policer rules for all domains. + * If meter already initialized, this will replace all old rules with new ones. * * @param[in] dev * Pointer to Ethernet device. @@ -13762,41 +13813,107 @@ flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, * 0 on success, -1 otherwise. */ static int -flow_dv_create_policer_rules(struct rte_eth_dev *dev, - struct mlx5_flow_meter *fm, - const struct rte_flow_attr *attr) +flow_dv_prepare_policer_rules(struct rte_eth_dev *dev, + struct mlx5_flow_meter *fm, + const struct rte_flow_attr *attr) { - struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_meter_domains_infos *mtb = fm->mfts; + bool initialized = false; + struct mlx5_flow_counter *cnt; + void *egress_drop_rule = NULL; + void *egress_green_rule = NULL; + void *ingress_drop_rule = NULL; + void *ingress_green_rule = NULL; + void *transfer_drop_rule = NULL; + void *transfer_green_rule = NULL; int ret; + /* Get the statistics counters for green/drop. */ + if (fm->policer_stats.cnt[RTE_COLOR_GREEN]) { + cnt = flow_dv_counter_get_by_idx(dev, + fm->policer_stats.cnt[RTE_COLOR_GREEN], + NULL); + mtb->green_count = cnt->action; + } else { + mtb->green_count = NULL; + } + if (fm->policer_stats.cnt[RTE_MTR_DROPPED]) { + cnt = flow_dv_counter_get_by_idx(dev, + fm->policer_stats.cnt[RTE_MTR_DROPPED], + NULL); + mtb->drop_count = cnt->action; + } else { + mtb->drop_count = NULL; + } + /** + * If flow meter has been initialized, all policer rules + * are created. So can get if meter initialized by checking + * any policer rule. + */ + if (mtb->egress.drop_rule) + initialized = true; if (attr->egress) { - ret = flow_dv_create_policer_forward_rule(fm, &mtb->egress, - priv->mtr_color_reg); + ret = flow_dv_create_policer_forward_rule(dev, + fm, &mtb->egress, + &egress_drop_rule, &egress_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create egress policer."); goto error; } } if (attr->ingress) { - ret = flow_dv_create_policer_forward_rule(fm, &mtb->ingress, - priv->mtr_color_reg); + ret = flow_dv_create_policer_forward_rule(dev, + fm, &mtb->ingress, + &ingress_drop_rule, &ingress_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create ingress policer."); goto error; } } if (attr->transfer) { - ret = flow_dv_create_policer_forward_rule(fm, &mtb->transfer, - priv->mtr_color_reg); + ret = flow_dv_create_policer_forward_rule(dev, + fm, &mtb->transfer, + &transfer_drop_rule, &transfer_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create transfer policer."); goto error; } } + /* Replace old flows if existing. */ + if (mtb->egress.drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->egress.drop_rule)); + if (mtb->egress.green_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->egress.green_rule)); + if (mtb->ingress.drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->ingress.drop_rule)); + if (mtb->ingress.green_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->ingress.green_rule)); + if (mtb->transfer.drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->transfer.drop_rule)); + if (mtb->transfer.green_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->transfer.green_rule)); + mtb->egress.drop_rule = egress_drop_rule; + mtb->egress.green_rule = egress_green_rule; + mtb->ingress.drop_rule = ingress_drop_rule; + mtb->ingress.green_rule = ingress_green_rule; + mtb->transfer.drop_rule = transfer_drop_rule; + mtb->transfer.green_rule = transfer_green_rule; return 0; error: - flow_dv_destroy_policer_rules(dev, fm, attr); + if (egress_drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(egress_drop_rule)); + if (egress_green_rule) + claim_zero(mlx5_flow_os_destroy_flow(egress_green_rule)); + if (ingress_drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(ingress_drop_rule)); + if (ingress_green_rule) + claim_zero(mlx5_flow_os_destroy_flow(ingress_green_rule)); + if (transfer_drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(transfer_drop_rule)); + if (transfer_green_rule) + claim_zero(mlx5_flow_os_destroy_flow(transfer_green_rule)); + if (!initialized) + flow_dv_destroy_policer_rules(dev, fm, attr); return -1; } @@ -14093,7 +14210,7 @@ const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = { .query = flow_dv_query, .create_mtr_tbls = flow_dv_create_mtr_tbl, .destroy_mtr_tbls = flow_dv_destroy_mtr_tbl, - .create_policer_rules = flow_dv_create_policer_rules, + .prepare_policer_rules = flow_dv_prepare_policer_rules, .destroy_policer_rules = flow_dv_destroy_policer_rules, .counter_alloc = flow_dv_counter_allocate, .counter_free = flow_dv_counter_free, diff --git a/drivers/net/mlx5/mlx5_flow_meter.c b/drivers/net/mlx5/mlx5_flow_meter.c index dbc574b508..7f7693b698 100644 --- a/drivers/net/mlx5/mlx5_flow_meter.c +++ b/drivers/net/mlx5/mlx5_flow_meter.c @@ -474,6 +474,12 @@ mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id, MTR_POLICER_ACTION_COLOR_RED }; int i; + /* Meter must use global drop action. */ + if (!priv->sh->dr_drop_action) + return -rte_mtr_error_set(error, ENOTSUP, + RTE_MTR_ERROR_TYPE_MTR_PARAMS, + NULL, + "No drop action ready for meter."); /* Meter params must not be NULL. */ if (params == NULL) return -rte_mtr_error_set(error, EINVAL, @@ -630,9 +636,18 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, .egress = 1, .transfer = priv->config.dv_esw_en ? 1 : 0, }; + struct mlx5_indexed_pool_config flow_ipool_cfg = { + .size = 0, + .trunk_size = 64, + .need_lock = 1, + .type = "mlx5_flow_mtr_flow_id_pool", + }; int ret; unsigned int i; uint32_t idx = 0; + uint8_t mtr_id_bits; + uint8_t mtr_reg_bits = priv->mtr_reg_share ? + MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; if (!priv->mtr_en) return -rte_mtr_error_set(error, ENOTSUP, @@ -654,6 +669,13 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, return -rte_mtr_error_set(error, ENOMEM, RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, "Memory alloc failed for meter."); + mtr_id_bits = MLX5_REG_BITS - __builtin_clz(idx); + if ((mtr_id_bits + priv->max_mtr_flow_bits) > mtr_reg_bits) { + DRV_LOG(ERR, "Meter number exceeds max limit."); + goto error; + } + if (mtr_id_bits > priv->max_mtr_bits) + priv->max_mtr_bits = mtr_id_bits; fm->idx = idx; /* Fill the flow meter parameters. */ fm->meter_id = meter_id; @@ -667,10 +689,10 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, if (!fm->policer_stats.cnt[i]) goto error; } - fm->mfts = mlx5_flow_create_mtr_tbls(dev, fm); + fm->mfts = mlx5_flow_create_mtr_tbls(dev); if (!fm->mfts) goto error; - ret = mlx5_flow_create_policer_rules(dev, fm, &attr); + ret = mlx5_flow_prepare_policer_rules(dev, fm, &attr); if (ret) goto error; /* Add to the flow meter list. */ @@ -679,6 +701,9 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, fm->shared = !!shared; fm->policer_stats.stats_mask = params->stats_mask; fm->profile->ref_cnt++; + fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg); + if (!fm->flow_ipool) + goto error; rte_spinlock_init(&fm->sl); return 0; error: @@ -749,6 +774,8 @@ mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id, if (fm->policer_stats.cnt[i]) mlx5_counter_free(dev, fm->policer_stats.cnt[i]); /* Free meter flow table */ + if (fm->flow_ipool) + mlx5_ipool_destroy(fm->flow_ipool); mlx5_flow_destroy_policer_rules(dev, fm, &attr); mlx5_flow_destroy_mtr_tbls(dev, fm->mfts); mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], fm->idx); @@ -1153,30 +1180,24 @@ mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id) * * @param [in] priv * Pointer to mlx5 private data. - * @param [in] meter_id - * Flow meter id. + * @param[in] fm + * Pointer to flow meter. * @param [in] attr * Pointer to flow attributes. * @param [out] error * Pointer to error structure. * - * @return the flow meter pointer, NULL otherwise. + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. */ -struct mlx5_flow_meter * -mlx5_flow_meter_attach(struct mlx5_priv *priv, uint32_t meter_id, +int +mlx5_flow_meter_attach(struct mlx5_priv *priv, + struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr, struct rte_flow_error *error) { - struct mlx5_flow_meter *fm; int ret = 0; - fm = mlx5_flow_meter_find(priv, meter_id); - if (fm == NULL) { - rte_flow_error_set(error, ENOENT, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "Meter object id not valid"); - return fm; - } rte_spinlock_lock(&fm->sl); if (fm->mfts->meter_action) { if (fm->shared && @@ -1210,7 +1231,7 @@ mlx5_flow_meter_attach(struct mlx5_priv *priv, uint32_t meter_id, fm->mfts->meter_action ? "Meter attr not match" : "Meter action create failed"); - return ret ? NULL : fm; + return ret ? -rte_errno : 0; } /** -- 2.27.0 ^ permalink raw reply [flat|nested] 8+ messages in thread
[parent not found: <20210415151135.2098674-1-lizh@nvidia.com>]
* [dpdk-stable] [PATCH v5 03/14] net/mlx5: fix meter statistics [not found] ` <20210415151135.2098674-1-lizh@nvidia.com> @ 2021-04-15 15:11 ` Li Zhang 0 siblings, 0 replies; 8+ messages in thread From: Li Zhang @ 2021-04-15 15:11 UTC (permalink / raw) To: dekelp, orika, viacheslavo, matan, shahafs, Suanming Mou Cc: dev, thomas, rasland, roniba, Shun Hao, stable From: Shun Hao <shunh@nvidia.com> Currently, packets after meter will be steered to a global policer table, which includes green/red color rules for every meter, so as to have counter statistics of each color in every meter. There's a bug that all the rules in global policer table are matching only color criteria, so all packets will be counted to one meter only, and other meter statistics are always zero. This patch does these: 1. The rules in policer table matches both meter index and color, so packet after meter could be counted to the correct meter counter. 2. The meter index and flow index are now sharing the available register bits dynamically. Meter index starts from lsb, and flow index starts from msb. Fixes: 46a5e6bc6a ("net/mlx5: prepare meter flow tables") Cc: stable@dpdk.org Signed-off-by: Shun Hao <shunh@nvidia.com> Acked-by: Matan Azrad <matan@nvidia.com> --- drivers/net/mlx5/mlx5.c | 7 +- drivers/net/mlx5/mlx5.h | 15 +- drivers/net/mlx5/mlx5_flow.c | 183 +++++++---- drivers/net/mlx5/mlx5_flow.h | 40 +-- drivers/net/mlx5/mlx5_flow_dv.c | 489 ++++++++++++++++++----------- drivers/net/mlx5/mlx5_flow_meter.c | 53 +++- 6 files changed, 496 insertions(+), 291 deletions(-) diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index 02cc2c781e..a6cc2d3108 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -277,10 +277,13 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = { }, #endif [MLX5_IPOOL_MTR] = { + /** + * The ipool index should grow continually from small to big, + * for meter idx, so not set grow_trunk to avoid meter index + * not jump continually. + */ .size = sizeof(struct mlx5_flow_meter), .trunk_size = 64, - .grow_trunk = 3, - .grow_shift = 2, .need_lock = 1, .release_mem_en = 1, .malloc = mlx5_malloc, diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index e0f7101dbb..c4a04716f7 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -944,9 +944,9 @@ struct mlx5_priv { unsigned int representor:1; /* Device is a port representor. */ unsigned int master:1; /* Device is a E-Switch master. */ unsigned int txpp_en:1; /* Tx packet pacing enabled. */ + unsigned int sampler_en:1; /* Whether support sampler. */ unsigned int mtr_en:1; /* Whether support meter. */ unsigned int mtr_reg_share:1; /* Whether support meter REG_C share. */ - unsigned int sampler_en:1; /* Whether support sampler. */ uint16_t domain_id; /* Switch domain identifier. */ uint16_t vport_id; /* Associated VF vport index (if any). */ uint32_t vport_meta_tag; /* Used for vport index match ove VF LAG. */ @@ -1003,6 +1003,10 @@ struct mlx5_priv { uint32_t rss_shared_actions; /* RSS shared actions. */ struct mlx5_devx_obj *q_counters; /* DevX queue counter object. */ uint32_t counter_set_id; /* Queue counter ID to set in DevX objects. */ + uint8_t max_mtr_bits; + /* Indicate how many bits are used by meter id at the most. */ + uint8_t max_mtr_flow_bits; + /* Indicate how many bits are used by meter flow id at the most. */ }; #define PORT_ID(priv) ((priv)->dev_data->port_id) @@ -1271,11 +1275,10 @@ int mlx5_pmd_socket_init(void); int mlx5_flow_meter_ops_get(struct rte_eth_dev *dev, void *arg); struct mlx5_flow_meter *mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id); -struct mlx5_flow_meter *mlx5_flow_meter_attach - (struct mlx5_priv *priv, - uint32_t meter_id, - const struct rte_flow_attr *attr, - struct rte_flow_error *error); +int mlx5_flow_meter_attach(struct mlx5_priv *priv, + struct mlx5_flow_meter *fm, + const struct rte_flow_attr *attr, + struct rte_flow_error *error); void mlx5_flow_meter_detach(struct mlx5_flow_meter *fm); /* mlx5_os.c */ diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c index 84463074a5..3ef98b24c4 100644 --- a/drivers/net/mlx5/mlx5_flow.c +++ b/drivers/net/mlx5/mlx5_flow.c @@ -749,9 +749,9 @@ mlx5_flow_get_reg_id(struct rte_eth_dev *dev, return REG_C_0; } break; - case MLX5_MTR_SFX: + case MLX5_MTR_ID: /* - * If meter color and flow match share one register, flow match + * If meter color and meter id share one register, flow match * should use the meter color register for match. */ if (priv->mtr_reg_share) @@ -3036,7 +3036,8 @@ flow_mreg_split_qrss_release(struct rte_eth_dev *dev, SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, handle_idx, dev_handle, next) - if (dev_handle->split_flow_id) + if (dev_handle->split_flow_id && + !dev_handle->is_meter_flow_id) mlx5_ipool_free(priv->sh->ipool [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], dev_handle->split_flow_id); @@ -3675,23 +3676,30 @@ flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], * * @param[in] actions * Pointer to the list of actions. - * @param[out] mtr + * @param[out] has_mtr * Pointer to the meter exist flag. + * @param[out] meter_id + * Pointer to the meter id. * * @return * Total number of actions. */ static int -flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr) +flow_check_meter_action(const struct rte_flow_action actions[], + bool *has_mtr, + uint32_t *meter_id) { + const struct rte_flow_action_meter *mtr = NULL; int actions_n = 0; - MLX5_ASSERT(mtr); - *mtr = 0; + MLX5_ASSERT(has_mtr); + *has_mtr = false; for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { switch (actions->type) { case RTE_FLOW_ACTION_TYPE_METER: - *mtr = 1; + mtr = actions->conf; + *meter_id = mtr->mtr_id; + *has_mtr = true; break; default: break; @@ -4348,8 +4356,10 @@ flow_create_split_inner(struct rte_eth_dev *dev, * header will be in the prefix sub flow, as not to take the * L3 tunnel header into account. * - * @param dev + * @param[in] dev * Pointer to Ethernet device. + * @param[in] fm + * Pointer to flow meter structure. * @param[in] items * Pattern specification (list terminated by the END pattern item). * @param[out] sfx_items @@ -4364,29 +4374,39 @@ flow_create_split_inner(struct rte_eth_dev *dev, * The pattern items for the suffix flow. * @param[out] tag_sfx * Pointer to suffix flow tag. + * @param[out] error + * Perform verbose error reporting if not NULL. * * @return - * 0 on success. + * The flow id, 0 otherwise and rte_errno is set. */ -static int +static uint32_t flow_meter_split_prep(struct rte_eth_dev *dev, - const struct rte_flow_item items[], - struct rte_flow_item sfx_items[], - const struct rte_flow_action actions[], - struct rte_flow_action actions_sfx[], - struct rte_flow_action actions_pre[]) + struct mlx5_flow_meter *fm, + const struct rte_flow_item items[], + struct rte_flow_item sfx_items[], + const struct rte_flow_action actions[], + struct rte_flow_action actions_sfx[], + struct rte_flow_action actions_pre[], + struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; struct rte_flow_action *tag_action = NULL; struct rte_flow_item *tag_item; struct mlx5_rte_flow_action_set_tag *set_tag; - struct rte_flow_error error; const struct rte_flow_action_raw_encap *raw_encap; const struct rte_flow_action_raw_decap *raw_decap; - struct mlx5_rte_flow_item_tag *tag_spec; - struct mlx5_rte_flow_item_tag *tag_mask; + struct mlx5_rte_flow_item_tag *tag_item_spec; + struct mlx5_rte_flow_item_tag *tag_item_mask; uint32_t tag_id = 0; bool copy_vlan = false; + uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; + uint8_t mtr_reg_bits = priv->mtr_reg_share ? + MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; + uint32_t flow_id = 0; + uint32_t flow_id_reversed = 0; + uint8_t flow_id_bits = 0; + int shift; /* Prepare the actions for prefix and suffix flow. */ for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { @@ -4395,10 +4415,7 @@ flow_meter_split_prep(struct rte_eth_dev *dev, switch (actions->type) { case RTE_FLOW_ACTION_TYPE_METER: /* Add the extra tag action first. */ - tag_action = actions_pre; - tag_action->type = (enum rte_flow_action_type) - MLX5_RTE_FLOW_ACTION_TYPE_TAG; - actions_pre++; + tag_action = actions_pre++; action_cur = &actions_pre; break; case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: @@ -4431,23 +4448,22 @@ flow_meter_split_prep(struct rte_eth_dev *dev, actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; actions_pre->type = RTE_FLOW_ACTION_TYPE_END; actions_pre++; - /* Set the tag. */ - set_tag = (void *)actions_pre; - set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); - mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], - &tag_id); - if (tag_id >= (1 << (sizeof(tag_id) * 8 - MLX5_MTR_COLOR_BITS))) { - DRV_LOG(ERR, "Port %u meter flow id exceed max limit.", - dev->data->port_id); - mlx5_ipool_free(priv->sh->ipool - [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], tag_id); - return 0; - } else if (!tag_id) { - return 0; + mlx5_ipool_malloc(fm->flow_ipool, &tag_id); + if (!tag_id) + return rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Failed to allocate meter flow id."); + flow_id = tag_id - 1; + flow_id_bits = MLX5_REG_BITS - __builtin_clz(flow_id); + flow_id_bits = flow_id_bits ? flow_id_bits : 1; + if ((flow_id_bits + priv->max_mtr_bits) > mtr_reg_bits) { + mlx5_ipool_free(fm->flow_ipool, tag_id); + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Meter flow id exceeds max limit."); } - set_tag->data = tag_id << MLX5_MTR_COLOR_BITS; - assert(tag_action); - tag_action->conf = set_tag; + if (flow_id_bits > priv->max_mtr_flow_bits) + priv->max_mtr_flow_bits = flow_id_bits; /* Prepare the suffix subflow items. */ tag_item = sfx_items++; for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { @@ -4476,16 +4492,34 @@ flow_meter_split_prep(struct rte_eth_dev *dev, } sfx_items->type = RTE_FLOW_ITEM_TYPE_END; sfx_items++; - tag_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; - tag_spec->data = tag_id << MLX5_MTR_COLOR_BITS; - tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); - tag_mask = tag_spec + 1; - tag_mask->data = 0xffffff00; + /* Build tag actions and items for meter_id/meter flow_id. */ + assert(tag_action); + set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; + tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; + tag_item_mask = tag_item_spec + 1; + /* + * The color Reg bits used by flow_id are growing from + * msb to lsb, so must do bit reverse for flow_id val in RegC. + */ + for (shift = 0; shift < flow_id_bits; shift++) + flow_id_reversed = (flow_id_reversed << 1) | + ((flow_id >> shift) & 0x1); + /* Both flow_id and meter_id share the same register. */ + set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 0, error); + set_tag->data = + (fm->idx | (flow_id_reversed << (mtr_reg_bits - flow_id_bits))) + << mtr_id_offset; + tag_item_spec->id = set_tag->id; + tag_item_spec->data = set_tag->data; + tag_item_mask->data = UINT32_MAX << mtr_id_offset; + tag_action->type = (enum rte_flow_action_type) + MLX5_RTE_FLOW_ACTION_TYPE_TAG; + tag_action->conf = set_tag; tag_item->type = (enum rte_flow_item_type) - MLX5_RTE_FLOW_ITEM_TYPE_TAG; - tag_item->spec = tag_spec; + MLX5_RTE_FLOW_ITEM_TYPE_TAG; + tag_item->spec = tag_item_spec; tag_item->last = NULL; - tag_item->mask = tag_mask; + tag_item->mask = tag_item_mask; return tag_id; } @@ -5185,25 +5219,49 @@ flow_create_split_meter(struct rte_eth_dev *dev, struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); struct rte_flow_action *sfx_actions = NULL; struct rte_flow_action *pre_actions = NULL; struct rte_flow_item *sfx_items = NULL; struct mlx5_flow *dev_flow = NULL; struct rte_flow_attr sfx_attr = *attr; - uint32_t mtr = 0; + struct mlx5_flow_meter *fm = NULL; + bool has_mtr = false; + uint32_t meter_id; uint32_t mtr_tag_id = 0; size_t act_size; size_t item_size; int actions_n = 0; - int ret; + int ret = 0; if (priv->mtr_en) - actions_n = flow_check_meter_action(actions, &mtr); - if (mtr) { - /* The five prefix actions: meter, decap, encap, tag, end. */ + actions_n = flow_check_meter_action(actions, &has_mtr, + &meter_id); + if (has_mtr) { + if (flow->meter) { + fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], + flow->meter); + if (!fm) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "Meter not found."); + } else { + fm = mlx5_flow_meter_find(priv, meter_id); + if (!fm) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "Meter not found."); + ret = mlx5_flow_meter_attach(priv, fm, + &sfx_attr, error); + if (ret) + return -rte_errno; + flow->meter = fm->idx; + } + wks->fm = fm; + /* The prefix actions: meter, decap, encap, tag, end. */ act_size = sizeof(struct rte_flow_action) * (actions_n + 5) + sizeof(struct mlx5_rte_flow_action_set_tag); - /* tag, vlan, port id, end. */ + /* The suffix items: tag, vlan, port id, end. */ #define METER_SUFFIX_ITEM 4 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + sizeof(struct mlx5_rte_flow_item_tag) * 2; @@ -5217,9 +5275,9 @@ flow_create_split_meter(struct rte_eth_dev *dev, sfx_items = (struct rte_flow_item *)((char *)sfx_actions + act_size); pre_actions = sfx_actions + actions_n; - mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items, + mtr_tag_id = flow_meter_split_prep(dev, fm, items, sfx_items, actions, sfx_actions, - pre_actions); + pre_actions, error); if (!mtr_tag_id) { ret = -rte_errno; goto exit; @@ -5230,10 +5288,12 @@ flow_create_split_meter(struct rte_eth_dev *dev, attr, items, pre_actions, flow_split_info, error); if (ret) { + mlx5_ipool_free(fm->flow_ipool, mtr_tag_id); ret = -rte_errno; goto exit; } dev_flow->handle->split_flow_id = mtr_tag_id; + dev_flow->handle->is_meter_flow_id = 1; /* Setting the sfx group atrr. */ sfx_attr.group = sfx_attr.transfer ? (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : @@ -6505,20 +6565,17 @@ mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, * * @param[in] dev * Pointer to Ethernet device. - * @param[in] fm - * Pointer to the flow meter. * * @return * Pointer to table set on success, NULL otherwise. */ struct mlx5_meter_domains_infos * -mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm) +mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev) { const struct mlx5_flow_driver_ops *fops; fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); - return fops->create_mtr_tbls(dev, fm); + return fops->create_mtr_tbls(dev); } /** @@ -6543,7 +6600,7 @@ mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, } /** - * Create policer rules. + * Prepare policer rules. * * @param[in] dev * Pointer to Ethernet device. @@ -6556,14 +6613,14 @@ mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, * 0 on success, -1 otherwise. */ int -mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, +mlx5_flow_prepare_policer_rules(struct rte_eth_dev *dev, struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr) { const struct mlx5_flow_driver_ops *fops; fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); - return fops->create_policer_rules(dev, fm, attr); + return fops->prepare_policer_rules(dev, fm, attr); } /** diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index ec673c29ab..4482a456f0 100644 --- a/drivers/net/mlx5/mlx5_flow.h +++ b/drivers/net/mlx5/mlx5_flow.h @@ -79,7 +79,7 @@ enum mlx5_feature_name { MLX5_APP_TAG, MLX5_COPY_MARK, MLX5_MTR_COLOR, - MLX5_MTR_SFX, + MLX5_MTR_ID, MLX5_ASO_FLOW_HIT, }; @@ -655,7 +655,8 @@ struct mlx5_flow_handle { uint64_t layers; /**< Bit-fields of present layers, see MLX5_FLOW_LAYER_*. */ void *drv_flow; /**< pointer to driver flow object. */ - uint32_t split_flow_id:28; /**< Sub flow unique match flow id. */ + uint32_t split_flow_id:27; /**< Sub flow unique match flow id. */ + uint32_t is_meter_flow_id:1; /**< Indate if flow_id is for meter. */ uint32_t mark:1; /**< Metadate rxq mark flag. */ uint32_t fate_action:3; /**< Fate action type. */ union { @@ -842,14 +843,16 @@ struct mlx5_meter_domain_info { /**< Meter table. */ struct mlx5_flow_tbl_resource *sfx_tbl; /**< Meter suffix table. */ - void *any_matcher; - /**< Meter color not match default criteria. */ - void *color_matcher; - /**< Meter color match criteria. */ + struct mlx5_flow_dv_matcher *drop_matcher; + /**< Matcher for Drop. */ + struct mlx5_flow_dv_matcher *color_matcher; + /**< Matcher for Color. */ void *jump_actn; /**< Meter match action. */ - void *policer_rules[RTE_MTR_DROPPED + 1]; - /**< Meter policer for the match. */ + void *green_rule; + /**< Meter green rule. */ + void *drop_rule; + /**< Meter drop rule. */ }; /* Meter table set for TX RX FDB. */ @@ -862,10 +865,10 @@ struct mlx5_meter_domains_infos { /**< RX meter table. */ struct mlx5_meter_domain_info transfer; /**< FDB meter table. */ - void *drop_actn; - /**< Drop action as not matched. */ - void *count_actns[RTE_MTR_DROPPED + 1]; - /**< Counters for match and unmatched statistics. */ + void *green_count; + /**< Counters for green rule. */ + void *drop_count; + /**< Counters for green rule. */ uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)]; /**< Flow meter parameter. */ size_t fmp_size; @@ -928,6 +931,8 @@ struct mlx5_flow_meter { /**< Meter state. */ uint32_t shared:1; /**< Meter shared or not. */ + struct mlx5_indexed_pool *flow_ipool; + /**< Index pool for flow id. */ }; /* RFC2697 parameter structure. */ @@ -1166,6 +1171,7 @@ struct mlx5_flow_workspace { struct mlx5_flow_rss_desc rss_desc; uint32_t rssq_num; /* Allocated queue num in rss_desc. */ uint32_t flow_idx; /* Intermediate device flow index. */ + struct mlx5_flow_meter *fm; /* Pointer to the meter in flow. */ }; struct mlx5_flow_split_info { @@ -1206,8 +1212,7 @@ typedef int (*mlx5_flow_query_t)(struct rte_eth_dev *dev, void *data, struct rte_flow_error *error); typedef struct mlx5_meter_domains_infos *(*mlx5_flow_create_mtr_tbls_t) - (struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm); + (struct rte_eth_dev *dev); typedef int (*mlx5_flow_destroy_mtr_tbls_t)(struct rte_eth_dev *dev, struct mlx5_meter_domains_infos *tbls); typedef int (*mlx5_flow_create_policer_rules_t) @@ -1270,7 +1275,7 @@ struct mlx5_flow_driver_ops { mlx5_flow_query_t query; mlx5_flow_create_mtr_tbls_t create_mtr_tbls; mlx5_flow_destroy_mtr_tbls_t destroy_mtr_tbls; - mlx5_flow_create_policer_rules_t create_policer_rules; + mlx5_flow_create_policer_rules_t prepare_policer_rules; mlx5_flow_destroy_policer_rules_t destroy_policer_rules; mlx5_flow_counter_alloc_t counter_alloc; mlx5_flow_counter_free_t counter_free; @@ -1470,11 +1475,10 @@ int mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, const struct rte_flow_item_ecpri *acc_mask, struct rte_flow_error *error); struct mlx5_meter_domains_infos *mlx5_flow_create_mtr_tbls - (struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm); + (struct rte_eth_dev *dev); int mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, struct mlx5_meter_domains_infos *tbl); -int mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, +int mlx5_flow_prepare_policer_rules(struct rte_eth_dev *dev, struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr); int mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index e65cc13bd6..feac8bca34 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -10982,14 +10982,12 @@ flow_dv_translate(struct rte_eth_dev *dev, const struct rte_flow_action_rss *rss; const struct rte_flow_action *action = actions; const uint8_t *rss_key; - const struct rte_flow_action_meter *mtr; struct mlx5_flow_tbl_resource *tbl; struct mlx5_aso_age_action *age_act; uint32_t port_id = 0; struct mlx5_flow_dv_port_id_action_resource port_id_resource; int action_type = actions->type; const struct rte_flow_action *found_action = NULL; - struct mlx5_flow_meter *fm = NULL; uint32_t jump_group = 0; if (!mlx5_flow_os_action_supported(action_type)) @@ -11414,33 +11412,13 @@ flow_dv_translate(struct rte_eth_dev *dev, MLX5_FLOW_FATE_DEFAULT_MISS; break; case RTE_FLOW_ACTION_TYPE_METER: - mtr = actions->conf; - if (!flow->meter) { - fm = mlx5_flow_meter_attach(priv, mtr->mtr_id, - attr, error); - if (!fm) - return rte_flow_error_set(error, - rte_errno, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "meter not found " - "or invalid parameters"); - flow->meter = fm->idx; - } + if (!wks->fm) + return rte_flow_error_set(error, rte_errno, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "Failed to get meter in flow."); /* Set the meter action. */ - if (!fm) { - fm = mlx5_ipool_get(priv->sh->ipool - [MLX5_IPOOL_MTR], flow->meter); - if (!fm) - return rte_flow_error_set(error, - rte_errno, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "meter not found " - "or invalid parameters"); - } dev_flow->dv.actions[actions_n++] = - fm->mfts->meter_action; + wks->fm->mfts->meter_action; action_flags |= MLX5_FLOW_ACTION_METER; break; case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: @@ -12589,6 +12567,7 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) { struct mlx5_flow_handle *dev_handle; struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_meter *fm = NULL; uint32_t srss = 0; if (!flow) @@ -12599,8 +12578,6 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) flow->counter = 0; } if (flow->meter) { - struct mlx5_flow_meter *fm; - fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], flow->meter); if (fm) @@ -12642,6 +12619,10 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) flow_dv_fate_resource_release(dev, dev_handle); else if (!srss) srss = dev_handle->rix_srss; + if (fm && dev_handle->is_meter_flow_id && + dev_handle->split_flow_id) + mlx5_ipool_free(fm->flow_ipool, + dev_handle->split_flow_id); mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], tmp_idx); } @@ -13407,49 +13388,20 @@ flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev, if (!mtd || !priv->config.dv_flow_en) return 0; - if (mtd->ingress.policer_rules[RTE_MTR_DROPPED]) - claim_zero(mlx5_flow_os_destroy_flow - (mtd->ingress.policer_rules[RTE_MTR_DROPPED])); - if (mtd->egress.policer_rules[RTE_MTR_DROPPED]) - claim_zero(mlx5_flow_os_destroy_flow - (mtd->egress.policer_rules[RTE_MTR_DROPPED])); - if (mtd->transfer.policer_rules[RTE_MTR_DROPPED]) - claim_zero(mlx5_flow_os_destroy_flow - (mtd->transfer.policer_rules[RTE_MTR_DROPPED])); - if (mtd->egress.color_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->egress.color_matcher)); - if (mtd->egress.any_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->egress.any_matcher)); if (mtd->egress.tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->egress.tbl); if (mtd->egress.sfx_tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->egress.sfx_tbl); - if (mtd->ingress.color_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->ingress.color_matcher)); - if (mtd->ingress.any_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->ingress.any_matcher)); if (mtd->ingress.tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->ingress.tbl); if (mtd->ingress.sfx_tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->ingress.sfx_tbl); - if (mtd->transfer.color_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->transfer.color_matcher)); - if (mtd->transfer.any_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->transfer.any_matcher)); if (mtd->transfer.tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->transfer.tbl); if (mtd->transfer.sfx_tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->transfer.sfx_tbl); - if (mtd->drop_actn) - claim_zero(mlx5_flow_os_destroy_flow_action(mtd->drop_actn)); mlx5_free(mtd); return 0; } @@ -13468,37 +13420,17 @@ flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev, * Table attribute. * @param[in] transfer * Table attribute. - * @param[in] color_reg_c_idx - * Reg C index for color match. * * @return - * 0 on success, -1 otherwise and rte_errno is set. + * 0 on success, -1 otherwise. */ static int flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, struct mlx5_meter_domains_infos *mtb, - uint8_t egress, uint8_t transfer, - uint32_t color_reg_c_idx) + uint8_t egress, uint8_t transfer) { - struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_dev_ctx_shared *sh = priv->sh; - struct mlx5_flow_dv_match_params mask = { - .size = sizeof(mask.buf), - }; - struct mlx5_flow_dv_match_params value = { - .size = sizeof(value.buf), - }; - struct mlx5dv_flow_matcher_attr dv_attr = { - .type = IBV_FLOW_ATTR_NORMAL, - .priority = 0, - .match_criteria_enable = 0, - .match_mask = (void *)&mask, - }; - void *actions[METER_ACTIONS]; - struct mlx5_meter_domain_info *dtb; struct rte_flow_error error; - int i = 0; - int ret; + struct mlx5_meter_domain_info *dtb; if (transfer) dtb = &mtb->transfer; @@ -13523,41 +13455,7 @@ flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, DRV_LOG(ERR, "Failed to create meter suffix table."); return -1; } - /* Create matchers, Any and Color. */ - dv_attr.priority = 3; - dv_attr.match_criteria_enable = 0; - ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj, - &dtb->any_matcher); - if (ret) { - DRV_LOG(ERR, "Failed to create meter" - " policer default matcher."); - goto error_exit; - } - dv_attr.priority = 0; - dv_attr.match_criteria_enable = - 1 << MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT; - flow_dv_match_meta_reg(mask.buf, value.buf, color_reg_c_idx, - rte_col_2_mlx5_col(RTE_COLORS), UINT8_MAX); - ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj, - &dtb->color_matcher); - if (ret) { - DRV_LOG(ERR, "Failed to create meter policer color matcher."); - goto error_exit; - } - if (mtb->count_actns[RTE_MTR_DROPPED]) - actions[i++] = mtb->count_actns[RTE_MTR_DROPPED]; - actions[i++] = mtb->drop_actn; - /* Default rule: lowest priority, match any, actions: drop. */ - ret = mlx5_flow_os_create_flow(dtb->any_matcher, (void *)&value, i, - actions, - &dtb->policer_rules[RTE_MTR_DROPPED]); - if (ret) { - DRV_LOG(ERR, "Failed to create meter policer drop rule."); - goto error_exit; - } return 0; -error_exit: - return -1; } /** @@ -13566,20 +13464,16 @@ flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, * * @param[in] dev * Pointer to Ethernet device. - * @param[in] fm - * Pointer to the flow meter. * * @return * Pointer to table set on success, NULL otherwise and rte_errno is set. */ static struct mlx5_meter_domains_infos * -flow_dv_create_mtr_tbl(struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm) +flow_dv_create_mtr_tbl(struct rte_eth_dev *dev) { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_meter_domains_infos *mtb; int ret; - int i; if (!priv->mtr_en) { rte_errno = ENOTSUP; @@ -13590,37 +13484,21 @@ flow_dv_create_mtr_tbl(struct rte_eth_dev *dev, DRV_LOG(ERR, "Failed to allocate memory for meter."); return NULL; } - /* Create meter count actions */ - for (i = 0; i <= RTE_MTR_DROPPED; i++) { - struct mlx5_flow_counter *cnt; - if (!fm->policer_stats.cnt[i]) - continue; - cnt = flow_dv_counter_get_by_idx(dev, - fm->policer_stats.cnt[i], NULL); - mtb->count_actns[i] = cnt->action; - } - /* Create drop action. */ - ret = mlx5_flow_os_create_flow_action_drop(&mtb->drop_actn); - if (ret) { - DRV_LOG(ERR, "Failed to create drop action."); - goto error_exit; - } /* Egress meter table. */ - ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0, priv->mtr_color_reg); + ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0); if (ret) { DRV_LOG(ERR, "Failed to prepare egress meter table."); goto error_exit; } /* Ingress meter table. */ - ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0, priv->mtr_color_reg); + ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0); if (ret) { DRV_LOG(ERR, "Failed to prepare ingress meter table."); goto error_exit; } /* FDB meter table. */ if (priv->config.dv_esw_en) { - ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1, - priv->mtr_color_reg); + ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1); if (ret) { DRV_LOG(ERR, "Failed to prepare fdb meter table."); goto error_exit; @@ -13632,24 +13510,152 @@ flow_dv_create_mtr_tbl(struct rte_eth_dev *dev, return NULL; } +/** + * Destroy the meter table matchers. + * Lock free, (mutex should be acquired by caller). + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in,out] dtb + * Pointer to DV meter table. + * + * @return + * Always 0. + */ +static int +flow_dv_destroy_mtr_matchers(struct rte_eth_dev *dev, + struct mlx5_meter_domain_info *dtb) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_tbl_data_entry *tbl; + + if (!priv->config.dv_flow_en) + return 0; + if (dtb->drop_matcher) { + tbl = container_of(dtb->drop_matcher->tbl, typeof(*tbl), tbl); + mlx5_cache_unregister(&tbl->matchers, + &dtb->drop_matcher->entry); + dtb->drop_matcher = NULL; + } + if (dtb->color_matcher) { + tbl = container_of(dtb->color_matcher->tbl, typeof(*tbl), tbl); + mlx5_cache_unregister(&tbl->matchers, + &dtb->color_matcher->entry); + dtb->color_matcher = NULL; + } + return 0; +} + +/** + * Create the matchers for meter table. + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in] color_reg_c_idx + * Reg C index for color match. + * @param[in] mtr_id_reg_c_idx + * Reg C index for meter_id match. + * @param[in] mtr_id_mask + * Mask for meter_id match criteria. + * @param[in,out] dtb + * Pointer to DV meter table. + * @param[out] error + * Perform verbose error reporting if not NULL. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_prepare_mtr_matchers(struct rte_eth_dev *dev, + uint32_t color_reg_c_idx, + uint32_t mtr_id_reg_c_idx, + uint32_t mtr_id_mask, + struct mlx5_meter_domain_info *dtb, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_tbl_data_entry *tbl_data; + struct mlx5_cache_entry *entry; + struct mlx5_flow_dv_matcher matcher = { + .mask = { + .size = sizeof(matcher.mask.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), + }, + .tbl = dtb->tbl, + }; + struct mlx5_flow_dv_match_params value = { + .size = sizeof(value.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), + }; + struct mlx5_flow_cb_ctx ctx = { + .error = error, + .data = &matcher, + }; + uint32_t color_mask = (UINT32_C(1) << MLX5_MTR_COLOR_BITS) - 1; + + tbl_data = container_of(dtb->tbl, struct mlx5_flow_tbl_data_entry, tbl); + if (!dtb->drop_matcher) { + /* Create matchers for Drop. */ + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + mtr_id_reg_c_idx, 0, mtr_id_mask); + matcher.priority = MLX5_REG_BITS * 2 - priv->max_mtr_bits; + matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf, + matcher.mask.size); + entry = mlx5_cache_register(&tbl_data->matchers, &ctx); + if (!entry) { + DRV_LOG(ERR, "Failed to register meter drop matcher."); + return -1; + } + dtb->drop_matcher = + container_of(entry, struct mlx5_flow_dv_matcher, entry); + } + if (!dtb->color_matcher) { + /* Create matchers for Color + meter_id. */ + if (priv->mtr_reg_share) { + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + color_reg_c_idx, 0, + (mtr_id_mask | color_mask)); + } else { + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + color_reg_c_idx, 0, color_mask); + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + mtr_id_reg_c_idx, 0, mtr_id_mask); + } + matcher.priority = MLX5_REG_BITS - priv->max_mtr_bits; + matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf, + matcher.mask.size); + entry = mlx5_cache_register(&tbl_data->matchers, &ctx); + if (!entry) { + DRV_LOG(ERR, "Failed to register meter color matcher."); + return -1; + } + dtb->color_matcher = + container_of(entry, struct mlx5_flow_dv_matcher, entry); + } + return 0; +} + /** * Destroy domain policer rule. * + * @param[in] dev + * Pointer to Ethernet device. * @param[in] dt * Pointer to domain table. */ static void -flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt) +flow_dv_destroy_domain_policer_rule(struct rte_eth_dev *dev, + struct mlx5_meter_domain_info *dt) { - int i; - - for (i = 0; i < RTE_MTR_DROPPED; i++) { - if (dt->policer_rules[i]) { - claim_zero(mlx5_flow_os_destroy_flow - (dt->policer_rules[i])); - dt->policer_rules[i] = NULL; - } + if (dt->drop_rule) { + claim_zero(mlx5_flow_os_destroy_flow(dt->drop_rule)); + dt->drop_rule = NULL; } + if (dt->green_rule) { + claim_zero(mlx5_flow_os_destroy_flow(dt->green_rule)); + dt->green_rule = NULL; + } + flow_dv_destroy_mtr_matchers(dev, dt); if (dt->jump_actn) { claim_zero(mlx5_flow_os_destroy_flow_action(dt->jump_actn)); dt->jump_actn = NULL; @@ -13670,7 +13676,7 @@ flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt) * Always 0. */ static int -flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused, +flow_dv_destroy_policer_rules(struct rte_eth_dev *dev, const struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr) { @@ -13679,39 +13685,56 @@ flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused, if (!mtb) return 0; if (attr->egress) - flow_dv_destroy_domain_policer_rule(&mtb->egress); + flow_dv_destroy_domain_policer_rule(dev, &mtb->egress); if (attr->ingress) - flow_dv_destroy_domain_policer_rule(&mtb->ingress); + flow_dv_destroy_domain_policer_rule(dev, &mtb->ingress); if (attr->transfer) - flow_dv_destroy_domain_policer_rule(&mtb->transfer); + flow_dv_destroy_domain_policer_rule(dev, &mtb->transfer); return 0; } /** * Create specify domain meter policer rule. * + * @param[in] dev + * Pointer to Ethernet device. * @param[in] fm * Pointer to flow meter structure. * @param[in] mtb * Pointer to DV meter table set. - * @param[in] mtr_reg_c - * Color match REG_C. + * @param[out] drop_rule + * The address of pointer saving drop rule. + * @param[out] color_rule + * The address of pointer saving green rule. * * @return * 0 on success, -1 otherwise. */ static int -flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, +flow_dv_create_policer_forward_rule(struct rte_eth_dev *dev, + struct mlx5_flow_meter *fm, struct mlx5_meter_domain_info *dtb, - uint8_t mtr_reg_c) + void **drop_rule, + void **green_rule) { + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flow_dv_match_params matcher = { - .size = sizeof(matcher.buf), + .size = sizeof(matcher.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), }; struct mlx5_flow_dv_match_params value = { - .size = sizeof(value.buf), + .size = sizeof(value.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), }; struct mlx5_meter_domains_infos *mtb = fm->mfts; + struct rte_flow_error error; + uint32_t color_reg_c = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, + 0, &error); + uint32_t mtr_id_reg_c = mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, + 0, &error); + uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; + uint32_t mtr_id_mask = + ((UINT32_C(1) << priv->max_mtr_bits) - 1) << mtr_id_offset; void *actions[METER_ACTIONS]; int i; int ret = 0; @@ -13724,25 +13747,52 @@ flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, DRV_LOG(ERR, "Failed to create policer jump action."); goto error; } - for (i = 0; i < RTE_MTR_DROPPED; i++) { - int j = 0; - - flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_reg_c, - rte_col_2_mlx5_col(i), UINT8_MAX); - if (mtb->count_actns[i]) - actions[j++] = mtb->count_actns[i]; - if (fm->action[i] == MTR_POLICER_ACTION_DROP) - actions[j++] = mtb->drop_actn; - else - actions[j++] = dtb->jump_actn; - ret = mlx5_flow_os_create_flow(dtb->color_matcher, - (void *)&value, j, actions, - &dtb->policer_rules[i]); + /* Prepare matchers. */ + if (!dtb->drop_matcher || !dtb->color_matcher) { + ret = flow_dv_prepare_mtr_matchers(dev, color_reg_c, + mtr_id_reg_c, mtr_id_mask, + dtb, &error); if (ret) { - DRV_LOG(ERR, "Failed to create policer rule."); + DRV_LOG(ERR, "Failed to setup matchers for mtr table."); goto error; } } + /* Create Drop flow, matching meter_id only. */ + i = 0; + flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_id_reg_c, + (fm->idx << mtr_id_offset), UINT32_MAX); + if (mtb->drop_count) + actions[i++] = mtb->drop_count; + actions[i++] = priv->sh->dr_drop_action; + ret = mlx5_flow_os_create_flow(dtb->drop_matcher->matcher_object, + (void *)&value, i, actions, drop_rule); + if (ret) { + DRV_LOG(ERR, "Failed to create meter policer drop rule."); + goto error; + } + /* Create flow matching Green color + meter_id. */ + i = 0; + if (priv->mtr_reg_share) { + flow_dv_match_meta_reg(matcher.buf, value.buf, color_reg_c, + ((fm->idx << mtr_id_offset) | + rte_col_2_mlx5_col(RTE_COLOR_GREEN)), + UINT32_MAX); + } else { + flow_dv_match_meta_reg(matcher.buf, value.buf, color_reg_c, + rte_col_2_mlx5_col(RTE_COLOR_GREEN), + UINT32_MAX); + flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_id_reg_c, + fm->idx, UINT32_MAX); + } + if (mtb->green_count) + actions[i++] = mtb->green_count; + actions[i++] = dtb->jump_actn; + ret = mlx5_flow_os_create_flow(dtb->color_matcher->matcher_object, + (void *)&value, i, actions, green_rule); + if (ret) { + DRV_LOG(ERR, "Failed to create meter policer color rule."); + goto error; + } return 0; error: rte_errno = errno; @@ -13750,7 +13800,8 @@ flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, } /** - * Create policer rules. + * Prepare policer rules for all domains. + * If meter already initialized, this will replace all old rules with new ones. * * @param[in] dev * Pointer to Ethernet device. @@ -13763,41 +13814,107 @@ flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, * 0 on success, -1 otherwise. */ static int -flow_dv_create_policer_rules(struct rte_eth_dev *dev, - struct mlx5_flow_meter *fm, - const struct rte_flow_attr *attr) +flow_dv_prepare_policer_rules(struct rte_eth_dev *dev, + struct mlx5_flow_meter *fm, + const struct rte_flow_attr *attr) { - struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_meter_domains_infos *mtb = fm->mfts; + bool initialized = false; + struct mlx5_flow_counter *cnt; + void *egress_drop_rule = NULL; + void *egress_green_rule = NULL; + void *ingress_drop_rule = NULL; + void *ingress_green_rule = NULL; + void *transfer_drop_rule = NULL; + void *transfer_green_rule = NULL; int ret; + /* Get the statistics counters for green/drop. */ + if (fm->policer_stats.cnt[RTE_COLOR_GREEN]) { + cnt = flow_dv_counter_get_by_idx(dev, + fm->policer_stats.cnt[RTE_COLOR_GREEN], + NULL); + mtb->green_count = cnt->action; + } else { + mtb->green_count = NULL; + } + if (fm->policer_stats.cnt[RTE_MTR_DROPPED]) { + cnt = flow_dv_counter_get_by_idx(dev, + fm->policer_stats.cnt[RTE_MTR_DROPPED], + NULL); + mtb->drop_count = cnt->action; + } else { + mtb->drop_count = NULL; + } + /** + * If flow meter has been initialized, all policer rules + * are created. So can get if meter initialized by checking + * any policer rule. + */ + if (mtb->egress.drop_rule) + initialized = true; if (attr->egress) { - ret = flow_dv_create_policer_forward_rule(fm, &mtb->egress, - priv->mtr_color_reg); + ret = flow_dv_create_policer_forward_rule(dev, + fm, &mtb->egress, + &egress_drop_rule, &egress_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create egress policer."); goto error; } } if (attr->ingress) { - ret = flow_dv_create_policer_forward_rule(fm, &mtb->ingress, - priv->mtr_color_reg); + ret = flow_dv_create_policer_forward_rule(dev, + fm, &mtb->ingress, + &ingress_drop_rule, &ingress_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create ingress policer."); goto error; } } if (attr->transfer) { - ret = flow_dv_create_policer_forward_rule(fm, &mtb->transfer, - priv->mtr_color_reg); + ret = flow_dv_create_policer_forward_rule(dev, + fm, &mtb->transfer, + &transfer_drop_rule, &transfer_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create transfer policer."); goto error; } } + /* Replace old flows if existing. */ + if (mtb->egress.drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->egress.drop_rule)); + if (mtb->egress.green_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->egress.green_rule)); + if (mtb->ingress.drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->ingress.drop_rule)); + if (mtb->ingress.green_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->ingress.green_rule)); + if (mtb->transfer.drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->transfer.drop_rule)); + if (mtb->transfer.green_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->transfer.green_rule)); + mtb->egress.drop_rule = egress_drop_rule; + mtb->egress.green_rule = egress_green_rule; + mtb->ingress.drop_rule = ingress_drop_rule; + mtb->ingress.green_rule = ingress_green_rule; + mtb->transfer.drop_rule = transfer_drop_rule; + mtb->transfer.green_rule = transfer_green_rule; return 0; error: - flow_dv_destroy_policer_rules(dev, fm, attr); + if (egress_drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(egress_drop_rule)); + if (egress_green_rule) + claim_zero(mlx5_flow_os_destroy_flow(egress_green_rule)); + if (ingress_drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(ingress_drop_rule)); + if (ingress_green_rule) + claim_zero(mlx5_flow_os_destroy_flow(ingress_green_rule)); + if (transfer_drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(transfer_drop_rule)); + if (transfer_green_rule) + claim_zero(mlx5_flow_os_destroy_flow(transfer_green_rule)); + if (!initialized) + flow_dv_destroy_policer_rules(dev, fm, attr); return -1; } @@ -14094,7 +14211,7 @@ const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = { .query = flow_dv_query, .create_mtr_tbls = flow_dv_create_mtr_tbl, .destroy_mtr_tbls = flow_dv_destroy_mtr_tbl, - .create_policer_rules = flow_dv_create_policer_rules, + .prepare_policer_rules = flow_dv_prepare_policer_rules, .destroy_policer_rules = flow_dv_destroy_policer_rules, .counter_alloc = flow_dv_counter_allocate, .counter_free = flow_dv_counter_free, diff --git a/drivers/net/mlx5/mlx5_flow_meter.c b/drivers/net/mlx5/mlx5_flow_meter.c index dbc574b508..7f7693b698 100644 --- a/drivers/net/mlx5/mlx5_flow_meter.c +++ b/drivers/net/mlx5/mlx5_flow_meter.c @@ -474,6 +474,12 @@ mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id, MTR_POLICER_ACTION_COLOR_RED }; int i; + /* Meter must use global drop action. */ + if (!priv->sh->dr_drop_action) + return -rte_mtr_error_set(error, ENOTSUP, + RTE_MTR_ERROR_TYPE_MTR_PARAMS, + NULL, + "No drop action ready for meter."); /* Meter params must not be NULL. */ if (params == NULL) return -rte_mtr_error_set(error, EINVAL, @@ -630,9 +636,18 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, .egress = 1, .transfer = priv->config.dv_esw_en ? 1 : 0, }; + struct mlx5_indexed_pool_config flow_ipool_cfg = { + .size = 0, + .trunk_size = 64, + .need_lock = 1, + .type = "mlx5_flow_mtr_flow_id_pool", + }; int ret; unsigned int i; uint32_t idx = 0; + uint8_t mtr_id_bits; + uint8_t mtr_reg_bits = priv->mtr_reg_share ? + MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; if (!priv->mtr_en) return -rte_mtr_error_set(error, ENOTSUP, @@ -654,6 +669,13 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, return -rte_mtr_error_set(error, ENOMEM, RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, "Memory alloc failed for meter."); + mtr_id_bits = MLX5_REG_BITS - __builtin_clz(idx); + if ((mtr_id_bits + priv->max_mtr_flow_bits) > mtr_reg_bits) { + DRV_LOG(ERR, "Meter number exceeds max limit."); + goto error; + } + if (mtr_id_bits > priv->max_mtr_bits) + priv->max_mtr_bits = mtr_id_bits; fm->idx = idx; /* Fill the flow meter parameters. */ fm->meter_id = meter_id; @@ -667,10 +689,10 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, if (!fm->policer_stats.cnt[i]) goto error; } - fm->mfts = mlx5_flow_create_mtr_tbls(dev, fm); + fm->mfts = mlx5_flow_create_mtr_tbls(dev); if (!fm->mfts) goto error; - ret = mlx5_flow_create_policer_rules(dev, fm, &attr); + ret = mlx5_flow_prepare_policer_rules(dev, fm, &attr); if (ret) goto error; /* Add to the flow meter list. */ @@ -679,6 +701,9 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, fm->shared = !!shared; fm->policer_stats.stats_mask = params->stats_mask; fm->profile->ref_cnt++; + fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg); + if (!fm->flow_ipool) + goto error; rte_spinlock_init(&fm->sl); return 0; error: @@ -749,6 +774,8 @@ mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id, if (fm->policer_stats.cnt[i]) mlx5_counter_free(dev, fm->policer_stats.cnt[i]); /* Free meter flow table */ + if (fm->flow_ipool) + mlx5_ipool_destroy(fm->flow_ipool); mlx5_flow_destroy_policer_rules(dev, fm, &attr); mlx5_flow_destroy_mtr_tbls(dev, fm->mfts); mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], fm->idx); @@ -1153,30 +1180,24 @@ mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id) * * @param [in] priv * Pointer to mlx5 private data. - * @param [in] meter_id - * Flow meter id. + * @param[in] fm + * Pointer to flow meter. * @param [in] attr * Pointer to flow attributes. * @param [out] error * Pointer to error structure. * - * @return the flow meter pointer, NULL otherwise. + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. */ -struct mlx5_flow_meter * -mlx5_flow_meter_attach(struct mlx5_priv *priv, uint32_t meter_id, +int +mlx5_flow_meter_attach(struct mlx5_priv *priv, + struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr, struct rte_flow_error *error) { - struct mlx5_flow_meter *fm; int ret = 0; - fm = mlx5_flow_meter_find(priv, meter_id); - if (fm == NULL) { - rte_flow_error_set(error, ENOENT, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "Meter object id not valid"); - return fm; - } rte_spinlock_lock(&fm->sl); if (fm->mfts->meter_action) { if (fm->shared && @@ -1210,7 +1231,7 @@ mlx5_flow_meter_attach(struct mlx5_priv *priv, uint32_t meter_id, fm->mfts->meter_action ? "Meter attr not match" : "Meter action create failed"); - return ret ? NULL : fm; + return ret ? -rte_errno : 0; } /** -- 2.27.0 ^ permalink raw reply [flat|nested] 8+ messages in thread
[parent not found: <1618916122-181792-1-git-send-email-jiaweiw@nvidia.com>]
* [dpdk-stable] [PATCH v6 03/15] net/mlx5: fix meter statistics [not found] ` <1618916122-181792-1-git-send-email-jiaweiw@nvidia.com> @ 2021-04-20 10:55 ` Jiawei Wang 0 siblings, 0 replies; 8+ messages in thread From: Jiawei Wang @ 2021-04-20 10:55 UTC (permalink / raw) To: matan, orika, viacheslavo, ferruh.yigit, thomas, Shahaf Shuler, Suanming Mou Cc: dev, rasland, asafp, Shun Hao, stable From: Shun Hao <shunh@nvidia.com> Currently, packets after meter will be steered to a global policer table, which includes green/red color rules for every meter, so as to have counter statistics of each color in every meter. There's a bug that all the rules in global policer table are matching only color criteria, so all packets will be counted to one meter only, and other meter statistics are always zero. This patch does these: 1. The rules in policer table matches both meter index and color, so packet after meter could be counted to the correct meter counter. 2. The meter index and flow index are now sharing the available register bits dynamically. Meter index starts from lsb, and flow index starts from msb. Fixes: 46a5e6bc6a ("net/mlx5: prepare meter flow tables") Cc: stable@dpdk.org Signed-off-by: Shun Hao <shunh@nvidia.com> Acked-by: Matan Azrad <matan@nvidia.com> --- drivers/net/mlx5/mlx5.c | 7 +- drivers/net/mlx5/mlx5.h | 15 +- drivers/net/mlx5/mlx5_flow.c | 183 +++++++++----- drivers/net/mlx5/mlx5_flow.h | 40 +-- drivers/net/mlx5/mlx5_flow_dv.c | 489 +++++++++++++++++++++++-------------- drivers/net/mlx5/mlx5_flow_meter.c | 53 ++-- 6 files changed, 496 insertions(+), 291 deletions(-) diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index 3bf224c..cc756ca 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -277,10 +277,13 @@ static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list = }, #endif [MLX5_IPOOL_MTR] = { + /** + * The ipool index should grow continually from small to big, + * for meter idx, so not set grow_trunk to avoid meter index + * not jump continually. + */ .size = sizeof(struct mlx5_flow_meter), .trunk_size = 64, - .grow_trunk = 3, - .grow_shift = 2, .need_lock = 1, .release_mem_en = 1, .malloc = mlx5_malloc, diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index c61f8ff..d208efb 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -953,9 +953,9 @@ struct mlx5_priv { unsigned int representor:1; /* Device is a port representor. */ unsigned int master:1; /* Device is a E-Switch master. */ unsigned int txpp_en:1; /* Tx packet pacing enabled. */ + unsigned int sampler_en:1; /* Whether support sampler. */ unsigned int mtr_en:1; /* Whether support meter. */ unsigned int mtr_reg_share:1; /* Whether support meter REG_C share. */ - unsigned int sampler_en:1; /* Whether support sampler. */ uint16_t domain_id; /* Switch domain identifier. */ uint16_t vport_id; /* Associated VF vport index (if any). */ uint32_t vport_meta_tag; /* Used for vport index match ove VF LAG. */ @@ -1012,6 +1012,10 @@ struct mlx5_priv { uint32_t rss_shared_actions; /* RSS shared actions. */ struct mlx5_devx_obj *q_counters; /* DevX queue counter object. */ uint32_t counter_set_id; /* Queue counter ID to set in DevX objects. */ + uint8_t max_mtr_bits; + /* Indicate how many bits are used by meter id at the most. */ + uint8_t max_mtr_flow_bits; + /* Indicate how many bits are used by meter flow id at the most. */ }; #define PORT_ID(priv) ((priv)->dev_data->port_id) @@ -1280,11 +1284,10 @@ int mlx5_mp_os_req_queue_control(struct rte_eth_dev *dev, uint16_t queue_id, int mlx5_flow_meter_ops_get(struct rte_eth_dev *dev, void *arg); struct mlx5_flow_meter *mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id); -struct mlx5_flow_meter *mlx5_flow_meter_attach - (struct mlx5_priv *priv, - uint32_t meter_id, - const struct rte_flow_attr *attr, - struct rte_flow_error *error); +int mlx5_flow_meter_attach(struct mlx5_priv *priv, + struct mlx5_flow_meter *fm, + const struct rte_flow_attr *attr, + struct rte_flow_error *error); void mlx5_flow_meter_detach(struct mlx5_flow_meter *fm); /* mlx5_os.c */ diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c index bc27b45..3bb04d5 100644 --- a/drivers/net/mlx5/mlx5_flow.c +++ b/drivers/net/mlx5/mlx5_flow.c @@ -748,9 +748,9 @@ struct mlx5_flow_tunnel_info { return REG_C_0; } break; - case MLX5_MTR_SFX: + case MLX5_MTR_ID: /* - * If meter color and flow match share one register, flow match + * If meter color and meter id share one register, flow match * should use the meter color register for match. */ if (priv->mtr_reg_share) @@ -3035,7 +3035,8 @@ struct mlx5_flow_tunnel_info { SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, handle_idx, dev_handle, next) - if (dev_handle->split_flow_id) + if (dev_handle->split_flow_id && + !dev_handle->is_meter_flow_id) mlx5_ipool_free(priv->sh->ipool [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], dev_handle->split_flow_id); @@ -3674,23 +3675,30 @@ struct mlx5_translated_action_handle { * * @param[in] actions * Pointer to the list of actions. - * @param[out] mtr + * @param[out] has_mtr * Pointer to the meter exist flag. + * @param[out] meter_id + * Pointer to the meter id. * * @return * Total number of actions. */ static int -flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr) +flow_check_meter_action(const struct rte_flow_action actions[], + bool *has_mtr, + uint32_t *meter_id) { + const struct rte_flow_action_meter *mtr = NULL; int actions_n = 0; - MLX5_ASSERT(mtr); - *mtr = 0; + MLX5_ASSERT(has_mtr); + *has_mtr = false; for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { switch (actions->type) { case RTE_FLOW_ACTION_TYPE_METER: - *mtr = 1; + mtr = actions->conf; + *meter_id = mtr->mtr_id; + *has_mtr = true; break; default: break; @@ -4347,8 +4355,10 @@ struct mlx5_hlist_entry * * header will be in the prefix sub flow, as not to take the * L3 tunnel header into account. * - * @param dev + * @param[in] dev * Pointer to Ethernet device. + * @param[in] fm + * Pointer to flow meter structure. * @param[in] items * Pattern specification (list terminated by the END pattern item). * @param[out] sfx_items @@ -4363,29 +4373,39 @@ struct mlx5_hlist_entry * * The pattern items for the suffix flow. * @param[out] tag_sfx * Pointer to suffix flow tag. + * @param[out] error + * Perform verbose error reporting if not NULL. * * @return - * 0 on success. + * The flow id, 0 otherwise and rte_errno is set. */ -static int +static uint32_t flow_meter_split_prep(struct rte_eth_dev *dev, - const struct rte_flow_item items[], - struct rte_flow_item sfx_items[], - const struct rte_flow_action actions[], - struct rte_flow_action actions_sfx[], - struct rte_flow_action actions_pre[]) + struct mlx5_flow_meter *fm, + const struct rte_flow_item items[], + struct rte_flow_item sfx_items[], + const struct rte_flow_action actions[], + struct rte_flow_action actions_sfx[], + struct rte_flow_action actions_pre[], + struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; struct rte_flow_action *tag_action = NULL; struct rte_flow_item *tag_item; struct mlx5_rte_flow_action_set_tag *set_tag; - struct rte_flow_error error; const struct rte_flow_action_raw_encap *raw_encap; const struct rte_flow_action_raw_decap *raw_decap; - struct mlx5_rte_flow_item_tag *tag_spec; - struct mlx5_rte_flow_item_tag *tag_mask; + struct mlx5_rte_flow_item_tag *tag_item_spec; + struct mlx5_rte_flow_item_tag *tag_item_mask; uint32_t tag_id = 0; bool copy_vlan = false; + uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; + uint8_t mtr_reg_bits = priv->mtr_reg_share ? + MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; + uint32_t flow_id = 0; + uint32_t flow_id_reversed = 0; + uint8_t flow_id_bits = 0; + int shift; /* Prepare the actions for prefix and suffix flow. */ for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { @@ -4394,10 +4414,7 @@ struct mlx5_hlist_entry * switch (actions->type) { case RTE_FLOW_ACTION_TYPE_METER: /* Add the extra tag action first. */ - tag_action = actions_pre; - tag_action->type = (enum rte_flow_action_type) - MLX5_RTE_FLOW_ACTION_TYPE_TAG; - actions_pre++; + tag_action = actions_pre++; action_cur = &actions_pre; break; case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: @@ -4430,23 +4447,22 @@ struct mlx5_hlist_entry * actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; actions_pre->type = RTE_FLOW_ACTION_TYPE_END; actions_pre++; - /* Set the tag. */ - set_tag = (void *)actions_pre; - set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); - mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], - &tag_id); - if (tag_id >= (1 << (sizeof(tag_id) * 8 - MLX5_MTR_COLOR_BITS))) { - DRV_LOG(ERR, "Port %u meter flow id exceed max limit.", - dev->data->port_id); - mlx5_ipool_free(priv->sh->ipool - [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], tag_id); - return 0; - } else if (!tag_id) { - return 0; + mlx5_ipool_malloc(fm->flow_ipool, &tag_id); + if (!tag_id) + return rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Failed to allocate meter flow id."); + flow_id = tag_id - 1; + flow_id_bits = MLX5_REG_BITS - __builtin_clz(flow_id); + flow_id_bits = flow_id_bits ? flow_id_bits : 1; + if ((flow_id_bits + priv->max_mtr_bits) > mtr_reg_bits) { + mlx5_ipool_free(fm->flow_ipool, tag_id); + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Meter flow id exceeds max limit."); } - set_tag->data = tag_id << MLX5_MTR_COLOR_BITS; - assert(tag_action); - tag_action->conf = set_tag; + if (flow_id_bits > priv->max_mtr_flow_bits) + priv->max_mtr_flow_bits = flow_id_bits; /* Prepare the suffix subflow items. */ tag_item = sfx_items++; for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { @@ -4475,16 +4491,34 @@ struct mlx5_hlist_entry * } sfx_items->type = RTE_FLOW_ITEM_TYPE_END; sfx_items++; - tag_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; - tag_spec->data = tag_id << MLX5_MTR_COLOR_BITS; - tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); - tag_mask = tag_spec + 1; - tag_mask->data = 0xffffff00; + /* Build tag actions and items for meter_id/meter flow_id. */ + assert(tag_action); + set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; + tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; + tag_item_mask = tag_item_spec + 1; + /* + * The color Reg bits used by flow_id are growing from + * msb to lsb, so must do bit reverse for flow_id val in RegC. + */ + for (shift = 0; shift < flow_id_bits; shift++) + flow_id_reversed = (flow_id_reversed << 1) | + ((flow_id >> shift) & 0x1); + /* Both flow_id and meter_id share the same register. */ + set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 0, error); + set_tag->data = + (fm->idx | (flow_id_reversed << (mtr_reg_bits - flow_id_bits))) + << mtr_id_offset; + tag_item_spec->id = set_tag->id; + tag_item_spec->data = set_tag->data; + tag_item_mask->data = UINT32_MAX << mtr_id_offset; + tag_action->type = (enum rte_flow_action_type) + MLX5_RTE_FLOW_ACTION_TYPE_TAG; + tag_action->conf = set_tag; tag_item->type = (enum rte_flow_item_type) - MLX5_RTE_FLOW_ITEM_TYPE_TAG; - tag_item->spec = tag_spec; + MLX5_RTE_FLOW_ITEM_TYPE_TAG; + tag_item->spec = tag_item_spec; tag_item->last = NULL; - tag_item->mask = tag_mask; + tag_item->mask = tag_item_mask; return tag_id; } @@ -5192,25 +5226,49 @@ struct mlx5_hlist_entry * struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); struct rte_flow_action *sfx_actions = NULL; struct rte_flow_action *pre_actions = NULL; struct rte_flow_item *sfx_items = NULL; struct mlx5_flow *dev_flow = NULL; struct rte_flow_attr sfx_attr = *attr; - uint32_t mtr = 0; + struct mlx5_flow_meter *fm = NULL; + bool has_mtr = false; + uint32_t meter_id; uint32_t mtr_tag_id = 0; size_t act_size; size_t item_size; int actions_n = 0; - int ret; + int ret = 0; if (priv->mtr_en) - actions_n = flow_check_meter_action(actions, &mtr); - if (mtr) { - /* The five prefix actions: meter, decap, encap, tag, end. */ + actions_n = flow_check_meter_action(actions, &has_mtr, + &meter_id); + if (has_mtr) { + if (flow->meter) { + fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], + flow->meter); + if (!fm) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "Meter not found."); + } else { + fm = mlx5_flow_meter_find(priv, meter_id); + if (!fm) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "Meter not found."); + ret = mlx5_flow_meter_attach(priv, fm, + &sfx_attr, error); + if (ret) + return -rte_errno; + flow->meter = fm->idx; + } + wks->fm = fm; + /* The prefix actions: meter, decap, encap, tag, end. */ act_size = sizeof(struct rte_flow_action) * (actions_n + 5) + sizeof(struct mlx5_rte_flow_action_set_tag); - /* tag, vlan, port id, end. */ + /* The suffix items: tag, vlan, port id, end. */ #define METER_SUFFIX_ITEM 4 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + sizeof(struct mlx5_rte_flow_item_tag) * 2; @@ -5224,9 +5282,9 @@ struct mlx5_hlist_entry * sfx_items = (struct rte_flow_item *)((char *)sfx_actions + act_size); pre_actions = sfx_actions + actions_n; - mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items, + mtr_tag_id = flow_meter_split_prep(dev, fm, items, sfx_items, actions, sfx_actions, - pre_actions); + pre_actions, error); if (!mtr_tag_id) { ret = -rte_errno; goto exit; @@ -5237,10 +5295,12 @@ struct mlx5_hlist_entry * attr, items, pre_actions, flow_split_info, error); if (ret) { + mlx5_ipool_free(fm->flow_ipool, mtr_tag_id); ret = -rte_errno; goto exit; } dev_flow->handle->split_flow_id = mtr_tag_id; + dev_flow->handle->is_meter_flow_id = 1; /* Setting the sfx group atrr. */ sfx_attr.group = sfx_attr.transfer ? (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : @@ -6512,20 +6572,17 @@ struct mlx5_flow_workspace* * * @param[in] dev * Pointer to Ethernet device. - * @param[in] fm - * Pointer to the flow meter. * * @return * Pointer to table set on success, NULL otherwise. */ struct mlx5_meter_domains_infos * -mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm) +mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev) { const struct mlx5_flow_driver_ops *fops; fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); - return fops->create_mtr_tbls(dev, fm); + return fops->create_mtr_tbls(dev); } /** @@ -6550,7 +6607,7 @@ struct mlx5_meter_domains_infos * } /** - * Create policer rules. + * Prepare policer rules. * * @param[in] dev * Pointer to Ethernet device. @@ -6563,14 +6620,14 @@ struct mlx5_meter_domains_infos * * 0 on success, -1 otherwise. */ int -mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, +mlx5_flow_prepare_policer_rules(struct rte_eth_dev *dev, struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr) { const struct mlx5_flow_driver_ops *fops; fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); - return fops->create_policer_rules(dev, fm, attr); + return fops->prepare_policer_rules(dev, fm, attr); } /** diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index a26f11a..a209b81 100644 --- a/drivers/net/mlx5/mlx5_flow.h +++ b/drivers/net/mlx5/mlx5_flow.h @@ -78,7 +78,7 @@ enum mlx5_feature_name { MLX5_APP_TAG, MLX5_COPY_MARK, MLX5_MTR_COLOR, - MLX5_MTR_SFX, + MLX5_MTR_ID, MLX5_ASO_FLOW_HIT, }; @@ -654,7 +654,8 @@ struct mlx5_flow_handle { uint64_t layers; /**< Bit-fields of present layers, see MLX5_FLOW_LAYER_*. */ void *drv_flow; /**< pointer to driver flow object. */ - uint32_t split_flow_id:28; /**< Sub flow unique match flow id. */ + uint32_t split_flow_id:27; /**< Sub flow unique match flow id. */ + uint32_t is_meter_flow_id:1; /**< Indate if flow_id is for meter. */ uint32_t mark:1; /**< Metadate rxq mark flag. */ uint32_t fate_action:3; /**< Fate action type. */ union { @@ -841,14 +842,16 @@ struct mlx5_meter_domain_info { /**< Meter table. */ struct mlx5_flow_tbl_resource *sfx_tbl; /**< Meter suffix table. */ - void *any_matcher; - /**< Meter color not match default criteria. */ - void *color_matcher; - /**< Meter color match criteria. */ + struct mlx5_flow_dv_matcher *drop_matcher; + /**< Matcher for Drop. */ + struct mlx5_flow_dv_matcher *color_matcher; + /**< Matcher for Color. */ void *jump_actn; /**< Meter match action. */ - void *policer_rules[RTE_MTR_DROPPED + 1]; - /**< Meter policer for the match. */ + void *green_rule; + /**< Meter green rule. */ + void *drop_rule; + /**< Meter drop rule. */ }; /* Meter table set for TX RX FDB. */ @@ -861,10 +864,10 @@ struct mlx5_meter_domains_infos { /**< RX meter table. */ struct mlx5_meter_domain_info transfer; /**< FDB meter table. */ - void *drop_actn; - /**< Drop action as not matched. */ - void *count_actns[RTE_MTR_DROPPED + 1]; - /**< Counters for match and unmatched statistics. */ + void *green_count; + /**< Counters for green rule. */ + void *drop_count; + /**< Counters for green rule. */ uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)]; /**< Flow meter parameter. */ size_t fmp_size; @@ -927,6 +930,8 @@ struct mlx5_flow_meter { /**< Meter state. */ uint32_t shared:1; /**< Meter shared or not. */ + struct mlx5_indexed_pool *flow_ipool; + /**< Index pool for flow id. */ }; /* RFC2697 parameter structure. */ @@ -1165,6 +1170,7 @@ struct mlx5_flow_workspace { struct mlx5_flow_rss_desc rss_desc; uint32_t rssq_num; /* Allocated queue num in rss_desc. */ uint32_t flow_idx; /* Intermediate device flow index. */ + struct mlx5_flow_meter *fm; /* Pointer to the meter in flow. */ }; struct mlx5_flow_split_info { @@ -1205,8 +1211,7 @@ typedef int (*mlx5_flow_query_t)(struct rte_eth_dev *dev, void *data, struct rte_flow_error *error); typedef struct mlx5_meter_domains_infos *(*mlx5_flow_create_mtr_tbls_t) - (struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm); + (struct rte_eth_dev *dev); typedef int (*mlx5_flow_destroy_mtr_tbls_t)(struct rte_eth_dev *dev, struct mlx5_meter_domains_infos *tbls); typedef int (*mlx5_flow_create_policer_rules_t) @@ -1269,7 +1274,7 @@ struct mlx5_flow_driver_ops { mlx5_flow_query_t query; mlx5_flow_create_mtr_tbls_t create_mtr_tbls; mlx5_flow_destroy_mtr_tbls_t destroy_mtr_tbls; - mlx5_flow_create_policer_rules_t create_policer_rules; + mlx5_flow_create_policer_rules_t prepare_policer_rules; mlx5_flow_destroy_policer_rules_t destroy_policer_rules; mlx5_flow_counter_alloc_t counter_alloc; mlx5_flow_counter_free_t counter_free; @@ -1469,11 +1474,10 @@ int mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, const struct rte_flow_item_ecpri *acc_mask, struct rte_flow_error *error); struct mlx5_meter_domains_infos *mlx5_flow_create_mtr_tbls - (struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm); + (struct rte_eth_dev *dev); int mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, struct mlx5_meter_domains_infos *tbl); -int mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, +int mlx5_flow_prepare_policer_rules(struct rte_eth_dev *dev, struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr); int mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 0f215df..d4d88aa 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -10997,14 +10997,12 @@ struct mlx5_cache_entry * const struct rte_flow_action_rss *rss; const struct rte_flow_action *action = actions; const uint8_t *rss_key; - const struct rte_flow_action_meter *mtr; struct mlx5_flow_tbl_resource *tbl; struct mlx5_aso_age_action *age_act; uint32_t port_id = 0; struct mlx5_flow_dv_port_id_action_resource port_id_resource; int action_type = actions->type; const struct rte_flow_action *found_action = NULL; - struct mlx5_flow_meter *fm = NULL; uint32_t jump_group = 0; if (!mlx5_flow_os_action_supported(action_type)) @@ -11429,33 +11427,13 @@ struct mlx5_cache_entry * MLX5_FLOW_FATE_DEFAULT_MISS; break; case RTE_FLOW_ACTION_TYPE_METER: - mtr = actions->conf; - if (!flow->meter) { - fm = mlx5_flow_meter_attach(priv, mtr->mtr_id, - attr, error); - if (!fm) - return rte_flow_error_set(error, - rte_errno, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "meter not found " - "or invalid parameters"); - flow->meter = fm->idx; - } + if (!wks->fm) + return rte_flow_error_set(error, rte_errno, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "Failed to get meter in flow."); /* Set the meter action. */ - if (!fm) { - fm = mlx5_ipool_get(priv->sh->ipool - [MLX5_IPOOL_MTR], flow->meter); - if (!fm) - return rte_flow_error_set(error, - rte_errno, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "meter not found " - "or invalid parameters"); - } dev_flow->dv.actions[actions_n++] = - fm->mfts->meter_action; + wks->fm->mfts->meter_action; action_flags |= MLX5_FLOW_ACTION_METER; break; case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: @@ -12604,6 +12582,7 @@ struct mlx5_cache_entry * { struct mlx5_flow_handle *dev_handle; struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_meter *fm = NULL; uint32_t srss = 0; if (!flow) @@ -12614,8 +12593,6 @@ struct mlx5_cache_entry * flow->counter = 0; } if (flow->meter) { - struct mlx5_flow_meter *fm; - fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], flow->meter); if (fm) @@ -12657,6 +12634,10 @@ struct mlx5_cache_entry * flow_dv_fate_resource_release(dev, dev_handle); else if (!srss) srss = dev_handle->rix_srss; + if (fm && dev_handle->is_meter_flow_id && + dev_handle->split_flow_id) + mlx5_ipool_free(fm->flow_ipool, + dev_handle->split_flow_id); mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], tmp_idx); } @@ -13425,49 +13406,20 @@ struct mlx5_cache_entry * if (!mtd || !priv->config.dv_flow_en) return 0; - if (mtd->ingress.policer_rules[RTE_MTR_DROPPED]) - claim_zero(mlx5_flow_os_destroy_flow - (mtd->ingress.policer_rules[RTE_MTR_DROPPED])); - if (mtd->egress.policer_rules[RTE_MTR_DROPPED]) - claim_zero(mlx5_flow_os_destroy_flow - (mtd->egress.policer_rules[RTE_MTR_DROPPED])); - if (mtd->transfer.policer_rules[RTE_MTR_DROPPED]) - claim_zero(mlx5_flow_os_destroy_flow - (mtd->transfer.policer_rules[RTE_MTR_DROPPED])); - if (mtd->egress.color_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->egress.color_matcher)); - if (mtd->egress.any_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->egress.any_matcher)); if (mtd->egress.tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->egress.tbl); if (mtd->egress.sfx_tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->egress.sfx_tbl); - if (mtd->ingress.color_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->ingress.color_matcher)); - if (mtd->ingress.any_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->ingress.any_matcher)); if (mtd->ingress.tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->ingress.tbl); if (mtd->ingress.sfx_tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->ingress.sfx_tbl); - if (mtd->transfer.color_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->transfer.color_matcher)); - if (mtd->transfer.any_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->transfer.any_matcher)); if (mtd->transfer.tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->transfer.tbl); if (mtd->transfer.sfx_tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->transfer.sfx_tbl); - if (mtd->drop_actn) - claim_zero(mlx5_flow_os_destroy_flow_action(mtd->drop_actn)); mlx5_free(mtd); return 0; } @@ -13486,37 +13438,17 @@ struct mlx5_cache_entry * * Table attribute. * @param[in] transfer * Table attribute. - * @param[in] color_reg_c_idx - * Reg C index for color match. * * @return - * 0 on success, -1 otherwise and rte_errno is set. + * 0 on success, -1 otherwise. */ static int flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, struct mlx5_meter_domains_infos *mtb, - uint8_t egress, uint8_t transfer, - uint32_t color_reg_c_idx) + uint8_t egress, uint8_t transfer) { - struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_dev_ctx_shared *sh = priv->sh; - struct mlx5_flow_dv_match_params mask = { - .size = sizeof(mask.buf), - }; - struct mlx5_flow_dv_match_params value = { - .size = sizeof(value.buf), - }; - struct mlx5dv_flow_matcher_attr dv_attr = { - .type = IBV_FLOW_ATTR_NORMAL, - .priority = 0, - .match_criteria_enable = 0, - .match_mask = (void *)&mask, - }; - void *actions[METER_ACTIONS]; - struct mlx5_meter_domain_info *dtb; struct rte_flow_error error; - int i = 0; - int ret; + struct mlx5_meter_domain_info *dtb; if (transfer) dtb = &mtb->transfer; @@ -13541,41 +13473,7 @@ struct mlx5_cache_entry * DRV_LOG(ERR, "Failed to create meter suffix table."); return -1; } - /* Create matchers, Any and Color. */ - dv_attr.priority = 3; - dv_attr.match_criteria_enable = 0; - ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj, - &dtb->any_matcher); - if (ret) { - DRV_LOG(ERR, "Failed to create meter" - " policer default matcher."); - goto error_exit; - } - dv_attr.priority = 0; - dv_attr.match_criteria_enable = - 1 << MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT; - flow_dv_match_meta_reg(mask.buf, value.buf, color_reg_c_idx, - rte_col_2_mlx5_col(RTE_COLORS), UINT8_MAX); - ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj, - &dtb->color_matcher); - if (ret) { - DRV_LOG(ERR, "Failed to create meter policer color matcher."); - goto error_exit; - } - if (mtb->count_actns[RTE_MTR_DROPPED]) - actions[i++] = mtb->count_actns[RTE_MTR_DROPPED]; - actions[i++] = mtb->drop_actn; - /* Default rule: lowest priority, match any, actions: drop. */ - ret = mlx5_flow_os_create_flow(dtb->any_matcher, (void *)&value, i, - actions, - &dtb->policer_rules[RTE_MTR_DROPPED]); - if (ret) { - DRV_LOG(ERR, "Failed to create meter policer drop rule."); - goto error_exit; - } return 0; -error_exit: - return -1; } /** @@ -13584,20 +13482,16 @@ struct mlx5_cache_entry * * * @param[in] dev * Pointer to Ethernet device. - * @param[in] fm - * Pointer to the flow meter. * * @return * Pointer to table set on success, NULL otherwise and rte_errno is set. */ static struct mlx5_meter_domains_infos * -flow_dv_create_mtr_tbl(struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm) +flow_dv_create_mtr_tbl(struct rte_eth_dev *dev) { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_meter_domains_infos *mtb; int ret; - int i; if (!priv->mtr_en) { rte_errno = ENOTSUP; @@ -13608,37 +13502,21 @@ struct mlx5_cache_entry * DRV_LOG(ERR, "Failed to allocate memory for meter."); return NULL; } - /* Create meter count actions */ - for (i = 0; i <= RTE_MTR_DROPPED; i++) { - struct mlx5_flow_counter *cnt; - if (!fm->policer_stats.cnt[i]) - continue; - cnt = flow_dv_counter_get_by_idx(dev, - fm->policer_stats.cnt[i], NULL); - mtb->count_actns[i] = cnt->action; - } - /* Create drop action. */ - ret = mlx5_flow_os_create_flow_action_drop(&mtb->drop_actn); - if (ret) { - DRV_LOG(ERR, "Failed to create drop action."); - goto error_exit; - } /* Egress meter table. */ - ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0, priv->mtr_color_reg); + ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0); if (ret) { DRV_LOG(ERR, "Failed to prepare egress meter table."); goto error_exit; } /* Ingress meter table. */ - ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0, priv->mtr_color_reg); + ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0); if (ret) { DRV_LOG(ERR, "Failed to prepare ingress meter table."); goto error_exit; } /* FDB meter table. */ if (priv->config.dv_esw_en) { - ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1, - priv->mtr_color_reg); + ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1); if (ret) { DRV_LOG(ERR, "Failed to prepare fdb meter table."); goto error_exit; @@ -13651,23 +13529,151 @@ struct mlx5_cache_entry * } /** + * Destroy the meter table matchers. + * Lock free, (mutex should be acquired by caller). + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in,out] dtb + * Pointer to DV meter table. + * + * @return + * Always 0. + */ +static int +flow_dv_destroy_mtr_matchers(struct rte_eth_dev *dev, + struct mlx5_meter_domain_info *dtb) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_tbl_data_entry *tbl; + + if (!priv->config.dv_flow_en) + return 0; + if (dtb->drop_matcher) { + tbl = container_of(dtb->drop_matcher->tbl, typeof(*tbl), tbl); + mlx5_cache_unregister(&tbl->matchers, + &dtb->drop_matcher->entry); + dtb->drop_matcher = NULL; + } + if (dtb->color_matcher) { + tbl = container_of(dtb->color_matcher->tbl, typeof(*tbl), tbl); + mlx5_cache_unregister(&tbl->matchers, + &dtb->color_matcher->entry); + dtb->color_matcher = NULL; + } + return 0; +} + +/** + * Create the matchers for meter table. + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in] color_reg_c_idx + * Reg C index for color match. + * @param[in] mtr_id_reg_c_idx + * Reg C index for meter_id match. + * @param[in] mtr_id_mask + * Mask for meter_id match criteria. + * @param[in,out] dtb + * Pointer to DV meter table. + * @param[out] error + * Perform verbose error reporting if not NULL. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_prepare_mtr_matchers(struct rte_eth_dev *dev, + uint32_t color_reg_c_idx, + uint32_t mtr_id_reg_c_idx, + uint32_t mtr_id_mask, + struct mlx5_meter_domain_info *dtb, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_tbl_data_entry *tbl_data; + struct mlx5_cache_entry *entry; + struct mlx5_flow_dv_matcher matcher = { + .mask = { + .size = sizeof(matcher.mask.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), + }, + .tbl = dtb->tbl, + }; + struct mlx5_flow_dv_match_params value = { + .size = sizeof(value.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), + }; + struct mlx5_flow_cb_ctx ctx = { + .error = error, + .data = &matcher, + }; + uint32_t color_mask = (UINT32_C(1) << MLX5_MTR_COLOR_BITS) - 1; + + tbl_data = container_of(dtb->tbl, struct mlx5_flow_tbl_data_entry, tbl); + if (!dtb->drop_matcher) { + /* Create matchers for Drop. */ + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + mtr_id_reg_c_idx, 0, mtr_id_mask); + matcher.priority = MLX5_REG_BITS * 2 - priv->max_mtr_bits; + matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf, + matcher.mask.size); + entry = mlx5_cache_register(&tbl_data->matchers, &ctx); + if (!entry) { + DRV_LOG(ERR, "Failed to register meter drop matcher."); + return -1; + } + dtb->drop_matcher = + container_of(entry, struct mlx5_flow_dv_matcher, entry); + } + if (!dtb->color_matcher) { + /* Create matchers for Color + meter_id. */ + if (priv->mtr_reg_share) { + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + color_reg_c_idx, 0, + (mtr_id_mask | color_mask)); + } else { + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + color_reg_c_idx, 0, color_mask); + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + mtr_id_reg_c_idx, 0, mtr_id_mask); + } + matcher.priority = MLX5_REG_BITS - priv->max_mtr_bits; + matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf, + matcher.mask.size); + entry = mlx5_cache_register(&tbl_data->matchers, &ctx); + if (!entry) { + DRV_LOG(ERR, "Failed to register meter color matcher."); + return -1; + } + dtb->color_matcher = + container_of(entry, struct mlx5_flow_dv_matcher, entry); + } + return 0; +} + +/** * Destroy domain policer rule. * + * @param[in] dev + * Pointer to Ethernet device. * @param[in] dt * Pointer to domain table. */ static void -flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt) +flow_dv_destroy_domain_policer_rule(struct rte_eth_dev *dev, + struct mlx5_meter_domain_info *dt) { - int i; - - for (i = 0; i < RTE_MTR_DROPPED; i++) { - if (dt->policer_rules[i]) { - claim_zero(mlx5_flow_os_destroy_flow - (dt->policer_rules[i])); - dt->policer_rules[i] = NULL; - } + if (dt->drop_rule) { + claim_zero(mlx5_flow_os_destroy_flow(dt->drop_rule)); + dt->drop_rule = NULL; } + if (dt->green_rule) { + claim_zero(mlx5_flow_os_destroy_flow(dt->green_rule)); + dt->green_rule = NULL; + } + flow_dv_destroy_mtr_matchers(dev, dt); if (dt->jump_actn) { claim_zero(mlx5_flow_os_destroy_flow_action(dt->jump_actn)); dt->jump_actn = NULL; @@ -13688,7 +13694,7 @@ struct mlx5_cache_entry * * Always 0. */ static int -flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused, +flow_dv_destroy_policer_rules(struct rte_eth_dev *dev, const struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr) { @@ -13697,39 +13703,56 @@ struct mlx5_cache_entry * if (!mtb) return 0; if (attr->egress) - flow_dv_destroy_domain_policer_rule(&mtb->egress); + flow_dv_destroy_domain_policer_rule(dev, &mtb->egress); if (attr->ingress) - flow_dv_destroy_domain_policer_rule(&mtb->ingress); + flow_dv_destroy_domain_policer_rule(dev, &mtb->ingress); if (attr->transfer) - flow_dv_destroy_domain_policer_rule(&mtb->transfer); + flow_dv_destroy_domain_policer_rule(dev, &mtb->transfer); return 0; } /** * Create specify domain meter policer rule. * + * @param[in] dev + * Pointer to Ethernet device. * @param[in] fm * Pointer to flow meter structure. * @param[in] mtb * Pointer to DV meter table set. - * @param[in] mtr_reg_c - * Color match REG_C. + * @param[out] drop_rule + * The address of pointer saving drop rule. + * @param[out] color_rule + * The address of pointer saving green rule. * * @return * 0 on success, -1 otherwise. */ static int -flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, +flow_dv_create_policer_forward_rule(struct rte_eth_dev *dev, + struct mlx5_flow_meter *fm, struct mlx5_meter_domain_info *dtb, - uint8_t mtr_reg_c) + void **drop_rule, + void **green_rule) { + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flow_dv_match_params matcher = { - .size = sizeof(matcher.buf), + .size = sizeof(matcher.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), }; struct mlx5_flow_dv_match_params value = { - .size = sizeof(value.buf), + .size = sizeof(value.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), }; struct mlx5_meter_domains_infos *mtb = fm->mfts; + struct rte_flow_error error; + uint32_t color_reg_c = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, + 0, &error); + uint32_t mtr_id_reg_c = mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, + 0, &error); + uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; + uint32_t mtr_id_mask = + ((UINT32_C(1) << priv->max_mtr_bits) - 1) << mtr_id_offset; void *actions[METER_ACTIONS]; int i; int ret = 0; @@ -13742,25 +13765,52 @@ struct mlx5_cache_entry * DRV_LOG(ERR, "Failed to create policer jump action."); goto error; } - for (i = 0; i < RTE_MTR_DROPPED; i++) { - int j = 0; - - flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_reg_c, - rte_col_2_mlx5_col(i), UINT8_MAX); - if (mtb->count_actns[i]) - actions[j++] = mtb->count_actns[i]; - if (fm->action[i] == MTR_POLICER_ACTION_DROP) - actions[j++] = mtb->drop_actn; - else - actions[j++] = dtb->jump_actn; - ret = mlx5_flow_os_create_flow(dtb->color_matcher, - (void *)&value, j, actions, - &dtb->policer_rules[i]); + /* Prepare matchers. */ + if (!dtb->drop_matcher || !dtb->color_matcher) { + ret = flow_dv_prepare_mtr_matchers(dev, color_reg_c, + mtr_id_reg_c, mtr_id_mask, + dtb, &error); if (ret) { - DRV_LOG(ERR, "Failed to create policer rule."); + DRV_LOG(ERR, "Failed to setup matchers for mtr table."); goto error; } } + /* Create Drop flow, matching meter_id only. */ + i = 0; + flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_id_reg_c, + (fm->idx << mtr_id_offset), UINT32_MAX); + if (mtb->drop_count) + actions[i++] = mtb->drop_count; + actions[i++] = priv->sh->dr_drop_action; + ret = mlx5_flow_os_create_flow(dtb->drop_matcher->matcher_object, + (void *)&value, i, actions, drop_rule); + if (ret) { + DRV_LOG(ERR, "Failed to create meter policer drop rule."); + goto error; + } + /* Create flow matching Green color + meter_id. */ + i = 0; + if (priv->mtr_reg_share) { + flow_dv_match_meta_reg(matcher.buf, value.buf, color_reg_c, + ((fm->idx << mtr_id_offset) | + rte_col_2_mlx5_col(RTE_COLOR_GREEN)), + UINT32_MAX); + } else { + flow_dv_match_meta_reg(matcher.buf, value.buf, color_reg_c, + rte_col_2_mlx5_col(RTE_COLOR_GREEN), + UINT32_MAX); + flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_id_reg_c, + fm->idx, UINT32_MAX); + } + if (mtb->green_count) + actions[i++] = mtb->green_count; + actions[i++] = dtb->jump_actn; + ret = mlx5_flow_os_create_flow(dtb->color_matcher->matcher_object, + (void *)&value, i, actions, green_rule); + if (ret) { + DRV_LOG(ERR, "Failed to create meter policer color rule."); + goto error; + } return 0; error: rte_errno = errno; @@ -13768,7 +13818,8 @@ struct mlx5_cache_entry * } /** - * Create policer rules. + * Prepare policer rules for all domains. + * If meter already initialized, this will replace all old rules with new ones. * * @param[in] dev * Pointer to Ethernet device. @@ -13781,41 +13832,107 @@ struct mlx5_cache_entry * * 0 on success, -1 otherwise. */ static int -flow_dv_create_policer_rules(struct rte_eth_dev *dev, - struct mlx5_flow_meter *fm, - const struct rte_flow_attr *attr) +flow_dv_prepare_policer_rules(struct rte_eth_dev *dev, + struct mlx5_flow_meter *fm, + const struct rte_flow_attr *attr) { - struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_meter_domains_infos *mtb = fm->mfts; + bool initialized = false; + struct mlx5_flow_counter *cnt; + void *egress_drop_rule = NULL; + void *egress_green_rule = NULL; + void *ingress_drop_rule = NULL; + void *ingress_green_rule = NULL; + void *transfer_drop_rule = NULL; + void *transfer_green_rule = NULL; int ret; + /* Get the statistics counters for green/drop. */ + if (fm->policer_stats.cnt[RTE_COLOR_GREEN]) { + cnt = flow_dv_counter_get_by_idx(dev, + fm->policer_stats.cnt[RTE_COLOR_GREEN], + NULL); + mtb->green_count = cnt->action; + } else { + mtb->green_count = NULL; + } + if (fm->policer_stats.cnt[RTE_MTR_DROPPED]) { + cnt = flow_dv_counter_get_by_idx(dev, + fm->policer_stats.cnt[RTE_MTR_DROPPED], + NULL); + mtb->drop_count = cnt->action; + } else { + mtb->drop_count = NULL; + } + /** + * If flow meter has been initialized, all policer rules + * are created. So can get if meter initialized by checking + * any policer rule. + */ + if (mtb->egress.drop_rule) + initialized = true; if (attr->egress) { - ret = flow_dv_create_policer_forward_rule(fm, &mtb->egress, - priv->mtr_color_reg); + ret = flow_dv_create_policer_forward_rule(dev, + fm, &mtb->egress, + &egress_drop_rule, &egress_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create egress policer."); goto error; } } if (attr->ingress) { - ret = flow_dv_create_policer_forward_rule(fm, &mtb->ingress, - priv->mtr_color_reg); + ret = flow_dv_create_policer_forward_rule(dev, + fm, &mtb->ingress, + &ingress_drop_rule, &ingress_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create ingress policer."); goto error; } } if (attr->transfer) { - ret = flow_dv_create_policer_forward_rule(fm, &mtb->transfer, - priv->mtr_color_reg); + ret = flow_dv_create_policer_forward_rule(dev, + fm, &mtb->transfer, + &transfer_drop_rule, &transfer_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create transfer policer."); goto error; } } + /* Replace old flows if existing. */ + if (mtb->egress.drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->egress.drop_rule)); + if (mtb->egress.green_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->egress.green_rule)); + if (mtb->ingress.drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->ingress.drop_rule)); + if (mtb->ingress.green_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->ingress.green_rule)); + if (mtb->transfer.drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->transfer.drop_rule)); + if (mtb->transfer.green_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->transfer.green_rule)); + mtb->egress.drop_rule = egress_drop_rule; + mtb->egress.green_rule = egress_green_rule; + mtb->ingress.drop_rule = ingress_drop_rule; + mtb->ingress.green_rule = ingress_green_rule; + mtb->transfer.drop_rule = transfer_drop_rule; + mtb->transfer.green_rule = transfer_green_rule; return 0; error: - flow_dv_destroy_policer_rules(dev, fm, attr); + if (egress_drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(egress_drop_rule)); + if (egress_green_rule) + claim_zero(mlx5_flow_os_destroy_flow(egress_green_rule)); + if (ingress_drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(ingress_drop_rule)); + if (ingress_green_rule) + claim_zero(mlx5_flow_os_destroy_flow(ingress_green_rule)); + if (transfer_drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(transfer_drop_rule)); + if (transfer_green_rule) + claim_zero(mlx5_flow_os_destroy_flow(transfer_green_rule)); + if (!initialized) + flow_dv_destroy_policer_rules(dev, fm, attr); return -1; } @@ -14112,7 +14229,7 @@ struct mlx5_cache_entry * .query = flow_dv_query, .create_mtr_tbls = flow_dv_create_mtr_tbl, .destroy_mtr_tbls = flow_dv_destroy_mtr_tbl, - .create_policer_rules = flow_dv_create_policer_rules, + .prepare_policer_rules = flow_dv_prepare_policer_rules, .destroy_policer_rules = flow_dv_destroy_policer_rules, .counter_alloc = flow_dv_counter_allocate, .counter_free = flow_dv_counter_free, diff --git a/drivers/net/mlx5/mlx5_flow_meter.c b/drivers/net/mlx5/mlx5_flow_meter.c index dbc574b..7f7693b 100644 --- a/drivers/net/mlx5/mlx5_flow_meter.c +++ b/drivers/net/mlx5/mlx5_flow_meter.c @@ -474,6 +474,12 @@ MTR_POLICER_ACTION_COLOR_RED }; int i; + /* Meter must use global drop action. */ + if (!priv->sh->dr_drop_action) + return -rte_mtr_error_set(error, ENOTSUP, + RTE_MTR_ERROR_TYPE_MTR_PARAMS, + NULL, + "No drop action ready for meter."); /* Meter params must not be NULL. */ if (params == NULL) return -rte_mtr_error_set(error, EINVAL, @@ -630,9 +636,18 @@ .egress = 1, .transfer = priv->config.dv_esw_en ? 1 : 0, }; + struct mlx5_indexed_pool_config flow_ipool_cfg = { + .size = 0, + .trunk_size = 64, + .need_lock = 1, + .type = "mlx5_flow_mtr_flow_id_pool", + }; int ret; unsigned int i; uint32_t idx = 0; + uint8_t mtr_id_bits; + uint8_t mtr_reg_bits = priv->mtr_reg_share ? + MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; if (!priv->mtr_en) return -rte_mtr_error_set(error, ENOTSUP, @@ -654,6 +669,13 @@ return -rte_mtr_error_set(error, ENOMEM, RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, "Memory alloc failed for meter."); + mtr_id_bits = MLX5_REG_BITS - __builtin_clz(idx); + if ((mtr_id_bits + priv->max_mtr_flow_bits) > mtr_reg_bits) { + DRV_LOG(ERR, "Meter number exceeds max limit."); + goto error; + } + if (mtr_id_bits > priv->max_mtr_bits) + priv->max_mtr_bits = mtr_id_bits; fm->idx = idx; /* Fill the flow meter parameters. */ fm->meter_id = meter_id; @@ -667,10 +689,10 @@ if (!fm->policer_stats.cnt[i]) goto error; } - fm->mfts = mlx5_flow_create_mtr_tbls(dev, fm); + fm->mfts = mlx5_flow_create_mtr_tbls(dev); if (!fm->mfts) goto error; - ret = mlx5_flow_create_policer_rules(dev, fm, &attr); + ret = mlx5_flow_prepare_policer_rules(dev, fm, &attr); if (ret) goto error; /* Add to the flow meter list. */ @@ -679,6 +701,9 @@ fm->shared = !!shared; fm->policer_stats.stats_mask = params->stats_mask; fm->profile->ref_cnt++; + fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg); + if (!fm->flow_ipool) + goto error; rte_spinlock_init(&fm->sl); return 0; error: @@ -749,6 +774,8 @@ if (fm->policer_stats.cnt[i]) mlx5_counter_free(dev, fm->policer_stats.cnt[i]); /* Free meter flow table */ + if (fm->flow_ipool) + mlx5_ipool_destroy(fm->flow_ipool); mlx5_flow_destroy_policer_rules(dev, fm, &attr); mlx5_flow_destroy_mtr_tbls(dev, fm->mfts); mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], fm->idx); @@ -1153,30 +1180,24 @@ struct mlx5_flow_meter * * * @param [in] priv * Pointer to mlx5 private data. - * @param [in] meter_id - * Flow meter id. + * @param[in] fm + * Pointer to flow meter. * @param [in] attr * Pointer to flow attributes. * @param [out] error * Pointer to error structure. * - * @return the flow meter pointer, NULL otherwise. + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. */ -struct mlx5_flow_meter * -mlx5_flow_meter_attach(struct mlx5_priv *priv, uint32_t meter_id, +int +mlx5_flow_meter_attach(struct mlx5_priv *priv, + struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr, struct rte_flow_error *error) { - struct mlx5_flow_meter *fm; int ret = 0; - fm = mlx5_flow_meter_find(priv, meter_id); - if (fm == NULL) { - rte_flow_error_set(error, ENOENT, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "Meter object id not valid"); - return fm; - } rte_spinlock_lock(&fm->sl); if (fm->mfts->meter_action) { if (fm->shared && @@ -1210,7 +1231,7 @@ struct mlx5_flow_meter * fm->mfts->meter_action ? "Meter attr not match" : "Meter action create failed"); - return ret ? NULL : fm; + return ret ? -rte_errno : 0; } /** -- 1.8.3.1 ^ permalink raw reply [flat|nested] 8+ messages in thread
end of thread, other threads:[~2021-04-20 10:55 UTC | newest]
Thread overview: 8+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
[not found] <20210331073632.1443011-1-lizh@nvidia.com>
2021-03-31 7:36 ` [dpdk-stable] [PATCH 02/13] net/mlx5: fix meter statistics Li Zhang
[not found] ` <20210402151430.1531529-1-lizh@nvidia.com>
2021-04-02 15:14 ` [dpdk-stable] [PATCH v2 " Li Zhang
[not found] ` <20210402151627.1531623-1-lizh@nvidia.com>
2021-04-02 15:16 ` Li Zhang
[not found] ` <20210412160817.1986716-1-lizh@nvidia.com>
2021-04-12 16:08 ` [dpdk-stable] [PATCH v3 03/14] " Li Zhang
[not found] ` <20210413001033.1999355-1-lizh@nvidia.com>
2021-04-13 0:10 ` Li Zhang
[not found] ` <20210414025736.31142-1-lizh@nvidia.com>
2021-04-14 2:57 ` [dpdk-stable] [PATCH v4 " Li Zhang
[not found] ` <20210415151135.2098674-1-lizh@nvidia.com>
2021-04-15 15:11 ` [dpdk-stable] [PATCH v5 " Li Zhang
[not found] ` <1618916122-181792-1-git-send-email-jiaweiw@nvidia.com>
2021-04-20 10:55 ` [dpdk-stable] [PATCH v6 03/15] " Jiawei Wang
This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inbox; as well as URLs for NNTP newsgroup(s).