From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 74FB6A034F; Wed, 31 Mar 2021 09:38:05 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id E2113406FF; Wed, 31 Mar 2021 09:37:00 +0200 (CEST) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by mails.dpdk.org (Postfix) with ESMTP id 755C9140DED for ; Wed, 31 Mar 2021 09:36:46 +0200 (CEST) Received: from Internal Mail-Server by MTLPINE1 (envelope-from lizh@nvidia.com) with SMTP; 31 Mar 2021 10:36:40 +0300 Received: from nvidia.com (c-235-17-1-009.mtl.labs.mlnx [10.235.17.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 12V7adwA001778; Wed, 31 Mar 2021 10:36:40 +0300 From: Li Zhang To: dekelp@nvidia.com, orika@nvidia.com, viacheslavo@nvidia.com, matan@nvidia.com, shahafs@nvidia.com, Suanming Mou Cc: dev@dpdk.org, thomas@monjalon.net, rasland@nvidia.com, roniba@nvidia.com, Shun Hao , stable@dpdk.org Date: Wed, 31 Mar 2021 10:36:20 +0300 Message-Id: <20210331073632.1443011-3-lizh@nvidia.com> X-Mailer: git-send-email 2.21.0 In-Reply-To: <20210331073632.1443011-1-lizh@nvidia.com> References: <20210331073632.1443011-1-lizh@nvidia.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Subject: [dpdk-dev] [PATCH 02/13] net/mlx5: fix meter statistics X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" From: Shun Hao This fixes the meter statistics issue that when using multiple meters, only one meter has stats value. To match the correct meter in policer table, now the meter_id is also used in its match criteria, so only one color and one drop matcher are needed. And both meter_id and flow_id will be written to related registers in meter prefix flow. Fixes: 46a5e6bc6a ("net/mlx5: prepare meter flow tables") Cc: stable@dpdk.org Signed-off-by: Shun Hao --- drivers/net/mlx5/linux/mlx5_os.c | 9 +- drivers/net/mlx5/mlx5.c | 7 +- drivers/net/mlx5/mlx5.h | 25 +- drivers/net/mlx5/mlx5_flow.c | 182 +++++++---- drivers/net/mlx5/mlx5_flow.h | 40 +-- drivers/net/mlx5/mlx5_flow_dv.c | 489 ++++++++++++++++++----------- drivers/net/mlx5/mlx5_flow_meter.c | 53 +++- 7 files changed, 512 insertions(+), 293 deletions(-) diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c index 2d5bcab4cf..b7eca9d153 100644 --- a/drivers/net/mlx5/linux/mlx5_os.c +++ b/drivers/net/mlx5/linux/mlx5_os.c @@ -316,7 +316,13 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv) } sh->tx_domain = domain; #ifdef HAVE_MLX5DV_DR_ESWITCH + sh->esw_drop_action = mlx5_glue->dr_create_flow_action_drop(); if (priv->config.dv_esw_en) { + if (!sh->esw_drop_action) { + DRV_LOG(ERR, "Failed to create eswitch drop action"); + err = errno; + goto error; + } domain = mlx5_glue->dr_create_domain (sh->ctx, MLX5DV_DR_DOMAIN_TYPE_FDB); if (!domain) { @@ -325,7 +331,6 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv) goto error; } sh->fdb_domain = domain; - sh->esw_drop_action = mlx5_glue->dr_create_flow_action_drop(); } #endif if (!sh->tunnel_hub) @@ -1273,7 +1278,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev, - 1 + REG_C_0; priv->mtr_en = 1; priv->mtr_reg_share = - config->hca_attr.qos.flow_meter; + config->hca_attr.qos.flow_meter; DRV_LOG(DEBUG, "The REG_C meter uses is %d", priv->mtr_color_reg); } diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index 3538cc8c20..d41c098f65 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -275,10 +275,13 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = { }, #endif [MLX5_IPOOL_MTR] = { + /** + * The ipool index should grow continually from small to big, + * for meter idx, so not set grow_trunk to avoid meter index + * not jump continually. + */ .size = sizeof(struct mlx5_flow_meter), .trunk_size = 64, - .grow_trunk = 3, - .grow_shift = 2, .need_lock = 1, .release_mem_en = 1, .malloc = mlx5_malloc, diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index 6faba4fbb1..95469c605a 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -188,6 +188,16 @@ struct mlx5_stats_ctrl { /* Maximal number of segments to split. */ #define MLX5_MAX_RXQ_NSEG (1u << MLX5_MAX_LOG_RQ_SEGS) +/* The bit size of one register. */ +#define MLX5_REG_BITS 32 + +/* Idle bits for non-color usage in color register. */ +#define MLX5_MTR_IDLE_BITS_IN_COLOR_REG (MLX5_REG_BITS - MLX5_MTR_COLOR_BITS) + +#define UINT32_T(x) ((uint32_t)(x)) + +#define LS32_MASK(bits) ((UINT32_T(1) << (bits)) - 1) + /* LRO configurations structure. */ struct mlx5_lro_config { uint32_t supported:1; /* Whether LRO is supported. */ @@ -944,9 +954,9 @@ struct mlx5_priv { unsigned int representor:1; /* Device is a port representor. */ unsigned int master:1; /* Device is a E-Switch master. */ unsigned int txpp_en:1; /* Tx packet pacing enabled. */ + unsigned int sampler_en:1; /* Whether support sampler. */ unsigned int mtr_en:1; /* Whether support meter. */ unsigned int mtr_reg_share:1; /* Whether support meter REG_C share. */ - unsigned int sampler_en:1; /* Whether support sampler. */ uint16_t domain_id; /* Switch domain identifier. */ uint16_t vport_id; /* Associated VF vport index (if any). */ uint32_t vport_meta_tag; /* Used for vport index match ove VF LAG. */ @@ -1003,6 +1013,10 @@ struct mlx5_priv { uint32_t rss_shared_actions; /* RSS shared actions. */ struct mlx5_devx_obj *q_counters; /* DevX queue counter object. */ uint32_t counter_set_id; /* Queue counter ID to set in DevX objects. */ + uint8_t max_mtr_bits; + /* Indicate how many bits are used by meter id at the most. */ + uint8_t max_mtr_flow_bits; + /* Indicate how many bits are used by meter flow id at the most. */ }; #define PORT_ID(priv) ((priv)->dev_data->port_id) @@ -1271,11 +1285,10 @@ int mlx5_pmd_socket_init(void); int mlx5_flow_meter_ops_get(struct rte_eth_dev *dev, void *arg); struct mlx5_flow_meter *mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id); -struct mlx5_flow_meter *mlx5_flow_meter_attach - (struct mlx5_priv *priv, - uint32_t meter_id, - const struct rte_flow_attr *attr, - struct rte_flow_error *error); +int mlx5_flow_meter_attach(struct mlx5_priv *priv, + struct mlx5_flow_meter *fm, + const struct rte_flow_attr *attr, + struct rte_flow_error *error); void mlx5_flow_meter_detach(struct mlx5_flow_meter *fm); /* mlx5_os.c */ diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c index c347f8130e..cffd6129e8 100644 --- a/drivers/net/mlx5/mlx5_flow.c +++ b/drivers/net/mlx5/mlx5_flow.c @@ -764,9 +764,9 @@ mlx5_flow_get_reg_id(struct rte_eth_dev *dev, return REG_C_0; } break; - case MLX5_MTR_SFX: + case MLX5_MTR_ID: /* - * If meter color and flow match share one register, flow match + * If meter color and meter id share one register, flow match * should use the meter color register for match. */ if (priv->mtr_reg_share) @@ -3051,7 +3051,8 @@ flow_mreg_split_qrss_release(struct rte_eth_dev *dev, SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, handle_idx, dev_handle, next) - if (dev_handle->split_flow_id) + if (dev_handle->split_flow_id && + !dev_handle->is_meter_flow_id) mlx5_ipool_free(priv->sh->ipool [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], dev_handle->split_flow_id); @@ -3690,23 +3691,30 @@ flow_parse_metadata_split_actions_info(const struct rte_flow_action actions[], * * @param[in] actions * Pointer to the list of actions. - * @param[out] mtr + * @param[out] has_mtr * Pointer to the meter exist flag. + * @param[out] meter_id + * Pointer to the meter id. * * @return * Total number of actions. */ static int -flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr) +flow_check_meter_action(const struct rte_flow_action actions[], + bool *has_mtr, + uint32_t *meter_id) { + const struct rte_flow_action_meter *mtr = NULL; int actions_n = 0; - MLX5_ASSERT(mtr); - *mtr = 0; + MLX5_ASSERT(has_mtr); + *has_mtr = false; for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { switch (actions->type) { case RTE_FLOW_ACTION_TYPE_METER: - *mtr = 1; + mtr = actions->conf; + *meter_id = mtr->mtr_id; + *has_mtr = true; break; default: break; @@ -4363,8 +4371,10 @@ flow_create_split_inner(struct rte_eth_dev *dev, * header will be in the prefix sub flow, as not to take the * L3 tunnel header into account. * - * @param dev + * @param[in] dev * Pointer to Ethernet device. + * @param[in] fm + * Pointer to flow meter structure. * @param[in] items * Pattern specification (list terminated by the END pattern item). * @param[out] sfx_items @@ -4379,29 +4389,38 @@ flow_create_split_inner(struct rte_eth_dev *dev, * The pattern items for the suffix flow. * @param[out] tag_sfx * Pointer to suffix flow tag. + * @param[out] error + * Perform verbose error reporting if not NULL. * * @return - * 0 on success. + * The flow id, 0 otherwise and rte_errno is set. */ -static int +static uint32_t flow_meter_split_prep(struct rte_eth_dev *dev, - const struct rte_flow_item items[], - struct rte_flow_item sfx_items[], - const struct rte_flow_action actions[], - struct rte_flow_action actions_sfx[], - struct rte_flow_action actions_pre[]) + struct mlx5_flow_meter *fm, + const struct rte_flow_item items[], + struct rte_flow_item sfx_items[], + const struct rte_flow_action actions[], + struct rte_flow_action actions_sfx[], + struct rte_flow_action actions_pre[], + struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; struct rte_flow_action *tag_action = NULL; struct rte_flow_item *tag_item; struct mlx5_rte_flow_action_set_tag *set_tag; - struct rte_flow_error error; const struct rte_flow_action_raw_encap *raw_encap; const struct rte_flow_action_raw_decap *raw_decap; - struct mlx5_rte_flow_item_tag *tag_spec; - struct mlx5_rte_flow_item_tag *tag_mask; + struct mlx5_rte_flow_item_tag *tag_item_spec; + struct mlx5_rte_flow_item_tag *tag_item_mask; uint32_t tag_id = 0; bool copy_vlan = false; + uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; + uint8_t mtr_reg_bits = priv->mtr_reg_share ? + MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; + uint8_t flow_id_bits = 0; + int shift; + uint32_t flow_id_val = 0; /* Prepare the actions for prefix and suffix flow. */ for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { @@ -4410,10 +4429,7 @@ flow_meter_split_prep(struct rte_eth_dev *dev, switch (actions->type) { case RTE_FLOW_ACTION_TYPE_METER: /* Add the extra tag action first. */ - tag_action = actions_pre; - tag_action->type = (enum rte_flow_action_type) - MLX5_RTE_FLOW_ACTION_TYPE_TAG; - actions_pre++; + tag_action = actions_pre++; action_cur = &actions_pre; break; case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: @@ -4446,23 +4462,22 @@ flow_meter_split_prep(struct rte_eth_dev *dev, actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; actions_pre->type = RTE_FLOW_ACTION_TYPE_END; actions_pre++; - /* Set the tag. */ - set_tag = (void *)actions_pre; - set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); - mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], - &tag_id); - if (tag_id >= (1 << (sizeof(tag_id) * 8 - MLX5_MTR_COLOR_BITS))) { - DRV_LOG(ERR, "Port %u meter flow id exceed max limit.", - dev->data->port_id); - mlx5_ipool_free(priv->sh->ipool - [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], tag_id); - return 0; - } else if (!tag_id) { - return 0; + /* Generate meter flow_id only if support multiple flows per meter. */ + mlx5_ipool_malloc(fm->flow_ipool, &tag_id); + if (!tag_id) + return rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Failed to allocate meter flow id."); + flow_id_bits = MLX5_REG_BITS - __builtin_clz(tag_id - 1); + flow_id_bits = flow_id_bits ? flow_id_bits : 1; + if ((flow_id_bits + priv->max_mtr_bits) > mtr_reg_bits) { + mlx5_ipool_free(fm->flow_ipool, tag_id); + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Meter flow id exceeds max limit."); } - set_tag->data = tag_id << MLX5_MTR_COLOR_BITS; - assert(tag_action); - tag_action->conf = set_tag; + if (flow_id_bits > priv->max_mtr_flow_bits) + priv->max_mtr_flow_bits = flow_id_bits; /* Prepare the suffix subflow items. */ tag_item = sfx_items++; for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { @@ -4491,16 +4506,34 @@ flow_meter_split_prep(struct rte_eth_dev *dev, } sfx_items->type = RTE_FLOW_ITEM_TYPE_END; sfx_items++; - tag_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; - tag_spec->data = tag_id << MLX5_MTR_COLOR_BITS; - tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); - tag_mask = tag_spec + 1; - tag_mask->data = 0xffffff00; + /* Build tag actions and items for meter_id/meter flow_id. */ + assert(tag_action); + set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; + tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; + tag_item_mask = tag_item_spec + 1; + /* + * The color Reg bits used by flow_id are growing from + * msb to lsb, so must do bit reverse for flow_id val in RegC. + */ + for (shift = 0; shift < flow_id_bits; shift++) + flow_id_val = (flow_id_val << 1) | + (((tag_id - 1) >> shift) & 0x1); + /* Both flow_id and meter_id share the same register. */ + set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 0, error); + set_tag->data = + (fm->idx | (flow_id_val << (mtr_reg_bits - flow_id_bits))) + << mtr_id_offset; + tag_item_spec->id = set_tag->id; + tag_item_spec->data = set_tag->data; + tag_item_mask->data = UINT32_MAX << mtr_id_offset; + tag_action->type = (enum rte_flow_action_type) + MLX5_RTE_FLOW_ACTION_TYPE_TAG; + tag_action->conf = set_tag; tag_item->type = (enum rte_flow_item_type) - MLX5_RTE_FLOW_ITEM_TYPE_TAG; - tag_item->spec = tag_spec; + MLX5_RTE_FLOW_ITEM_TYPE_TAG; + tag_item->spec = tag_item_spec; tag_item->last = NULL; - tag_item->mask = tag_mask; + tag_item->mask = tag_item_mask; return tag_id; } @@ -5200,25 +5233,49 @@ flow_create_split_meter(struct rte_eth_dev *dev, struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); struct rte_flow_action *sfx_actions = NULL; struct rte_flow_action *pre_actions = NULL; struct rte_flow_item *sfx_items = NULL; struct mlx5_flow *dev_flow = NULL; struct rte_flow_attr sfx_attr = *attr; - uint32_t mtr = 0; + struct mlx5_flow_meter *fm = NULL; + bool has_mtr = false; + uint32_t meter_id; uint32_t mtr_tag_id = 0; size_t act_size; size_t item_size; int actions_n = 0; - int ret; + int ret = 0; if (priv->mtr_en) - actions_n = flow_check_meter_action(actions, &mtr); - if (mtr) { - /* The five prefix actions: meter, decap, encap, tag, end. */ + actions_n = flow_check_meter_action(actions, &has_mtr, + &meter_id); + if (has_mtr) { + if (flow->meter) { + fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], + flow->meter); + if (!fm) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "Meter not found."); + } else { + fm = mlx5_flow_meter_find(priv, meter_id); + if (!fm) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "Meter not found."); + ret = mlx5_flow_meter_attach(priv, fm, + &sfx_attr, error); + if (ret) + return -rte_errno; + flow->meter = fm->idx; + } + wks->fm = fm; + /* The prefix actions: meter, decap, encap, tag, end. */ act_size = sizeof(struct rte_flow_action) * (actions_n + 5) + sizeof(struct mlx5_rte_flow_action_set_tag); - /* tag, vlan, port id, end. */ + /* The suffix items: tag, vlan, port id, end. */ #define METER_SUFFIX_ITEM 4 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + sizeof(struct mlx5_rte_flow_item_tag) * 2; @@ -5232,9 +5289,9 @@ flow_create_split_meter(struct rte_eth_dev *dev, sfx_items = (struct rte_flow_item *)((char *)sfx_actions + act_size); pre_actions = sfx_actions + actions_n; - mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items, + mtr_tag_id = flow_meter_split_prep(dev, fm, items, sfx_items, actions, sfx_actions, - pre_actions); + pre_actions, error); if (!mtr_tag_id) { ret = -rte_errno; goto exit; @@ -5245,10 +5302,12 @@ flow_create_split_meter(struct rte_eth_dev *dev, attr, items, pre_actions, flow_split_info, error); if (ret) { + mlx5_ipool_free(fm->flow_ipool, mtr_tag_id); ret = -rte_errno; goto exit; } dev_flow->handle->split_flow_id = mtr_tag_id; + dev_flow->handle->is_meter_flow_id = 1; /* Setting the sfx group atrr. */ sfx_attr.group = sfx_attr.transfer ? (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : @@ -6520,20 +6579,17 @@ mlx5_flow_ops_get(struct rte_eth_dev *dev __rte_unused, * * @param[in] dev * Pointer to Ethernet device. - * @param[in] fm - * Pointer to the flow meter. * * @return * Pointer to table set on success, NULL otherwise. */ struct mlx5_meter_domains_infos * -mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm) +mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev) { const struct mlx5_flow_driver_ops *fops; fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); - return fops->create_mtr_tbls(dev, fm); + return fops->create_mtr_tbls(dev); } /** @@ -6558,7 +6614,7 @@ mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, } /** - * Create policer rules. + * Prepare policer rules. * * @param[in] dev * Pointer to Ethernet device. @@ -6571,14 +6627,14 @@ mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, * 0 on success, -1 otherwise. */ int -mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, +mlx5_flow_prepare_policer_rules(struct rte_eth_dev *dev, struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr) { const struct mlx5_flow_driver_ops *fops; fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); - return fops->create_policer_rules(dev, fm, attr); + return fops->prepare_policer_rules(dev, fm, attr); } /** diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index 8324e188e1..c92e746a04 100644 --- a/drivers/net/mlx5/mlx5_flow.h +++ b/drivers/net/mlx5/mlx5_flow.h @@ -79,7 +79,7 @@ enum mlx5_feature_name { MLX5_APP_TAG, MLX5_COPY_MARK, MLX5_MTR_COLOR, - MLX5_MTR_SFX, + MLX5_MTR_ID, MLX5_ASO_FLOW_HIT, }; @@ -655,7 +655,8 @@ struct mlx5_flow_handle { uint64_t layers; /**< Bit-fields of present layers, see MLX5_FLOW_LAYER_*. */ void *drv_flow; /**< pointer to driver flow object. */ - uint32_t split_flow_id:28; /**< Sub flow unique match flow id. */ + uint32_t split_flow_id:27; /**< Sub flow unique match flow id. */ + uint32_t is_meter_flow_id:1; /**< Indate if flow_id is for meter. */ uint32_t mark:1; /**< Metadate rxq mark flag. */ uint32_t fate_action:3; /**< Fate action type. */ union { @@ -842,14 +843,16 @@ struct mlx5_meter_domain_info { /**< Meter table. */ struct mlx5_flow_tbl_resource *sfx_tbl; /**< Meter suffix table. */ - void *any_matcher; - /**< Meter color not match default criteria. */ - void *color_matcher; - /**< Meter color match criteria. */ + struct mlx5_flow_dv_matcher *drop_matcher; + /**< Matcher for Drop. */ + struct mlx5_flow_dv_matcher *color_matcher; + /**< Matcher for Color. */ void *jump_actn; /**< Meter match action. */ - void *policer_rules[RTE_MTR_DROPPED + 1]; - /**< Meter policer for the match. */ + void *green_rule; + /**< Meter green rule. */ + void *drop_rule; + /**< Meter drop rule. */ }; /* Meter table set for TX RX FDB. */ @@ -862,10 +865,10 @@ struct mlx5_meter_domains_infos { /**< RX meter table. */ struct mlx5_meter_domain_info transfer; /**< FDB meter table. */ - void *drop_actn; - /**< Drop action as not matched. */ - void *count_actns[RTE_MTR_DROPPED + 1]; - /**< Counters for match and unmatched statistics. */ + void *green_count; + /**< Counters for green rule. */ + void *drop_count; + /**< Counters for green rule. */ uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)]; /**< Flow meter parameter. */ size_t fmp_size; @@ -928,6 +931,8 @@ struct mlx5_flow_meter { /**< Meter state. */ uint32_t shared:1; /**< Meter shared or not. */ + struct mlx5_indexed_pool *flow_ipool; + /**< Index pool for flow id. */ }; /* RFC2697 parameter structure. */ @@ -1148,6 +1153,7 @@ struct mlx5_flow_workspace { struct mlx5_flow_rss_desc rss_desc; uint32_t rssq_num; /* Allocated queue num in rss_desc. */ uint32_t flow_idx; /* Intermediate device flow index. */ + struct mlx5_flow_meter *fm; /* Pointer to the meter in flow. */ }; struct mlx5_flow_split_info { @@ -1188,8 +1194,7 @@ typedef int (*mlx5_flow_query_t)(struct rte_eth_dev *dev, void *data, struct rte_flow_error *error); typedef struct mlx5_meter_domains_infos *(*mlx5_flow_create_mtr_tbls_t) - (struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm); + (struct rte_eth_dev *dev); typedef int (*mlx5_flow_destroy_mtr_tbls_t)(struct rte_eth_dev *dev, struct mlx5_meter_domains_infos *tbls); typedef int (*mlx5_flow_create_policer_rules_t) @@ -1252,7 +1257,7 @@ struct mlx5_flow_driver_ops { mlx5_flow_query_t query; mlx5_flow_create_mtr_tbls_t create_mtr_tbls; mlx5_flow_destroy_mtr_tbls_t destroy_mtr_tbls; - mlx5_flow_create_policer_rules_t create_policer_rules; + mlx5_flow_create_policer_rules_t prepare_policer_rules; mlx5_flow_destroy_policer_rules_t destroy_policer_rules; mlx5_flow_counter_alloc_t counter_alloc; mlx5_flow_counter_free_t counter_free; @@ -1452,11 +1457,10 @@ int mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, const struct rte_flow_item_ecpri *acc_mask, struct rte_flow_error *error); struct mlx5_meter_domains_infos *mlx5_flow_create_mtr_tbls - (struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm); + (struct rte_eth_dev *dev); int mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, struct mlx5_meter_domains_infos *tbl); -int mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, +int mlx5_flow_prepare_policer_rules(struct rte_eth_dev *dev, struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr); int mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 23e5849783..ce9857d3f7 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -10982,14 +10982,12 @@ flow_dv_translate(struct rte_eth_dev *dev, const struct rte_flow_action_rss *rss; const struct rte_flow_action *action = actions; const uint8_t *rss_key; - const struct rte_flow_action_meter *mtr; struct mlx5_flow_tbl_resource *tbl; struct mlx5_aso_age_action *age_act; uint32_t port_id = 0; struct mlx5_flow_dv_port_id_action_resource port_id_resource; int action_type = actions->type; const struct rte_flow_action *found_action = NULL; - struct mlx5_flow_meter *fm = NULL; uint32_t jump_group = 0; if (!mlx5_flow_os_action_supported(action_type)) @@ -11414,33 +11412,13 @@ flow_dv_translate(struct rte_eth_dev *dev, MLX5_FLOW_FATE_DEFAULT_MISS; break; case RTE_FLOW_ACTION_TYPE_METER: - mtr = actions->conf; - if (!flow->meter) { - fm = mlx5_flow_meter_attach(priv, mtr->mtr_id, - attr, error); - if (!fm) - return rte_flow_error_set(error, - rte_errno, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "meter not found " - "or invalid parameters"); - flow->meter = fm->idx; - } + if (!wks->fm) + return rte_flow_error_set(error, rte_errno, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "Failed to get meter in flow."); /* Set the meter action. */ - if (!fm) { - fm = mlx5_ipool_get(priv->sh->ipool - [MLX5_IPOOL_MTR], flow->meter); - if (!fm) - return rte_flow_error_set(error, - rte_errno, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "meter not found " - "or invalid parameters"); - } dev_flow->dv.actions[actions_n++] = - fm->mfts->meter_action; + wks->fm->mfts->meter_action; action_flags |= MLX5_FLOW_ACTION_METER; break; case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: @@ -12536,6 +12514,7 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) { struct mlx5_flow_handle *dev_handle; struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_meter *fm = NULL; uint32_t srss = 0; if (!flow) @@ -12546,8 +12525,6 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) flow->counter = 0; } if (flow->meter) { - struct mlx5_flow_meter *fm; - fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], flow->meter); if (fm) @@ -12589,6 +12566,10 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) flow_dv_fate_resource_release(dev, dev_handle); else if (!srss) srss = dev_handle->rix_srss; + if (fm && dev_handle->is_meter_flow_id && + dev_handle->split_flow_id) + mlx5_ipool_free(fm->flow_ipool, + dev_handle->split_flow_id); mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], tmp_idx); } @@ -13274,49 +13255,20 @@ flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev, if (!mtd || !priv->config.dv_flow_en) return 0; - if (mtd->ingress.policer_rules[RTE_MTR_DROPPED]) - claim_zero(mlx5_flow_os_destroy_flow - (mtd->ingress.policer_rules[RTE_MTR_DROPPED])); - if (mtd->egress.policer_rules[RTE_MTR_DROPPED]) - claim_zero(mlx5_flow_os_destroy_flow - (mtd->egress.policer_rules[RTE_MTR_DROPPED])); - if (mtd->transfer.policer_rules[RTE_MTR_DROPPED]) - claim_zero(mlx5_flow_os_destroy_flow - (mtd->transfer.policer_rules[RTE_MTR_DROPPED])); - if (mtd->egress.color_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->egress.color_matcher)); - if (mtd->egress.any_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->egress.any_matcher)); if (mtd->egress.tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->egress.tbl); if (mtd->egress.sfx_tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->egress.sfx_tbl); - if (mtd->ingress.color_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->ingress.color_matcher)); - if (mtd->ingress.any_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->ingress.any_matcher)); if (mtd->ingress.tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->ingress.tbl); if (mtd->ingress.sfx_tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->ingress.sfx_tbl); - if (mtd->transfer.color_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->transfer.color_matcher)); - if (mtd->transfer.any_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->transfer.any_matcher)); if (mtd->transfer.tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->transfer.tbl); if (mtd->transfer.sfx_tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->transfer.sfx_tbl); - if (mtd->drop_actn) - claim_zero(mlx5_flow_os_destroy_flow_action(mtd->drop_actn)); mlx5_free(mtd); return 0; } @@ -13335,37 +13287,17 @@ flow_dv_destroy_mtr_tbl(struct rte_eth_dev *dev, * Table attribute. * @param[in] transfer * Table attribute. - * @param[in] color_reg_c_idx - * Reg C index for color match. * * @return - * 0 on success, -1 otherwise and rte_errno is set. + * 0 on success, -1 otherwise. */ static int flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, struct mlx5_meter_domains_infos *mtb, - uint8_t egress, uint8_t transfer, - uint32_t color_reg_c_idx) + uint8_t egress, uint8_t transfer) { - struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_dev_ctx_shared *sh = priv->sh; - struct mlx5_flow_dv_match_params mask = { - .size = sizeof(mask.buf), - }; - struct mlx5_flow_dv_match_params value = { - .size = sizeof(value.buf), - }; - struct mlx5dv_flow_matcher_attr dv_attr = { - .type = IBV_FLOW_ATTR_NORMAL, - .priority = 0, - .match_criteria_enable = 0, - .match_mask = (void *)&mask, - }; - void *actions[METER_ACTIONS]; - struct mlx5_meter_domain_info *dtb; struct rte_flow_error error; - int i = 0; - int ret; + struct mlx5_meter_domain_info *dtb; if (transfer) dtb = &mtb->transfer; @@ -13390,41 +13322,7 @@ flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, DRV_LOG(ERR, "Failed to create meter suffix table."); return -1; } - /* Create matchers, Any and Color. */ - dv_attr.priority = 3; - dv_attr.match_criteria_enable = 0; - ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj, - &dtb->any_matcher); - if (ret) { - DRV_LOG(ERR, "Failed to create meter" - " policer default matcher."); - goto error_exit; - } - dv_attr.priority = 0; - dv_attr.match_criteria_enable = - 1 << MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT; - flow_dv_match_meta_reg(mask.buf, value.buf, color_reg_c_idx, - rte_col_2_mlx5_col(RTE_COLORS), UINT8_MAX); - ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj, - &dtb->color_matcher); - if (ret) { - DRV_LOG(ERR, "Failed to create meter policer color matcher."); - goto error_exit; - } - if (mtb->count_actns[RTE_MTR_DROPPED]) - actions[i++] = mtb->count_actns[RTE_MTR_DROPPED]; - actions[i++] = mtb->drop_actn; - /* Default rule: lowest priority, match any, actions: drop. */ - ret = mlx5_flow_os_create_flow(dtb->any_matcher, (void *)&value, i, - actions, - &dtb->policer_rules[RTE_MTR_DROPPED]); - if (ret) { - DRV_LOG(ERR, "Failed to create meter policer drop rule."); - goto error_exit; - } return 0; -error_exit: - return -1; } /** @@ -13433,20 +13331,16 @@ flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, * * @param[in] dev * Pointer to Ethernet device. - * @param[in] fm - * Pointer to the flow meter. * * @return * Pointer to table set on success, NULL otherwise and rte_errno is set. */ static struct mlx5_meter_domains_infos * -flow_dv_create_mtr_tbl(struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm) +flow_dv_create_mtr_tbl(struct rte_eth_dev *dev) { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_meter_domains_infos *mtb; int ret; - int i; if (!priv->mtr_en) { rte_errno = ENOTSUP; @@ -13457,37 +13351,21 @@ flow_dv_create_mtr_tbl(struct rte_eth_dev *dev, DRV_LOG(ERR, "Failed to allocate memory for meter."); return NULL; } - /* Create meter count actions */ - for (i = 0; i <= RTE_MTR_DROPPED; i++) { - struct mlx5_flow_counter *cnt; - if (!fm->policer_stats.cnt[i]) - continue; - cnt = flow_dv_counter_get_by_idx(dev, - fm->policer_stats.cnt[i], NULL); - mtb->count_actns[i] = cnt->action; - } - /* Create drop action. */ - ret = mlx5_flow_os_create_flow_action_drop(&mtb->drop_actn); - if (ret) { - DRV_LOG(ERR, "Failed to create drop action."); - goto error_exit; - } /* Egress meter table. */ - ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0, priv->mtr_color_reg); + ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0); if (ret) { DRV_LOG(ERR, "Failed to prepare egress meter table."); goto error_exit; } /* Ingress meter table. */ - ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0, priv->mtr_color_reg); + ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0); if (ret) { DRV_LOG(ERR, "Failed to prepare ingress meter table."); goto error_exit; } /* FDB meter table. */ if (priv->config.dv_esw_en) { - ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1, - priv->mtr_color_reg); + ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1); if (ret) { DRV_LOG(ERR, "Failed to prepare fdb meter table."); goto error_exit; @@ -13499,24 +13377,153 @@ flow_dv_create_mtr_tbl(struct rte_eth_dev *dev, return NULL; } +/** + * Destroy the meter table matchers. + * Lock free, (mutex should be acquired by caller). + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in,out] dtb + * Pointer to DV meter table. + * + * @return + * Always 0. + */ +static int +flow_dv_destroy_mtr_matchers(struct rte_eth_dev *dev, + struct mlx5_meter_domain_info *dtb) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_tbl_data_entry *tbl; + + if (!priv->config.dv_flow_en) + return 0; + if (dtb->drop_matcher) { + tbl = container_of(dtb->drop_matcher->tbl, typeof(*tbl), tbl); + mlx5_cache_unregister(&tbl->matchers, + &dtb->drop_matcher->entry); + dtb->drop_matcher = NULL; + } + if (dtb->color_matcher) { + tbl = container_of(dtb->color_matcher->tbl, typeof(*tbl), tbl); + mlx5_cache_unregister(&tbl->matchers, + &dtb->color_matcher->entry); + dtb->color_matcher = NULL; + } + return 0; +} + +/** + * Create the matchers for meter table. + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in] color_reg_c_idx + * Reg C index for color match. + * @param[in] mtr_id_reg_c_idx + * Reg C index for meter_id match. + * @param[in] mtr_id_mask + * Mask for meter_id match criteria. + * @param[in,out] dtb + * Pointer to DV meter table. + * @param[out] error + * Perform verbose error reporting if not NULL. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_prepare_mtr_matchers(struct rte_eth_dev *dev, + uint32_t color_reg_c_idx, + uint32_t mtr_id_reg_c_idx, + uint32_t mtr_id_mask, + struct mlx5_meter_domain_info *dtb, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_tbl_data_entry *tbl_data; + struct mlx5_cache_entry *entry; + struct mlx5_flow_dv_matcher matcher = { + .mask = { + .size = sizeof(matcher.mask.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), + }, + .tbl = dtb->tbl, + }; + struct mlx5_flow_dv_match_params value = { + .size = sizeof(value.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), + }; + struct mlx5_flow_cb_ctx ctx = { + .error = error, + .data = &matcher, + }; + + tbl_data = container_of(dtb->tbl, struct mlx5_flow_tbl_data_entry, tbl); + if (!dtb->drop_matcher) { + /* Create matchers for Drop. */ + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + mtr_id_reg_c_idx, 0, mtr_id_mask); + matcher.priority = MLX5_REG_BITS * 2 - priv->max_mtr_bits; + matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf, + matcher.mask.size); + entry = mlx5_cache_register(&tbl_data->matchers, &ctx); + if (!entry) { + DRV_LOG(ERR, "Failed to register meter drop matcher."); + return -1; + } + dtb->drop_matcher = + container_of(entry, struct mlx5_flow_dv_matcher, entry); + } + if (!dtb->color_matcher) { + /* Create matchers for Color + meter_id. */ + if (priv->mtr_reg_share) { + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + color_reg_c_idx, 0, + (mtr_id_mask | + LS32_MASK(MLX5_MTR_COLOR_BITS))); + } else { + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + color_reg_c_idx, 0, + LS32_MASK(MLX5_MTR_COLOR_BITS)); + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + mtr_id_reg_c_idx, 0, mtr_id_mask); + } + matcher.priority = MLX5_REG_BITS - priv->max_mtr_bits; + matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf, + matcher.mask.size); + entry = mlx5_cache_register(&tbl_data->matchers, &ctx); + if (!entry) { + DRV_LOG(ERR, "Failed to register meter color matcher."); + return -1; + } + dtb->color_matcher = + container_of(entry, struct mlx5_flow_dv_matcher, entry); + } + return 0; +} + /** * Destroy domain policer rule. * + * @param[in] dev + * Pointer to Ethernet device. * @param[in] dt * Pointer to domain table. */ static void -flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt) +flow_dv_destroy_domain_policer_rule(struct rte_eth_dev *dev, + struct mlx5_meter_domain_info *dt) { - int i; - - for (i = 0; i < RTE_MTR_DROPPED; i++) { - if (dt->policer_rules[i]) { - claim_zero(mlx5_flow_os_destroy_flow - (dt->policer_rules[i])); - dt->policer_rules[i] = NULL; - } + if (dt->drop_rule) { + claim_zero(mlx5_flow_os_destroy_flow(dt->drop_rule)); + dt->drop_rule = NULL; + } + if (dt->green_rule) { + claim_zero(mlx5_flow_os_destroy_flow(dt->green_rule)); + dt->green_rule = NULL; } + flow_dv_destroy_mtr_matchers(dev, dt); if (dt->jump_actn) { claim_zero(mlx5_flow_os_destroy_flow_action(dt->jump_actn)); dt->jump_actn = NULL; @@ -13537,7 +13544,7 @@ flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt) * Always 0. */ static int -flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused, +flow_dv_destroy_policer_rules(struct rte_eth_dev *dev, const struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr) { @@ -13546,39 +13553,55 @@ flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused, if (!mtb) return 0; if (attr->egress) - flow_dv_destroy_domain_policer_rule(&mtb->egress); + flow_dv_destroy_domain_policer_rule(dev, &mtb->egress); if (attr->ingress) - flow_dv_destroy_domain_policer_rule(&mtb->ingress); + flow_dv_destroy_domain_policer_rule(dev, &mtb->ingress); if (attr->transfer) - flow_dv_destroy_domain_policer_rule(&mtb->transfer); + flow_dv_destroy_domain_policer_rule(dev, &mtb->transfer); return 0; } /** * Create specify domain meter policer rule. * + * @param[in] dev + * Pointer to Ethernet device. * @param[in] fm * Pointer to flow meter structure. * @param[in] mtb * Pointer to DV meter table set. - * @param[in] mtr_reg_c - * Color match REG_C. + * @param[out] drop_rule + * The address of pointer saving drop rule. + * @param[out] color_rule + * The address of pointer saving green rule. * * @return * 0 on success, -1 otherwise. */ static int -flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, +flow_dv_create_policer_forward_rule(struct rte_eth_dev *dev, + struct mlx5_flow_meter *fm, struct mlx5_meter_domain_info *dtb, - uint8_t mtr_reg_c) + void **drop_rule, + void **green_rule) { + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flow_dv_match_params matcher = { - .size = sizeof(matcher.buf), + .size = sizeof(matcher.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), }; struct mlx5_flow_dv_match_params value = { - .size = sizeof(value.buf), + .size = sizeof(value.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), }; struct mlx5_meter_domains_infos *mtb = fm->mfts; + struct rte_flow_error error; + uint32_t color_reg_c = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, + 0, &error); + uint32_t mtr_id_reg_c = mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, + 0, &error); + uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; + uint32_t mtr_id_mask = LS32_MASK(priv->max_mtr_bits) << mtr_id_offset; void *actions[METER_ACTIONS]; int i; int ret = 0; @@ -13591,25 +13614,52 @@ flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, DRV_LOG(ERR, "Failed to create policer jump action."); goto error; } - for (i = 0; i < RTE_MTR_DROPPED; i++) { - int j = 0; - - flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_reg_c, - rte_col_2_mlx5_col(i), UINT8_MAX); - if (mtb->count_actns[i]) - actions[j++] = mtb->count_actns[i]; - if (fm->action[i] == MTR_POLICER_ACTION_DROP) - actions[j++] = mtb->drop_actn; - else - actions[j++] = dtb->jump_actn; - ret = mlx5_flow_os_create_flow(dtb->color_matcher, - (void *)&value, j, actions, - &dtb->policer_rules[i]); + /* Prepare matchers. */ + if (!dtb->drop_matcher || !dtb->color_matcher) { + ret = flow_dv_prepare_mtr_matchers(dev, color_reg_c, + mtr_id_reg_c, mtr_id_mask, + dtb, &error); if (ret) { - DRV_LOG(ERR, "Failed to create policer rule."); + DRV_LOG(ERR, "Failed to setup matchers for mtr table."); goto error; } } + /* Create Drop flow, matching meter_id only. */ + i = 0; + flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_id_reg_c, + (fm->idx << mtr_id_offset), UINT32_MAX); + if (mtb->drop_count) + actions[i++] = mtb->drop_count; + actions[i++] = priv->sh->esw_drop_action; + ret = mlx5_flow_os_create_flow(dtb->drop_matcher->matcher_object, + (void *)&value, i, actions, drop_rule); + if (ret) { + DRV_LOG(ERR, "Failed to create meter policer drop rule."); + goto error; + } + /* Create flow matching Green color + meter_id. */ + i = 0; + if (priv->mtr_reg_share) { + flow_dv_match_meta_reg(matcher.buf, value.buf, color_reg_c, + ((fm->idx << mtr_id_offset) | + rte_col_2_mlx5_col(RTE_COLOR_GREEN)), + UINT32_MAX); + } else { + flow_dv_match_meta_reg(matcher.buf, value.buf, color_reg_c, + rte_col_2_mlx5_col(RTE_COLOR_GREEN), + UINT32_MAX); + flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_id_reg_c, + fm->idx, UINT32_MAX); + } + if (mtb->green_count) + actions[i++] = mtb->green_count; + actions[i++] = dtb->jump_actn; + ret = mlx5_flow_os_create_flow(dtb->color_matcher->matcher_object, + (void *)&value, i, actions, green_rule); + if (ret) { + DRV_LOG(ERR, "Failed to create meter policer color rule."); + goto error; + } return 0; error: rte_errno = errno; @@ -13617,7 +13667,8 @@ flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, } /** - * Create policer rules. + * Prepare policer rules for all domains. + * If meter already initialized, this will replace all old rules with new ones. * * @param[in] dev * Pointer to Ethernet device. @@ -13630,41 +13681,107 @@ flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, * 0 on success, -1 otherwise. */ static int -flow_dv_create_policer_rules(struct rte_eth_dev *dev, - struct mlx5_flow_meter *fm, - const struct rte_flow_attr *attr) +flow_dv_prepare_policer_rules(struct rte_eth_dev *dev, + struct mlx5_flow_meter *fm, + const struct rte_flow_attr *attr) { - struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_meter_domains_infos *mtb = fm->mfts; + bool initialized = false; + struct mlx5_flow_counter *cnt; + void *egress_drop_rule = NULL; + void *egress_green_rule = NULL; + void *ingress_drop_rule = NULL; + void *ingress_green_rule = NULL; + void *transfer_drop_rule = NULL; + void *transfer_green_rule = NULL; int ret; + /* Get the statistics counters for green/drop. */ + if (fm->policer_stats.cnt[RTE_COLOR_GREEN]) { + cnt = flow_dv_counter_get_by_idx(dev, + fm->policer_stats.cnt[RTE_COLOR_GREEN], + NULL); + mtb->green_count = cnt->action; + } else { + mtb->green_count = NULL; + } + if (fm->policer_stats.cnt[RTE_MTR_DROPPED]) { + cnt = flow_dv_counter_get_by_idx(dev, + fm->policer_stats.cnt[RTE_MTR_DROPPED], + NULL); + mtb->drop_count = cnt->action; + } else { + mtb->drop_count = NULL; + } + /** + * If flow meter has been initilized, all policer rules + * are created. So can get if meter initialized by checking + * any policer rule. + */ + if (mtb->egress.drop_rule) + initialized = true; if (attr->egress) { - ret = flow_dv_create_policer_forward_rule(fm, &mtb->egress, - priv->mtr_color_reg); + ret = flow_dv_create_policer_forward_rule(dev, + fm, &mtb->egress, + &egress_drop_rule, &egress_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create egress policer."); goto error; } } if (attr->ingress) { - ret = flow_dv_create_policer_forward_rule(fm, &mtb->ingress, - priv->mtr_color_reg); + ret = flow_dv_create_policer_forward_rule(dev, + fm, &mtb->ingress, + &ingress_drop_rule, &ingress_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create ingress policer."); goto error; } } if (attr->transfer) { - ret = flow_dv_create_policer_forward_rule(fm, &mtb->transfer, - priv->mtr_color_reg); + ret = flow_dv_create_policer_forward_rule(dev, + fm, &mtb->transfer, + &transfer_drop_rule, &transfer_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create transfer policer."); goto error; } } + /* Replace old flows if existing. */ + if (mtb->egress.drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->egress.drop_rule)); + if (mtb->egress.green_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->egress.green_rule)); + if (mtb->ingress.drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->ingress.drop_rule)); + if (mtb->ingress.green_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->ingress.green_rule)); + if (mtb->transfer.drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->transfer.drop_rule)); + if (mtb->transfer.green_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->transfer.green_rule)); + mtb->egress.drop_rule = egress_drop_rule; + mtb->egress.green_rule = egress_green_rule; + mtb->ingress.drop_rule = ingress_drop_rule; + mtb->ingress.green_rule = ingress_green_rule; + mtb->transfer.drop_rule = transfer_drop_rule; + mtb->transfer.green_rule = transfer_green_rule; return 0; error: - flow_dv_destroy_policer_rules(dev, fm, attr); + if (egress_drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(egress_drop_rule)); + if (egress_green_rule) + claim_zero(mlx5_flow_os_destroy_flow(egress_green_rule)); + if (ingress_drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(ingress_drop_rule)); + if (ingress_green_rule) + claim_zero(mlx5_flow_os_destroy_flow(ingress_green_rule)); + if (transfer_drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(transfer_drop_rule)); + if (transfer_green_rule) + claim_zero(mlx5_flow_os_destroy_flow(transfer_green_rule)); + if (!initialized) + flow_dv_destroy_policer_rules(dev, fm, attr); return -1; } @@ -13960,7 +14077,7 @@ const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = { .query = flow_dv_query, .create_mtr_tbls = flow_dv_create_mtr_tbl, .destroy_mtr_tbls = flow_dv_destroy_mtr_tbl, - .create_policer_rules = flow_dv_create_policer_rules, + .prepare_policer_rules = flow_dv_prepare_policer_rules, .destroy_policer_rules = flow_dv_destroy_policer_rules, .counter_alloc = flow_dv_counter_allocate, .counter_free = flow_dv_counter_free, diff --git a/drivers/net/mlx5/mlx5_flow_meter.c b/drivers/net/mlx5/mlx5_flow_meter.c index dbc574b508..68f5c344cf 100644 --- a/drivers/net/mlx5/mlx5_flow_meter.c +++ b/drivers/net/mlx5/mlx5_flow_meter.c @@ -474,6 +474,12 @@ mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id, MTR_POLICER_ACTION_COLOR_RED }; int i; + /* Meter must use global drop action. */ + if (!priv->sh->esw_drop_action) + return -rte_mtr_error_set(error, ENOTSUP, + RTE_MTR_ERROR_TYPE_MTR_PARAMS, + NULL, + "No drop action ready for meter."); /* Meter params must not be NULL. */ if (params == NULL) return -rte_mtr_error_set(error, EINVAL, @@ -630,9 +636,18 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, .egress = 1, .transfer = priv->config.dv_esw_en ? 1 : 0, }; + struct mlx5_indexed_pool_config flow_ipool_cfg = { + .size = 0, + .trunk_size = 64, + .need_lock = 1, + .type = "mlx5_flow_mtr_flow_id_pool", + }; int ret; unsigned int i; uint32_t idx = 0; + uint8_t mtr_id_bits; + uint8_t mtr_reg_bits = priv->mtr_reg_share ? + MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; if (!priv->mtr_en) return -rte_mtr_error_set(error, ENOTSUP, @@ -654,6 +669,13 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, return -rte_mtr_error_set(error, ENOMEM, RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, "Memory alloc failed for meter."); + mtr_id_bits = MLX5_REG_BITS - __builtin_clz(idx); + if ((mtr_id_bits + priv->max_mtr_flow_bits) > mtr_reg_bits) { + DRV_LOG(ERR, "Meter number exceeds max limit."); + goto error; + } + if (mtr_id_bits > priv->max_mtr_bits) + priv->max_mtr_bits = mtr_id_bits; fm->idx = idx; /* Fill the flow meter parameters. */ fm->meter_id = meter_id; @@ -667,10 +689,10 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, if (!fm->policer_stats.cnt[i]) goto error; } - fm->mfts = mlx5_flow_create_mtr_tbls(dev, fm); + fm->mfts = mlx5_flow_create_mtr_tbls(dev); if (!fm->mfts) goto error; - ret = mlx5_flow_create_policer_rules(dev, fm, &attr); + ret = mlx5_flow_prepare_policer_rules(dev, fm, &attr); if (ret) goto error; /* Add to the flow meter list. */ @@ -679,6 +701,9 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, fm->shared = !!shared; fm->policer_stats.stats_mask = params->stats_mask; fm->profile->ref_cnt++; + fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg); + if (!fm->flow_ipool) + goto error; rte_spinlock_init(&fm->sl); return 0; error: @@ -749,6 +774,8 @@ mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id, if (fm->policer_stats.cnt[i]) mlx5_counter_free(dev, fm->policer_stats.cnt[i]); /* Free meter flow table */ + if (fm->flow_ipool) + mlx5_ipool_destroy(fm->flow_ipool); mlx5_flow_destroy_policer_rules(dev, fm, &attr); mlx5_flow_destroy_mtr_tbls(dev, fm->mfts); mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], fm->idx); @@ -1153,30 +1180,24 @@ mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id) * * @param [in] priv * Pointer to mlx5 private data. - * @param [in] meter_id - * Flow meter id. + * @param[in] fm + * Pointer to flow meter. * @param [in] attr * Pointer to flow attributes. * @param [out] error * Pointer to error structure. * - * @return the flow meter pointer, NULL otherwise. + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. */ -struct mlx5_flow_meter * -mlx5_flow_meter_attach(struct mlx5_priv *priv, uint32_t meter_id, +int +mlx5_flow_meter_attach(struct mlx5_priv *priv, + struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr, struct rte_flow_error *error) { - struct mlx5_flow_meter *fm; int ret = 0; - fm = mlx5_flow_meter_find(priv, meter_id); - if (fm == NULL) { - rte_flow_error_set(error, ENOENT, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "Meter object id not valid"); - return fm; - } rte_spinlock_lock(&fm->sl); if (fm->mfts->meter_action) { if (fm->shared && @@ -1210,7 +1231,7 @@ mlx5_flow_meter_attach(struct mlx5_priv *priv, uint32_t meter_id, fm->mfts->meter_action ? "Meter attr not match" : "Meter action create failed"); - return ret ? NULL : fm; + return ret ? -rte_errno : 0; } /** -- 2.27.0