From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 8359FA0548 for ; Tue, 20 Apr 2021 12:55:39 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 6E7D741771; Tue, 20 Apr 2021 12:55:39 +0200 (CEST) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by mails.dpdk.org (Postfix) with ESMTP id ECE5241726 for ; Tue, 20 Apr 2021 12:55:25 +0200 (CEST) Received: from Internal Mail-Server by MTLPINE1 (envelope-from jiaweiw@nvidia.com) with SMTP; 20 Apr 2021 13:55:23 +0300 Received: from nvidia.com (gen-l-vrt-281.mtl.labs.mlnx [10.237.44.1]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 13KAtMRp009943; Tue, 20 Apr 2021 13:55:23 +0300 From: Jiawei Wang To: matan@nvidia.com, orika@nvidia.com, viacheslavo@nvidia.com, ferruh.yigit@intel.com, thomas@monjalon.net, Shahaf Shuler , Suanming Mou Cc: dev@dpdk.org, rasland@nvidia.com, asafp@nvidia.com, Shun Hao , stable@dpdk.org Date: Tue, 20 Apr 2021 13:55:10 +0300 Message-Id: <1618916122-181792-4-git-send-email-jiaweiw@nvidia.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1618916122-181792-1-git-send-email-jiaweiw@nvidia.com> References: <20210331073632.1443011-1-lizh@nvidia.com> <1618916122-181792-1-git-send-email-jiaweiw@nvidia.com> Subject: [dpdk-stable] [PATCH v6 03/15] net/mlx5: fix meter statistics X-BeenThere: stable@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: patches for DPDK stable branches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: stable-bounces@dpdk.org Sender: "stable" From: Shun Hao Currently, packets after meter will be steered to a global policer table, which includes green/red color rules for every meter, so as to have counter statistics of each color in every meter. There's a bug that all the rules in global policer table are matching only color criteria, so all packets will be counted to one meter only, and other meter statistics are always zero. This patch does these: 1. The rules in policer table matches both meter index and color, so packet after meter could be counted to the correct meter counter. 2. The meter index and flow index are now sharing the available register bits dynamically. Meter index starts from lsb, and flow index starts from msb. Fixes: 46a5e6bc6a ("net/mlx5: prepare meter flow tables") Cc: stable@dpdk.org Signed-off-by: Shun Hao Acked-by: Matan Azrad --- drivers/net/mlx5/mlx5.c | 7 +- drivers/net/mlx5/mlx5.h | 15 +- drivers/net/mlx5/mlx5_flow.c | 183 +++++++++----- drivers/net/mlx5/mlx5_flow.h | 40 +-- drivers/net/mlx5/mlx5_flow_dv.c | 489 +++++++++++++++++++++++-------------- drivers/net/mlx5/mlx5_flow_meter.c | 53 ++-- 6 files changed, 496 insertions(+), 291 deletions(-) diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index 3bf224c..cc756ca 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -277,10 +277,13 @@ static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list = }, #endif [MLX5_IPOOL_MTR] = { + /** + * The ipool index should grow continually from small to big, + * for meter idx, so not set grow_trunk to avoid meter index + * not jump continually. + */ .size = sizeof(struct mlx5_flow_meter), .trunk_size = 64, - .grow_trunk = 3, - .grow_shift = 2, .need_lock = 1, .release_mem_en = 1, .malloc = mlx5_malloc, diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index c61f8ff..d208efb 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -953,9 +953,9 @@ struct mlx5_priv { unsigned int representor:1; /* Device is a port representor. */ unsigned int master:1; /* Device is a E-Switch master. */ unsigned int txpp_en:1; /* Tx packet pacing enabled. */ + unsigned int sampler_en:1; /* Whether support sampler. */ unsigned int mtr_en:1; /* Whether support meter. */ unsigned int mtr_reg_share:1; /* Whether support meter REG_C share. */ - unsigned int sampler_en:1; /* Whether support sampler. */ uint16_t domain_id; /* Switch domain identifier. */ uint16_t vport_id; /* Associated VF vport index (if any). */ uint32_t vport_meta_tag; /* Used for vport index match ove VF LAG. */ @@ -1012,6 +1012,10 @@ struct mlx5_priv { uint32_t rss_shared_actions; /* RSS shared actions. */ struct mlx5_devx_obj *q_counters; /* DevX queue counter object. */ uint32_t counter_set_id; /* Queue counter ID to set in DevX objects. */ + uint8_t max_mtr_bits; + /* Indicate how many bits are used by meter id at the most. */ + uint8_t max_mtr_flow_bits; + /* Indicate how many bits are used by meter flow id at the most. */ }; #define PORT_ID(priv) ((priv)->dev_data->port_id) @@ -1280,11 +1284,10 @@ int mlx5_mp_os_req_queue_control(struct rte_eth_dev *dev, uint16_t queue_id, int mlx5_flow_meter_ops_get(struct rte_eth_dev *dev, void *arg); struct mlx5_flow_meter *mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id); -struct mlx5_flow_meter *mlx5_flow_meter_attach - (struct mlx5_priv *priv, - uint32_t meter_id, - const struct rte_flow_attr *attr, - struct rte_flow_error *error); +int mlx5_flow_meter_attach(struct mlx5_priv *priv, + struct mlx5_flow_meter *fm, + const struct rte_flow_attr *attr, + struct rte_flow_error *error); void mlx5_flow_meter_detach(struct mlx5_flow_meter *fm); /* mlx5_os.c */ diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c index bc27b45..3bb04d5 100644 --- a/drivers/net/mlx5/mlx5_flow.c +++ b/drivers/net/mlx5/mlx5_flow.c @@ -748,9 +748,9 @@ struct mlx5_flow_tunnel_info { return REG_C_0; } break; - case MLX5_MTR_SFX: + case MLX5_MTR_ID: /* - * If meter color and flow match share one register, flow match + * If meter color and meter id share one register, flow match * should use the meter color register for match. */ if (priv->mtr_reg_share) @@ -3035,7 +3035,8 @@ struct mlx5_flow_tunnel_info { SILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], flow->dev_handles, handle_idx, dev_handle, next) - if (dev_handle->split_flow_id) + if (dev_handle->split_flow_id && + !dev_handle->is_meter_flow_id) mlx5_ipool_free(priv->sh->ipool [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], dev_handle->split_flow_id); @@ -3674,23 +3675,30 @@ struct mlx5_translated_action_handle { * * @param[in] actions * Pointer to the list of actions. - * @param[out] mtr + * @param[out] has_mtr * Pointer to the meter exist flag. + * @param[out] meter_id + * Pointer to the meter id. * * @return * Total number of actions. */ static int -flow_check_meter_action(const struct rte_flow_action actions[], uint32_t *mtr) +flow_check_meter_action(const struct rte_flow_action actions[], + bool *has_mtr, + uint32_t *meter_id) { + const struct rte_flow_action_meter *mtr = NULL; int actions_n = 0; - MLX5_ASSERT(mtr); - *mtr = 0; + MLX5_ASSERT(has_mtr); + *has_mtr = false; for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { switch (actions->type) { case RTE_FLOW_ACTION_TYPE_METER: - *mtr = 1; + mtr = actions->conf; + *meter_id = mtr->mtr_id; + *has_mtr = true; break; default: break; @@ -4347,8 +4355,10 @@ struct mlx5_hlist_entry * * header will be in the prefix sub flow, as not to take the * L3 tunnel header into account. * - * @param dev + * @param[in] dev * Pointer to Ethernet device. + * @param[in] fm + * Pointer to flow meter structure. * @param[in] items * Pattern specification (list terminated by the END pattern item). * @param[out] sfx_items @@ -4363,29 +4373,39 @@ struct mlx5_hlist_entry * * The pattern items for the suffix flow. * @param[out] tag_sfx * Pointer to suffix flow tag. + * @param[out] error + * Perform verbose error reporting if not NULL. * * @return - * 0 on success. + * The flow id, 0 otherwise and rte_errno is set. */ -static int +static uint32_t flow_meter_split_prep(struct rte_eth_dev *dev, - const struct rte_flow_item items[], - struct rte_flow_item sfx_items[], - const struct rte_flow_action actions[], - struct rte_flow_action actions_sfx[], - struct rte_flow_action actions_pre[]) + struct mlx5_flow_meter *fm, + const struct rte_flow_item items[], + struct rte_flow_item sfx_items[], + const struct rte_flow_action actions[], + struct rte_flow_action actions_sfx[], + struct rte_flow_action actions_pre[], + struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; struct rte_flow_action *tag_action = NULL; struct rte_flow_item *tag_item; struct mlx5_rte_flow_action_set_tag *set_tag; - struct rte_flow_error error; const struct rte_flow_action_raw_encap *raw_encap; const struct rte_flow_action_raw_decap *raw_decap; - struct mlx5_rte_flow_item_tag *tag_spec; - struct mlx5_rte_flow_item_tag *tag_mask; + struct mlx5_rte_flow_item_tag *tag_item_spec; + struct mlx5_rte_flow_item_tag *tag_item_mask; uint32_t tag_id = 0; bool copy_vlan = false; + uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; + uint8_t mtr_reg_bits = priv->mtr_reg_share ? + MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; + uint32_t flow_id = 0; + uint32_t flow_id_reversed = 0; + uint8_t flow_id_bits = 0; + int shift; /* Prepare the actions for prefix and suffix flow. */ for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { @@ -4394,10 +4414,7 @@ struct mlx5_hlist_entry * switch (actions->type) { case RTE_FLOW_ACTION_TYPE_METER: /* Add the extra tag action first. */ - tag_action = actions_pre; - tag_action->type = (enum rte_flow_action_type) - MLX5_RTE_FLOW_ACTION_TYPE_TAG; - actions_pre++; + tag_action = actions_pre++; action_cur = &actions_pre; break; case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: @@ -4430,23 +4447,22 @@ struct mlx5_hlist_entry * actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; actions_pre->type = RTE_FLOW_ACTION_TYPE_END; actions_pre++; - /* Set the tag. */ - set_tag = (void *)actions_pre; - set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); - mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], - &tag_id); - if (tag_id >= (1 << (sizeof(tag_id) * 8 - MLX5_MTR_COLOR_BITS))) { - DRV_LOG(ERR, "Port %u meter flow id exceed max limit.", - dev->data->port_id); - mlx5_ipool_free(priv->sh->ipool - [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], tag_id); - return 0; - } else if (!tag_id) { - return 0; + mlx5_ipool_malloc(fm->flow_ipool, &tag_id); + if (!tag_id) + return rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Failed to allocate meter flow id."); + flow_id = tag_id - 1; + flow_id_bits = MLX5_REG_BITS - __builtin_clz(flow_id); + flow_id_bits = flow_id_bits ? flow_id_bits : 1; + if ((flow_id_bits + priv->max_mtr_bits) > mtr_reg_bits) { + mlx5_ipool_free(fm->flow_ipool, tag_id); + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Meter flow id exceeds max limit."); } - set_tag->data = tag_id << MLX5_MTR_COLOR_BITS; - assert(tag_action); - tag_action->conf = set_tag; + if (flow_id_bits > priv->max_mtr_flow_bits) + priv->max_mtr_flow_bits = flow_id_bits; /* Prepare the suffix subflow items. */ tag_item = sfx_items++; for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) { @@ -4475,16 +4491,34 @@ struct mlx5_hlist_entry * } sfx_items->type = RTE_FLOW_ITEM_TYPE_END; sfx_items++; - tag_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; - tag_spec->data = tag_id << MLX5_MTR_COLOR_BITS; - tag_spec->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); - tag_mask = tag_spec + 1; - tag_mask->data = 0xffffff00; + /* Build tag actions and items for meter_id/meter flow_id. */ + assert(tag_action); + set_tag = (struct mlx5_rte_flow_action_set_tag *)actions_pre; + tag_item_spec = (struct mlx5_rte_flow_item_tag *)sfx_items; + tag_item_mask = tag_item_spec + 1; + /* + * The color Reg bits used by flow_id are growing from + * msb to lsb, so must do bit reverse for flow_id val in RegC. + */ + for (shift = 0; shift < flow_id_bits; shift++) + flow_id_reversed = (flow_id_reversed << 1) | + ((flow_id >> shift) & 0x1); + /* Both flow_id and meter_id share the same register. */ + set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, 0, error); + set_tag->data = + (fm->idx | (flow_id_reversed << (mtr_reg_bits - flow_id_bits))) + << mtr_id_offset; + tag_item_spec->id = set_tag->id; + tag_item_spec->data = set_tag->data; + tag_item_mask->data = UINT32_MAX << mtr_id_offset; + tag_action->type = (enum rte_flow_action_type) + MLX5_RTE_FLOW_ACTION_TYPE_TAG; + tag_action->conf = set_tag; tag_item->type = (enum rte_flow_item_type) - MLX5_RTE_FLOW_ITEM_TYPE_TAG; - tag_item->spec = tag_spec; + MLX5_RTE_FLOW_ITEM_TYPE_TAG; + tag_item->spec = tag_item_spec; tag_item->last = NULL; - tag_item->mask = tag_mask; + tag_item->mask = tag_item_mask; return tag_id; } @@ -5192,25 +5226,49 @@ struct mlx5_hlist_entry * struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace(); struct rte_flow_action *sfx_actions = NULL; struct rte_flow_action *pre_actions = NULL; struct rte_flow_item *sfx_items = NULL; struct mlx5_flow *dev_flow = NULL; struct rte_flow_attr sfx_attr = *attr; - uint32_t mtr = 0; + struct mlx5_flow_meter *fm = NULL; + bool has_mtr = false; + uint32_t meter_id; uint32_t mtr_tag_id = 0; size_t act_size; size_t item_size; int actions_n = 0; - int ret; + int ret = 0; if (priv->mtr_en) - actions_n = flow_check_meter_action(actions, &mtr); - if (mtr) { - /* The five prefix actions: meter, decap, encap, tag, end. */ + actions_n = flow_check_meter_action(actions, &has_mtr, + &meter_id); + if (has_mtr) { + if (flow->meter) { + fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], + flow->meter); + if (!fm) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "Meter not found."); + } else { + fm = mlx5_flow_meter_find(priv, meter_id); + if (!fm) + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "Meter not found."); + ret = mlx5_flow_meter_attach(priv, fm, + &sfx_attr, error); + if (ret) + return -rte_errno; + flow->meter = fm->idx; + } + wks->fm = fm; + /* The prefix actions: meter, decap, encap, tag, end. */ act_size = sizeof(struct rte_flow_action) * (actions_n + 5) + sizeof(struct mlx5_rte_flow_action_set_tag); - /* tag, vlan, port id, end. */ + /* The suffix items: tag, vlan, port id, end. */ #define METER_SUFFIX_ITEM 4 item_size = sizeof(struct rte_flow_item) * METER_SUFFIX_ITEM + sizeof(struct mlx5_rte_flow_item_tag) * 2; @@ -5224,9 +5282,9 @@ struct mlx5_hlist_entry * sfx_items = (struct rte_flow_item *)((char *)sfx_actions + act_size); pre_actions = sfx_actions + actions_n; - mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items, + mtr_tag_id = flow_meter_split_prep(dev, fm, items, sfx_items, actions, sfx_actions, - pre_actions); + pre_actions, error); if (!mtr_tag_id) { ret = -rte_errno; goto exit; @@ -5237,10 +5295,12 @@ struct mlx5_hlist_entry * attr, items, pre_actions, flow_split_info, error); if (ret) { + mlx5_ipool_free(fm->flow_ipool, mtr_tag_id); ret = -rte_errno; goto exit; } dev_flow->handle->split_flow_id = mtr_tag_id; + dev_flow->handle->is_meter_flow_id = 1; /* Setting the sfx group atrr. */ sfx_attr.group = sfx_attr.transfer ? (MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) : @@ -6512,20 +6572,17 @@ struct mlx5_flow_workspace* * * @param[in] dev * Pointer to Ethernet device. - * @param[in] fm - * Pointer to the flow meter. * * @return * Pointer to table set on success, NULL otherwise. */ struct mlx5_meter_domains_infos * -mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm) +mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev) { const struct mlx5_flow_driver_ops *fops; fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); - return fops->create_mtr_tbls(dev, fm); + return fops->create_mtr_tbls(dev); } /** @@ -6550,7 +6607,7 @@ struct mlx5_meter_domains_infos * } /** - * Create policer rules. + * Prepare policer rules. * * @param[in] dev * Pointer to Ethernet device. @@ -6563,14 +6620,14 @@ struct mlx5_meter_domains_infos * * 0 on success, -1 otherwise. */ int -mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, +mlx5_flow_prepare_policer_rules(struct rte_eth_dev *dev, struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr) { const struct mlx5_flow_driver_ops *fops; fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); - return fops->create_policer_rules(dev, fm, attr); + return fops->prepare_policer_rules(dev, fm, attr); } /** diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index a26f11a..a209b81 100644 --- a/drivers/net/mlx5/mlx5_flow.h +++ b/drivers/net/mlx5/mlx5_flow.h @@ -78,7 +78,7 @@ enum mlx5_feature_name { MLX5_APP_TAG, MLX5_COPY_MARK, MLX5_MTR_COLOR, - MLX5_MTR_SFX, + MLX5_MTR_ID, MLX5_ASO_FLOW_HIT, }; @@ -654,7 +654,8 @@ struct mlx5_flow_handle { uint64_t layers; /**< Bit-fields of present layers, see MLX5_FLOW_LAYER_*. */ void *drv_flow; /**< pointer to driver flow object. */ - uint32_t split_flow_id:28; /**< Sub flow unique match flow id. */ + uint32_t split_flow_id:27; /**< Sub flow unique match flow id. */ + uint32_t is_meter_flow_id:1; /**< Indate if flow_id is for meter. */ uint32_t mark:1; /**< Metadate rxq mark flag. */ uint32_t fate_action:3; /**< Fate action type. */ union { @@ -841,14 +842,16 @@ struct mlx5_meter_domain_info { /**< Meter table. */ struct mlx5_flow_tbl_resource *sfx_tbl; /**< Meter suffix table. */ - void *any_matcher; - /**< Meter color not match default criteria. */ - void *color_matcher; - /**< Meter color match criteria. */ + struct mlx5_flow_dv_matcher *drop_matcher; + /**< Matcher for Drop. */ + struct mlx5_flow_dv_matcher *color_matcher; + /**< Matcher for Color. */ void *jump_actn; /**< Meter match action. */ - void *policer_rules[RTE_MTR_DROPPED + 1]; - /**< Meter policer for the match. */ + void *green_rule; + /**< Meter green rule. */ + void *drop_rule; + /**< Meter drop rule. */ }; /* Meter table set for TX RX FDB. */ @@ -861,10 +864,10 @@ struct mlx5_meter_domains_infos { /**< RX meter table. */ struct mlx5_meter_domain_info transfer; /**< FDB meter table. */ - void *drop_actn; - /**< Drop action as not matched. */ - void *count_actns[RTE_MTR_DROPPED + 1]; - /**< Counters for match and unmatched statistics. */ + void *green_count; + /**< Counters for green rule. */ + void *drop_count; + /**< Counters for green rule. */ uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)]; /**< Flow meter parameter. */ size_t fmp_size; @@ -927,6 +930,8 @@ struct mlx5_flow_meter { /**< Meter state. */ uint32_t shared:1; /**< Meter shared or not. */ + struct mlx5_indexed_pool *flow_ipool; + /**< Index pool for flow id. */ }; /* RFC2697 parameter structure. */ @@ -1165,6 +1170,7 @@ struct mlx5_flow_workspace { struct mlx5_flow_rss_desc rss_desc; uint32_t rssq_num; /* Allocated queue num in rss_desc. */ uint32_t flow_idx; /* Intermediate device flow index. */ + struct mlx5_flow_meter *fm; /* Pointer to the meter in flow. */ }; struct mlx5_flow_split_info { @@ -1205,8 +1211,7 @@ typedef int (*mlx5_flow_query_t)(struct rte_eth_dev *dev, void *data, struct rte_flow_error *error); typedef struct mlx5_meter_domains_infos *(*mlx5_flow_create_mtr_tbls_t) - (struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm); + (struct rte_eth_dev *dev); typedef int (*mlx5_flow_destroy_mtr_tbls_t)(struct rte_eth_dev *dev, struct mlx5_meter_domains_infos *tbls); typedef int (*mlx5_flow_create_policer_rules_t) @@ -1269,7 +1274,7 @@ struct mlx5_flow_driver_ops { mlx5_flow_query_t query; mlx5_flow_create_mtr_tbls_t create_mtr_tbls; mlx5_flow_destroy_mtr_tbls_t destroy_mtr_tbls; - mlx5_flow_create_policer_rules_t create_policer_rules; + mlx5_flow_create_policer_rules_t prepare_policer_rules; mlx5_flow_destroy_policer_rules_t destroy_policer_rules; mlx5_flow_counter_alloc_t counter_alloc; mlx5_flow_counter_free_t counter_free; @@ -1469,11 +1474,10 @@ int mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, const struct rte_flow_item_ecpri *acc_mask, struct rte_flow_error *error); struct mlx5_meter_domains_infos *mlx5_flow_create_mtr_tbls - (struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm); + (struct rte_eth_dev *dev); int mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, struct mlx5_meter_domains_infos *tbl); -int mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, +int mlx5_flow_prepare_policer_rules(struct rte_eth_dev *dev, struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr); int mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 0f215df..d4d88aa 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -10997,14 +10997,12 @@ struct mlx5_cache_entry * const struct rte_flow_action_rss *rss; const struct rte_flow_action *action = actions; const uint8_t *rss_key; - const struct rte_flow_action_meter *mtr; struct mlx5_flow_tbl_resource *tbl; struct mlx5_aso_age_action *age_act; uint32_t port_id = 0; struct mlx5_flow_dv_port_id_action_resource port_id_resource; int action_type = actions->type; const struct rte_flow_action *found_action = NULL; - struct mlx5_flow_meter *fm = NULL; uint32_t jump_group = 0; if (!mlx5_flow_os_action_supported(action_type)) @@ -11429,33 +11427,13 @@ struct mlx5_cache_entry * MLX5_FLOW_FATE_DEFAULT_MISS; break; case RTE_FLOW_ACTION_TYPE_METER: - mtr = actions->conf; - if (!flow->meter) { - fm = mlx5_flow_meter_attach(priv, mtr->mtr_id, - attr, error); - if (!fm) - return rte_flow_error_set(error, - rte_errno, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "meter not found " - "or invalid parameters"); - flow->meter = fm->idx; - } + if (!wks->fm) + return rte_flow_error_set(error, rte_errno, + RTE_FLOW_ERROR_TYPE_ACTION, + NULL, "Failed to get meter in flow."); /* Set the meter action. */ - if (!fm) { - fm = mlx5_ipool_get(priv->sh->ipool - [MLX5_IPOOL_MTR], flow->meter); - if (!fm) - return rte_flow_error_set(error, - rte_errno, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "meter not found " - "or invalid parameters"); - } dev_flow->dv.actions[actions_n++] = - fm->mfts->meter_action; + wks->fm->mfts->meter_action; action_flags |= MLX5_FLOW_ACTION_METER; break; case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: @@ -12604,6 +12582,7 @@ struct mlx5_cache_entry * { struct mlx5_flow_handle *dev_handle; struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_meter *fm = NULL; uint32_t srss = 0; if (!flow) @@ -12614,8 +12593,6 @@ struct mlx5_cache_entry * flow->counter = 0; } if (flow->meter) { - struct mlx5_flow_meter *fm; - fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], flow->meter); if (fm) @@ -12657,6 +12634,10 @@ struct mlx5_cache_entry * flow_dv_fate_resource_release(dev, dev_handle); else if (!srss) srss = dev_handle->rix_srss; + if (fm && dev_handle->is_meter_flow_id && + dev_handle->split_flow_id) + mlx5_ipool_free(fm->flow_ipool, + dev_handle->split_flow_id); mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW], tmp_idx); } @@ -13425,49 +13406,20 @@ struct mlx5_cache_entry * if (!mtd || !priv->config.dv_flow_en) return 0; - if (mtd->ingress.policer_rules[RTE_MTR_DROPPED]) - claim_zero(mlx5_flow_os_destroy_flow - (mtd->ingress.policer_rules[RTE_MTR_DROPPED])); - if (mtd->egress.policer_rules[RTE_MTR_DROPPED]) - claim_zero(mlx5_flow_os_destroy_flow - (mtd->egress.policer_rules[RTE_MTR_DROPPED])); - if (mtd->transfer.policer_rules[RTE_MTR_DROPPED]) - claim_zero(mlx5_flow_os_destroy_flow - (mtd->transfer.policer_rules[RTE_MTR_DROPPED])); - if (mtd->egress.color_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->egress.color_matcher)); - if (mtd->egress.any_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->egress.any_matcher)); if (mtd->egress.tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->egress.tbl); if (mtd->egress.sfx_tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->egress.sfx_tbl); - if (mtd->ingress.color_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->ingress.color_matcher)); - if (mtd->ingress.any_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->ingress.any_matcher)); if (mtd->ingress.tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->ingress.tbl); if (mtd->ingress.sfx_tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->ingress.sfx_tbl); - if (mtd->transfer.color_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->transfer.color_matcher)); - if (mtd->transfer.any_matcher) - claim_zero(mlx5_flow_os_destroy_flow_matcher - (mtd->transfer.any_matcher)); if (mtd->transfer.tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->transfer.tbl); if (mtd->transfer.sfx_tbl) flow_dv_tbl_resource_release(MLX5_SH(dev), mtd->transfer.sfx_tbl); - if (mtd->drop_actn) - claim_zero(mlx5_flow_os_destroy_flow_action(mtd->drop_actn)); mlx5_free(mtd); return 0; } @@ -13486,37 +13438,17 @@ struct mlx5_cache_entry * * Table attribute. * @param[in] transfer * Table attribute. - * @param[in] color_reg_c_idx - * Reg C index for color match. * * @return - * 0 on success, -1 otherwise and rte_errno is set. + * 0 on success, -1 otherwise. */ static int flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, struct mlx5_meter_domains_infos *mtb, - uint8_t egress, uint8_t transfer, - uint32_t color_reg_c_idx) + uint8_t egress, uint8_t transfer) { - struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_dev_ctx_shared *sh = priv->sh; - struct mlx5_flow_dv_match_params mask = { - .size = sizeof(mask.buf), - }; - struct mlx5_flow_dv_match_params value = { - .size = sizeof(value.buf), - }; - struct mlx5dv_flow_matcher_attr dv_attr = { - .type = IBV_FLOW_ATTR_NORMAL, - .priority = 0, - .match_criteria_enable = 0, - .match_mask = (void *)&mask, - }; - void *actions[METER_ACTIONS]; - struct mlx5_meter_domain_info *dtb; struct rte_flow_error error; - int i = 0; - int ret; + struct mlx5_meter_domain_info *dtb; if (transfer) dtb = &mtb->transfer; @@ -13541,41 +13473,7 @@ struct mlx5_cache_entry * DRV_LOG(ERR, "Failed to create meter suffix table."); return -1; } - /* Create matchers, Any and Color. */ - dv_attr.priority = 3; - dv_attr.match_criteria_enable = 0; - ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj, - &dtb->any_matcher); - if (ret) { - DRV_LOG(ERR, "Failed to create meter" - " policer default matcher."); - goto error_exit; - } - dv_attr.priority = 0; - dv_attr.match_criteria_enable = - 1 << MLX5_MATCH_CRITERIA_ENABLE_MISC2_BIT; - flow_dv_match_meta_reg(mask.buf, value.buf, color_reg_c_idx, - rte_col_2_mlx5_col(RTE_COLORS), UINT8_MAX); - ret = mlx5_flow_os_create_flow_matcher(sh->ctx, &dv_attr, dtb->tbl->obj, - &dtb->color_matcher); - if (ret) { - DRV_LOG(ERR, "Failed to create meter policer color matcher."); - goto error_exit; - } - if (mtb->count_actns[RTE_MTR_DROPPED]) - actions[i++] = mtb->count_actns[RTE_MTR_DROPPED]; - actions[i++] = mtb->drop_actn; - /* Default rule: lowest priority, match any, actions: drop. */ - ret = mlx5_flow_os_create_flow(dtb->any_matcher, (void *)&value, i, - actions, - &dtb->policer_rules[RTE_MTR_DROPPED]); - if (ret) { - DRV_LOG(ERR, "Failed to create meter policer drop rule."); - goto error_exit; - } return 0; -error_exit: - return -1; } /** @@ -13584,20 +13482,16 @@ struct mlx5_cache_entry * * * @param[in] dev * Pointer to Ethernet device. - * @param[in] fm - * Pointer to the flow meter. * * @return * Pointer to table set on success, NULL otherwise and rte_errno is set. */ static struct mlx5_meter_domains_infos * -flow_dv_create_mtr_tbl(struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm) +flow_dv_create_mtr_tbl(struct rte_eth_dev *dev) { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_meter_domains_infos *mtb; int ret; - int i; if (!priv->mtr_en) { rte_errno = ENOTSUP; @@ -13608,37 +13502,21 @@ struct mlx5_cache_entry * DRV_LOG(ERR, "Failed to allocate memory for meter."); return NULL; } - /* Create meter count actions */ - for (i = 0; i <= RTE_MTR_DROPPED; i++) { - struct mlx5_flow_counter *cnt; - if (!fm->policer_stats.cnt[i]) - continue; - cnt = flow_dv_counter_get_by_idx(dev, - fm->policer_stats.cnt[i], NULL); - mtb->count_actns[i] = cnt->action; - } - /* Create drop action. */ - ret = mlx5_flow_os_create_flow_action_drop(&mtb->drop_actn); - if (ret) { - DRV_LOG(ERR, "Failed to create drop action."); - goto error_exit; - } /* Egress meter table. */ - ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0, priv->mtr_color_reg); + ret = flow_dv_prepare_mtr_tables(dev, mtb, 1, 0); if (ret) { DRV_LOG(ERR, "Failed to prepare egress meter table."); goto error_exit; } /* Ingress meter table. */ - ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0, priv->mtr_color_reg); + ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 0); if (ret) { DRV_LOG(ERR, "Failed to prepare ingress meter table."); goto error_exit; } /* FDB meter table. */ if (priv->config.dv_esw_en) { - ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1, - priv->mtr_color_reg); + ret = flow_dv_prepare_mtr_tables(dev, mtb, 0, 1); if (ret) { DRV_LOG(ERR, "Failed to prepare fdb meter table."); goto error_exit; @@ -13651,23 +13529,151 @@ struct mlx5_cache_entry * } /** + * Destroy the meter table matchers. + * Lock free, (mutex should be acquired by caller). + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in,out] dtb + * Pointer to DV meter table. + * + * @return + * Always 0. + */ +static int +flow_dv_destroy_mtr_matchers(struct rte_eth_dev *dev, + struct mlx5_meter_domain_info *dtb) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_tbl_data_entry *tbl; + + if (!priv->config.dv_flow_en) + return 0; + if (dtb->drop_matcher) { + tbl = container_of(dtb->drop_matcher->tbl, typeof(*tbl), tbl); + mlx5_cache_unregister(&tbl->matchers, + &dtb->drop_matcher->entry); + dtb->drop_matcher = NULL; + } + if (dtb->color_matcher) { + tbl = container_of(dtb->color_matcher->tbl, typeof(*tbl), tbl); + mlx5_cache_unregister(&tbl->matchers, + &dtb->color_matcher->entry); + dtb->color_matcher = NULL; + } + return 0; +} + +/** + * Create the matchers for meter table. + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in] color_reg_c_idx + * Reg C index for color match. + * @param[in] mtr_id_reg_c_idx + * Reg C index for meter_id match. + * @param[in] mtr_id_mask + * Mask for meter_id match criteria. + * @param[in,out] dtb + * Pointer to DV meter table. + * @param[out] error + * Perform verbose error reporting if not NULL. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +flow_dv_prepare_mtr_matchers(struct rte_eth_dev *dev, + uint32_t color_reg_c_idx, + uint32_t mtr_id_reg_c_idx, + uint32_t mtr_id_mask, + struct mlx5_meter_domain_info *dtb, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_flow_tbl_data_entry *tbl_data; + struct mlx5_cache_entry *entry; + struct mlx5_flow_dv_matcher matcher = { + .mask = { + .size = sizeof(matcher.mask.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), + }, + .tbl = dtb->tbl, + }; + struct mlx5_flow_dv_match_params value = { + .size = sizeof(value.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), + }; + struct mlx5_flow_cb_ctx ctx = { + .error = error, + .data = &matcher, + }; + uint32_t color_mask = (UINT32_C(1) << MLX5_MTR_COLOR_BITS) - 1; + + tbl_data = container_of(dtb->tbl, struct mlx5_flow_tbl_data_entry, tbl); + if (!dtb->drop_matcher) { + /* Create matchers for Drop. */ + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + mtr_id_reg_c_idx, 0, mtr_id_mask); + matcher.priority = MLX5_REG_BITS * 2 - priv->max_mtr_bits; + matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf, + matcher.mask.size); + entry = mlx5_cache_register(&tbl_data->matchers, &ctx); + if (!entry) { + DRV_LOG(ERR, "Failed to register meter drop matcher."); + return -1; + } + dtb->drop_matcher = + container_of(entry, struct mlx5_flow_dv_matcher, entry); + } + if (!dtb->color_matcher) { + /* Create matchers for Color + meter_id. */ + if (priv->mtr_reg_share) { + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + color_reg_c_idx, 0, + (mtr_id_mask | color_mask)); + } else { + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + color_reg_c_idx, 0, color_mask); + flow_dv_match_meta_reg(matcher.mask.buf, value.buf, + mtr_id_reg_c_idx, 0, mtr_id_mask); + } + matcher.priority = MLX5_REG_BITS - priv->max_mtr_bits; + matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf, + matcher.mask.size); + entry = mlx5_cache_register(&tbl_data->matchers, &ctx); + if (!entry) { + DRV_LOG(ERR, "Failed to register meter color matcher."); + return -1; + } + dtb->color_matcher = + container_of(entry, struct mlx5_flow_dv_matcher, entry); + } + return 0; +} + +/** * Destroy domain policer rule. * + * @param[in] dev + * Pointer to Ethernet device. * @param[in] dt * Pointer to domain table. */ static void -flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt) +flow_dv_destroy_domain_policer_rule(struct rte_eth_dev *dev, + struct mlx5_meter_domain_info *dt) { - int i; - - for (i = 0; i < RTE_MTR_DROPPED; i++) { - if (dt->policer_rules[i]) { - claim_zero(mlx5_flow_os_destroy_flow - (dt->policer_rules[i])); - dt->policer_rules[i] = NULL; - } + if (dt->drop_rule) { + claim_zero(mlx5_flow_os_destroy_flow(dt->drop_rule)); + dt->drop_rule = NULL; } + if (dt->green_rule) { + claim_zero(mlx5_flow_os_destroy_flow(dt->green_rule)); + dt->green_rule = NULL; + } + flow_dv_destroy_mtr_matchers(dev, dt); if (dt->jump_actn) { claim_zero(mlx5_flow_os_destroy_flow_action(dt->jump_actn)); dt->jump_actn = NULL; @@ -13688,7 +13694,7 @@ struct mlx5_cache_entry * * Always 0. */ static int -flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused, +flow_dv_destroy_policer_rules(struct rte_eth_dev *dev, const struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr) { @@ -13697,39 +13703,56 @@ struct mlx5_cache_entry * if (!mtb) return 0; if (attr->egress) - flow_dv_destroy_domain_policer_rule(&mtb->egress); + flow_dv_destroy_domain_policer_rule(dev, &mtb->egress); if (attr->ingress) - flow_dv_destroy_domain_policer_rule(&mtb->ingress); + flow_dv_destroy_domain_policer_rule(dev, &mtb->ingress); if (attr->transfer) - flow_dv_destroy_domain_policer_rule(&mtb->transfer); + flow_dv_destroy_domain_policer_rule(dev, &mtb->transfer); return 0; } /** * Create specify domain meter policer rule. * + * @param[in] dev + * Pointer to Ethernet device. * @param[in] fm * Pointer to flow meter structure. * @param[in] mtb * Pointer to DV meter table set. - * @param[in] mtr_reg_c - * Color match REG_C. + * @param[out] drop_rule + * The address of pointer saving drop rule. + * @param[out] color_rule + * The address of pointer saving green rule. * * @return * 0 on success, -1 otherwise. */ static int -flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, +flow_dv_create_policer_forward_rule(struct rte_eth_dev *dev, + struct mlx5_flow_meter *fm, struct mlx5_meter_domain_info *dtb, - uint8_t mtr_reg_c) + void **drop_rule, + void **green_rule) { + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flow_dv_match_params matcher = { - .size = sizeof(matcher.buf), + .size = sizeof(matcher.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), }; struct mlx5_flow_dv_match_params value = { - .size = sizeof(value.buf), + .size = sizeof(value.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), }; struct mlx5_meter_domains_infos *mtb = fm->mfts; + struct rte_flow_error error; + uint32_t color_reg_c = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, + 0, &error); + uint32_t mtr_id_reg_c = mlx5_flow_get_reg_id(dev, MLX5_MTR_ID, + 0, &error); + uint8_t mtr_id_offset = priv->mtr_reg_share ? MLX5_MTR_COLOR_BITS : 0; + uint32_t mtr_id_mask = + ((UINT32_C(1) << priv->max_mtr_bits) - 1) << mtr_id_offset; void *actions[METER_ACTIONS]; int i; int ret = 0; @@ -13742,25 +13765,52 @@ struct mlx5_cache_entry * DRV_LOG(ERR, "Failed to create policer jump action."); goto error; } - for (i = 0; i < RTE_MTR_DROPPED; i++) { - int j = 0; - - flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_reg_c, - rte_col_2_mlx5_col(i), UINT8_MAX); - if (mtb->count_actns[i]) - actions[j++] = mtb->count_actns[i]; - if (fm->action[i] == MTR_POLICER_ACTION_DROP) - actions[j++] = mtb->drop_actn; - else - actions[j++] = dtb->jump_actn; - ret = mlx5_flow_os_create_flow(dtb->color_matcher, - (void *)&value, j, actions, - &dtb->policer_rules[i]); + /* Prepare matchers. */ + if (!dtb->drop_matcher || !dtb->color_matcher) { + ret = flow_dv_prepare_mtr_matchers(dev, color_reg_c, + mtr_id_reg_c, mtr_id_mask, + dtb, &error); if (ret) { - DRV_LOG(ERR, "Failed to create policer rule."); + DRV_LOG(ERR, "Failed to setup matchers for mtr table."); goto error; } } + /* Create Drop flow, matching meter_id only. */ + i = 0; + flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_id_reg_c, + (fm->idx << mtr_id_offset), UINT32_MAX); + if (mtb->drop_count) + actions[i++] = mtb->drop_count; + actions[i++] = priv->sh->dr_drop_action; + ret = mlx5_flow_os_create_flow(dtb->drop_matcher->matcher_object, + (void *)&value, i, actions, drop_rule); + if (ret) { + DRV_LOG(ERR, "Failed to create meter policer drop rule."); + goto error; + } + /* Create flow matching Green color + meter_id. */ + i = 0; + if (priv->mtr_reg_share) { + flow_dv_match_meta_reg(matcher.buf, value.buf, color_reg_c, + ((fm->idx << mtr_id_offset) | + rte_col_2_mlx5_col(RTE_COLOR_GREEN)), + UINT32_MAX); + } else { + flow_dv_match_meta_reg(matcher.buf, value.buf, color_reg_c, + rte_col_2_mlx5_col(RTE_COLOR_GREEN), + UINT32_MAX); + flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_id_reg_c, + fm->idx, UINT32_MAX); + } + if (mtb->green_count) + actions[i++] = mtb->green_count; + actions[i++] = dtb->jump_actn; + ret = mlx5_flow_os_create_flow(dtb->color_matcher->matcher_object, + (void *)&value, i, actions, green_rule); + if (ret) { + DRV_LOG(ERR, "Failed to create meter policer color rule."); + goto error; + } return 0; error: rte_errno = errno; @@ -13768,7 +13818,8 @@ struct mlx5_cache_entry * } /** - * Create policer rules. + * Prepare policer rules for all domains. + * If meter already initialized, this will replace all old rules with new ones. * * @param[in] dev * Pointer to Ethernet device. @@ -13781,41 +13832,107 @@ struct mlx5_cache_entry * * 0 on success, -1 otherwise. */ static int -flow_dv_create_policer_rules(struct rte_eth_dev *dev, - struct mlx5_flow_meter *fm, - const struct rte_flow_attr *attr) +flow_dv_prepare_policer_rules(struct rte_eth_dev *dev, + struct mlx5_flow_meter *fm, + const struct rte_flow_attr *attr) { - struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_meter_domains_infos *mtb = fm->mfts; + bool initialized = false; + struct mlx5_flow_counter *cnt; + void *egress_drop_rule = NULL; + void *egress_green_rule = NULL; + void *ingress_drop_rule = NULL; + void *ingress_green_rule = NULL; + void *transfer_drop_rule = NULL; + void *transfer_green_rule = NULL; int ret; + /* Get the statistics counters for green/drop. */ + if (fm->policer_stats.cnt[RTE_COLOR_GREEN]) { + cnt = flow_dv_counter_get_by_idx(dev, + fm->policer_stats.cnt[RTE_COLOR_GREEN], + NULL); + mtb->green_count = cnt->action; + } else { + mtb->green_count = NULL; + } + if (fm->policer_stats.cnt[RTE_MTR_DROPPED]) { + cnt = flow_dv_counter_get_by_idx(dev, + fm->policer_stats.cnt[RTE_MTR_DROPPED], + NULL); + mtb->drop_count = cnt->action; + } else { + mtb->drop_count = NULL; + } + /** + * If flow meter has been initialized, all policer rules + * are created. So can get if meter initialized by checking + * any policer rule. + */ + if (mtb->egress.drop_rule) + initialized = true; if (attr->egress) { - ret = flow_dv_create_policer_forward_rule(fm, &mtb->egress, - priv->mtr_color_reg); + ret = flow_dv_create_policer_forward_rule(dev, + fm, &mtb->egress, + &egress_drop_rule, &egress_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create egress policer."); goto error; } } if (attr->ingress) { - ret = flow_dv_create_policer_forward_rule(fm, &mtb->ingress, - priv->mtr_color_reg); + ret = flow_dv_create_policer_forward_rule(dev, + fm, &mtb->ingress, + &ingress_drop_rule, &ingress_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create ingress policer."); goto error; } } if (attr->transfer) { - ret = flow_dv_create_policer_forward_rule(fm, &mtb->transfer, - priv->mtr_color_reg); + ret = flow_dv_create_policer_forward_rule(dev, + fm, &mtb->transfer, + &transfer_drop_rule, &transfer_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create transfer policer."); goto error; } } + /* Replace old flows if existing. */ + if (mtb->egress.drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->egress.drop_rule)); + if (mtb->egress.green_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->egress.green_rule)); + if (mtb->ingress.drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->ingress.drop_rule)); + if (mtb->ingress.green_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->ingress.green_rule)); + if (mtb->transfer.drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->transfer.drop_rule)); + if (mtb->transfer.green_rule) + claim_zero(mlx5_flow_os_destroy_flow(mtb->transfer.green_rule)); + mtb->egress.drop_rule = egress_drop_rule; + mtb->egress.green_rule = egress_green_rule; + mtb->ingress.drop_rule = ingress_drop_rule; + mtb->ingress.green_rule = ingress_green_rule; + mtb->transfer.drop_rule = transfer_drop_rule; + mtb->transfer.green_rule = transfer_green_rule; return 0; error: - flow_dv_destroy_policer_rules(dev, fm, attr); + if (egress_drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(egress_drop_rule)); + if (egress_green_rule) + claim_zero(mlx5_flow_os_destroy_flow(egress_green_rule)); + if (ingress_drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(ingress_drop_rule)); + if (ingress_green_rule) + claim_zero(mlx5_flow_os_destroy_flow(ingress_green_rule)); + if (transfer_drop_rule) + claim_zero(mlx5_flow_os_destroy_flow(transfer_drop_rule)); + if (transfer_green_rule) + claim_zero(mlx5_flow_os_destroy_flow(transfer_green_rule)); + if (!initialized) + flow_dv_destroy_policer_rules(dev, fm, attr); return -1; } @@ -14112,7 +14229,7 @@ struct mlx5_cache_entry * .query = flow_dv_query, .create_mtr_tbls = flow_dv_create_mtr_tbl, .destroy_mtr_tbls = flow_dv_destroy_mtr_tbl, - .create_policer_rules = flow_dv_create_policer_rules, + .prepare_policer_rules = flow_dv_prepare_policer_rules, .destroy_policer_rules = flow_dv_destroy_policer_rules, .counter_alloc = flow_dv_counter_allocate, .counter_free = flow_dv_counter_free, diff --git a/drivers/net/mlx5/mlx5_flow_meter.c b/drivers/net/mlx5/mlx5_flow_meter.c index dbc574b..7f7693b 100644 --- a/drivers/net/mlx5/mlx5_flow_meter.c +++ b/drivers/net/mlx5/mlx5_flow_meter.c @@ -474,6 +474,12 @@ MTR_POLICER_ACTION_COLOR_RED }; int i; + /* Meter must use global drop action. */ + if (!priv->sh->dr_drop_action) + return -rte_mtr_error_set(error, ENOTSUP, + RTE_MTR_ERROR_TYPE_MTR_PARAMS, + NULL, + "No drop action ready for meter."); /* Meter params must not be NULL. */ if (params == NULL) return -rte_mtr_error_set(error, EINVAL, @@ -630,9 +636,18 @@ .egress = 1, .transfer = priv->config.dv_esw_en ? 1 : 0, }; + struct mlx5_indexed_pool_config flow_ipool_cfg = { + .size = 0, + .trunk_size = 64, + .need_lock = 1, + .type = "mlx5_flow_mtr_flow_id_pool", + }; int ret; unsigned int i; uint32_t idx = 0; + uint8_t mtr_id_bits; + uint8_t mtr_reg_bits = priv->mtr_reg_share ? + MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; if (!priv->mtr_en) return -rte_mtr_error_set(error, ENOTSUP, @@ -654,6 +669,13 @@ return -rte_mtr_error_set(error, ENOMEM, RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, "Memory alloc failed for meter."); + mtr_id_bits = MLX5_REG_BITS - __builtin_clz(idx); + if ((mtr_id_bits + priv->max_mtr_flow_bits) > mtr_reg_bits) { + DRV_LOG(ERR, "Meter number exceeds max limit."); + goto error; + } + if (mtr_id_bits > priv->max_mtr_bits) + priv->max_mtr_bits = mtr_id_bits; fm->idx = idx; /* Fill the flow meter parameters. */ fm->meter_id = meter_id; @@ -667,10 +689,10 @@ if (!fm->policer_stats.cnt[i]) goto error; } - fm->mfts = mlx5_flow_create_mtr_tbls(dev, fm); + fm->mfts = mlx5_flow_create_mtr_tbls(dev); if (!fm->mfts) goto error; - ret = mlx5_flow_create_policer_rules(dev, fm, &attr); + ret = mlx5_flow_prepare_policer_rules(dev, fm, &attr); if (ret) goto error; /* Add to the flow meter list. */ @@ -679,6 +701,9 @@ fm->shared = !!shared; fm->policer_stats.stats_mask = params->stats_mask; fm->profile->ref_cnt++; + fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg); + if (!fm->flow_ipool) + goto error; rte_spinlock_init(&fm->sl); return 0; error: @@ -749,6 +774,8 @@ if (fm->policer_stats.cnt[i]) mlx5_counter_free(dev, fm->policer_stats.cnt[i]); /* Free meter flow table */ + if (fm->flow_ipool) + mlx5_ipool_destroy(fm->flow_ipool); mlx5_flow_destroy_policer_rules(dev, fm, &attr); mlx5_flow_destroy_mtr_tbls(dev, fm->mfts); mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], fm->idx); @@ -1153,30 +1180,24 @@ struct mlx5_flow_meter * * * @param [in] priv * Pointer to mlx5 private data. - * @param [in] meter_id - * Flow meter id. + * @param[in] fm + * Pointer to flow meter. * @param [in] attr * Pointer to flow attributes. * @param [out] error * Pointer to error structure. * - * @return the flow meter pointer, NULL otherwise. + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. */ -struct mlx5_flow_meter * -mlx5_flow_meter_attach(struct mlx5_priv *priv, uint32_t meter_id, +int +mlx5_flow_meter_attach(struct mlx5_priv *priv, + struct mlx5_flow_meter *fm, const struct rte_flow_attr *attr, struct rte_flow_error *error) { - struct mlx5_flow_meter *fm; int ret = 0; - fm = mlx5_flow_meter_find(priv, meter_id); - if (fm == NULL) { - rte_flow_error_set(error, ENOENT, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "Meter object id not valid"); - return fm; - } rte_spinlock_lock(&fm->sl); if (fm->mfts->meter_action) { if (fm->shared && @@ -1210,7 +1231,7 @@ struct mlx5_flow_meter * fm->mfts->meter_action ? "Meter attr not match" : "Meter action create failed"); - return ret ? NULL : fm; + return ret ? -rte_errno : 0; } /** -- 1.8.3.1