From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 2826BA0548; Fri, 2 Apr 2021 17:16:55 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 4D76E1410A3; Fri, 2 Apr 2021 17:16:40 +0200 (CEST) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by mails.dpdk.org (Postfix) with ESMTP id 695E414107B for ; Fri, 2 Apr 2021 17:16:35 +0200 (CEST) Received: from Internal Mail-Server by MTLPINE1 (envelope-from lizh@nvidia.com) with SMTP; 2 Apr 2021 18:16:33 +0300 Received: from nvidia.com (c-235-17-1-009.mtl.labs.mlnx [10.235.17.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 132FGWdQ028272; Fri, 2 Apr 2021 18:16:33 +0300 From: Li Zhang To: dekelp@nvidia.com, orika@nvidia.com, viacheslavo@nvidia.com, matan@nvidia.com, shahafs@nvidia.com Cc: dev@dpdk.org, thomas@monjalon.net, rasland@nvidia.com, roniba@nvidia.com Date: Fri, 2 Apr 2021 18:16:17 +0300 Message-Id: <20210402151627.1531623-4-lizh@nvidia.com> X-Mailer: git-send-email 2.21.0 In-Reply-To: <20210402151627.1531623-1-lizh@nvidia.com> References: <20210331073632.1443011-1-lizh@nvidia.com> <20210402151627.1531623-1-lizh@nvidia.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Subject: [dpdk-dev] [PATCH v2 03/13] net/mlx5: optimize meter statistics X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" 1. It does not support yellow counter and return 0. 2. All the meter colors with drop action will be counted only by the global drop statistics. 3. Red color must be with drop action. Signed-off-by: Li Zhang --- doc/guides/nics/mlx5.rst | 6 + drivers/net/mlx5/mlx5_flow.h | 24 ++-- drivers/net/mlx5/mlx5_flow_dv.c | 8 +- drivers/net/mlx5/mlx5_flow_meter.c | 217 +++++++++++++++++++---------- 4 files changed, 173 insertions(+), 82 deletions(-) diff --git a/doc/guides/nics/mlx5.rst b/doc/guides/nics/mlx5.rst index a2cfc51b2a..fa63e62a5d 100644 --- a/doc/guides/nics/mlx5.rst +++ b/doc/guides/nics/mlx5.rst @@ -398,6 +398,12 @@ Limitations - Hairpin between two ports could only manual binding and explicit Tx flow mode. For single port hairpin, all the combinations of auto/manual binding and explicit/implicit Tx flow mode could be supported. - Hairpin in switchdev SR-IOV mode is not supported till now. +- Meter: + - All the meter colors with drop action will be counted only by the global drop statistics. + - Green color is not supported with drop action. + - Yellow detection is not supported. + - Red color must be with drop action. + Statistics ---------- diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index c92e746a04..76870b8061 100644 --- a/drivers/net/mlx5/mlx5_flow.h +++ b/drivers/net/mlx5/mlx5_flow.h @@ -831,10 +831,10 @@ struct mlx5_flow { /* Meter policer statistics */ struct mlx5_flow_policer_stats { - uint32_t cnt[RTE_COLORS + 1]; - /**< Color counter, extra for drop. */ - uint64_t stats_mask; - /**< Statistics mask for the colors. */ + uint32_t pass_cnt; + /**< Color counter for pass. */ + uint32_t drop_cnt; + /**< Color counter for drop. */ }; /* Meter table structure. */ @@ -892,10 +892,18 @@ struct mlx5_flow_meter { /** Policer actions (per meter output color). */ enum rte_mtr_policer_action action[RTE_COLORS]; - /** Set of stats counters to be enabled. - * @see enum rte_mtr_stats_type - */ - uint64_t stats_mask; + uint32_t green_bytes:1; + /** Set green bytes stats to be enabled. */ + uint32_t green_pkts:1; + /** Set green packets stats to be enabled. */ + uint32_t red_bytes:1; + /** Set red bytes stats to be enabled. */ + uint32_t red_pkts:1; + /** Set red packets stats to be enabled. */ + uint32_t bytes_dropped:1; + /** Set bytes dropped stats to be enabled. */ + uint32_t pkts_dropped:1; + /** Set packets dropped stats to be enabled. */ /**< Rule applies to ingress traffic. */ uint32_t ingress:1; diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index ce9857d3f7..d5c8f32038 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -13697,17 +13697,17 @@ flow_dv_prepare_policer_rules(struct rte_eth_dev *dev, int ret; /* Get the statistics counters for green/drop. */ - if (fm->policer_stats.cnt[RTE_COLOR_GREEN]) { + if (fm->policer_stats.pass_cnt) { cnt = flow_dv_counter_get_by_idx(dev, - fm->policer_stats.cnt[RTE_COLOR_GREEN], + fm->policer_stats.pass_cnt, NULL); mtb->green_count = cnt->action; } else { mtb->green_count = NULL; } - if (fm->policer_stats.cnt[RTE_MTR_DROPPED]) { + if (fm->policer_stats.drop_cnt) { cnt = flow_dv_counter_get_by_idx(dev, - fm->policer_stats.cnt[RTE_MTR_DROPPED], + fm->policer_stats.drop_cnt, NULL); mtb->drop_count = cnt->action; } else { diff --git a/drivers/net/mlx5/mlx5_flow_meter.c b/drivers/net/mlx5/mlx5_flow_meter.c index 68f5c344cf..73b6e771d9 100644 --- a/drivers/net/mlx5/mlx5_flow_meter.c +++ b/drivers/net/mlx5/mlx5_flow_meter.c @@ -467,13 +467,6 @@ mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id, struct rte_mtr_params *params, struct rte_mtr_error *error) { - static enum rte_mtr_policer_action - valid_recol_action[RTE_COLORS] = { - MTR_POLICER_ACTION_COLOR_GREEN, - MTR_POLICER_ACTION_COLOR_YELLOW, - MTR_POLICER_ACTION_COLOR_RED }; - int i; - /* Meter must use global drop action. */ if (!priv->sh->esw_drop_action) return -rte_mtr_error_set(error, ENOTSUP, @@ -493,13 +486,18 @@ mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id, "Previous meter color " "not supported."); /* Validate policer settings. */ - for (i = 0; i < RTE_COLORS; i++) - if (params->action[i] != valid_recol_action[i] && - params->action[i] != MTR_POLICER_ACTION_DROP) - return -rte_mtr_error_set - (error, ENOTSUP, - action2error(params->action[i]), NULL, - "Recolor action not supported."); + if (params->action[RTE_COLOR_RED] != MTR_POLICER_ACTION_DROP) + return -rte_mtr_error_set + (error, ENOTSUP, + action2error(params->action[RTE_COLOR_RED]), + NULL, + "Red color only supports drop action."); + if (params->action[RTE_COLOR_GREEN] != MTR_POLICER_ACTION_COLOR_GREEN) + return -rte_mtr_error_set + (error, ENOTSUP, + action2error(params->action[RTE_COLOR_GREEN]), + NULL, + "Green color only supports recolor green action."); /* Validate meter id. */ if (mlx5_flow_meter_find(priv, meter_id)) return -rte_mtr_error_set(error, EEXIST, @@ -605,6 +603,19 @@ mlx5_flow_meter_action_modify(struct mlx5_priv *priv, #endif } +static void +mlx5_flow_meter_stats_enable_update(struct mlx5_flow_meter *fm, + uint64_t stats_mask) +{ + fm->green_bytes = (stats_mask & RTE_MTR_STATS_N_BYTES_GREEN) ? 1 : 0; + fm->green_pkts = (stats_mask & RTE_MTR_STATS_N_PKTS_GREEN) ? 1 : 0; + fm->red_bytes = (stats_mask & RTE_MTR_STATS_N_BYTES_RED) ? 1 : 0; + fm->red_pkts = (stats_mask & RTE_MTR_STATS_N_PKTS_RED) ? 1 : 0; + fm->bytes_dropped = + (stats_mask & RTE_MTR_STATS_N_BYTES_DROPPED) ? 1 : 0; + fm->pkts_dropped = (stats_mask & RTE_MTR_STATS_N_PKTS_DROPPED) ? 1 : 0; +} + /** * Create meter rules. * @@ -643,7 +654,6 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, .type = "mlx5_flow_mtr_flow_id_pool", }; int ret; - unsigned int i; uint32_t idx = 0; uint8_t mtr_id_bits; uint8_t mtr_reg_bits = priv->mtr_reg_share ? @@ -681,12 +691,18 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, fm->meter_id = meter_id; fm->profile = fmp; memcpy(fm->action, params->action, sizeof(params->action)); - fm->stats_mask = params->stats_mask; + mlx5_flow_meter_stats_enable_update(fm, params->stats_mask); /* Alloc policer counters. */ - for (i = 0; i < RTE_DIM(fm->policer_stats.cnt); i++) { - fm->policer_stats.cnt[i] = mlx5_counter_alloc(dev); - if (!fm->policer_stats.cnt[i]) + if (fm->green_bytes || fm->green_pkts) { + fm->policer_stats.pass_cnt = mlx5_counter_alloc(dev); + if (!fm->policer_stats.pass_cnt) + goto error; + } + if (fm->red_bytes || fm->red_pkts || + fm->bytes_dropped || fm->pkts_dropped) { + fm->policer_stats.drop_cnt = mlx5_counter_alloc(dev); + if (!fm->policer_stats.drop_cnt) goto error; } fm->mfts = mlx5_flow_create_mtr_tbls(dev); @@ -699,7 +715,6 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, TAILQ_INSERT_TAIL(fms, fm, next); fm->active_state = 1; /* Config meter starts as active. */ fm->shared = !!shared; - fm->policer_stats.stats_mask = params->stats_mask; fm->profile->ref_cnt++; fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg); if (!fm->flow_ipool) @@ -710,9 +725,10 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, mlx5_flow_destroy_policer_rules(dev, fm, &attr); mlx5_flow_destroy_mtr_tbls(dev, fm->mfts); /* Free policer counters. */ - for (i = 0; i < RTE_DIM(fm->policer_stats.cnt); i++) - if (fm->policer_stats.cnt[i]) - mlx5_counter_free(dev, fm->policer_stats.cnt[i]); + if (fm->policer_stats.pass_cnt) + mlx5_counter_free(dev, fm->policer_stats.pass_cnt); + if (fm->policer_stats.drop_cnt) + mlx5_counter_free(dev, fm->policer_stats.drop_cnt); mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], idx); return -rte_mtr_error_set(error, -ret, RTE_MTR_ERROR_TYPE_UNSPECIFIED, @@ -745,7 +761,6 @@ mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id, .egress = 1, .transfer = priv->config.dv_esw_en ? 1 : 0, }; - unsigned int i; if (!priv->mtr_en) return -rte_mtr_error_set(error, ENOTSUP, @@ -770,9 +785,10 @@ mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id, /* Remove from the flow meter list. */ TAILQ_REMOVE(fms, fm, next); /* Free policer counters. */ - for (i = 0; i < RTE_DIM(fm->policer_stats.cnt); i++) - if (fm->policer_stats.cnt[i]) - mlx5_counter_free(dev, fm->policer_stats.cnt[i]); + if (fm->policer_stats.pass_cnt) + mlx5_counter_free(dev, fm->policer_stats.pass_cnt); + if (fm->policer_stats.drop_cnt) + mlx5_counter_free(dev, fm->policer_stats.drop_cnt); /* Free meter flow table */ if (fm->flow_ipool) mlx5_ipool_destroy(fm->flow_ipool); @@ -1005,6 +1021,13 @@ mlx5_flow_meter_stats_update(struct rte_eth_dev *dev, { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flow_meter *fm; + const struct rte_flow_attr attr = { + .ingress = 1, + .egress = 1, + .transfer = priv->config.dv_esw_en ? 1 : 0, + }; + bool need_updated = false; + struct mlx5_flow_policer_stats old_policer_stats; if (!priv->mtr_en) return -rte_mtr_error_set(error, ENOTSUP, @@ -1016,7 +1039,70 @@ mlx5_flow_meter_stats_update(struct rte_eth_dev *dev, return -rte_mtr_error_set(error, ENOENT, RTE_MTR_ERROR_TYPE_MTR_ID, NULL, "Meter object id not valid."); - fm->policer_stats.stats_mask = stats_mask; + old_policer_stats.pass_cnt = 0; + old_policer_stats.drop_cnt = 0; + if (!!((RTE_MTR_STATS_N_PKTS_GREEN | + RTE_MTR_STATS_N_BYTES_GREEN) & stats_mask) != + !!fm->policer_stats.pass_cnt) { + need_updated = true; + if (fm->policer_stats.pass_cnt) { + old_policer_stats.pass_cnt = fm->policer_stats.pass_cnt; + fm->policer_stats.pass_cnt = 0; + } else { + fm->policer_stats.pass_cnt = + mlx5_counter_alloc(dev); + if (!fm->policer_stats.pass_cnt) + return -rte_mtr_error_set(error, ENOMEM, + RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, + "Counter alloc failed for meter."); + } + } + if (!!((RTE_MTR_STATS_N_PKTS_RED | RTE_MTR_STATS_N_BYTES_RED | + RTE_MTR_STATS_N_PKTS_DROPPED | RTE_MTR_STATS_N_BYTES_DROPPED) & + stats_mask) != + !!fm->policer_stats.drop_cnt) { + need_updated = true; + if (fm->policer_stats.drop_cnt) { + old_policer_stats.drop_cnt = fm->policer_stats.drop_cnt; + fm->policer_stats.drop_cnt = 0; + } else { + fm->policer_stats.drop_cnt = + mlx5_counter_alloc(dev); + if (!fm->policer_stats.drop_cnt) + return -rte_mtr_error_set(error, ENOMEM, + RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, + "Counter alloc failed for meter."); + } + } + if (need_updated) { + if (mlx5_flow_prepare_policer_rules(dev, fm, &attr)) { + if (fm->policer_stats.pass_cnt && + fm->policer_stats.pass_cnt != + old_policer_stats.pass_cnt) + mlx5_counter_free(dev, + fm->policer_stats.pass_cnt); + fm->policer_stats.pass_cnt = + old_policer_stats.pass_cnt; + if (fm->policer_stats.drop_cnt && + fm->policer_stats.drop_cnt != + old_policer_stats.drop_cnt) + mlx5_counter_free(dev, + fm->policer_stats.drop_cnt); + fm->policer_stats.pass_cnt = + old_policer_stats.pass_cnt; + return -rte_mtr_error_set(error, ENOTSUP, + RTE_MTR_ERROR_TYPE_UNSPECIFIED, + NULL, "Failed to create meter policer rules."); + } + /* Free old policer counters. */ + if (old_policer_stats.pass_cnt) + mlx5_counter_free(dev, + old_policer_stats.pass_cnt); + if (old_policer_stats.drop_cnt) + mlx5_counter_free(dev, + old_policer_stats.drop_cnt); + } + mlx5_flow_meter_stats_enable_update(fm, stats_mask); return 0; } @@ -1047,20 +1133,11 @@ mlx5_flow_meter_stats_read(struct rte_eth_dev *dev, int clear, struct rte_mtr_error *error) { - static uint64_t meter2mask[RTE_MTR_DROPPED + 1] = { - RTE_MTR_STATS_N_PKTS_GREEN | RTE_MTR_STATS_N_BYTES_GREEN, - RTE_MTR_STATS_N_PKTS_YELLOW | RTE_MTR_STATS_N_BYTES_YELLOW, - RTE_MTR_STATS_N_PKTS_RED | RTE_MTR_STATS_N_BYTES_RED, - RTE_MTR_STATS_N_PKTS_DROPPED | RTE_MTR_STATS_N_BYTES_DROPPED - }; struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flow_meter *fm; struct mlx5_flow_policer_stats *ps; - uint64_t pkts_dropped = 0; - uint64_t bytes_dropped = 0; uint64_t pkts; uint64_t bytes; - int i; int ret = 0; if (!priv->mtr_en) @@ -1074,41 +1151,42 @@ mlx5_flow_meter_stats_read(struct rte_eth_dev *dev, RTE_MTR_ERROR_TYPE_MTR_ID, NULL, "Meter object id not valid."); ps = &fm->policer_stats; - *stats_mask = ps->stats_mask; - for (i = 0; i < RTE_MTR_DROPPED; i++) { - if (*stats_mask & meter2mask[i]) { - ret = mlx5_counter_query(dev, ps->cnt[i], clear, &pkts, + *stats_mask = 0; + if (fm->green_bytes) + *stats_mask |= RTE_MTR_STATS_N_BYTES_GREEN; + if (fm->green_pkts) + *stats_mask |= RTE_MTR_STATS_N_PKTS_GREEN; + if (fm->red_bytes) + *stats_mask |= RTE_MTR_STATS_N_BYTES_RED; + if (fm->red_pkts) + *stats_mask |= RTE_MTR_STATS_N_PKTS_RED; + if (fm->bytes_dropped) + *stats_mask |= RTE_MTR_STATS_N_BYTES_DROPPED; + if (fm->pkts_dropped) + *stats_mask |= RTE_MTR_STATS_N_PKTS_DROPPED; + memset(stats, 0, sizeof(*stats)); + if (ps->pass_cnt) { + ret = mlx5_counter_query(dev, ps->pass_cnt, clear, &pkts, &bytes); - if (ret) - goto error; - if (fm->action[i] == MTR_POLICER_ACTION_DROP) { - pkts_dropped += pkts; - bytes_dropped += bytes; - } - /* If need to read the packets, set it. */ - if ((1 << i) & (*stats_mask & meter2mask[i])) - stats->n_pkts[i] = pkts; - /* If need to read the bytes, set it. */ - if ((1 << (RTE_MTR_DROPPED + 1 + i)) & - (*stats_mask & meter2mask[i])) - stats->n_bytes[i] = bytes; - } + if (ret) + goto error; + /* If need to read the packets, set it. */ + if (fm->green_pkts) + stats->n_pkts[RTE_COLOR_GREEN] = pkts; + /* If need to read the bytes, set it. */ + if (fm->green_bytes) + stats->n_bytes[RTE_COLOR_GREEN] = bytes; } - /* Dropped packets/bytes are treated differently. */ - if (*stats_mask & meter2mask[i]) { - ret = mlx5_counter_query(dev, ps->cnt[i], clear, &pkts, - &bytes); + if (ps->drop_cnt) { + ret = mlx5_counter_query(dev, ps->drop_cnt, clear, &pkts, + &bytes); if (ret) goto error; - pkts += pkts_dropped; - bytes += bytes_dropped; /* If need to read the packets, set it. */ - if ((*stats_mask & meter2mask[i]) & - RTE_MTR_STATS_N_PKTS_DROPPED) + if (fm->pkts_dropped) stats->n_pkts_dropped = pkts; /* If need to read the bytes, set it. */ - if ((*stats_mask & meter2mask[i]) & - RTE_MTR_STATS_N_BYTES_DROPPED) + if (fm->bytes_dropped) stats->n_bytes_dropped = bytes; } return 0; @@ -1284,7 +1362,6 @@ mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error) .transfer = priv->config.dv_esw_en ? 1 : 0, }; void *tmp; - uint32_t i; TAILQ_FOREACH_SAFE(fm, fms, next, tmp) { /* Meter object must not have any owner. */ @@ -1300,10 +1377,10 @@ mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error) /* Remove from list. */ TAILQ_REMOVE(fms, fm, next); /* Free policer counters. */ - for (i = 0; i < RTE_DIM(fm->policer_stats.cnt); i++) - if (fm->policer_stats.cnt[i]) - mlx5_counter_free(dev, - fm->policer_stats.cnt[i]); + if (fm->policer_stats.pass_cnt) + mlx5_counter_free(dev, fm->policer_stats.pass_cnt); + if (fm->policer_stats.drop_cnt) + mlx5_counter_free(dev, fm->policer_stats.drop_cnt); /* Free meter flow table. */ mlx5_flow_destroy_policer_rules(dev, fm, &attr); mlx5_flow_destroy_mtr_tbls(dev, fm->mfts); -- 2.27.0