From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 36F39A0548; Tue, 20 Apr 2021 12:56:23 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 9BCB641775; Tue, 20 Apr 2021 12:55:39 +0200 (CEST) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by mails.dpdk.org (Postfix) with ESMTP id F08F541728 for ; Tue, 20 Apr 2021 12:55:25 +0200 (CEST) Received: from Internal Mail-Server by MTLPINE1 (envelope-from jiaweiw@nvidia.com) with SMTP; 20 Apr 2021 13:55:24 +0300 Received: from nvidia.com (gen-l-vrt-281.mtl.labs.mlnx [10.237.44.1]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 13KAtMRv009943; Tue, 20 Apr 2021 13:55:24 +0300 From: Jiawei Wang To: matan@nvidia.com, orika@nvidia.com, viacheslavo@nvidia.com, ferruh.yigit@intel.com, thomas@monjalon.net, Shahaf Shuler Cc: dev@dpdk.org, rasland@nvidia.com, asafp@nvidia.com, Li Zhang Date: Tue, 20 Apr 2021 13:55:16 +0300 Message-Id: <1618916122-181792-10-git-send-email-jiaweiw@nvidia.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1618916122-181792-1-git-send-email-jiaweiw@nvidia.com> References: <20210331073632.1443011-1-lizh@nvidia.com> <1618916122-181792-1-git-send-email-jiaweiw@nvidia.com> Subject: [dpdk-dev] [PATCH v6 09/15] net/mlx5: flow meter pool to manage meter object X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" From: Li Zhang Add ASO flow meter pool to manage meter object Signed-off-by: Li Zhang Acked-by: Matan Azrad --- drivers/net/mlx5/mlx5.c | 2 +- drivers/net/mlx5/mlx5.h | 207 ++++++++++++++++- drivers/net/mlx5/mlx5_flow.c | 70 ++++-- drivers/net/mlx5/mlx5_flow.h | 196 ++++------------ drivers/net/mlx5/mlx5_flow_dv.c | 203 +++++++++++++++-- drivers/net/mlx5/mlx5_flow_meter.c | 450 +++++++++++++++++++++++-------------- 6 files changed, 769 insertions(+), 359 deletions(-) diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index cc756ca..1b5b5cb 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -282,7 +282,7 @@ static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list = * for meter idx, so not set grow_trunk to avoid meter index * not jump continually. */ - .size = sizeof(struct mlx5_flow_meter), + .size = sizeof(struct mlx5_legacy_flow_meter), .trunk_size = 64, .need_lock = 1, .release_mem_en = 1, diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index d208efb..2e93dda 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -582,6 +583,193 @@ struct mlx5_dev_shared_port { /* Aging information for per port. */ }; +/*ASO flow meter structures*/ +/* Modify this value if enum rte_mtr_color changes. */ +#define RTE_MTR_DROPPED RTE_COLORS + +/* Meter policer statistics */ +struct mlx5_flow_policer_stats { + uint32_t pass_cnt; + /**< Color counter for pass. */ + uint32_t drop_cnt; + /**< Color counter for drop. */ +}; + +/* Meter table structure. */ +struct mlx5_meter_domain_info { + struct mlx5_flow_tbl_resource *tbl; + /**< Meter table. */ + struct mlx5_flow_tbl_resource *sfx_tbl; + /**< Meter suffix table. */ + struct mlx5_flow_dv_matcher *drop_matcher; + /**< Matcher for Drop. */ + struct mlx5_flow_dv_matcher *color_matcher; + /**< Matcher for Color. */ + void *jump_actn; + /**< Meter match action. */ + void *green_rule; + /**< Meter green rule. */ + void *drop_rule; + /**< Meter drop rule. */ +}; + +/* Meter table set for TX RX FDB. */ +struct mlx5_meter_domains_infos { + uint32_t ref_cnt; + /**< Table user count. */ + struct mlx5_meter_domain_info egress; + /**< TX meter table. */ + struct mlx5_meter_domain_info ingress; + /**< RX meter table. */ + struct mlx5_meter_domain_info transfer; + /**< FDB meter table. */ + void *drop_actn; + /**< Drop action as not matched. */ + void *green_count; + /**< Counters for green rule. */ + void *drop_count; + /**< Counters for green rule. */ + void *meter_action; + /**< Flow meter action. */ +}; + +/* Meter parameter structure. */ +struct mlx5_flow_meter_info { + uint32_t meter_id; + /**< Meter id. */ + struct mlx5_flow_meter_profile *profile; + /**< Meter profile parameters. */ + rte_spinlock_t sl; /**< Meter action spinlock. */ + /** Policer actions (per meter output color). */ + enum rte_mtr_policer_action action[RTE_COLORS]; + /** Set of stats counters to be enabled. + * @see enum rte_mtr_stats_type + */ + uint32_t green_bytes:1; + /** Set green bytes stats to be enabled. */ + uint32_t green_pkts:1; + /** Set green packets stats to be enabled. */ + uint32_t red_bytes:1; + /** Set red bytes stats to be enabled. */ + uint32_t red_pkts:1; + /** Set red packets stats to be enabled. */ + uint32_t bytes_dropped:1; + /** Set bytes dropped stats to be enabled. */ + uint32_t pkts_dropped:1; + /** Set packets dropped stats to be enabled. */ + uint32_t active_state:1; + /**< Meter hw active state. */ + uint32_t shared:1; + /**< Meter shared or not. */ + uint32_t is_enable:1; + /**< Meter disable/enable state. */ + uint32_t ingress:1; + /**< Rule applies to egress traffic. */ + uint32_t egress:1; + /** + * Instead of simply matching the properties of traffic as it would + * appear on a given DPDK port ID, enabling this attribute transfers + * a flow rule to the lowest possible level of any device endpoints + * found in the pattern. + * + * When supported, this effectively enables an application to + * re-route traffic not necessarily intended for it (e.g. coming + * from or addressed to different physical ports, VFs or + * applications) at the device level. + * + * It complements the behavior of some pattern items such as + * RTE_FLOW_ITEM_TYPE_PHY_PORT and is meaningless without them. + * + * When transferring flow rules, ingress and egress attributes keep + * their original meaning, as if processing traffic emitted or + * received by the application. + */ + uint32_t transfer:1; + struct mlx5_meter_domains_infos *mfts; + /**< Flow table created for this meter. */ + struct mlx5_flow_policer_stats policer_stats; + /**< Meter policer statistics. */ + uint32_t ref_cnt; + /**< Use count. */ + struct mlx5_indexed_pool *flow_ipool; + /**< Index pool for flow id. */ +}; + +/* RFC2697 parameter structure. */ +struct mlx5_flow_meter_srtcm_rfc2697_prm { + rte_be32_t cbs_cir; + /* + * bit 24-28: cbs_exponent, bit 16-23 cbs_mantissa, + * bit 8-12: cir_exponent, bit 0-7 cir_mantissa. + */ + rte_be32_t ebs_eir; + /* + * bit 24-28: ebs_exponent, bit 16-23 ebs_mantissa, + * bit 8-12: eir_exponent, bit 0-7 eir_mantissa. + */ +}; + +/* Flow meter profile structure. */ +struct mlx5_flow_meter_profile { + TAILQ_ENTRY(mlx5_flow_meter_profile) next; + /**< Pointer to the next flow meter structure. */ + uint32_t id; /**< Profile id. */ + struct rte_mtr_meter_profile profile; /**< Profile detail. */ + union { + struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm_prm; + /**< srtcm_rfc2697 struct. */ + }; + uint32_t ref_cnt; /**< Use count. */ +}; + +/* 2 meters in each ASO cache line */ +#define MLX5_MTRS_CONTAINER_RESIZE 64 +/* + * The pool index and offset of meter in the pool array makes up the + * meter index. In case the meter is from pool 0 and offset 0, it + * should plus 1 to avoid index 0, since 0 means invalid meter index + * currently. + */ +#define MLX5_MAKE_MTR_IDX(pi, offset) \ + ((pi) * MLX5_ASO_MTRS_PER_POOL + (offset) + 1) + +/*aso flow meter state*/ +enum mlx5_aso_mtr_state { + ASO_METER_FREE, /* In free list. */ + ASO_METER_WAIT, /* ACCESS_ASO WQE in progress. */ + ASO_METER_READY, /* CQE received. */ +}; + +/* Generic aso_flow_meter information. */ +struct mlx5_aso_mtr { + LIST_ENTRY(mlx5_aso_mtr) next; + struct mlx5_flow_meter_info fm; + /**< Pointer to the next aso flow meter structure. */ + uint8_t state; /**< ASO flow meter state. */ + uint8_t offset; +}; + +/* Generic aso_flow_meter pool structure. */ +struct mlx5_aso_mtr_pool { + struct mlx5_aso_mtr mtrs[MLX5_ASO_MTRS_PER_POOL]; + /*Must be the first in pool*/ + struct mlx5_devx_obj *devx_obj; + /* The devx object of the minimum aso flow meter ID. */ + uint32_t index; /* Pool index in management structure. */ +}; + +LIST_HEAD(aso_meter_list, mlx5_aso_mtr); +/* Pools management structure for ASO flow meter pools. */ +struct mlx5_aso_mtr_pools_mng { + volatile uint16_t n_valid; /* Number of valid pools. */ + uint16_t n; /* Number of pools. */ + rte_spinlock_t mtrsl; /* The ASO flow meter free list lock. */ + struct mlx5_l3t_tbl *mtr_idx_tbl; /* Meter index lookup table. */ + struct aso_meter_list meters; /* Free ASO flow meter list. */ + struct mlx5_aso_sq sq; /*SQ using by ASO flow meter. */ + struct mlx5_aso_mtr_pool **pools; /* ASO flow meter pool array. */ +}; + /* Table key of the hash organization. */ union mlx5_flow_tbl_key { struct { @@ -708,6 +896,7 @@ struct mlx5_dev_ctx_shared { uint32_t rq_ts_format:2; /* RQ timestamp formats supported. */ uint32_t sq_ts_format:2; /* SQ timestamp formats supported. */ uint32_t qp_ts_format:2; /* QP timestamp formats supported. */ + uint32_t meter_aso_en:1; /* Flow Meter ASO is supported. */ uint32_t max_port; /* Maximal IB device port index. */ struct mlx5_bond_info bond; /* Bonding information. */ void *ctx; /* Verbs/DV/DevX context. */ @@ -768,6 +957,8 @@ struct mlx5_dev_ctx_shared { struct mlx5_geneve_tlv_option_resource *geneve_tlv_option_resource; /* Management structure for geneve tlv option */ rte_spinlock_t geneve_tlv_opt_sl; /* Lock for geneve tlv resource */ + struct mlx5_aso_mtr_pools_mng *mtrmng; + /* Meter pools management structure. */ struct mlx5_dev_shared_port port[]; /* per device port data array. */ }; @@ -785,7 +976,7 @@ struct mlx5_proc_priv { /* MTR profile list. */ TAILQ_HEAD(mlx5_mtr_profiles, mlx5_flow_meter_profile); /* MTR list. */ -TAILQ_HEAD(mlx5_flow_meters, mlx5_flow_meter); +TAILQ_HEAD(mlx5_legacy_flow_meters, mlx5_legacy_flow_meter); /* RSS description. */ struct mlx5_flow_rss_desc { @@ -1003,7 +1194,7 @@ struct mlx5_priv { uint8_t mtr_sfx_reg; /* Meter prefix-suffix flow match REG_C. */ uint8_t mtr_color_reg; /* Meter color match REG_C. */ struct mlx5_mtr_profiles flow_meter_profiles; /* MTR profile list. */ - struct mlx5_flow_meters flow_meters; /* MTR list. */ + struct mlx5_legacy_flow_meters flow_meters; /* MTR list. */ uint8_t skip_default_rss_reta; /* Skip configuration of default reta. */ uint8_t fdb_def_rule; /* Whether fdb jump to table 1 is configured. */ struct mlx5_mp_id mp_id; /* ID of a multi-process process */ @@ -1282,13 +1473,15 @@ int mlx5_mp_os_req_queue_control(struct rte_eth_dev *dev, uint16_t queue_id, /* mlx5_flow_meter.c */ int mlx5_flow_meter_ops_get(struct rte_eth_dev *dev, void *arg); -struct mlx5_flow_meter *mlx5_flow_meter_find(struct mlx5_priv *priv, - uint32_t meter_id); +struct mlx5_flow_meter_info *mlx5_flow_meter_find(struct mlx5_priv *priv, + uint32_t meter_id, uint32_t *mtr_idx); +struct mlx5_flow_meter_info * +flow_dv_meter_find_by_idx(struct mlx5_priv *priv, uint32_t idx); int mlx5_flow_meter_attach(struct mlx5_priv *priv, - struct mlx5_flow_meter *fm, + struct mlx5_flow_meter_info *fm, const struct rte_flow_attr *attr, struct rte_flow_error *error); -void mlx5_flow_meter_detach(struct mlx5_flow_meter *fm); +void mlx5_flow_meter_detach(struct mlx5_flow_meter_info *fm); /* mlx5_os.c */ struct rte_pci_driver; @@ -1333,7 +1526,7 @@ int mlx5_txpp_xstats_get_names(struct rte_eth_dev *dev, eth_tx_burst_t mlx5_select_tx_function(struct rte_eth_dev *dev); -/* mlx5_flow_age.c */ +/* mlx5_flow_aso.c */ int mlx5_aso_queue_init(struct mlx5_dev_ctx_shared *sh); int mlx5_aso_queue_start(struct mlx5_dev_ctx_shared *sh); diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c index 1afddf7..7a6d6f8 100644 --- a/drivers/net/mlx5/mlx5_flow.c +++ b/drivers/net/mlx5/mlx5_flow.c @@ -4359,6 +4359,8 @@ struct mlx5_hlist_entry * * * @param[in] dev * Pointer to Ethernet device. + * @param[in] flow + * Parent flow structure pointer. * @param[in] fm * Pointer to flow meter structure. * @param[in] items @@ -4371,10 +4373,6 @@ struct mlx5_hlist_entry * * Suffix flow actions. * @param[out] actions_pre * Prefix flow actions. - * @param[out] pattern_sfx - * The pattern items for the suffix flow. - * @param[out] tag_sfx - * Pointer to suffix flow tag. * @param[out] error * Perform verbose error reporting if not NULL. * @@ -4383,7 +4381,8 @@ struct mlx5_hlist_entry * */ static uint32_t flow_meter_split_prep(struct rte_eth_dev *dev, - struct mlx5_flow_meter *fm, + struct rte_flow *flow, + struct mlx5_flow_meter_info *fm, const struct rte_flow_item items[], struct rte_flow_item sfx_items[], const struct rte_flow_action actions[], @@ -4504,7 +4503,7 @@ struct mlx5_hlist_entry * 0, error), .offset = mtr_id_offset, .length = mtr_reg_bits, - .data = fm->idx, + .data = flow->meter, }; /* * The color Reg bits used by flow_id are growing from @@ -5240,9 +5239,10 @@ struct mlx5_hlist_entry * struct rte_flow_item *sfx_items = NULL; struct mlx5_flow *dev_flow = NULL; struct rte_flow_attr sfx_attr = *attr; - struct mlx5_flow_meter *fm = NULL; + struct mlx5_flow_meter_info *fm = NULL; bool has_mtr = false; uint32_t meter_id; + uint32_t mtr_idx = 0; uint32_t mtr_tag_id = 0; size_t act_size; size_t item_size; @@ -5254,14 +5254,13 @@ struct mlx5_hlist_entry * &meter_id); if (has_mtr) { if (flow->meter) { - fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], - flow->meter); + fm = flow_dv_meter_find_by_idx(priv, flow->meter); if (!fm) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "Meter not found."); } else { - fm = mlx5_flow_meter_find(priv, meter_id); + fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); if (!fm) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, @@ -5270,7 +5269,7 @@ struct mlx5_hlist_entry * &sfx_attr, error); if (ret) return -rte_errno; - flow->meter = fm->idx; + flow->meter = mtr_idx; } wks->fm = fm; /* The prefix actions: meter, decap, encap, tag, end. */ @@ -5290,9 +5289,10 @@ struct mlx5_hlist_entry * sfx_items = (struct rte_flow_item *)((char *)sfx_actions + act_size); pre_actions = sfx_actions + actions_n; - mtr_tag_id = flow_meter_split_prep(dev, fm, items, sfx_items, - actions, sfx_actions, - pre_actions, error); + mtr_tag_id = flow_meter_split_prep(dev, flow, fm, items, + sfx_items, actions, + sfx_actions, pre_actions, + error); if (!mtr_tag_id) { ret = -rte_errno; goto exit; @@ -6629,7 +6629,7 @@ struct mlx5_meter_domains_infos * */ int mlx5_flow_prepare_policer_rules(struct rte_eth_dev *dev, - struct mlx5_flow_meter *fm, + struct mlx5_flow_meter_info *fm, const struct rte_flow_attr *attr) { const struct mlx5_flow_driver_ops *fops; @@ -6651,7 +6651,7 @@ struct mlx5_meter_domains_infos * */ int mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, - struct mlx5_flow_meter *fm, + struct mlx5_flow_meter_info *fm, const struct rte_flow_attr *attr) { const struct mlx5_flow_driver_ops *fops; @@ -6661,6 +6661,44 @@ struct mlx5_meter_domains_infos * } /** + * Allocate the needed aso flow meter id. + * + * @param[in] dev + * Pointer to Ethernet device. + * + * @return + * Index to aso flow meter on success, NULL otherwise. + */ +uint32_t +mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) +{ + const struct mlx5_flow_driver_ops *fops; + + fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); + return fops->create_meter(dev); +} + +/** + * Free the aso flow meter id. + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in] mtr_idx + * Index to aso flow meter to be free. + * + * @return + * 0 on success. + */ +void +mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) +{ + const struct mlx5_flow_driver_ops *fops; + + fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); + fops->free_meter(dev, mtr_idx); +} + +/** * Allocate a counter. * * @param[in] dev diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index 0806407..b6ec727 100644 --- a/drivers/net/mlx5/mlx5_flow.h +++ b/drivers/net/mlx5/mlx5_flow.h @@ -826,148 +826,17 @@ struct mlx5_flow { #define MLX5_FLOW_METER_DISABLE 0 #define MLX5_FLOW_METER_ENABLE 1 -#define MLX5_MAN_WIDTH 8 -/* Modify this value if enum rte_mtr_color changes. */ -#define RTE_MTR_DROPPED RTE_COLORS - -/* Meter policer statistics */ -struct mlx5_flow_policer_stats { - uint32_t pass_cnt; - /**< Color counter for pass. */ - uint32_t drop_cnt; - /**< Color counter for drop. */ -}; - -/* Meter table structure. */ -struct mlx5_meter_domain_info { - struct mlx5_flow_tbl_resource *tbl; - /**< Meter table. */ - struct mlx5_flow_tbl_resource *sfx_tbl; - /**< Meter suffix table. */ - struct mlx5_flow_dv_matcher *drop_matcher; - /**< Matcher for Drop. */ - struct mlx5_flow_dv_matcher *color_matcher; - /**< Matcher for Color. */ - void *jump_actn; - /**< Meter match action. */ - void *green_rule; - /**< Meter green rule. */ - void *drop_rule; - /**< Meter drop rule. */ -}; - -/* Meter table set for TX RX FDB. */ -struct mlx5_meter_domains_infos { - uint32_t ref_cnt; - /**< Table user count. */ - struct mlx5_meter_domain_info egress; - /**< TX meter table. */ - struct mlx5_meter_domain_info ingress; - /**< RX meter table. */ - struct mlx5_meter_domain_info transfer; - /**< FDB meter table. */ - void *green_count; - /**< Counters for green rule. */ - void *drop_count; - /**< Counters for green rule. */ - uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)]; - /**< Flow meter parameter. */ - size_t fmp_size; - /**< Flow meter parameter size. */ - void *meter_action; - /**< Flow meter action. */ -}; +#define MLX5_ASO_CQE_RESPONSE_DELAY 10 +#define MLX5_MTR_POLL_CQE_TIMES 100000u -/* Meter parameter structure. */ -struct mlx5_flow_meter { - TAILQ_ENTRY(mlx5_flow_meter) next; +#define MLX5_MAN_WIDTH 8 +/* Legacy Meter parameter structure. */ +struct mlx5_legacy_flow_meter { + struct mlx5_flow_meter_info fm; + /* Must be the first in struct. */ + TAILQ_ENTRY(mlx5_legacy_flow_meter) next; /**< Pointer to the next flow meter structure. */ uint32_t idx; /* Index to meter object. */ - uint32_t meter_id; - /**< Meter id. */ - struct mlx5_flow_meter_profile *profile; - /**< Meter profile parameters. */ - - rte_spinlock_t sl; /**< Meter action spinlock. */ - - /** Policer actions (per meter output color). */ - enum rte_mtr_policer_action action[RTE_COLORS]; - - uint32_t green_bytes:1; - /** Set green bytes stats to be enabled. */ - uint32_t green_pkts:1; - /** Set green packets stats to be enabled. */ - uint32_t red_bytes:1; - /** Set red bytes stats to be enabled. */ - uint32_t red_pkts:1; - /** Set red packets stats to be enabled. */ - uint32_t bytes_dropped:1; - /** Set bytes dropped stats to be enabled. */ - uint32_t pkts_dropped:1; - /** Set packets dropped stats to be enabled. */ - - /**< Rule applies to ingress traffic. */ - uint32_t ingress:1; - - /**< Rule applies to egress traffic. */ - uint32_t egress:1; - /** - * Instead of simply matching the properties of traffic as it would - * appear on a given DPDK port ID, enabling this attribute transfers - * a flow rule to the lowest possible level of any device endpoints - * found in the pattern. - * - * When supported, this effectively enables an application to - * re-route traffic not necessarily intended for it (e.g. coming - * from or addressed to different physical ports, VFs or - * applications) at the device level. - * - * It complements the behavior of some pattern items such as - * RTE_FLOW_ITEM_TYPE_PHY_PORT and is meaningless without them. - * - * When transferring flow rules, ingress and egress attributes keep - * their original meaning, as if processing traffic emitted or - * received by the application. - */ - uint32_t transfer:1; - struct mlx5_meter_domains_infos *mfts; - /**< Flow table created for this meter. */ - struct mlx5_flow_policer_stats policer_stats; - /**< Meter policer statistics. */ - uint32_t ref_cnt; - /**< Use count. */ - uint32_t active_state:1; - /**< Meter state. */ - uint32_t shared:1; - /**< Meter shared or not. */ - struct mlx5_indexed_pool *flow_ipool; - /**< Index pool for flow id. */ -}; - -/* RFC2697 parameter structure. */ -struct mlx5_flow_meter_srtcm_rfc2697_prm { - /* green_saturation_value = cbs_mantissa * 2^cbs_exponent */ - uint32_t cbs_exponent:5; - uint32_t cbs_mantissa:8; - /* cir = 8G * cir_mantissa * 1/(2^cir_exponent) Bytes/Sec */ - uint32_t cir_exponent:5; - uint32_t cir_mantissa:8; - /* yellow _saturation_value = ebs_mantissa * 2^ebs_exponent */ - uint32_t ebs_exponent:5; - uint32_t ebs_mantissa:8; -}; - -/* Flow meter profile structure. */ -struct mlx5_flow_meter_profile { - TAILQ_ENTRY(mlx5_flow_meter_profile) next; - /**< Pointer to the next flow meter structure. */ - uint32_t meter_profile_id; /**< Profile id. */ - struct rte_mtr_meter_profile profile; /**< Profile detail. */ - union { - struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm_prm; - /**< srtcm_rfc2697 struct. */ - }; - uint32_t ref_cnt; /**< Use count. */ }; #define MLX5_MAX_TUNNELS 256 @@ -1093,7 +962,7 @@ struct rte_flow { /**< Device flow handles that are part of the flow. */ uint32_t drv_type:2; /**< Driver type. */ uint32_t tunnel:1; - uint32_t meter:16; /**< Holds flow meter id. */ + uint32_t meter:24; /**< Holds flow meter id. */ uint32_t rix_mreg_copy; /**< Index to metadata register copy table resource. */ uint32_t counter; /**< Holds flow counter. */ @@ -1180,7 +1049,7 @@ struct mlx5_flow_workspace { struct mlx5_flow_rss_desc rss_desc; uint32_t rssq_num; /* Allocated queue num in rss_desc. */ uint32_t flow_idx; /* Intermediate device flow index. */ - struct mlx5_flow_meter *fm; /* Pointer to the meter in flow. */ + struct mlx5_flow_meter_info *fm; /* Pointer to the meter in flow. */ }; struct mlx5_flow_split_info { @@ -1226,12 +1095,16 @@ typedef int (*mlx5_flow_destroy_mtr_tbls_t)(struct rte_eth_dev *dev, struct mlx5_meter_domains_infos *tbls); typedef int (*mlx5_flow_create_policer_rules_t) (struct rte_eth_dev *dev, - struct mlx5_flow_meter *fm, + struct mlx5_flow_meter_info *fm, const struct rte_flow_attr *attr); typedef int (*mlx5_flow_destroy_policer_rules_t) (struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm, + const struct mlx5_flow_meter_info *fm, const struct rte_flow_attr *attr); +typedef uint32_t (*mlx5_flow_mtr_alloc_t) + (struct rte_eth_dev *dev); +typedef void (*mlx5_flow_mtr_free_t)(struct rte_eth_dev *dev, + uint32_t mtr_idx); typedef uint32_t (*mlx5_flow_counter_alloc_t) (struct rte_eth_dev *dev); typedef void (*mlx5_flow_counter_free_t)(struct rte_eth_dev *dev, @@ -1286,6 +1159,8 @@ struct mlx5_flow_driver_ops { mlx5_flow_destroy_mtr_tbls_t destroy_mtr_tbls; mlx5_flow_create_policer_rules_t prepare_policer_rules; mlx5_flow_destroy_policer_rules_t destroy_policer_rules; + mlx5_flow_mtr_alloc_t create_meter; + mlx5_flow_mtr_free_t free_meter; mlx5_flow_counter_alloc_t counter_alloc; mlx5_flow_counter_free_t counter_free; mlx5_flow_counter_query_t counter_query; @@ -1345,6 +1220,32 @@ struct flow_grp_info { return verdict; } +/** + * Get DV flow aso meter by index. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] idx + * mlx5 flow aso meter index in the container. + * @param[out] ppool + * mlx5 flow aso meter pool in the container, + * + * @return + * Pointer to the aso meter, NULL otherwise. + */ +static inline struct mlx5_aso_mtr * +mlx5_aso_meter_by_idx(struct mlx5_priv *priv, uint32_t idx) +{ + struct mlx5_aso_mtr_pool *pool; + struct mlx5_aso_mtr_pools_mng *mtrmng = priv->sh->mtrmng; + + /* Decrease to original index. */ + idx--; + MLX5_ASSERT(idx / MLX5_ASO_MTRS_PER_POOL < mtrmng->n); + pool = mtrmng->pools[idx / MLX5_ASO_MTRS_PER_POOL]; + return &pool->mtrs[idx % MLX5_ASO_MTRS_PER_POOL]; +} + int mlx5_flow_group_to_table(struct rte_eth_dev *dev, const struct mlx5_flow_tunnel *tunnel, uint32_t group, uint32_t *table, @@ -1488,10 +1389,10 @@ struct mlx5_meter_domains_infos *mlx5_flow_create_mtr_tbls int mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, struct mlx5_meter_domains_infos *tbl); int mlx5_flow_prepare_policer_rules(struct rte_eth_dev *dev, - struct mlx5_flow_meter *fm, + struct mlx5_flow_meter_info *fm, const struct rte_flow_attr *attr); int mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, - struct mlx5_flow_meter *fm, + struct mlx5_flow_meter_info *fm, const struct rte_flow_attr *attr); int mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error); @@ -1587,12 +1488,11 @@ struct mlx5_aso_age_action *flow_aso_age_get_by_idx(struct rte_eth_dev *dev, int flow_dev_geneve_tlv_option_resource_register(struct rte_eth_dev *dev, const struct rte_flow_item *item, struct rte_flow_error *error); - void flow_release_workspace(void *data); int mlx5_flow_os_init_workspace_once(void); void *mlx5_flow_os_get_specific_workspace(void); int mlx5_flow_os_set_specific_workspace(struct mlx5_flow_workspace *data); void mlx5_flow_os_release_workspace(void); - - +uint32_t mlx5_flow_mtr_alloc(struct rte_eth_dev *dev); +void mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx); #endif /* RTE_PMD_MLX5_FLOW_H_ */ diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 4b6b62c..4b2a272 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -4866,7 +4866,7 @@ struct mlx5_cache_entry * { struct mlx5_priv *priv = dev->data->dev_private; const struct rte_flow_action_meter *am = action->conf; - struct mlx5_flow_meter *fm; + struct mlx5_flow_meter_info *fm; if (!am) return rte_flow_error_set(error, EINVAL, @@ -4886,7 +4886,7 @@ struct mlx5_cache_entry * RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "meter action not supported"); - fm = mlx5_flow_meter_find(priv, am->mtr_id); + fm = mlx5_flow_meter_find(priv, am->mtr_id, NULL); if (!fm) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, @@ -5940,6 +5940,161 @@ struct mlx5_hlist_entry * } /** + * Resize a meter id container. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * + * @return + * 0 on success, otherwise negative errno value and rte_errno is set. + */ +static int +flow_dv_mtr_container_resize(struct rte_eth_dev *dev) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_aso_mtr_pools_mng *mtrmng = priv->sh->mtrmng; + void *old_pools = mtrmng->pools; + uint32_t resize = mtrmng->n + MLX5_MTRS_CONTAINER_RESIZE; + uint32_t mem_size = sizeof(struct mlx5_aso_mtr_pool *) * resize; + void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY); + + if (!pools) { + rte_errno = ENOMEM; + return -ENOMEM; + } + if (old_pools) + memcpy(pools, old_pools, mtrmng->n * + sizeof(struct mlx5_aso_mtr_pool *)); + mtrmng->n = resize; + mtrmng->pools = pools; + if (old_pools) + mlx5_free(old_pools); + return 0; +} + +/** + * Prepare a new meter and/or a new meter pool. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[out] mtr_free + * Where to put the pointer of a new meter.g. + * + * @return + * The meter pool pointer and @mtr_free is set on success, + * NULL otherwise and rte_errno is set. + */ +static struct mlx5_aso_mtr_pool * +flow_dv_mtr_pool_create(struct rte_eth_dev *dev, + struct mlx5_aso_mtr **mtr_free) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_aso_mtr_pools_mng *mtrmng = priv->sh->mtrmng; + struct mlx5_aso_mtr_pool *pool = NULL; + struct mlx5_devx_obj *dcs = NULL; + uint32_t i; + uint32_t log_obj_size; + + log_obj_size = rte_log2_u32(MLX5_ASO_MTRS_PER_POOL >> 1); + dcs = mlx5_devx_cmd_create_flow_meter_aso_obj(priv->sh->ctx, + priv->sh->pdn, log_obj_size); + if (!dcs) { + rte_errno = ENODATA; + return NULL; + } + pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool), 0, SOCKET_ID_ANY); + if (!pool) { + rte_errno = ENOMEM; + claim_zero(mlx5_devx_cmd_destroy(dcs)); + return NULL; + } + pool->devx_obj = dcs; + pool->index = mtrmng->n_valid; + if (pool->index == mtrmng->n && flow_dv_mtr_container_resize(dev)) { + mlx5_free(pool); + claim_zero(mlx5_devx_cmd_destroy(dcs)); + return NULL; + } + mtrmng->pools[pool->index] = pool; + mtrmng->n_valid++; + for (i = 1; i < MLX5_ASO_MTRS_PER_POOL; ++i) { + pool->mtrs[i].offset = i; + pool->mtrs[i].fm.meter_id = UINT32_MAX; + LIST_INSERT_HEAD(&mtrmng->meters, + &pool->mtrs[i], next); + } + pool->mtrs[0].offset = 0; + pool->mtrs[0].fm.meter_id = UINT32_MAX; + *mtr_free = &pool->mtrs[0]; + return pool; +} + +/** + * Release a flow meter into pool. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] mtr_idx + * Index to aso flow meter. + */ +static void +flow_dv_aso_mtr_release_to_pool(struct rte_eth_dev *dev, uint32_t mtr_idx) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_aso_mtr_pools_mng *mtrmng = priv->sh->mtrmng; + struct mlx5_aso_mtr *aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx); + + MLX5_ASSERT(aso_mtr); + rte_spinlock_lock(&mtrmng->mtrsl); + memset(&aso_mtr->fm, 0, sizeof(struct mlx5_flow_meter_info)); + aso_mtr->state = ASO_METER_FREE; + aso_mtr->fm.meter_id = UINT32_MAX; + LIST_INSERT_HEAD(&mtrmng->meters, aso_mtr, next); + rte_spinlock_unlock(&mtrmng->mtrsl); +} + +/** + * Allocate a aso flow meter. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * + * @return + * Index to aso flow meter on success, 0 otherwise and rte_errno is set. + */ +static uint32_t +flow_dv_mtr_alloc(struct rte_eth_dev *dev) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_aso_mtr *mtr_free = NULL; + struct mlx5_aso_mtr_pools_mng *mtrmng = priv->sh->mtrmng; + struct mlx5_aso_mtr_pool *pool; + uint32_t mtr_idx = 0; + + if (!priv->config.devx) { + rte_errno = ENOTSUP; + return 0; + } + /* Allocate the flow meter memory. */ + /* Get free meters from management. */ + rte_spinlock_lock(&mtrmng->mtrsl); + mtr_free = LIST_FIRST(&mtrmng->meters); + if (mtr_free) + LIST_REMOVE(mtr_free, next); + if (!mtr_free && !flow_dv_mtr_pool_create(dev, &mtr_free)) { + rte_spinlock_unlock(&mtrmng->mtrsl); + return 0; + } + mtr_free->state = ASO_METER_WAIT; + rte_spinlock_unlock(&mtrmng->mtrsl); + pool = container_of(mtr_free, + struct mlx5_aso_mtr_pool, + mtrs[mtr_free->offset]); + mtr_idx = MLX5_MAKE_MTR_IDX(pool->index, mtr_free->offset); + return mtr_idx; +} + +/** * Verify the @p attributes will be correctly understood by the NIC and store * them in the @p flow if everything is correct. * @@ -12584,7 +12739,7 @@ struct mlx5_cache_entry * { struct mlx5_flow_handle *dev_handle; struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_flow_meter *fm = NULL; + struct mlx5_flow_meter_info *fm = NULL; uint32_t srss = 0; if (!flow) @@ -12595,8 +12750,7 @@ struct mlx5_cache_entry * flow->counter = 0; } if (flow->meter) { - fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], - flow->meter); + fm = flow_dv_meter_find_by_idx(priv, flow->meter); if (fm) mlx5_flow_meter_detach(fm); flow->meter = 0; @@ -13697,7 +13851,7 @@ struct mlx5_cache_entry * */ static int flow_dv_destroy_policer_rules(struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm, + const struct mlx5_flow_meter_info *fm, const struct rte_flow_attr *attr) { struct mlx5_meter_domains_infos *mtb = fm ? fm->mfts : NULL; @@ -13720,6 +13874,8 @@ struct mlx5_cache_entry * * Pointer to Ethernet device. * @param[in] fm * Pointer to flow meter structure. + * @param[in] mtr_idx + * meter index. * @param[in] mtb * Pointer to DV meter table set. * @param[out] drop_rule @@ -13732,7 +13888,8 @@ struct mlx5_cache_entry * */ static int flow_dv_create_policer_forward_rule(struct rte_eth_dev *dev, - struct mlx5_flow_meter *fm, + struct mlx5_flow_meter_info *fm, + uint32_t mtr_idx, struct mlx5_meter_domain_info *dtb, void **drop_rule, void **green_rule) @@ -13780,7 +13937,7 @@ struct mlx5_cache_entry * /* Create Drop flow, matching meter_id only. */ i = 0; flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_id_reg_c, - (fm->idx << mtr_id_offset), UINT32_MAX); + (mtr_idx << mtr_id_offset), UINT32_MAX); if (mtb->drop_count) actions[i++] = mtb->drop_count; actions[i++] = priv->sh->dr_drop_action; @@ -13794,7 +13951,7 @@ struct mlx5_cache_entry * i = 0; if (priv->mtr_reg_share) { flow_dv_match_meta_reg(matcher.buf, value.buf, color_reg_c, - ((fm->idx << mtr_id_offset) | + ((mtr_idx << mtr_id_offset) | rte_col_2_mlx5_col(RTE_COLOR_GREEN)), UINT32_MAX); } else { @@ -13802,7 +13959,7 @@ struct mlx5_cache_entry * rte_col_2_mlx5_col(RTE_COLOR_GREEN), UINT32_MAX); flow_dv_match_meta_reg(matcher.buf, value.buf, mtr_id_reg_c, - fm->idx, UINT32_MAX); + mtr_idx, UINT32_MAX); } if (mtb->green_count) actions[i++] = mtb->green_count; @@ -13835,9 +13992,10 @@ struct mlx5_cache_entry * */ static int flow_dv_prepare_policer_rules(struct rte_eth_dev *dev, - struct mlx5_flow_meter *fm, + struct mlx5_flow_meter_info *fm, const struct rte_flow_attr *attr) { + struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_meter_domains_infos *mtb = fm->mfts; bool initialized = false; struct mlx5_flow_counter *cnt; @@ -13847,6 +14005,7 @@ struct mlx5_cache_entry * void *ingress_green_rule = NULL; void *transfer_drop_rule = NULL; void *transfer_green_rule = NULL; + uint32_t mtr_idx; int ret; /* Get the statistics counters for green/drop. */ @@ -13873,9 +14032,23 @@ struct mlx5_cache_entry * */ if (mtb->egress.drop_rule) initialized = true; + if (priv->sh->meter_aso_en) { + struct mlx5_aso_mtr *aso_mtr = NULL; + struct mlx5_aso_mtr_pool *pool; + + aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); + pool = container_of(aso_mtr, struct mlx5_aso_mtr_pool, + mtrs[aso_mtr->offset]); + mtr_idx = MLX5_MAKE_MTR_IDX(pool->index, aso_mtr->offset); + } else { + struct mlx5_legacy_flow_meter *legacy_fm; + + legacy_fm = container_of(fm, struct mlx5_legacy_flow_meter, fm); + mtr_idx = legacy_fm->idx; + } if (attr->egress) { ret = flow_dv_create_policer_forward_rule(dev, - fm, &mtb->egress, + fm, mtr_idx, &mtb->egress, &egress_drop_rule, &egress_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create egress policer."); @@ -13884,7 +14057,7 @@ struct mlx5_cache_entry * } if (attr->ingress) { ret = flow_dv_create_policer_forward_rule(dev, - fm, &mtb->ingress, + fm, mtr_idx, &mtb->ingress, &ingress_drop_rule, &ingress_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create ingress policer."); @@ -13893,7 +14066,7 @@ struct mlx5_cache_entry * } if (attr->transfer) { ret = flow_dv_create_policer_forward_rule(dev, - fm, &mtb->transfer, + fm, mtr_idx, &mtb->transfer, &transfer_drop_rule, &transfer_green_rule); if (ret) { DRV_LOG(ERR, "Failed to create transfer policer."); @@ -14233,6 +14406,8 @@ struct mlx5_cache_entry * .destroy_mtr_tbls = flow_dv_destroy_mtr_tbl, .prepare_policer_rules = flow_dv_prepare_policer_rules, .destroy_policer_rules = flow_dv_destroy_policer_rules, + .create_meter = flow_dv_mtr_alloc, + .free_meter = flow_dv_aso_mtr_release_to_pool, .counter_alloc = flow_dv_counter_allocate, .counter_free = flow_dv_counter_free, .counter_query = flow_dv_counter_query, diff --git a/drivers/net/mlx5/mlx5_flow_meter.c b/drivers/net/mlx5/mlx5_flow_meter.c index 2a37dec..956a6c3 100644 --- a/drivers/net/mlx5/mlx5_flow_meter.c +++ b/drivers/net/mlx5/mlx5_flow_meter.c @@ -28,40 +28,43 @@ */ static void * mlx5_flow_meter_action_create(struct mlx5_priv *priv, - struct mlx5_flow_meter *fm) + struct mlx5_flow_meter_info *fm) { #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER struct mlx5dv_dr_flow_meter_attr mtr_init; - void *attr = fm->mfts->fmp; + uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)]; struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = &fm->profile->srtcm_prm; + uint32_t cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir); + uint32_t ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir); + uint32_t val; - fm->mfts->fmp_size = MLX5_ST_SZ_BYTES(flow_meter_parameters); - memset(attr, 0, fm->mfts->fmp_size); - MLX5_SET(flow_meter_parameters, attr, valid, 1); - MLX5_SET(flow_meter_parameters, attr, bucket_overflow, 1); - MLX5_SET(flow_meter_parameters, attr, - start_color, MLX5_FLOW_COLOR_GREEN); - MLX5_SET(flow_meter_parameters, attr, both_buckets_on_green, 0); - MLX5_SET(flow_meter_parameters, - attr, cbs_exponent, srtcm->cbs_exponent); - MLX5_SET(flow_meter_parameters, - attr, cbs_mantissa, srtcm->cbs_mantissa); - MLX5_SET(flow_meter_parameters, - attr, cir_exponent, srtcm->cir_exponent); - MLX5_SET(flow_meter_parameters, - attr, cir_mantissa, srtcm->cir_mantissa); - MLX5_SET(flow_meter_parameters, - attr, ebs_exponent, srtcm->ebs_exponent); - MLX5_SET(flow_meter_parameters, - attr, ebs_mantissa, srtcm->ebs_mantissa); + memset(fmp, 0, MLX5_ST_SZ_BYTES(flow_meter_parameters)); + MLX5_SET(flow_meter_parameters, fmp, valid, 1); + MLX5_SET(flow_meter_parameters, fmp, bucket_overflow, 1); + MLX5_SET(flow_meter_parameters, fmp, + start_color, MLX5_FLOW_COLOR_GREEN); + MLX5_SET(flow_meter_parameters, fmp, both_buckets_on_green, 0); + val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK; + MLX5_SET(flow_meter_parameters, fmp, cbs_exponent, val); + val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK; + MLX5_SET(flow_meter_parameters, fmp, cbs_mantissa, val); + val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK; + MLX5_SET(flow_meter_parameters, fmp, cir_exponent, val); + val = (cbs_cir & ASO_DSEG_MAN_MASK); + MLX5_SET(flow_meter_parameters, fmp, cir_mantissa, val); + val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK; + MLX5_SET(flow_meter_parameters, fmp, ebs_exponent, val); + val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK; + MLX5_SET(flow_meter_parameters, fmp, ebs_mantissa, val); mtr_init.next_table = fm->transfer ? fm->mfts->transfer.tbl->obj : - fm->egress ? fm->mfts->egress.tbl->obj : - fm->mfts->ingress.tbl->obj; + fm->egress ? fm->mfts->egress.tbl->obj : + fm->mfts->ingress.tbl->obj; mtr_init.reg_c_index = priv->mtr_color_reg - REG_C_0; - mtr_init.flow_meter_parameter = fm->mfts->fmp; - mtr_init.flow_meter_parameter_sz = fm->mfts->fmp_size; + mtr_init.flow_meter_parameter = fmp; + mtr_init.flow_meter_parameter_sz = + MLX5_ST_SZ_BYTES(flow_meter_parameters); mtr_init.active = fm->active_state; return mlx5_glue->dv_create_flow_action_meter(&mtr_init); #else @@ -89,7 +92,7 @@ struct mlx5_flow_meter_profile *fmp; TAILQ_FOREACH(fmp, fmps, next) - if (meter_profile_id == fmp->meter_profile_id) + if (meter_profile_id == fmp->id) return fmp; return NULL; } @@ -239,44 +242,51 @@ { struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = &fmp->srtcm_prm; uint8_t man, exp; + uint32_t cbs_exp, cbs_man, cir_exp, cir_man; + uint32_t ebs_exp, ebs_man; if (fmp->profile.alg != RTE_MTR_SRTCM_RFC2697) return -rte_mtr_error_set(error, ENOTSUP, RTE_MTR_ERROR_TYPE_METER_PROFILE, NULL, "Metering algorithm not supported."); + /* cir = 8G * cir_mantissa * 1/(2^cir_exponent)) Bytes/Sec */ + mlx5_flow_meter_cir_man_exp_calc(fmp->profile.srtcm_rfc2697.cir, + &man, &exp); + /* Check if cir mantissa is too large. */ + if (exp > ASO_DSEG_CIR_EXP_MASK) + return -rte_mtr_error_set(error, ENOTSUP, + RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, + "meter profile parameter cir is" + " not supported."); + cir_man = man; + cir_exp = exp; /* cbs = cbs_mantissa * 2^cbs_exponent */ mlx5_flow_meter_xbs_man_exp_calc(fmp->profile.srtcm_rfc2697.cbs, &man, &exp); - srtcm->cbs_mantissa = man; - srtcm->cbs_exponent = exp; /* Check if cbs mantissa is too large. */ - if (srtcm->cbs_exponent != exp) - return -rte_mtr_error_set(error, EINVAL, + if (exp > ASO_DSEG_EXP_MASK) + return -rte_mtr_error_set(error, ENOTSUP, RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, - "Metering profile parameter cbs is" - " invalid."); - /* ebs = ebs_mantissa * 2^ebs_exponent */ + "meter profile parameter cbs is" + " not supported."); + cbs_man = man; + cbs_exp = exp; + srtcm->cbs_cir = rte_cpu_to_be_32(cbs_exp << ASO_DSEG_CBS_EXP_OFFSET | + cbs_man << ASO_DSEG_CBS_MAN_OFFSET | + cir_exp << ASO_DSEG_CIR_EXP_OFFSET | + cir_man); mlx5_flow_meter_xbs_man_exp_calc(fmp->profile.srtcm_rfc2697.ebs, &man, &exp); - srtcm->ebs_mantissa = man; - srtcm->ebs_exponent = exp; /* Check if ebs mantissa is too large. */ - if (srtcm->ebs_exponent != exp) - return -rte_mtr_error_set(error, EINVAL, - RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, - "Metering profile parameter ebs is" - " invalid."); - /* cir = 8G * cir_mantissa * 1/(2^cir_exponent)) Bytes/Sec */ - mlx5_flow_meter_cir_man_exp_calc(fmp->profile.srtcm_rfc2697.cir, - &man, &exp); - srtcm->cir_mantissa = man; - srtcm->cir_exponent = exp; - /* Check if cir mantissa is too large. */ - if (srtcm->cir_exponent != exp) - return -rte_mtr_error_set(error, EINVAL, + if (exp > ASO_DSEG_EXP_MASK) + return -rte_mtr_error_set(error, ENOTSUP, RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, - "Metering profile parameter cir is" - " invalid."); + "meter profile parameter ebs is" + " not supported."); + ebs_man = man; + ebs_exp = exp; + srtcm->ebs_eir = rte_cpu_to_be_32(ebs_exp << ASO_DSEG_EBS_EXP_OFFSET | + ebs_man << ASO_DSEG_EBS_MAN_OFFSET); return 0; } @@ -306,7 +316,11 @@ RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, "Meter is not supported"); memset(cap, 0, sizeof(*cap)); - cap->n_max = 1 << qattr->log_max_flow_meter; + if (priv->sh->meter_aso_en) + /* 2 meters per one ASO cache line. */ + cap->n_max = 1 << (qattr->log_max_num_meter_aso + 1); + else + cap->n_max = 1 << qattr->log_max_flow_meter; cap->n_shared_max = cap->n_max; cap->identical = 1; cap->shared_identical = 1; @@ -365,7 +379,7 @@ NULL, "Meter profile memory " "alloc failed."); /* Fill profile info. */ - fmp->meter_profile_id = meter_profile_id; + fmp->id = meter_profile_id; fmp->profile = *profile; /* Fill the flow meter parameters for the PRM. */ ret = mlx5_flow_meter_param_fill(fmp, error); @@ -499,7 +513,7 @@ NULL, "Green color only supports recolor green action."); /* Validate meter id. */ - if (mlx5_flow_meter_find(priv, meter_id)) + if (mlx5_flow_meter_find(priv, meter_id, NULL)) return -rte_mtr_error_set(error, EEXIST, RTE_MTR_ERROR_TYPE_MTR_ID, NULL, "Meter object already exists."); @@ -524,7 +538,7 @@ */ static int mlx5_flow_meter_action_modify(struct mlx5_priv *priv, - struct mlx5_flow_meter *fm, + struct mlx5_flow_meter_info *fm, const struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm, uint64_t modify_bits, uint32_t active_state) { @@ -533,33 +547,37 @@ uint32_t *attr; struct mlx5dv_dr_flow_meter_attr mod_attr = { 0 }; int ret; + uint32_t cbs_cir, ebs_eir, val; /* Fill command parameters. */ mod_attr.reg_c_index = priv->mtr_color_reg - REG_C_0; mod_attr.flow_meter_parameter = in; - mod_attr.flow_meter_parameter_sz = fm->mfts->fmp_size; + mod_attr.flow_meter_parameter_sz = + MLX5_ST_SZ_BYTES(flow_meter_parameters); if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE) mod_attr.active = !!active_state; else mod_attr.active = 0; attr = in; + cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir); + ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir); if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) { - MLX5_SET(flow_meter_parameters, - attr, cbs_exponent, srtcm->cbs_exponent); - MLX5_SET(flow_meter_parameters, - attr, cbs_mantissa, srtcm->cbs_mantissa); + val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK; + MLX5_SET(flow_meter_parameters, attr, cbs_exponent, val); + val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK; + MLX5_SET(flow_meter_parameters, attr, cbs_mantissa, val); } if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) { - MLX5_SET(flow_meter_parameters, - attr, cir_exponent, srtcm->cir_exponent); - MLX5_SET(flow_meter_parameters, - attr, cir_mantissa, srtcm->cir_mantissa); + val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK; + MLX5_SET(flow_meter_parameters, attr, cir_exponent, val); + val = cbs_cir & ASO_DSEG_MAN_MASK; + MLX5_SET(flow_meter_parameters, attr, cir_mantissa, val); } if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) { - MLX5_SET(flow_meter_parameters, - attr, ebs_exponent, srtcm->ebs_exponent); - MLX5_SET(flow_meter_parameters, - attr, ebs_mantissa, srtcm->ebs_mantissa); + val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK; + MLX5_SET(flow_meter_parameters, attr, ebs_exponent, val); + val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK; + MLX5_SET(flow_meter_parameters, attr, ebs_mantissa, val); } /* Apply modifications to meter only if it was created. */ if (fm->mfts->meter_action) { @@ -572,26 +590,6 @@ /* Update succeedded modify meter parameters. */ if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE) fm->active_state = !!active_state; - attr = fm->mfts->fmp; - if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) { - MLX5_SET(flow_meter_parameters, - attr, cbs_exponent, srtcm->cbs_exponent); - MLX5_SET(flow_meter_parameters, - attr, cbs_mantissa, srtcm->cbs_mantissa); - } - if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) { - MLX5_SET(flow_meter_parameters, - attr, cir_exponent, srtcm->cir_exponent); - MLX5_SET(flow_meter_parameters, - attr, cir_mantissa, srtcm->cir_mantissa); - } - if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) { - MLX5_SET(flow_meter_parameters, - attr, ebs_exponent, srtcm->ebs_exponent); - MLX5_SET(flow_meter_parameters, - attr, ebs_mantissa, srtcm->ebs_mantissa); - } - return 0; #else (void)priv; @@ -604,7 +602,7 @@ } static void -mlx5_flow_meter_stats_enable_update(struct mlx5_flow_meter *fm, +mlx5_flow_meter_stats_enable_update(struct mlx5_flow_meter_info *fm, uint64_t stats_mask) { fm->green_bytes = (stats_mask & RTE_MTR_STATS_N_BYTES_GREEN) ? 1 : 0; @@ -639,9 +637,10 @@ struct rte_mtr_error *error) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_flow_meters *fms = &priv->flow_meters; + struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; struct mlx5_flow_meter_profile *fmp; - struct mlx5_flow_meter *fm; + struct mlx5_legacy_flow_meter *legacy_fm; + struct mlx5_flow_meter_info *fm; const struct rte_flow_attr attr = { .ingress = 1, .egress = 1, @@ -653,8 +652,9 @@ .need_lock = 1, .type = "mlx5_flow_mtr_flow_id_pool", }; + struct mlx5_aso_mtr *aso_mtr; + uint32_t mtr_idx; int ret; - uint32_t idx = 0; uint8_t mtr_id_bits; uint8_t mtr_reg_bits = priv->mtr_reg_share ? MLX5_MTR_IDLE_BITS_IN_COLOR_REG : MLX5_REG_BITS; @@ -674,19 +674,31 @@ RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, NULL, "Meter profile id not valid."); /* Allocate the flow meter memory. */ - fm = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MTR], &idx); - if (fm == NULL) - return -rte_mtr_error_set(error, ENOMEM, - RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, - "Memory alloc failed for meter."); - mtr_id_bits = MLX5_REG_BITS - __builtin_clz(idx); + if (priv->sh->meter_aso_en) { + mtr_idx = mlx5_flow_mtr_alloc(dev); + if (!mtr_idx) + return -rte_mtr_error_set(error, ENOMEM, + RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, + "Memory alloc failed for meter."); + aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx); + fm = &aso_mtr->fm; + } else { + legacy_fm = mlx5_ipool_zmalloc + (priv->sh->ipool[MLX5_IPOOL_MTR], &mtr_idx); + if (legacy_fm == NULL) + return -rte_mtr_error_set(error, ENOMEM, + RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, + "Memory alloc failed for meter."); + legacy_fm->idx = mtr_idx; + fm = &legacy_fm->fm; + } + mtr_id_bits = MLX5_REG_BITS - __builtin_clz(mtr_idx); if ((mtr_id_bits + priv->max_mtr_flow_bits) > mtr_reg_bits) { DRV_LOG(ERR, "Meter number exceeds max limit."); goto error; } if (mtr_id_bits > priv->max_mtr_bits) priv->max_mtr_bits = mtr_id_bits; - fm->idx = idx; /* Fill the flow meter parameters. */ fm->meter_id = meter_id; fm->profile = fmp; @@ -712,10 +724,11 @@ if (ret) goto error; /* Add to the flow meter list. */ - TAILQ_INSERT_TAIL(fms, fm, next); + if (!priv->sh->meter_aso_en) + TAILQ_INSERT_TAIL(fms, legacy_fm, next); fm->active_state = 1; /* Config meter starts as active. */ fm->shared = !!shared; - fm->profile->ref_cnt++; + __atomic_add_fetch(&fm->profile->ref_cnt, 1, __ATOMIC_RELAXED); fm->flow_ipool = mlx5_ipool_create(&flow_ipool_cfg); if (!fm->flow_ipool) goto error; @@ -729,12 +742,57 @@ mlx5_counter_free(dev, fm->policer_stats.pass_cnt); if (fm->policer_stats.drop_cnt) mlx5_counter_free(dev, fm->policer_stats.drop_cnt); - mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], idx); + if (priv->sh->meter_aso_en) + mlx5_flow_mtr_free(dev, mtr_idx); + else + mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], mtr_idx); return -rte_mtr_error_set(error, -ret, RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, "Failed to create devx meter."); } +static int +mlx5_flow_meter_params_flush(struct rte_eth_dev *dev, + struct mlx5_flow_meter_info *fm, + const struct rte_flow_attr *attr, + uint32_t mtr_idx) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; + struct mlx5_flow_meter_profile *fmp; + struct mlx5_legacy_flow_meter *legacy_fm = NULL; + + /* Meter object must not have any owner. */ + MLX5_ASSERT(!fm->ref_cnt); + /* Get meter profile. */ + fmp = fm->profile; + if (fmp == NULL) + return -1; + /* Update dependencies. */ + __atomic_sub_fetch(&fmp->ref_cnt, 1, __ATOMIC_RELAXED); + /* Remove from list. */ + if (!priv->sh->meter_aso_en) { + legacy_fm = container_of(fm, struct mlx5_legacy_flow_meter, fm); + TAILQ_REMOVE(fms, legacy_fm, next); + } + /* Free policer counters. */ + if (fm->policer_stats.pass_cnt) + mlx5_counter_free(dev, fm->policer_stats.pass_cnt); + if (fm->policer_stats.drop_cnt) + mlx5_counter_free(dev, fm->policer_stats.drop_cnt); + /* Free meter flow table. */ + if (fm->flow_ipool) + mlx5_ipool_destroy(fm->flow_ipool); + mlx5_flow_destroy_policer_rules(dev, fm, attr); + mlx5_flow_destroy_mtr_tbls(dev, fm->mfts); + if (priv->sh->meter_aso_en) + mlx5_flow_mtr_free(dev, mtr_idx); + else + mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], + legacy_fm->idx); + return 0; +} + /** * Destroy meter rules. * @@ -753,21 +811,21 @@ struct rte_mtr_error *error) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_flow_meters *fms = &priv->flow_meters; - struct mlx5_flow_meter_profile *fmp; - struct mlx5_flow_meter *fm; + struct mlx5_aso_mtr_pools_mng *mtrmng = priv->sh->mtrmng; + struct mlx5_flow_meter_info *fm; const struct rte_flow_attr attr = { .ingress = 1, .egress = 1, .transfer = priv->config.dv_esw_en ? 1 : 0, }; + uint32_t mtr_idx = 0; if (!priv->mtr_en) return -rte_mtr_error_set(error, ENOTSUP, RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, "Meter is not supported"); /* Meter object must exist. */ - fm = mlx5_flow_meter_find(priv, meter_id); + fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); if (fm == NULL) return -rte_mtr_error_set(error, ENOENT, RTE_MTR_ERROR_TYPE_MTR_ID, @@ -777,24 +835,17 @@ return -rte_mtr_error_set(error, EBUSY, RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, "Meter object is being used."); - /* Get the meter profile. */ - fmp = fm->profile; - MLX5_ASSERT(fmp); - /* Update dependencies. */ - fmp->ref_cnt--; - /* Remove from the flow meter list. */ - TAILQ_REMOVE(fms, fm, next); - /* Free policer counters. */ - if (fm->policer_stats.pass_cnt) - mlx5_counter_free(dev, fm->policer_stats.pass_cnt); - if (fm->policer_stats.drop_cnt) - mlx5_counter_free(dev, fm->policer_stats.drop_cnt); - /* Free meter flow table */ - if (fm->flow_ipool) - mlx5_ipool_destroy(fm->flow_ipool); - mlx5_flow_destroy_policer_rules(dev, fm, &attr); - mlx5_flow_destroy_mtr_tbls(dev, fm->mfts); - mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], fm->idx); + if (priv->sh->meter_aso_en) { + if (mlx5_l3t_clear_entry(mtrmng->mtr_idx_tbl, meter_id)) + return -rte_mtr_error_set(error, EBUSY, + RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, + "Fail to delete ASO Meter in index table."); + } + /* Destroy the meter profile. */ + if (mlx5_flow_meter_params_flush(dev, fm, &attr, mtr_idx)) + return -rte_mtr_error_set(error, EINVAL, + RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, + NULL, "MTR object meter profile invalid."); return 0; } @@ -815,17 +866,13 @@ */ static int mlx5_flow_meter_modify_state(struct mlx5_priv *priv, - struct mlx5_flow_meter *fm, + struct mlx5_flow_meter_info *fm, uint32_t new_state, struct rte_mtr_error *error) { static const struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm = { - .cbs_exponent = 20, - .cbs_mantissa = 191, - .cir_exponent = 0, - .cir_mantissa = 200, - .ebs_exponent = 0, - .ebs_mantissa = 0, + .cbs_cir = RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL), + .ebs_eir = 0, }; uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS | MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR; @@ -867,7 +914,7 @@ struct rte_mtr_error *error) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_flow_meter *fm; + struct mlx5_flow_meter_info *fm; int ret; if (!priv->mtr_en) @@ -875,7 +922,7 @@ RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, "Meter is not supported"); /* Meter object must exist. */ - fm = mlx5_flow_meter_find(priv, meter_id); + fm = mlx5_flow_meter_find(priv, meter_id, NULL); if (fm == NULL) return -rte_mtr_error_set(error, ENOENT, RTE_MTR_ERROR_TYPE_MTR_ID, @@ -908,7 +955,7 @@ struct rte_mtr_error *error) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_flow_meter *fm; + struct mlx5_flow_meter_info *fm; int ret; if (!priv->mtr_en) @@ -916,7 +963,7 @@ RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, "Meter is not supported"); /* Meter object must exist. */ - fm = mlx5_flow_meter_find(priv, meter_id); + fm = mlx5_flow_meter_find(priv, meter_id, NULL); if (fm == NULL) return -rte_mtr_error_set(error, ENOENT, RTE_MTR_ERROR_TYPE_MTR_ID, @@ -954,7 +1001,7 @@ struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flow_meter_profile *fmp; struct mlx5_flow_meter_profile *old_fmp; - struct mlx5_flow_meter *fm; + struct mlx5_flow_meter_info *fm; uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS | MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR; int ret; @@ -970,7 +1017,7 @@ RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, NULL, "Meter profile not found."); /* Meter object must exist. */ - fm = mlx5_flow_meter_find(priv, meter_id); + fm = mlx5_flow_meter_find(priv, meter_id, NULL); if (fm == NULL) return -rte_mtr_error_set(error, ENOENT, RTE_MTR_ERROR_TYPE_MTR_ID, @@ -1020,7 +1067,7 @@ struct rte_mtr_error *error) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_flow_meter *fm; + struct mlx5_flow_meter_info *fm; const struct rte_flow_attr attr = { .ingress = 1, .egress = 1, @@ -1034,7 +1081,7 @@ RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, "Meter is not supported"); /* Meter object must exist. */ - fm = mlx5_flow_meter_find(priv, meter_id); + fm = mlx5_flow_meter_find(priv, meter_id, NULL); if (fm == NULL) return -rte_mtr_error_set(error, ENOENT, RTE_MTR_ERROR_TYPE_MTR_ID, @@ -1134,7 +1181,7 @@ struct rte_mtr_error *error) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_flow_meter *fm; + struct mlx5_flow_meter_info *fm; struct mlx5_flow_policer_stats *ps; uint64_t pkts; uint64_t bytes; @@ -1145,7 +1192,7 @@ RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, "Meter is not supported"); /* Meter object must exist. */ - fm = mlx5_flow_meter_find(priv, meter_id); + fm = mlx5_flow_meter_find(priv, meter_id, NULL); if (fm == NULL) return -rte_mtr_error_set(error, ENOENT, RTE_MTR_ERROR_TYPE_MTR_ID, @@ -1239,19 +1286,69 @@ * @return * Pointer to the profile found on success, NULL otherwise. */ -struct mlx5_flow_meter * -mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id) +struct mlx5_flow_meter_info * +mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id, + uint32_t *mtr_idx) { - struct mlx5_flow_meters *fms = &priv->flow_meters; - struct mlx5_flow_meter *fm; + struct mlx5_legacy_flow_meter *legacy_fm; + struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; + struct mlx5_aso_mtr *aso_mtr; + struct mlx5_aso_mtr_pools_mng *mtrmng = priv->sh->mtrmng; + union mlx5_l3t_data data; - TAILQ_FOREACH(fm, fms, next) - if (meter_id == fm->meter_id) - return fm; + if (priv->sh->meter_aso_en) { + rte_spinlock_lock(&mtrmng->mtrsl); + if (!mtrmng->n_valid) { + rte_spinlock_unlock(&mtrmng->mtrsl); + return NULL; + } + if (mlx5_l3t_get_entry(mtrmng->mtr_idx_tbl, meter_id, &data) || + !data.dword) { + rte_spinlock_unlock(&mtrmng->mtrsl); + return NULL; + } + if (mtr_idx) + *mtr_idx = data.dword; + aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword); + mlx5_l3t_clear_entry(mtrmng->mtr_idx_tbl, meter_id); + if (meter_id == aso_mtr->fm.meter_id) { + rte_spinlock_unlock(&mtrmng->mtrsl); + return &aso_mtr->fm; + } + rte_spinlock_unlock(&mtrmng->mtrsl); + } else { + TAILQ_FOREACH(legacy_fm, fms, next) + if (meter_id == legacy_fm->fm.meter_id) + return &legacy_fm->fm; + } return NULL; } /** + * Find meter by index. + * + * @param priv + * Pointer to mlx5_priv. + * @param idx + * Meter index. + * + * @return + * Pointer to the profile found on success, NULL otherwise. + */ +struct mlx5_flow_meter_info * +flow_dv_meter_find_by_idx(struct mlx5_priv *priv, uint32_t idx) +{ + struct mlx5_aso_mtr *aso_mtr; + + if (priv->sh->meter_aso_en) { + aso_mtr = mlx5_aso_meter_by_idx(priv, idx); + return &aso_mtr->fm; + } else { + return mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], idx); + } +} + +/** * Attach meter to flow. * Unidirectional Meter creation can only be done * when flow direction is known, i.e. when calling meter_attach. @@ -1270,7 +1367,7 @@ struct mlx5_flow_meter * */ int mlx5_flow_meter_attach(struct mlx5_priv *priv, - struct mlx5_flow_meter *fm, + struct mlx5_flow_meter_info *fm, const struct rte_flow_attr *attr, struct rte_flow_error *error) { @@ -1319,7 +1416,7 @@ struct mlx5_flow_meter * * Pointer to flow meter. */ void -mlx5_flow_meter_detach(struct mlx5_flow_meter *fm) +mlx5_flow_meter_detach(struct mlx5_flow_meter_info *fm) { #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER rte_spinlock_lock(&fm->sl); @@ -1352,39 +1449,46 @@ struct mlx5_flow_meter * mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_flow_meters *fms = &priv->flow_meters; + struct mlx5_aso_mtr_pools_mng *mtrmng = priv->sh->mtrmng; + struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; struct mlx5_mtr_profiles *fmps = &priv->flow_meter_profiles; struct mlx5_flow_meter_profile *fmp; - struct mlx5_flow_meter *fm; + struct mlx5_legacy_flow_meter *legacy_fm; + struct mlx5_flow_meter_info *fm; + struct mlx5_aso_mtr_pool *mtr_pool; const struct rte_flow_attr attr = { .ingress = 1, .egress = 1, .transfer = priv->config.dv_esw_en ? 1 : 0, }; void *tmp; + uint32_t i, offset, mtr_idx; - TAILQ_FOREACH_SAFE(fm, fms, next, tmp) { - /* Meter object must not have any owner. */ - MLX5_ASSERT(!fm->ref_cnt); - /* Get meter profile. */ - fmp = fm->profile; - if (fmp == NULL) - return -rte_mtr_error_set(error, EINVAL, - RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, - NULL, "MTR object meter profile invalid."); - /* Update dependencies. */ - fmp->ref_cnt--; - /* Remove from list. */ - TAILQ_REMOVE(fms, fm, next); - /* Free policer counters. */ - if (fm->policer_stats.pass_cnt) - mlx5_counter_free(dev, fm->policer_stats.pass_cnt); - if (fm->policer_stats.drop_cnt) - mlx5_counter_free(dev, fm->policer_stats.drop_cnt); - /* Free meter flow table. */ - mlx5_flow_destroy_policer_rules(dev, fm, &attr); - mlx5_flow_destroy_mtr_tbls(dev, fm->mfts); - mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], fm->idx); + if (priv->sh->meter_aso_en) { + i = mtrmng->n_valid; + while (i--) { + mtr_pool = mtrmng->pools[i]; + for (offset = 0; offset < MLX5_ASO_MTRS_PER_POOL; + offset++) { + fm = &mtr_pool->mtrs[offset].fm; + mtr_idx = MLX5_MAKE_MTR_IDX(i, offset); + if (fm->meter_id != UINT32_MAX && + mlx5_flow_meter_params_flush(dev, + fm, &attr, mtr_idx)) + return -rte_mtr_error_set + (error, EINVAL, + RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, + NULL, "MTR object meter profile invalid."); + } + } + } else { + TAILQ_FOREACH_SAFE(legacy_fm, fms, next, tmp) { + fm = &legacy_fm->fm; + if (mlx5_flow_meter_params_flush(dev, fm, &attr, 0)) + return -rte_mtr_error_set(error, EINVAL, + RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, + NULL, "MTR object meter profile invalid."); + } } TAILQ_FOREACH_SAFE(fmp, fmps, next, tmp) { /* Check unused. */ -- 1.8.3.1