From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 04EDDA04DE; Fri, 23 Oct 2020 09:20:14 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id A3E49A904; Fri, 23 Oct 2020 09:15:57 +0200 (CEST) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by dpdk.org (Postfix) with ESMTP id 5D9B672E7 for ; Fri, 23 Oct 2020 09:15:30 +0200 (CEST) Received: from Internal Mail-Server by MTLPINE1 (envelope-from suanmingm@nvidia.com) with SMTP; 23 Oct 2020 10:15:27 +0300 Received: from nvidia.com (mtbc-r640-04.mtbc.labs.mlnx [10.75.70.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 09N7F2LW026736; Fri, 23 Oct 2020 10:15:26 +0300 From: Suanming Mou To: Matan Azrad , Shahaf Shuler , Viacheslav Ovsiienko Cc: dev@dpdk.org, Xueming Li Date: Fri, 23 Oct 2020 15:14:43 +0800 Message-Id: <1603437295-119083-14-git-send-email-suanmingm@nvidia.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1603437295-119083-1-git-send-email-suanmingm@nvidia.com> References: <1601984948-313027-1-git-send-email-suanmingm@nvidia.com> <1603437295-119083-1-git-send-email-suanmingm@nvidia.com> Subject: [dpdk-dev] [PATCH v2 13/25] net/mlx5: make flow table cache thread safe X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" From: Xueming Li To support multi-thread flow insertion/removal, this patch uses thread safe hash list API for flow table cache hash list. Signed-off-by: Xueming Li Acked-by: Matan Azrad --- drivers/net/mlx5/mlx5.c | 102 ++++--------------------- drivers/net/mlx5/mlx5.h | 2 +- drivers/net/mlx5/mlx5_flow.h | 17 +++++ drivers/net/mlx5/mlx5_flow_dv.c | 164 ++++++++++++++++++++-------------------- 4 files changed, 116 insertions(+), 169 deletions(-) diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index da043e2..fa769cd 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -1003,7 +1003,7 @@ struct mlx5_dev_ctx_shared * } /** - * Destroy table hash list and all the root entries per domain. + * Destroy table hash list. * * @param[in] priv * Pointer to the private device data structure. @@ -1012,46 +1012,9 @@ struct mlx5_dev_ctx_shared * mlx5_free_table_hash_list(struct mlx5_priv *priv) { struct mlx5_dev_ctx_shared *sh = priv->sh; - struct mlx5_flow_tbl_data_entry *tbl_data; - union mlx5_flow_tbl_key table_key = { - { - .table_id = 0, - .reserved = 0, - .domain = 0, - .direction = 0, - } - }; - struct mlx5_hlist_entry *pos; if (!sh->flow_tbls) return; - pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL); - if (pos) { - tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry, - entry); - MLX5_ASSERT(tbl_data); - mlx5_hlist_remove(sh->flow_tbls, pos); - mlx5_free(tbl_data); - } - table_key.direction = 1; - pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL); - if (pos) { - tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry, - entry); - MLX5_ASSERT(tbl_data); - mlx5_hlist_remove(sh->flow_tbls, pos); - mlx5_free(tbl_data); - } - table_key.direction = 0; - table_key.domain = 1; - pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL); - if (pos) { - tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry, - entry); - MLX5_ASSERT(tbl_data); - mlx5_hlist_remove(sh->flow_tbls, pos); - mlx5_free(tbl_data); - } mlx5_hlist_destroy(sh->flow_tbls); } @@ -1066,80 +1029,45 @@ struct mlx5_dev_ctx_shared * * Zero on success, positive error code otherwise. */ int -mlx5_alloc_table_hash_list(struct mlx5_priv *priv) +mlx5_alloc_table_hash_list(struct mlx5_priv *priv __rte_unused) { + int err = 0; + /* Tables are only used in DV and DR modes. */ +#ifdef HAVE_IBV_FLOW_DV_SUPPORT struct mlx5_dev_ctx_shared *sh = priv->sh; char s[MLX5_HLIST_NAMESIZE]; - int err = 0; MLX5_ASSERT(sh); snprintf(s, sizeof(s), "%s_flow_table", priv->sh->ibdev_name); sh->flow_tbls = mlx5_hlist_create(s, MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE, - 0, 0, NULL, NULL, NULL); + 0, 0, flow_dv_tbl_create_cb, NULL, + flow_dv_tbl_remove_cb); if (!sh->flow_tbls) { DRV_LOG(ERR, "flow tables with hash creation failed."); err = ENOMEM; return err; } + sh->flow_tbls->ctx = sh; #ifndef HAVE_MLX5DV_DR + struct rte_flow_error error; + struct rte_eth_dev *dev = &rte_eth_devices[priv->dev_data->port_id]; + /* * In case we have not DR support, the zero tables should be created * because DV expect to see them even if they cannot be created by * RDMA-CORE. */ - union mlx5_flow_tbl_key table_key = { - { - .table_id = 0, - .reserved = 0, - .domain = 0, - .direction = 0, - } - }; - struct mlx5_flow_tbl_data_entry *tbl_data = mlx5_malloc(MLX5_MEM_ZERO, - sizeof(*tbl_data), 0, - SOCKET_ID_ANY); - - if (!tbl_data) { - err = ENOMEM; - goto error; - } - tbl_data->entry.key = table_key.v64; - err = mlx5_hlist_insert(sh->flow_tbls, &tbl_data->entry); - if (err) - goto error; - rte_atomic32_init(&tbl_data->tbl.refcnt); - rte_atomic32_inc(&tbl_data->tbl.refcnt); - table_key.direction = 1; - tbl_data = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*tbl_data), 0, - SOCKET_ID_ANY); - if (!tbl_data) { + if (!flow_dv_tbl_resource_get(dev, 0, 0, 0, 1, &error) || + !flow_dv_tbl_resource_get(dev, 0, 1, 0, 1, &error) || + !flow_dv_tbl_resource_get(dev, 0, 0, 1, 1, &error)) { err = ENOMEM; goto error; } - tbl_data->entry.key = table_key.v64; - err = mlx5_hlist_insert(sh->flow_tbls, &tbl_data->entry); - if (err) - goto error; - rte_atomic32_init(&tbl_data->tbl.refcnt); - rte_atomic32_inc(&tbl_data->tbl.refcnt); - table_key.direction = 0; - table_key.domain = 1; - tbl_data = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*tbl_data), 0, - SOCKET_ID_ANY); - if (!tbl_data) { - err = ENOMEM; - goto error; - } - tbl_data->entry.key = table_key.v64; - err = mlx5_hlist_insert(sh->flow_tbls, &tbl_data->entry); - if (err) - goto error; - rte_atomic32_init(&tbl_data->tbl.refcnt); - rte_atomic32_inc(&tbl_data->tbl.refcnt); return err; error: mlx5_free_table_hash_list(priv); #endif /* HAVE_MLX5DV_DR */ +#endif return err; } diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index ed16b0d..ae6d37d 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -496,7 +496,7 @@ struct mlx5_dev_shared_port { struct { /* Table ID should be at the lowest address. */ uint32_t table_id; /**< ID of the table. */ - uint16_t reserved; /**< must be zero for comparison. */ + uint16_t dummy; /**< Dummy table for DV API. */ uint8_t domain; /**< 1 - FDB, 0 - NIC TX/RX. */ uint8_t direction; /**< 1 - egress, 0 - ingress. */ }; diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index 3d325b2..8d12b5d 100644 --- a/drivers/net/mlx5/mlx5_flow.h +++ b/drivers/net/mlx5/mlx5_flow.h @@ -381,6 +381,13 @@ enum mlx5_flow_fate_type { MLX5_FLOW_FATE_MAX, }; +/* Hash list callback context */ +struct mlx5_flow_cb_ctx { + struct rte_eth_dev *dev; + struct rte_flow_error *error; + void *data; +}; + /* Matcher PRM representation */ struct mlx5_flow_dv_match_params { size_t size; @@ -1149,4 +1156,14 @@ int mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, int mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error); int mlx5_flow_dv_discover_counter_offset_support(struct rte_eth_dev *dev); + +/* Hash list callbacks for flow tables: */ +struct mlx5_hlist_entry *flow_dv_tbl_create_cb(struct mlx5_hlist *list, + uint64_t key, void *entry_ctx); +void flow_dv_tbl_remove_cb(struct mlx5_hlist *list, + struct mlx5_hlist_entry *entry); +struct mlx5_flow_tbl_resource *flow_dv_tbl_resource_get(struct rte_eth_dev *dev, + uint32_t table_id, uint8_t egress, uint8_t transfer, + uint8_t dummy, struct rte_flow_error *error); + #endif /* RTE_PMD_MLX5_FLOW_H_ */ diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 43d16b4..18a52b0 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -7800,54 +7800,18 @@ struct field_modify_info modify_tcp[] = { } -/** - * Get a flow table. - * - * @param[in, out] dev - * Pointer to rte_eth_dev structure. - * @param[in] table_id - * Table id to use. - * @param[in] egress - * Direction of the table. - * @param[in] transfer - * E-Switch or NIC flow. - * @param[out] error - * pointer to error structure. - * - * @return - * Returns tables resource based on the index, NULL in case of failed. - */ -static struct mlx5_flow_tbl_resource * -flow_dv_tbl_resource_get(struct rte_eth_dev *dev, - uint32_t table_id, uint8_t egress, - uint8_t transfer, - struct rte_flow_error *error) +struct mlx5_hlist_entry * +flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *ctx) { - struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_dev_ctx_shared *sh = priv->sh; - struct mlx5_flow_tbl_resource *tbl; - union mlx5_flow_tbl_key table_key = { - { - .table_id = table_id, - .reserved = 0, - .domain = !!transfer, - .direction = !!egress, - } - }; - struct mlx5_hlist_entry *pos = mlx5_hlist_lookup(sh->flow_tbls, - table_key.v64, NULL); + struct mlx5_dev_ctx_shared *sh = list->ctx; struct mlx5_flow_tbl_data_entry *tbl_data; + struct rte_flow_error *error = ctx; + union mlx5_flow_tbl_key key = { .v64 = key64 }; + struct mlx5_flow_tbl_resource *tbl; + void *domain; uint32_t idx = 0; int ret; - void *domain; - if (pos) { - tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry, - entry); - tbl = &tbl_data->tbl; - rte_atomic32_inc(&tbl->refcnt); - return tbl; - } tbl_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_JUMP], &idx); if (!tbl_data) { rte_flow_error_set(error, ENOMEM, @@ -7858,14 +7822,15 @@ struct field_modify_info modify_tcp[] = { } tbl_data->idx = idx; tbl = &tbl_data->tbl; - pos = &tbl_data->entry; - if (transfer) + if (key.dummy) + return &tbl_data->entry; + if (key.domain) domain = sh->fdb_domain; - else if (egress) + else if (key.direction) domain = sh->tx_domain; else domain = sh->rx_domain; - ret = mlx5_flow_os_create_flow_tbl(domain, table_id, &tbl->obj); + ret = mlx5_flow_os_create_flow_tbl(domain, key.table_id, &tbl->obj); if (ret) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, @@ -7873,12 +7838,7 @@ struct field_modify_info modify_tcp[] = { mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx); return NULL; } - /* - * No multi-threads now, but still better to initialize the reference - * count before insert it into the hash list. - */ - rte_atomic32_init(&tbl->refcnt); - if (table_id) { + if (key.table_id) { ret = mlx5_flow_os_create_flow_action_dest_flow_tbl (tbl->obj, &tbl_data->jump.action); if (ret) { @@ -7891,17 +7851,69 @@ struct field_modify_info modify_tcp[] = { return NULL; } } - pos->key = table_key.v64; - ret = !mlx5_hlist_insert(sh->flow_tbls, pos); - if (ret < 0) { - rte_flow_error_set(error, -ret, + return &tbl_data->entry; +} + +/** + * Get a flow table. + * + * @param[in, out] dev + * Pointer to rte_eth_dev structure. + * @param[in] table_id + * Table id to use. + * @param[in] egress + * Direction of the table. + * @param[in] transfer + * E-Switch or NIC flow. + * @param[in] dummy + * Dummy entry for dv API. + * @param[out] error + * pointer to error structure. + * + * @return + * Returns tables resource based on the index, NULL in case of failed. + */ +struct mlx5_flow_tbl_resource * +flow_dv_tbl_resource_get(struct rte_eth_dev *dev, + uint32_t table_id, uint8_t egress, + uint8_t transfer, uint8_t dummy, + struct rte_flow_error *error) +{ + struct mlx5_priv *priv = dev->data->dev_private; + union mlx5_flow_tbl_key table_key = { + { + .table_id = table_id, + .dummy = dummy, + .domain = !!transfer, + .direction = !!egress, + } + }; + struct mlx5_hlist_entry *entry; + struct mlx5_flow_tbl_data_entry *tbl_data; + + entry = mlx5_hlist_register(priv->sh->flow_tbls, table_key.v64, NULL); + if (!entry) { + rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "cannot insert flow table data entry"); - mlx5_flow_os_destroy_flow_tbl(tbl->obj); - mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx); + "cannot get table"); + return NULL; } - rte_atomic32_inc(&tbl->refcnt); - return tbl; + tbl_data = container_of(entry, struct mlx5_flow_tbl_data_entry, entry); + return &tbl_data->tbl; +} + +void +flow_dv_tbl_remove_cb(struct mlx5_hlist *list, + struct mlx5_hlist_entry *entry) +{ + struct mlx5_dev_ctx_shared *sh = list->ctx; + struct mlx5_flow_tbl_data_entry *tbl_data = + container_of(entry, struct mlx5_flow_tbl_data_entry, entry); + + MLX5_ASSERT(entry && sh); + if (tbl_data->tbl.obj) + mlx5_flow_os_destroy_flow_tbl(tbl_data->tbl.obj); + mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx); } /** @@ -7926,18 +7938,7 @@ struct field_modify_info modify_tcp[] = { if (!tbl) return 0; - if (rte_atomic32_dec_and_test(&tbl->refcnt)) { - struct mlx5_hlist_entry *pos = &tbl_data->entry; - - mlx5_flow_os_destroy_flow_tbl(tbl->obj); - tbl->obj = NULL; - /* remove the entry from the hash list and free memory. */ - mlx5_hlist_remove(sh->flow_tbls, pos); - mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_JUMP], - tbl_data->idx); - return 0; - } - return 1; + return mlx5_hlist_unregister(sh->flow_tbls, &tbl_data->entry); } /** @@ -7976,7 +7977,7 @@ struct field_modify_info modify_tcp[] = { int ret; tbl = flow_dv_tbl_resource_get(dev, key->table_id, key->direction, - key->domain, error); + key->domain, 0, error); if (!tbl) return -rte_errno; /* No need to refill the error info */ tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl); @@ -8471,7 +8472,7 @@ struct field_modify_info modify_tcp[] = { *cache_resource = *resource; /* Create normal path table level */ tbl = flow_dv_tbl_resource_get(dev, next_ft_id, - attr->egress, attr->transfer, error); + attr->egress, attr->transfer, 0, error); if (!tbl) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, @@ -9377,7 +9378,8 @@ struct field_modify_info modify_tcp[] = { return ret; tbl = flow_dv_tbl_resource_get(dev, table, attr->egress, - attr->transfer, error); + attr->transfer, 0, + error); if (!tbl) return rte_flow_error_set (error, errno, @@ -10771,7 +10773,7 @@ struct field_modify_info modify_tcp[] = { dtb = &mtb->ingress; /* Create the meter table with METER level. */ dtb->tbl = flow_dv_tbl_resource_get(dev, MLX5_FLOW_TABLE_LEVEL_METER, - egress, transfer, &error); + egress, transfer, 0, &error); if (!dtb->tbl) { DRV_LOG(ERR, "Failed to create meter policer table."); return -1; @@ -10779,7 +10781,7 @@ struct field_modify_info modify_tcp[] = { /* Create the meter suffix table with SUFFIX level. */ dtb->sfx_tbl = flow_dv_tbl_resource_get(dev, MLX5_FLOW_TABLE_LEVEL_SUFFIX, - egress, transfer, &error); + egress, transfer, 0, &error); if (!dtb->sfx_tbl) { DRV_LOG(ERR, "Failed to create meter suffix table."); return -1; @@ -11098,10 +11100,10 @@ struct field_modify_info modify_tcp[] = { void *flow = NULL; int i, ret = -1; - tbl = flow_dv_tbl_resource_get(dev, 0, 0, 0, NULL); + tbl = flow_dv_tbl_resource_get(dev, 0, 0, 0, 0, NULL); if (!tbl) goto err; - dest_tbl = flow_dv_tbl_resource_get(dev, 1, 0, 0, NULL); + dest_tbl = flow_dv_tbl_resource_get(dev, 1, 0, 0, 0, NULL); if (!dest_tbl) goto err; dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4); -- 1.8.3.1