From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 0726DA04DD; Wed, 28 Oct 2020 10:42:07 +0100 (CET) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 54FC8C8E0; Wed, 28 Oct 2020 10:37:25 +0100 (CET) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by dpdk.org (Postfix) with ESMTP id 04BD4C8E4 for ; Wed, 28 Oct 2020 10:34:53 +0100 (CET) Received: from Internal Mail-Server by MTLPINE1 (envelope-from suanmingm@nvidia.com) with SMTP; 28 Oct 2020 11:34:49 +0200 Received: from nvidia.com (mtbc-r640-04.mtbc.labs.mlnx [10.75.70.9]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 09S9Y0S5014036; Wed, 28 Oct 2020 11:34:47 +0200 From: Suanming Mou To: Matan Azrad , Shahaf Shuler , Viacheslav Ovsiienko Cc: dev@dpdk.org, rasland@nvidia.com Date: Wed, 28 Oct 2020 17:33:42 +0800 Message-Id: <1603877633-293405-24-git-send-email-suanmingm@nvidia.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1603877633-293405-1-git-send-email-suanmingm@nvidia.com> References: <1601984948-313027-1-git-send-email-suanmingm@nvidia.com> <1603877633-293405-1-git-send-email-suanmingm@nvidia.com> Subject: [dpdk-dev] [PATCH v5 23/34] net/mlx5: make Rx queue thread safe X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" This commit applies the cache linked list to Rx queue to make it thread safe. Signed-off-by: Suanming Mou Acked-by: Matan Azrad --- drivers/net/mlx5/linux/mlx5_os.c | 5 + drivers/net/mlx5/mlx5.c | 1 + drivers/net/mlx5/mlx5.h | 28 +++- drivers/net/mlx5/mlx5_flow.h | 16 --- drivers/net/mlx5/mlx5_flow_dv.c | 74 ++++------ drivers/net/mlx5/mlx5_flow_verbs.c | 21 +-- drivers/net/mlx5/mlx5_rxq.c | 281 ++++++++++++++++++++----------------- drivers/net/mlx5/mlx5_rxtx.h | 20 +-- 8 files changed, 228 insertions(+), 218 deletions(-) diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c index d017c23..10fc7c5 100644 --- a/drivers/net/mlx5/linux/mlx5_os.c +++ b/drivers/net/mlx5/linux/mlx5_os.c @@ -1468,6 +1468,10 @@ err = ENOTSUP; goto error; } + mlx5_cache_list_init(&priv->hrxqs, "hrxq", 0, eth_dev, + mlx5_hrxq_create_cb, + mlx5_hrxq_match_cb, + mlx5_hrxq_remove_cb); /* Query availability of metadata reg_c's. */ err = mlx5_flow_discover_mreg_c(eth_dev); if (err < 0) { @@ -1520,6 +1524,7 @@ mlx5_drop_action_destroy(eth_dev); if (own_domain_id) claim_zero(rte_eth_switch_domain_free(priv->domain_id)); + mlx5_cache_list_destroy(&priv->hrxqs); mlx5_free(priv); if (eth_dev != NULL) eth_dev->data->dev_private = NULL; diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index 1d25a8e..862bd40 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -1286,6 +1286,7 @@ struct mlx5_dev_ctx_shared * if (ret) DRV_LOG(WARNING, "port %u some flows still remain", dev->data->port_id); + mlx5_cache_list_destroy(&priv->hrxqs); /* * Free the shared context in last turn, because the cleanup * routines above may use some shared fields, like diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index 9f96bd0..7804d3a 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -65,6 +65,13 @@ enum mlx5_reclaim_mem_mode { MLX5_RCM_AGGR, /* Reclaim PMD and rdma-core level. */ }; +/* Hash and cache list callback context. */ +struct mlx5_flow_cb_ctx { + struct rte_eth_dev *dev; + struct rte_flow_error *error; + void *data; +}; + /* Device attributes used in mlx5 PMD */ struct mlx5_dev_attr { uint64_t device_cap_flags_ex; @@ -684,6 +691,22 @@ struct mlx5_proc_priv { /* MTR list. */ TAILQ_HEAD(mlx5_flow_meters, mlx5_flow_meter); +/* RSS description. */ +struct mlx5_flow_rss_desc { + uint32_t level; + uint32_t queue_num; /**< Number of entries in @p queue. */ + uint64_t types; /**< Specific RSS hash types (see ETH_RSS_*). */ + uint64_t hash_fields; /* Verbs Hash fields. */ + uint8_t key[MLX5_RSS_HASH_KEY_LEN]; /**< RSS hash key. */ + uint32_t key_len; /**< RSS hash key len. */ + uint32_t tunnel; /**< Queue in tunnel. */ + union { + uint16_t *queue; /**< Destination queues. */ + const uint16_t *const_q; /**< Const pointer convert. */ + }; + bool standalone; /**< Queue is standalone or not. */ +}; + #define MLX5_PROC_PRIV(port_id) \ ((struct mlx5_proc_priv *)rte_eth_devices[port_id].process_private) @@ -723,7 +746,7 @@ struct mlx5_ind_table_obj { /* Hash Rx queue. */ __extension__ struct mlx5_hrxq { - ILIST_ENTRY(uint32_t)next; /* Index to the next element. */ + struct mlx5_cache_entry entry; /* Cache entry. */ uint32_t refcnt; /* Reference counter. */ uint32_t standalone:1; /* This object used in shared action. */ struct mlx5_ind_table_obj *ind_table; /* Indirection table. */ @@ -737,6 +760,7 @@ struct mlx5_hrxq { #endif uint64_t hash_fields; /* Verbs Hash fields. */ uint32_t rss_key_len; /* Hash key length in bytes. */ + uint32_t idx; /* Hash Rx queue index. */ uint8_t rss_key[]; /* Hash key. */ }; @@ -853,7 +877,7 @@ struct mlx5_priv { struct mlx5_obj_ops obj_ops; /* HW objects operations. */ LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */ LIST_HEAD(rxqobj, mlx5_rxq_obj) rxqsobj; /* Verbs/DevX Rx queues. */ - uint32_t hrxqs; /* Verbs Hash Rx queues. */ + struct mlx5_cache_list hrxqs; /* Hash Rx queues. */ LIST_HEAD(txq, mlx5_txq_ctrl) txqsctrl; /* DPDK Tx queues. */ LIST_HEAD(txqobj, mlx5_txq_obj) txqsobj; /* Verbs/DevX Tx queues. */ /* Indirection tables. */ diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index d0a1000..84988c6 100644 --- a/drivers/net/mlx5/mlx5_flow.h +++ b/drivers/net/mlx5/mlx5_flow.h @@ -384,13 +384,6 @@ enum mlx5_flow_fate_type { MLX5_FLOW_FATE_MAX, }; -/* Hash list callback context */ -struct mlx5_flow_cb_ctx { - struct rte_eth_dev *dev; - struct rte_flow_error *error; - void *data; -}; - /* Matcher PRM representation */ struct mlx5_flow_dv_match_params { size_t size; @@ -609,15 +602,6 @@ struct ibv_spec_header { uint16_t size; }; -/* RSS description. */ -struct mlx5_flow_rss_desc { - uint32_t level; - uint32_t queue_num; /**< Number of entries in @p queue. */ - uint64_t types; /**< Specific RSS hash types (see ETH_RSS_*). */ - uint8_t key[MLX5_RSS_HASH_KEY_LEN]; /**< RSS hash key. */ - uint16_t *queue; /**< Destination queues. */ -}; - /* PMD flow priority for tunnel */ #define MLX5_TUNNEL_PRIO_GET(rss_desc) \ ((rss_desc)->level >= 2 ? MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4) diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 3a55f76..78984f9 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -8527,7 +8527,7 @@ struct mlx5_hlist_entry * } /** - * Create an Rx Hash queue. + * Prepare an Rx Hash queue. * * @param dev * Pointer to Ethernet device. @@ -8542,29 +8542,23 @@ struct mlx5_hlist_entry * * The Verbs/DevX object initialised, NULL otherwise and rte_errno is set. */ static struct mlx5_hrxq * -flow_dv_handle_rx_queue(struct rte_eth_dev *dev, - struct mlx5_flow *dev_flow, - struct mlx5_flow_rss_desc *rss_desc, - uint32_t *hrxq_idx) +flow_dv_hrxq_prepare(struct rte_eth_dev *dev, + struct mlx5_flow *dev_flow, + struct mlx5_flow_rss_desc *rss_desc, + uint32_t *hrxq_idx) { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flow_handle *dh = dev_flow->handle; struct mlx5_hrxq *hrxq; MLX5_ASSERT(rss_desc->queue_num); - *hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key, MLX5_RSS_HASH_KEY_LEN, - dev_flow->hash_fields, - rss_desc->queue, rss_desc->queue_num); - if (!*hrxq_idx) { - *hrxq_idx = mlx5_hrxq_new - (dev, rss_desc->key, MLX5_RSS_HASH_KEY_LEN, - dev_flow->hash_fields, - rss_desc->queue, rss_desc->queue_num, - !!(dh->layers & MLX5_FLOW_LAYER_TUNNEL), - false); - if (!*hrxq_idx) - return NULL; - } + rss_desc->key_len = MLX5_RSS_HASH_KEY_LEN; + rss_desc->hash_fields = dev_flow->hash_fields; + rss_desc->tunnel = !!(dh->layers & MLX5_FLOW_LAYER_TUNNEL); + rss_desc->standalone = false; + *hrxq_idx = mlx5_hrxq_get(dev, rss_desc); + if (!*hrxq_idx) + return NULL; hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], *hrxq_idx); return hrxq; @@ -8927,8 +8921,8 @@ struct mlx5_hlist_entry * queue = sub_actions->conf; rss_desc->queue_num = 1; rss_desc->queue[0] = queue->index; - hrxq = flow_dv_handle_rx_queue(dev, dev_flow, - rss_desc, &hrxq_idx); + hrxq = flow_dv_hrxq_prepare(dev, dev_flow, + rss_desc, &hrxq_idx); if (!hrxq) return rte_flow_error_set (error, rte_errno, @@ -9125,8 +9119,8 @@ struct mlx5_hlist_entry * if (num_of_dest > 1) { if (sample_act->action_flags & MLX5_FLOW_ACTION_QUEUE) { /* Handle QP action for mirroring */ - hrxq = flow_dv_handle_rx_queue(dev, dev_flow, - rss_desc, &hrxq_idx); + hrxq = flow_dv_hrxq_prepare(dev, dev_flow, + rss_desc, &hrxq_idx); if (!hrxq) return rte_flow_error_set (error, rte_errno, @@ -10261,24 +10255,8 @@ struct mlx5_hlist_entry * struct mlx5_flow_rss_desc *rss_desc = &wks->rss_desc[!!wks->flow_nested_idx]; - MLX5_ASSERT(rss_desc->queue_num); - hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key, - MLX5_RSS_HASH_KEY_LEN, - dev_flow->hash_fields, - rss_desc->queue, rss_desc->queue_num); - if (!hrxq_idx) { - hrxq_idx = mlx5_hrxq_new(dev, - rss_desc->key, - MLX5_RSS_HASH_KEY_LEN, - dev_flow->hash_fields, - rss_desc->queue, - rss_desc->queue_num, - !!(dev_flow->handle->layers & - MLX5_FLOW_LAYER_TUNNEL), - false); - } - *hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], - hrxq_idx); + *hrxq = flow_dv_hrxq_prepare(dev, dev_flow, rss_desc, + &hrxq_idx); } return hrxq_idx; } @@ -10332,7 +10310,6 @@ struct mlx5_hlist_entry * struct mlx5_hrxq *hrxq = NULL; uint32_t hrxq_idx = __flow_dv_rss_get_hrxq (dev, flow, dev_flow, &hrxq); - if (!hrxq) { rte_flow_error_set (error, rte_errno, @@ -10954,21 +10931,24 @@ struct mlx5_hlist_entry * struct mlx5_shared_action_rss *action, struct rte_flow_error *error) { + struct mlx5_flow_rss_desc rss_desc = { 0 }; size_t i; int err; + memcpy(rss_desc.key, action->origin.key, MLX5_RSS_HASH_KEY_LEN); + rss_desc.key_len = MLX5_RSS_HASH_KEY_LEN; + rss_desc.const_q = action->origin.queue; + rss_desc.queue_num = action->origin.queue_num; + rss_desc.standalone = true; for (i = 0; i < MLX5_RSS_HASH_FIELDS_LEN; i++) { uint32_t hrxq_idx; uint64_t hash_fields = mlx5_rss_hash_fields[i]; int tunnel; for (tunnel = 0; tunnel < 2; tunnel++) { - hrxq_idx = mlx5_hrxq_new(dev, action->origin.key, - MLX5_RSS_HASH_KEY_LEN, - hash_fields, - action->origin.queue, - action->origin.queue_num, - tunnel, true); + rss_desc.tunnel = tunnel; + rss_desc.hash_fields = hash_fields; + hrxq_idx = mlx5_hrxq_get(dev, &rss_desc); if (!hrxq_idx) { rte_flow_error_set (error, rte_errno, diff --git a/drivers/net/mlx5/mlx5_flow_verbs.c b/drivers/net/mlx5/mlx5_flow_verbs.c index 3c04727..28fd120 100644 --- a/drivers/net/mlx5/mlx5_flow_verbs.c +++ b/drivers/net/mlx5/mlx5_flow_verbs.c @@ -1968,21 +1968,12 @@ &wks->rss_desc[!!wks->flow_nested_idx]; MLX5_ASSERT(rss_desc->queue_num); - hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key, - MLX5_RSS_HASH_KEY_LEN, - dev_flow->hash_fields, - rss_desc->queue, - rss_desc->queue_num); - if (!hrxq_idx) - hrxq_idx = mlx5_hrxq_new - (dev, rss_desc->key, - MLX5_RSS_HASH_KEY_LEN, - dev_flow->hash_fields, - rss_desc->queue, - rss_desc->queue_num, - !!(handle->layers & - MLX5_FLOW_LAYER_TUNNEL), - false); + rss_desc->key_len = MLX5_RSS_HASH_KEY_LEN; + rss_desc->hash_fields = dev_flow->hash_fields; + rss_desc->tunnel = !!(handle->layers & + MLX5_FLOW_LAYER_TUNNEL); + rss_desc->standalone = false; + hrxq_idx = mlx5_hrxq_get(dev, rss_desc); hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], hrxq_idx); if (!hrxq) { diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c index 7579407..a733eda 100644 --- a/drivers/net/mlx5/mlx5_rxq.c +++ b/drivers/net/mlx5/mlx5_rxq.c @@ -2039,54 +2039,38 @@ struct mlx5_ind_table_obj * } /** - * Get an Rx Hash queue. + * Match an Rx Hash queue. * - * @param dev - * Pointer to Ethernet device. - * @param rss_conf - * RSS configuration for the Rx hash queue. - * @param queues - * Queues entering in hash queue. In case of empty hash_fields only the - * first queue index will be taken for the indirection table. - * @param queues_n - * Number of queues. + * @param list + * Cache list pointer. + * @param entry + * Hash queue entry pointer. + * @param cb_ctx + * Context of the callback function. * * @return - * An hash Rx queue index on success. + * 0 if match, none zero if not match. */ -uint32_t -mlx5_hrxq_get(struct rte_eth_dev *dev, - const uint8_t *rss_key, uint32_t rss_key_len, - uint64_t hash_fields, - const uint16_t *queues, uint32_t queues_n) +int +mlx5_hrxq_match_cb(struct mlx5_cache_list *list, + struct mlx5_cache_entry *entry, + void *cb_ctx) { - struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_hrxq *hrxq; - uint32_t idx; - - queues_n = hash_fields ? queues_n : 1; - ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_HRXQ], priv->hrxqs, idx, - hrxq, next) { - struct mlx5_ind_table_obj *ind_tbl; + struct rte_eth_dev *dev = list->ctx; + struct mlx5_flow_cb_ctx *ctx = cb_ctx; + struct mlx5_flow_rss_desc *rss_desc = ctx->data; + struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry); + struct mlx5_ind_table_obj *ind_tbl; - if (hrxq->rss_key_len != rss_key_len) - continue; - if (memcmp(hrxq->rss_key, rss_key, rss_key_len)) - continue; - if (hrxq->hash_fields != hash_fields) - continue; - ind_tbl = mlx5_ind_table_obj_get(dev, queues, queues_n); - if (!ind_tbl) - continue; - if (ind_tbl != hrxq->ind_table) { - mlx5_ind_table_obj_release(dev, ind_tbl, - hrxq->standalone); - continue; - } - __atomic_fetch_add(&hrxq->refcnt, 1, __ATOMIC_RELAXED); - return idx; - } - return 0; + if (hrxq->rss_key_len != rss_desc->key_len || + memcmp(hrxq->rss_key, rss_desc->key, rss_desc->key_len) || + hrxq->hash_fields != rss_desc->hash_fields) + return 1; + ind_tbl = mlx5_ind_table_obj_get(dev, rss_desc->queue, + rss_desc->queue_num); + if (ind_tbl) + mlx5_ind_table_obj_release(dev, ind_tbl, hrxq->standalone); + return ind_tbl != hrxq->ind_table; } /** @@ -2171,114 +2155,163 @@ struct mlx5_ind_table_obj * return -rte_errno; } -/** - * Release the hash Rx queue. - * - * @param dev - * Pointer to Ethernet device. - * @param hrxq - * Index to Hash Rx queue to release. - * - * @return - * 1 while a reference on it exists, 0 when freed. - */ -int -mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hrxq_idx) +static void +__mlx5_hrxq_remove(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_hrxq *hrxq; - hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], hrxq_idx); - if (!hrxq) - return 0; - if (__atomic_sub_fetch(&hrxq->refcnt, 1, __ATOMIC_RELAXED) == 0) { #ifdef HAVE_IBV_FLOW_DV_SUPPORT - mlx5_glue->destroy_flow_action(hrxq->action); + mlx5_glue->destroy_flow_action(hrxq->action); #endif - priv->obj_ops.hrxq_destroy(hrxq); - mlx5_ind_table_obj_release(dev, hrxq->ind_table, - hrxq->standalone); - if (!hrxq->standalone) - ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_HRXQ], - &priv->hrxqs, hrxq_idx, hrxq, next); - mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_HRXQ], hrxq_idx); - return 0; - } - claim_nonzero(mlx5_ind_table_obj_release(dev, hrxq->ind_table, - hrxq->standalone)); - return 1; + priv->obj_ops.hrxq_destroy(hrxq); + mlx5_ind_table_obj_release(dev, hrxq->ind_table, hrxq->standalone); + mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_HRXQ], hrxq->idx); } /** - * Create an Rx Hash queue. + * Release the hash Rx queue. * * @param dev * Pointer to Ethernet device. - * @param rss_key - * RSS key for the Rx hash queue. - * @param rss_key_len - * RSS key length. - * @param hash_fields - * Verbs protocol hash field to make the RSS on. - * @param queues - * Queues entering in hash queue. In case of empty hash_fields only the - * first queue index will be taken for the indirection table. - * @param queues_n - * Number of queues. - * @param tunnel - * Tunnel type. - * @param standalone - * Object of Rx Hash queue will be used in standalone shared action or not. + * @param hrxq + * Index to Hash Rx queue to release. * - * @return - * The DevX object initialized index, 0 otherwise and rte_errno is set. + * @param list + * Cache list pointer. + * @param entry + * Hash queue entry pointer. */ -uint32_t -mlx5_hrxq_new(struct rte_eth_dev *dev, - const uint8_t *rss_key, uint32_t rss_key_len, - uint64_t hash_fields, - const uint16_t *queues, uint32_t queues_n, - int tunnel, bool standalone) +void +mlx5_hrxq_remove_cb(struct mlx5_cache_list *list, + struct mlx5_cache_entry *entry) +{ + struct rte_eth_dev *dev = list->ctx; + struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry); + + __mlx5_hrxq_remove(dev, hrxq); +} + +static struct mlx5_hrxq * +__mlx5_hrxq_create(struct rte_eth_dev *dev, + struct mlx5_flow_rss_desc *rss_desc) { struct mlx5_priv *priv = dev->data->dev_private; + const uint8_t *rss_key = rss_desc->key; + uint32_t rss_key_len = rss_desc->key_len; + const uint16_t *queues = + rss_desc->standalone ? rss_desc->const_q : rss_desc->queue; + uint32_t queues_n = rss_desc->queue_num; struct mlx5_hrxq *hrxq = NULL; uint32_t hrxq_idx = 0; struct mlx5_ind_table_obj *ind_tbl; int ret; - queues_n = hash_fields ? queues_n : 1; + queues_n = rss_desc->hash_fields ? queues_n : 1; ind_tbl = mlx5_ind_table_obj_get(dev, queues, queues_n); if (!ind_tbl) ind_tbl = mlx5_ind_table_obj_new(dev, queues, queues_n, - standalone); - if (!ind_tbl) { - rte_errno = ENOMEM; - return 0; - } + rss_desc->standalone); + if (!ind_tbl) + return NULL; hrxq = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_HRXQ], &hrxq_idx); if (!hrxq) goto error; - hrxq->standalone = !!standalone; + hrxq->standalone = rss_desc->standalone; + hrxq->idx = hrxq_idx; hrxq->ind_table = ind_tbl; hrxq->rss_key_len = rss_key_len; - hrxq->hash_fields = hash_fields; + hrxq->hash_fields = rss_desc->hash_fields; memcpy(hrxq->rss_key, rss_key, rss_key_len); - ret = priv->obj_ops.hrxq_new(dev, hrxq, tunnel); - if (ret < 0) { - rte_errno = errno; + ret = priv->obj_ops.hrxq_new(dev, hrxq, rss_desc->tunnel); + if (ret < 0) goto error; - } - __atomic_fetch_add(&hrxq->refcnt, 1, __ATOMIC_RELAXED); - if (!hrxq->standalone) - ILIST_INSERT(priv->sh->ipool[MLX5_IPOOL_HRXQ], &priv->hrxqs, - hrxq_idx, hrxq, next); - return hrxq_idx; + return hrxq; error: - ret = rte_errno; /* Save rte_errno before cleanup. */ - mlx5_ind_table_obj_release(dev, ind_tbl, standalone); + mlx5_ind_table_obj_release(dev, ind_tbl, rss_desc->standalone); if (hrxq) mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_HRXQ], hrxq_idx); - rte_errno = ret; /* Restore rte_errno. */ + return NULL; +} + +/** + * Create an Rx Hash queue. + * + * @param list + * Cache list pointer. + * @param entry + * Hash queue entry pointer. + * @param cb_ctx + * Context of the callback function. + * + * @return + * queue entry on success, NULL otherwise. + */ +struct mlx5_cache_entry * +mlx5_hrxq_create_cb(struct mlx5_cache_list *list, + struct mlx5_cache_entry *entry __rte_unused, + void *cb_ctx) +{ + struct rte_eth_dev *dev = list->ctx; + struct mlx5_flow_cb_ctx *ctx = cb_ctx; + struct mlx5_flow_rss_desc *rss_desc = ctx->data; + struct mlx5_hrxq *hrxq; + + hrxq = __mlx5_hrxq_create(dev, rss_desc); + return hrxq ? &hrxq->entry : NULL; +} + +/** + * Get an Rx Hash queue. + * + * @param dev + * Pointer to Ethernet device. + * @param rss_desc + * RSS configuration for the Rx hash queue. + * + * @return + * An hash Rx queue index on success. + */ +uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev, + struct mlx5_flow_rss_desc *rss_desc) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_hrxq *hrxq; + struct mlx5_cache_entry *entry; + struct mlx5_flow_cb_ctx ctx = { + .data = rss_desc, + }; + + if (rss_desc->standalone) { + hrxq = __mlx5_hrxq_create(dev, rss_desc); + } else { + entry = mlx5_cache_register(&priv->hrxqs, &ctx); + if (!entry) + return 0; + hrxq = container_of(entry, typeof(*hrxq), entry); + } + return hrxq->idx; +} + +/** + * Release the hash Rx queue. + * + * @param dev + * Pointer to Ethernet device. + * @param hrxq_idx + * Index to Hash Rx queue to release. + * + * @return + * 1 while a reference on it exists, 0 when freed. + */ +int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hrxq_idx) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_hrxq *hrxq; + + hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], hrxq_idx); + if (!hrxq->standalone) + return mlx5_cache_unregister(&priv->hrxqs, &hrxq->entry); + __mlx5_hrxq_remove(dev, hrxq); return 0; } @@ -2364,22 +2397,12 @@ struct mlx5_hrxq * * @return * The number of object not released. */ -int +uint32_t mlx5_hrxq_verify(struct rte_eth_dev *dev) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_hrxq *hrxq; - uint32_t idx; - int ret = 0; - ILIST_FOREACH(priv->sh->ipool[MLX5_IPOOL_HRXQ], priv->hrxqs, idx, - hrxq, next) { - DRV_LOG(DEBUG, - "port %u hash Rx queue %p still referenced", - dev->data->port_id, (void *)hrxq); - ++ret; - } - return ret; + return mlx5_cache_list_get_entry_num(&priv->hrxqs); } /** diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h index 8fe0473..c361bbd 100644 --- a/drivers/net/mlx5/mlx5_rxtx.h +++ b/drivers/net/mlx5/mlx5_rxtx.h @@ -349,17 +349,19 @@ struct mlx5_ind_table_obj *mlx5_ind_table_obj_get(struct rte_eth_dev *dev, int mlx5_ind_table_obj_release(struct rte_eth_dev *dev, struct mlx5_ind_table_obj *ind_tbl, bool standalone); -uint32_t mlx5_hrxq_new(struct rte_eth_dev *dev, - const uint8_t *rss_key, uint32_t rss_key_len, - uint64_t hash_fields, - const uint16_t *queues, uint32_t queues_n, - int tunnel, bool standalone); +struct mlx5_cache_entry *mlx5_hrxq_create_cb(struct mlx5_cache_list *list, + struct mlx5_cache_entry *entry __rte_unused, void *cb_ctx); +int mlx5_hrxq_match_cb(struct mlx5_cache_list *list, + struct mlx5_cache_entry *entry, + void *cb_ctx); +void mlx5_hrxq_remove_cb(struct mlx5_cache_list *list, + struct mlx5_cache_entry *entry); uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev, - const uint8_t *rss_key, uint32_t rss_key_len, - uint64_t hash_fields, - const uint16_t *queues, uint32_t queues_n); + struct mlx5_flow_rss_desc *rss_desc); int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hxrq_idx); -int mlx5_hrxq_verify(struct rte_eth_dev *dev); +uint32_t mlx5_hrxq_verify(struct rte_eth_dev *dev); + + enum mlx5_rxq_type mlx5_rxq_get_type(struct rte_eth_dev *dev, uint16_t idx); const struct rte_eth_hairpin_conf *mlx5_rxq_get_hairpin_conf (struct rte_eth_dev *dev, uint16_t idx); -- 1.8.3.1