From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 25FEDA0554; Sat, 27 Aug 2022 02:36:58 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 014C141147; Sat, 27 Aug 2022 02:35:39 +0200 (CEST) Received: from linux.microsoft.com (linux.microsoft.com [13.77.154.182]) by mails.dpdk.org (Postfix) with ESMTP id 03E0142BBF for ; Sat, 27 Aug 2022 02:35:37 +0200 (CEST) Received: by linux.microsoft.com (Postfix, from userid 1004) id 5F619236D99C; Fri, 26 Aug 2022 17:35:36 -0700 (PDT) DKIM-Filter: OpenDKIM Filter v2.11.0 linux.microsoft.com 5F619236D99C DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxonhyperv.com; s=default; t=1661560536; bh=lgDZIqFr2LJrK/bngESw86SqtDovAGuD/sv4+dunTNg=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=mc+YGm8NI0TOt3tlcjFvuEbYjOR3pGXFfGL7cuVzfUxOP3fLus3nlI8p12V/2BmaY J1l+HOJRkXxy2F+KcQM72uHDxz/ylUvCXdT7zH5McCO5del3CRKPHMa3PnTyNzsNE3 dmvVn208s5HjyNzYGVuhX9ydR7cmbJpvmuOYmc1k= From: longli@linuxonhyperv.com To: Ferruh Yigit Cc: dev@dpdk.org, Ajay Sharma , Stephen Hemminger , Long Li Subject: [Patch v5 18/18] net/mana: add function to support RX interrupts Date: Fri, 26 Aug 2022 17:35:09 -0700 Message-Id: <1661560509-11009-19-git-send-email-longli@linuxonhyperv.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1661560509-11009-1-git-send-email-longli@linuxonhyperv.com> References: <1661560509-11009-1-git-send-email-longli@linuxonhyperv.com> X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org From: Long Li mana can receive RX interrupts from kernel through RDMA verbs interface. Implement RX interrupts in the driver. Signed-off-by: Long Li --- Change log: v5: New patch added to the series doc/guides/nics/features/mana.ini | 1 + drivers/net/mana/gdma.c | 10 +-- drivers/net/mana/mana.c | 122 +++++++++++++++++++++++++----- drivers/net/mana/mana.h | 13 +++- drivers/net/mana/rx.c | 91 +++++++++++++++++++--- drivers/net/mana/tx.c | 3 +- 6 files changed, 204 insertions(+), 36 deletions(-) diff --git a/doc/guides/nics/features/mana.ini b/doc/guides/nics/features/mana.ini index b2729aba3a..42d78ac6b1 100644 --- a/doc/guides/nics/features/mana.ini +++ b/doc/guides/nics/features/mana.ini @@ -14,6 +14,7 @@ Multiprocess aware = Y Queue start/stop = Y Removal event = Y RSS hash = Y +Rx interrupt = Y Speed capabilities = P Stats per queue = Y Usage doc = Y diff --git a/drivers/net/mana/gdma.c b/drivers/net/mana/gdma.c index 077ac7744b..d701c4bd31 100644 --- a/drivers/net/mana/gdma.c +++ b/drivers/net/mana/gdma.c @@ -204,7 +204,7 @@ union gdma_doorbell_entry { #define DOORBELL_OFFSET_EQ 0xFF8 int mana_ring_doorbell(void *db_page, enum gdma_queue_types queue_type, - uint32_t queue_id, uint32_t tail) + uint32_t queue_id, uint32_t tail, uint8_t arm) { uint8_t *addr = db_page; union gdma_doorbell_entry e = {}; @@ -219,14 +219,14 @@ int mana_ring_doorbell(void *db_page, enum gdma_queue_types queue_type, case gdma_queue_receive: e.rq.id = queue_id; e.rq.tail_ptr = tail; - e.rq.wqe_cnt = 1; + e.rq.wqe_cnt = arm; addr += DOORBELL_OFFSET_RQ; break; case gdma_queue_completion: e.cq.id = queue_id; e.cq.tail_ptr = tail; - e.cq.arm = 1; + e.cq.arm = arm; addr += DOORBELL_OFFSET_CQ; break; @@ -236,8 +236,8 @@ int mana_ring_doorbell(void *db_page, enum gdma_queue_types queue_type, } rte_wmb(); - DRV_LOG(DEBUG, "db_page %p addr %p queue_id %u type %u tail %u", - db_page, addr, queue_id, queue_type, tail); + DRV_LOG(DEBUG, "db_page %p addr %p queue_id %u type %u tail %u arm %u", + db_page, addr, queue_id, queue_type, tail, arm); rte_write64(e.as_uint64, addr); return 0; diff --git a/drivers/net/mana/mana.c b/drivers/net/mana/mana.c index 581e705f5d..c179f312e2 100644 --- a/drivers/net/mana/mana.c +++ b/drivers/net/mana/mana.c @@ -95,7 +95,68 @@ static int mana_dev_configure(struct rte_eth_dev *dev) return 0; } -static int mana_intr_uninstall(struct mana_priv *priv); +static void rx_intr_vec_disable(struct mana_priv *priv) +{ + struct rte_intr_handle *intr_handle = priv->intr_handle; + + rte_intr_free_epoll_fd(intr_handle); + rte_intr_vec_list_free(intr_handle); + rte_intr_nb_efd_set(intr_handle, 0); +} + +static int rx_intr_vec_enable(struct mana_priv *priv) +{ + unsigned int i; + unsigned int rxqs_n = priv->dev_data->nb_rx_queues; + unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID); + struct rte_intr_handle *intr_handle = priv->intr_handle; + int ret; + + rx_intr_vec_disable(priv); + + if (rte_intr_vec_list_alloc(intr_handle, NULL, n)) { + DRV_LOG(ERR, "Failed to allocate memory for interrupt vector"); + return -ENOMEM; + } + + for (i = 0; i < n; i++) { + struct mana_rxq *rxq = priv->dev_data->rx_queues[i]; + + ret = rte_intr_vec_list_index_set(intr_handle, i, + RTE_INTR_VEC_RXTX_OFFSET + i); + if (ret) { + DRV_LOG(ERR, "Failed to set intr vec %u", i); + return ret; + } + + ret = rte_intr_efds_index_set(intr_handle, i, rxq->channel->fd); + if (ret) { + DRV_LOG(ERR, "Failed to set FD at intr %u", i); + return ret; + } + } + + return rte_intr_nb_efd_set(intr_handle, n); +} + +static void rxq_intr_disable(struct mana_priv *priv) +{ + int err = rte_errno; + + rx_intr_vec_disable(priv); + rte_errno = err; +} + +static int rxq_intr_enable(struct mana_priv *priv) +{ + const struct rte_eth_intr_conf *const intr_conf = + &priv->dev_data->dev_conf.intr_conf; + + if (!intr_conf->rxq) + return 0; + + return rx_intr_vec_enable(priv); +} static int mana_dev_start(struct rte_eth_dev *dev) @@ -134,13 +195,22 @@ mana_dev_start(struct rte_eth_dev *dev) /* Enable datapath for secondary processes */ mana_mp_req_on_rxtx(dev, MANA_MP_REQ_START_RXTX); + ret = rxq_intr_enable(priv); + if (ret) { + DRV_LOG(ERR, "Failed to enable RX interrupts"); + return ret; + } + return 0; } static int -mana_dev_stop(struct rte_eth_dev *dev __rte_unused) +mana_dev_stop(struct rte_eth_dev *dev) { int ret; + struct mana_priv *priv = dev->data->dev_private; + + rxq_intr_disable(priv); dev->tx_pkt_burst = mana_tx_burst_removed; dev->rx_pkt_burst = mana_rx_burst_removed; @@ -588,6 +658,8 @@ const struct eth_dev_ops mana_dev_ops = { .tx_queue_release = mana_dev_tx_queue_release, .rx_queue_setup = mana_dev_rx_queue_setup, .rx_queue_release = mana_dev_rx_queue_release, + .rx_queue_intr_enable = mana_rx_intr_enable, + .rx_queue_intr_disable = mana_rx_intr_disable, .link_update = mana_dev_link_update, .stats_get = mana_dev_stats_get, .stats_reset = mana_dev_stats_reset, @@ -775,7 +847,7 @@ static int mana_ibv_device_to_pci_addr(const struct ibv_device *device, return 0; } -static void mana_intr_handler(void *arg) +void mana_intr_handler(void *arg) { struct mana_priv *priv = arg; struct ibv_context *ctx = priv->ib_ctx; @@ -799,7 +871,7 @@ static void mana_intr_handler(void *arg) } } -static int mana_intr_uninstall(struct mana_priv *priv) +int mana_intr_uninstall(struct mana_priv *priv) { int ret; @@ -815,9 +887,20 @@ static int mana_intr_uninstall(struct mana_priv *priv) return 0; } -static int mana_intr_install(struct mana_priv *priv) +int mana_fd_set_non_blocking(int fd) +{ + int ret = fcntl(fd, F_GETFL); + + if (ret != -1 && !fcntl(fd, F_SETFL, ret | O_NONBLOCK)) + return 0; + + rte_errno = errno; + return -rte_errno; +} + +int mana_intr_install(struct rte_eth_dev *eth_dev, struct mana_priv *priv) { - int ret, flags; + int ret; struct ibv_context *ctx = priv->ib_ctx; priv->intr_handle = rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_SHARED); @@ -827,31 +910,35 @@ static int mana_intr_install(struct mana_priv *priv) return -ENOMEM; } - rte_intr_fd_set(priv->intr_handle, -1); + ret = rte_intr_fd_set(priv->intr_handle, -1); + if (ret) + goto free_intr; - flags = fcntl(ctx->async_fd, F_GETFL); - ret = fcntl(ctx->async_fd, F_SETFL, flags | O_NONBLOCK); + ret = mana_fd_set_non_blocking(ctx->async_fd); if (ret) { DRV_LOG(ERR, "Failed to change async_fd to NONBLOCK"); goto free_intr; } - rte_intr_fd_set(priv->intr_handle, ctx->async_fd); - rte_intr_type_set(priv->intr_handle, RTE_INTR_HANDLE_EXT); + ret = rte_intr_fd_set(priv->intr_handle, ctx->async_fd); + if (ret) + goto free_intr; + + ret = rte_intr_type_set(priv->intr_handle, RTE_INTR_HANDLE_EXT); + if (ret) + goto free_intr; ret = rte_intr_callback_register(priv->intr_handle, mana_intr_handler, priv); if (ret) { DRV_LOG(ERR, "Failed to register intr callback"); rte_intr_fd_set(priv->intr_handle, -1); - goto restore_fd; + goto free_intr; } + eth_dev->intr_handle = priv->intr_handle; return 0; -restore_fd: - fcntl(ctx->async_fd, F_SETFL, flags); - free_intr: rte_intr_instance_free(priv->intr_handle); priv->intr_handle = NULL; @@ -1175,8 +1262,10 @@ static int mana_pci_probe_mac(struct rte_pci_driver *pci_drv __rte_unused, name, priv->max_rx_queues, priv->max_rx_desc, priv->max_send_sge); + rte_eth_copy_pci_info(eth_dev, pci_dev); + /* Create async interrupt handler */ - ret = mana_intr_install(priv); + ret = mana_intr_install(eth_dev, priv); if (ret) { DRV_LOG(ERR, "Failed to install intr handler"); goto failed; @@ -1199,7 +1288,6 @@ static int mana_pci_probe_mac(struct rte_pci_driver *pci_drv __rte_unused, eth_dev->tx_pkt_burst = mana_tx_burst_removed; eth_dev->dev_ops = &mana_dev_ops; - rte_eth_copy_pci_info(eth_dev, pci_dev); rte_eth_dev_probing_finish(eth_dev); } diff --git a/drivers/net/mana/mana.h b/drivers/net/mana/mana.h index 0581d17e33..3b1ce20dd4 100644 --- a/drivers/net/mana/mana.h +++ b/drivers/net/mana/mana.h @@ -426,6 +426,7 @@ struct mana_rxq { uint32_t num_desc; struct rte_mempool *mp; struct ibv_cq *cq; + struct ibv_comp_channel *channel; struct ibv_wq *wq; /* For storing pending requests */ @@ -459,8 +460,8 @@ extern int mana_logtype_init; #define PMD_INIT_FUNC_TRACE() PMD_INIT_LOG(DEBUG, " >>") int mana_ring_doorbell(void *db_page, enum gdma_queue_types queue_type, - uint32_t queue_id, uint32_t tail); -int mana_rq_ring_doorbell(struct mana_rxq *rxq); + uint32_t queue_id, uint32_t tail, uint8_t arm); +int mana_rq_ring_doorbell(struct mana_rxq *rxq, uint8_t arm); int gdma_post_work_request(struct mana_gdma_queue *queue, struct gdma_work_request *work_req, @@ -495,6 +496,10 @@ int mana_new_pmd_mr(struct mana_mr_btree *local_tree, struct mana_priv *priv, void mana_remove_all_mr(struct mana_priv *priv); void mana_del_pmd_mr(struct mana_mr_cache *mr); +void mana_intr_handler(void *arg); +int mana_intr_install(struct rte_eth_dev *eth_dev, struct mana_priv *priv); +int mana_intr_uninstall(struct mana_priv *priv); + void mana_mempool_chunk_cb(struct rte_mempool *mp, void *opaque, struct rte_mempool_memhdr *memhdr, unsigned int idx); @@ -540,4 +545,8 @@ void mana_mp_req_on_rxtx(struct rte_eth_dev *dev, enum mana_mp_req_type type); void *mana_alloc_verbs_buf(size_t size, void *data); void mana_free_verbs_buf(void *ptr, void *data __rte_unused); +int mana_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id); +int mana_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id); +int mana_fd_set_non_blocking(int fd); + #endif diff --git a/drivers/net/mana/rx.c b/drivers/net/mana/rx.c index f2573a6d06..1a61fc59b1 100644 --- a/drivers/net/mana/rx.c +++ b/drivers/net/mana/rx.c @@ -21,7 +21,7 @@ static uint8_t mana_rss_hash_key_default[TOEPLITZ_HASH_KEY_SIZE_IN_BYTES] = { 0xfc, 0x1f, 0xdc, 0x2a, }; -int mana_rq_ring_doorbell(struct mana_rxq *rxq) +int mana_rq_ring_doorbell(struct mana_rxq *rxq, uint8_t arm) { struct mana_priv *priv = rxq->priv; int ret; @@ -36,9 +36,9 @@ int mana_rq_ring_doorbell(struct mana_rxq *rxq) } ret = mana_ring_doorbell(db_page, gdma_queue_receive, - rxq->gdma_rq.id, - rxq->gdma_rq.head * - GDMA_WQE_ALIGNMENT_UNIT_SIZE); + rxq->gdma_rq.id, + rxq->gdma_rq.head * GDMA_WQE_ALIGNMENT_UNIT_SIZE, + arm); if (ret) DRV_LOG(ERR, "failed to ring RX doorbell ret %d", ret); @@ -115,7 +115,7 @@ static int mana_alloc_and_post_rx_wqes(struct mana_rxq *rxq) } } - mana_rq_ring_doorbell(rxq); + mana_rq_ring_doorbell(rxq, rxq->num_desc); return ret; } @@ -156,6 +156,14 @@ int mana_stop_rx_queues(struct rte_eth_dev *dev) DRV_LOG(ERR, "rx_queue destroy_cq failed %d", ret); rxq->cq = NULL; + + if (rxq->channel) { + ret = ibv_destroy_comp_channel(rxq->channel); + if (ret) + DRV_LOG(ERR, "failed destroy comp %d", + ret); + rxq->channel = NULL; + } } /* Drain and free posted WQEs */ @@ -196,8 +204,24 @@ int mana_start_rx_queues(struct rte_eth_dev *dev) .data = (void *)(uintptr_t)rxq->socket, })); + if (dev->data->dev_conf.intr_conf.rxq) { + rxq->channel = ibv_create_comp_channel(priv->ib_ctx); + if (!rxq->channel) { + ret = -errno; + DRV_LOG(ERR, "Queue %d comp channel failed", i); + goto fail; + } + + ret = mana_fd_set_non_blocking(rxq->channel->fd); + if (ret) { + DRV_LOG(ERR, "Failed to set comp non-blocking"); + goto fail; + } + } + rxq->cq = ibv_create_cq(priv->ib_ctx, rxq->num_desc, - NULL, NULL, 0); + NULL, rxq->channel, + rxq->channel ? i : 0); if (!rxq->cq) { ret = -errno; DRV_LOG(ERR, "failed to create rx cq queue %d", i); @@ -347,7 +371,8 @@ int mana_start_rx_queues(struct rte_eth_dev *dev) uint16_t mana_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) { - uint16_t pkt_received = 0, cqe_processed = 0; + uint16_t pkt_received = 0; + uint8_t wqe_posted = 0; struct mana_rxq *rxq = dpdk_rxq; struct mana_priv *priv = rxq->priv; struct gdma_comp comp; @@ -433,18 +458,62 @@ uint16_t mana_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) if (rxq->desc_ring_tail >= rxq->num_desc) rxq->desc_ring_tail = 0; - cqe_processed++; - /* Post another request */ ret = mana_alloc_and_post_rx_wqe(rxq); if (ret) { DRV_LOG(ERR, "failed to post rx wqe ret=%d", ret); break; } + + wqe_posted++; } - if (cqe_processed) - mana_rq_ring_doorbell(rxq); + if (wqe_posted) + mana_rq_ring_doorbell(rxq, wqe_posted); return pkt_received; } + +static int mana_arm_cq(struct mana_rxq *rxq, uint8_t arm) +{ + struct mana_priv *priv = rxq->priv; + uint32_t head = rxq->gdma_cq.head % + (rxq->gdma_cq.count << COMPLETION_QUEUE_ENTRY_OWNER_BITS_SIZE); + + DRV_LOG(ERR, "Ringing completion queue ID %u head %u arm %d", + rxq->gdma_cq.id, head, arm); + + return mana_ring_doorbell(priv->db_page, gdma_queue_completion, + rxq->gdma_cq.id, head, arm); +} + +int mana_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id) +{ + struct mana_rxq *rxq = dev->data->rx_queues[rx_queue_id]; + + return mana_arm_cq(rxq, 1); +} + +int mana_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id) +{ + struct mana_rxq *rxq = dev->data->rx_queues[rx_queue_id]; + struct ibv_cq *ev_cq; + void *ev_ctx; + int ret; + + ret = ibv_get_cq_event(rxq->channel, &ev_cq, &ev_ctx); + if (ret) + ret = errno; + else if (ev_cq != rxq->cq) + ret = EINVAL; + + if (ret) { + if (ret != EAGAIN) + DRV_LOG(ERR, "Can't disable RX intr queue %d", + rx_queue_id); + } else { + ibv_ack_cq_events(rxq->cq, 1); + } + + return -ret; +} diff --git a/drivers/net/mana/tx.c b/drivers/net/mana/tx.c index 26340311c9..68db131868 100644 --- a/drivers/net/mana/tx.c +++ b/drivers/net/mana/tx.c @@ -390,7 +390,8 @@ uint16_t mana_tx_burst(void *dpdk_txq, struct rte_mbuf **tx_pkts, ret = mana_ring_doorbell(db_page, gdma_queue_send, txq->gdma_sq.id, txq->gdma_sq.head * - GDMA_WQE_ALIGNMENT_UNIT_SIZE); + GDMA_WQE_ALIGNMENT_UNIT_SIZE, + 0); if (ret) DRV_LOG(ERR, "mana_ring_doorbell failed ret %d", ret); -- 2.34.1