From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 3D16345F25; Mon, 23 Dec 2024 21:11:38 +0100 (CET) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 123EF402AE; Mon, 23 Dec 2024 21:11:38 +0100 (CET) Received: from mail-pj1-f73.google.com (mail-pj1-f73.google.com [209.85.216.73]) by mails.dpdk.org (Postfix) with ESMTP id EE3764029C for ; Mon, 23 Dec 2024 21:11:35 +0100 (CET) Received: by mail-pj1-f73.google.com with SMTP id 98e67ed59e1d1-2ee5668e09bso4336670a91.3 for ; Mon, 23 Dec 2024 12:11:35 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=20230601; t=1734984695; x=1735589495; darn=dpdk.org; h=cc:to:from:subject:message-id:mime-version:date:from:to:cc:subject :date:message-id:reply-to; bh=6W4ZIximH7EHjmoS2ImeTaw05CT8KdGYgBytOKvd+BA=; b=AgD3Ikz8hzI+k5U+qnz31qZ8ACkD8BQUnlSkg1KobR4LH24XUgERN36IXxfppcWc3D GqNFUjPV+lhDFBYKO6b/huHpCLb45Cf5bOhhlbbaTsrxfp2v5bxzXMbgeMkBgrkvjq5a lIu+tFXltn6paJb9v5SyGPjLsCApgZ1rqKdZwvzCNg5bn0o9d124zMkWz39WO1jL+oxx NTjk5DXVQTPBmrXyfJWycu2kiBkQcYNeiM+jxdRD4slcT8Nsy5ywhGLsYVcxAnQC7296 S5XljrF4N59wjAPjoRhGu4XtzK+it9xhALpW4FFaYlWtv+t3yVwyfrWwTikSbQMHUm82 h5+Q== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1734984695; x=1735589495; h=cc:to:from:subject:message-id:mime-version:date:x-gm-message-state :from:to:cc:subject:date:message-id:reply-to; bh=6W4ZIximH7EHjmoS2ImeTaw05CT8KdGYgBytOKvd+BA=; b=Dr8yta5ievtv4oiG3FB9raRqR/CeLGuvI8llA4v5Tv9IXOrlEgZpNB9KcDtHGrmc2E kbOwIVIJVBkKXu1T4uqnR3hXokE+VNIC5mhT5CR9KUV7wDd1vRMnPCD6ro6cBJTirw+A 1wFy758cDiJgSrch7yRHeUP8D5reKUCInsFEbaoAqhNuS7nrKZ5f4IHJ991rFhJMOkIQ MYKcLBG3j3pk0NWe+nLmVO20OHBs27ObAKGE3ciw7/s1LsvyM5l9/MYOpJz7X9F05sPj FyewuMwUtZF/B2V0E/AlQPSbtJIzGBdkUfvTMvEPmBMEvdWN77mG+Dpvy76U6HVpn9tD O1CQ== X-Gm-Message-State: AOJu0YytzykWq/G59YdGvs4TpdNq7hVZuKgGz8Mo6cbddPZNYnZyfKjg u91/rEGGTJNbdbSsurmrGv8Z4rWN3Brgh9sDF6kQqyRQ8apPT6GkhKxDESh0YB2o6rmTjNOzx5Q Q0MTOU9fi+g== X-Google-Smtp-Source: AGHT+IFzh1FMYij3awxxVy5eE6Ktcutlus/06KcTmihqU4blXa+QDkYQ+NbBRmO1jM7gEcbZ77GmbCzWXTZk9w== X-Received: from pfbbv12.prod.google.com ([2002:a05:6a00:414c:b0:725:e4b6:901f]) (user=joshwash job=prod-delivery.src-stubby-dispatcher) by 2002:a05:6a00:392a:b0:71e:1722:d019 with SMTP id d2e1a72fcca58-72abdeb7326mr19953342b3a.22.1734984695095; Mon, 23 Dec 2024 12:11:35 -0800 (PST) Date: Mon, 23 Dec 2024 12:11:33 -0800 Mime-Version: 1.0 X-Mailer: git-send-email 2.47.1.613.gc27f4b7a9f-goog Message-ID: <20241223201133.4185508-1-joshwash@google.com> Subject: [PATCH] net/gve: Allocate qpl pages using malloc if memzone allocation fails From: Joshua Washington To: Jeroen de Borst , Rushil Gupta , Joshua Washington Cc: dev@dpdk.org, Ferruh Yigit , Praveen Kaligineedi Content-Type: text/plain; charset="UTF-8" X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org From: Praveen Kaligineedi Allocating QPL for an RX queue might fail if enough contiguous IOVA memory cannot be allocated. However, the only requirement for QPL for RX is that each 4K buffer be IOVA contiguous, not the entire QPL. Therefore, use malloc to allocate 4K buffers if the allocation using memzone fails. Signed-off-by: Praveen Kaligineedi Reviewed-by: Joshua Washington --- drivers/net/gve/gve_ethdev.c | 105 ++++++++++++++++++++++++++++++++++++++-------- drivers/net/gve/gve_ethdev.h | 1 + drivers/net/gve/gve_rx.c | 2 +- 3 files changed, 89 insertions(+), 19 deletions(-) diff --git a/drivers/net/gve/gve_ethdev.c b/drivers/net/gve/gve_ethdev.c index db4ebe7..09304ef 100644 --- a/drivers/net/gve/gve_ethdev.c +++ b/drivers/net/gve/gve_ethdev.c @@ -22,42 +22,100 @@ gve_write_version(uint8_t *driver_version_register) writeb('\n', driver_version_register); } +static const struct rte_memzone * +gve_alloc_using_mz(const char *name, uint32_t num_pages) +{ + const struct rte_memzone *mz; + mz = rte_memzone_reserve_aligned(name, num_pages * PAGE_SIZE, + rte_socket_id(), + RTE_MEMZONE_IOVA_CONTIG, PAGE_SIZE); + if (mz == NULL) + PMD_DRV_LOG(ERR, "Failed to alloc memzone %s.", name); + return mz; +} + +static int +gve_alloc_using_malloc(void **bufs, uint32_t num_entries) +{ + uint32_t i; + + for (i = 0; i < num_entries; i++) { + bufs[i] = rte_malloc_socket(NULL, PAGE_SIZE, PAGE_SIZE, rte_socket_id()); + if (bufs[i] == NULL) { + PMD_DRV_LOG(ERR, "Failed to malloc"); + goto free_bufs; + } + } + return 0; + +free_bufs: + while (i > 0) + rte_free(bufs[--i]); + + return -ENOMEM; +} + static struct gve_queue_page_list * -gve_alloc_queue_page_list(const char *name, uint32_t num_pages) +gve_alloc_queue_page_list(const char *name, uint32_t num_pages, bool is_rx) { struct gve_queue_page_list *qpl; const struct rte_memzone *mz; - dma_addr_t page_bus; uint32_t i; qpl = rte_zmalloc("qpl struct", sizeof(struct gve_queue_page_list), 0); if (!qpl) return NULL; - mz = rte_memzone_reserve_aligned(name, num_pages * PAGE_SIZE, - rte_socket_id(), - RTE_MEMZONE_IOVA_CONTIG, PAGE_SIZE); - if (mz == NULL) { - PMD_DRV_LOG(ERR, "Failed to alloc %s.", name); - goto free_qpl_struct; - } qpl->page_buses = rte_zmalloc("qpl page buses", num_pages * sizeof(dma_addr_t), 0); if (qpl->page_buses == NULL) { PMD_DRV_LOG(ERR, "Failed to alloc qpl page buses"); - goto free_qpl_memzone; + goto free_qpl_struct; } - page_bus = mz->iova; - for (i = 0; i < num_pages; i++) { - qpl->page_buses[i] = page_bus; - page_bus += PAGE_SIZE; + + qpl->qpl_bufs = rte_zmalloc("qpl bufs", + num_pages * sizeof(void *), 0); + if (qpl->qpl_bufs == NULL) { + PMD_DRV_LOG(ERR, "Failed to alloc qpl bufs"); + goto free_qpl_page_buses; } - qpl->mz = mz; + + mz = gve_alloc_using_mz(name, num_pages); + if (mz) { + qpl->mz = mz; + + /* Populate the buffer addresses */ + for (i = 0; i < num_pages; i++) + qpl->qpl_bufs[i] = + (void *)((uint64_t)(mz->addr) + i * PAGE_SIZE); + + /* Populate the IOVA addresses */ + for (i = 0; i < num_pages; i++) + qpl->page_buses[i] = mz->iova + i * PAGE_SIZE; + } else { + /* Allocate 4K size buffers. + * Cannot use non-contiguous approach for tx fifo. + */ + if (!is_rx) + goto free_qpl_page_bufs; + + PMD_DRV_LOG(ERR, "Allocating bufs using malloc for %s ", name); + if (gve_alloc_using_malloc(qpl->qpl_bufs, num_pages)) + goto free_qpl_page_bufs; + + /* Populate the IOVA addresses */ + for (i = 0; i < num_pages; i++) + qpl->page_buses[i] = + rte_malloc_virt2iova(qpl->qpl_bufs[i]); + } + qpl->num_entries = num_pages; return qpl; -free_qpl_memzone: - rte_memzone_free(qpl->mz); +free_qpl_page_bufs: + rte_free(qpl->qpl_bufs); +free_qpl_page_buses: + rte_free(qpl->page_buses); free_qpl_struct: rte_free(qpl); return NULL; @@ -69,7 +127,18 @@ gve_free_queue_page_list(struct gve_queue_page_list *qpl) if (qpl->mz) { rte_memzone_free(qpl->mz); qpl->mz = NULL; + } else if (qpl->qpl_bufs) { + uint32_t i; + + for (i = 0; i < qpl->num_entries; i++) + rte_free(qpl->qpl_bufs[i]); + } + + if (qpl->qpl_bufs) { + rte_free(qpl->qpl_bufs); + qpl->qpl_bufs = NULL; } + if (qpl->page_buses) { rte_free(qpl->page_buses); qpl->page_buses = NULL; @@ -89,7 +158,7 @@ gve_setup_queue_page_list(struct gve_priv *priv, uint16_t queue_id, bool is_rx, /* Allocate a new QPL. */ snprintf(qpl_name, sizeof(qpl_name), "gve_%s_%s_qpl%d", priv->pci_dev->device.name, queue_type_string, queue_id); - qpl = gve_alloc_queue_page_list(qpl_name, num_pages); + qpl = gve_alloc_queue_page_list(qpl_name, num_pages, is_rx); if (!qpl) { PMD_DRV_LOG(ERR, "Failed to alloc %s qpl for queue %hu.", diff --git a/drivers/net/gve/gve_ethdev.h b/drivers/net/gve/gve_ethdev.h index c417a0b..a011a0c 100644 --- a/drivers/net/gve/gve_ethdev.h +++ b/drivers/net/gve/gve_ethdev.h @@ -63,6 +63,7 @@ struct gve_queue_page_list { uint32_t num_entries; dma_addr_t *page_buses; /* the dma addrs of the pages */ const struct rte_memzone *mz; + void **qpl_bufs; /* qpl-buffer list */ }; /* A TX desc ring entry */ diff --git a/drivers/net/gve/gve_rx.c b/drivers/net/gve/gve_rx.c index 89b6ef3..e73704c 100644 --- a/drivers/net/gve/gve_rx.c +++ b/drivers/net/gve/gve_rx.c @@ -117,7 +117,7 @@ gve_rx_mbuf(struct gve_rx_queue *rxq, struct rte_mbuf *rxe, uint16_t len, rxq->ctx.mbuf_tail = rxe; } if (rxq->is_gqi_qpl) { - addr = (uint64_t)(rxq->qpl->mz->addr) + rx_id * PAGE_SIZE + padding; + addr = (uint64_t)rxq->qpl->qpl_bufs[rx_id] + padding; rte_memcpy((void *)((size_t)rxe->buf_addr + rxe->data_off), (void *)(size_t)addr, len); } -- 2.47.1.613.gc27f4b7a9f-goog