Test-Label: iol-testing Test-Status: WARNING http://dpdk.org/patch/119093 _apply patch failure_ Submitter: Junfeng Guo Date: Tuesday, October 25 2022 09:07:29 Applied on: CommitID:f800c03de9f71a7b59111b5972676edb99c28b0a Apply patch set 119093-119100 failed: Checking patch drivers/net/gve/base/gve.h... error: drivers/net/gve/base/gve.h: already exists in index Checking patch drivers/net/gve/base/gve_adminq.c... error: drivers/net/gve/base/gve_adminq.c: already exists in index Checking patch drivers/net/gve/base/gve_adminq.h... error: drivers/net/gve/base/gve_adminq.h: already exists in index Checking patch drivers/net/gve/base/gve_desc.h... error: drivers/net/gve/base/gve_desc.h: already exists in index Checking patch drivers/net/gve/base/gve_desc_dqo.h... error: drivers/net/gve/base/gve_desc_dqo.h: already exists in index Checking patch drivers/net/gve/base/gve_register.h... error: drivers/net/gve/base/gve_register.h: already exists in index Checking patch drivers/net/gve/base/gve_adminq.h... error: while searching for: #ifndef _GVE_ADMINQ_H #define _GVE_ADMINQ_H /* Admin queue opcodes */ enum gve_adminq_opcodes { GVE_ADMINQ_DESCRIBE_DEVICE = 0x1, error: patch failed: drivers/net/gve/base/gve_adminq.h:6 Checking patch drivers/net/gve/base/gve_desc.h... error: while searching for: #ifndef _GVE_DESC_H_ #define _GVE_DESC_H_ /* A note on seg_addrs * * Base addresses encoded in seg_addr are not assumed to be physical error: patch failed: drivers/net/gve/base/gve_desc.h:8 Checking patch drivers/net/gve/base/gve_desc_dqo.h... error: while searching for: #ifndef _GVE_DESC_DQO_H_ #define _GVE_DESC_DQO_H_ #define GVE_TX_MAX_HDR_SIZE_DQO 255 #define GVE_TX_MIN_TSO_MSS_DQO 88 error: patch failed: drivers/net/gve/base/gve_desc_dqo.h:8 Checking patch drivers/net/gve/base/gve_osdep.h... error: drivers/net/gve/base/gve_osdep.h: already exists in index Checking patch drivers/net/gve/base/gve_register.h... error: while searching for: #ifndef _GVE_REGISTER_H_ #define _GVE_REGISTER_H_ /* Fixed Configuration Registers */ struct gve_registers { __be32 device_status; error: patch failed: drivers/net/gve/base/gve_register.h:6 Applying patch drivers/net/gve/base/gve_adminq.h with 1 reject... Rejected hunk #1. Applying patch drivers/net/gve/base/gve_desc.h with 1 reject... Rejected hunk #1. Applying patch drivers/net/gve/base/gve_desc_dqo.h with 1 reject... Rejected hunk #1. Applying patch drivers/net/gve/base/gve_register.h with 1 reject... Rejected hunk #1. diff a/drivers/net/gve/base/gve_adminq.h b/drivers/net/gve/base/gve_adminq.h (rejected hunks) @@ -6,6 +6,8 @@ #ifndef _GVE_ADMINQ_H #define _GVE_ADMINQ_H +#include "gve_osdep.h" + /* Admin queue opcodes */ enum gve_adminq_opcodes { GVE_ADMINQ_DESCRIBE_DEVICE = 0x1, diff a/drivers/net/gve/base/gve_desc.h b/drivers/net/gve/base/gve_desc.h (rejected hunks) @@ -8,6 +8,8 @@ #ifndef _GVE_DESC_H_ #define _GVE_DESC_H_ +#include "gve_osdep.h" + /* A note on seg_addrs * * Base addresses encoded in seg_addr are not assumed to be physical diff a/drivers/net/gve/base/gve_desc_dqo.h b/drivers/net/gve/base/gve_desc_dqo.h (rejected hunks) @@ -8,6 +8,8 @@ #ifndef _GVE_DESC_DQO_H_ #define _GVE_DESC_DQO_H_ +#include "gve_osdep.h" + #define GVE_TX_MAX_HDR_SIZE_DQO 255 #define GVE_TX_MIN_TSO_MSS_DQO 88 diff a/drivers/net/gve/base/gve_register.h b/drivers/net/gve/base/gve_register.h (rejected hunks) @@ -6,6 +6,8 @@ #ifndef _GVE_REGISTER_H_ #define _GVE_REGISTER_H_ +#include "gve_osdep.h" + /* Fixed Configuration Registers */ struct gve_registers { __be32 device_status; Checking patch MAINTAINERS... error: while searching for: F: doc/guides/nics/enic.rst F: doc/guides/nics/features/enic.ini Hisilicon hns3 M: Dongdong Liu M: Yisen Zhuang error: patch failed: MAINTAINERS:697 Checking patch doc/guides/nics/features/gve.ini... error: doc/guides/nics/features/gve.ini: already exists in index Checking patch doc/guides/nics/gve.rst... error: doc/guides/nics/gve.rst: already exists in index Checking patch doc/guides/nics/index.rst... error: while searching for: enetfec enic fm10k hinic hns3 i40e error: patch failed: doc/guides/nics/index.rst:29 Checking patch doc/guides/rel_notes/release_22_11.rst... error: while searching for: * Added Q-in-CMB feature controlled by devarg ionic_cmb. * Added optimized handlers for non-scattered Rx and Tx. * **Updated Intel iavf driver.** * Added flow subscription support. error: patch failed: doc/guides/rel_notes/release_22_11.rst:152 Checking patch drivers/net/gve/base/gve_adminq.c... error: while searching for: * Copyright (C) 2015-2022 Google, Inc. */ #include "gve_adminq.h" #include "gve_register.h" error: patch failed: drivers/net/gve/base/gve_adminq.c:3 Checking patch drivers/net/gve/gve_ethdev.c... error: drivers/net/gve/gve_ethdev.c: already exists in index Checking patch drivers/net/gve/gve_ethdev.h... error: drivers/net/gve/gve_ethdev.h: already exists in index Checking patch drivers/net/gve/gve_logs.h... error: drivers/net/gve/gve_logs.h: already exists in index Checking patch drivers/net/gve/meson.build... error: drivers/net/gve/meson.build: already exists in index Checking patch drivers/net/gve/version.map... error: drivers/net/gve/version.map: already exists in index Checking patch drivers/net/meson.build... error: while searching for: 'enic', 'failsafe', 'fm10k', 'hinic', 'hns3', 'i40e', error: patch failed: drivers/net/meson.build:23 Applying patch MAINTAINERS with 1 reject... Rejected hunk #1. Applying patch doc/guides/nics/index.rst with 1 reject... Rejected hunk #1. Applying patch doc/guides/rel_notes/release_22_11.rst with 1 reject... Rejected hunk #1. Applying patch drivers/net/gve/base/gve_adminq.c with 1 reject... Rejected hunk #1. Applying patch drivers/net/meson.build with 1 reject... Rejected hunk #1. diff a/MAINTAINERS b/MAINTAINERS (rejected hunks) @@ -697,6 +697,12 @@ F: drivers/net/enic/ F: doc/guides/nics/enic.rst F: doc/guides/nics/features/enic.ini +Google Virtual Ethernet +M: Junfeng Guo +F: drivers/net/gve/ +F: doc/guides/nics/gve.rst +F: doc/guides/nics/features/gve.ini + Hisilicon hns3 M: Dongdong Liu M: Yisen Zhuang diff a/doc/guides/nics/index.rst b/doc/guides/nics/index.rst (rejected hunks) @@ -29,6 +29,7 @@ Network Interface Controller Drivers enetfec enic fm10k + gve hinic hns3 i40e diff a/doc/guides/rel_notes/release_22_11.rst b/doc/guides/rel_notes/release_22_11.rst (rejected hunks) @@ -152,6 +152,11 @@ New Features * Added Q-in-CMB feature controlled by devarg ionic_cmb. * Added optimized handlers for non-scattered Rx and Tx. +* **Added GVE net PMD** + + * Added the new ``gve`` net driver for Google Virtual Ethernet devices. + * See the :doc:`../nics/gve` NIC guide for more details on this new driver. + * **Updated Intel iavf driver.** * Added flow subscription support. diff a/drivers/net/gve/base/gve_adminq.c b/drivers/net/gve/base/gve_adminq.c (rejected hunks) @@ -3,6 +3,7 @@ * Copyright (C) 2015-2022 Google, Inc. */ +#include "../gve_ethdev.h" #include "gve_adminq.h" #include "gve_register.h" diff a/drivers/net/meson.build b/drivers/net/meson.build (rejected hunks) @@ -23,6 +23,7 @@ drivers = [ 'enic', 'failsafe', 'fm10k', + 'gve', 'hinic', 'hns3', 'i40e', Checking patch doc/guides/nics/features/gve.ini... error: while searching for: ; Refer to default.ini for the full list of available PMD features. ; [Features] Linux = Y x86-32 = Y x86-64 = Y error: patch failed: doc/guides/nics/features/gve.ini:4 Checking patch doc/guides/nics/gve.rst... error: while searching for: In this release, the GVE PMD provides the basic functionality of packet reception and transmission. Currently, only GQI_QPL and GQI_RDA queue format are supported in PMD. Jumbo Frame is not supported in PMD for now. It'll be added in the future error: patch failed: doc/guides/nics/gve.rst:60 Checking patch drivers/net/gve/gve_ethdev.c... error: while searching for: return 0; } static int gve_dev_start(struct rte_eth_dev *dev) { dev->data->dev_started = 1; return 0; } error: patch failed: drivers/net/gve/gve_ethdev.c:34 error: while searching for: .dev_start = gve_dev_start, .dev_stop = gve_dev_stop, .dev_close = gve_dev_close, }; static void error: patch failed: drivers/net/gve/gve_ethdev.c:72 Applying patch doc/guides/nics/features/gve.ini with 1 reject... Rejected hunk #1. Applying patch doc/guides/nics/gve.rst with 1 reject... Rejected hunk #1. Applying patch drivers/net/gve/gve_ethdev.c with 2 rejects... Rejected hunk #1. Rejected hunk #2. diff a/doc/guides/nics/features/gve.ini b/doc/guides/nics/features/gve.ini (rejected hunks) @@ -4,6 +4,7 @@ ; Refer to default.ini for the full list of available PMD features. ; [Features] +Link status = Y Linux = Y x86-32 = Y x86-64 = Y diff a/doc/guides/nics/gve.rst b/doc/guides/nics/gve.rst (rejected hunks) @@ -60,6 +60,9 @@ Features and Limitations In this release, the GVE PMD provides the basic functionality of packet reception and transmission. +Supported features of the GVE PMD are: + +- Link state information Currently, only GQI_QPL and GQI_RDA queue format are supported in PMD. Jumbo Frame is not supported in PMD for now. It'll be added in the future diff a/drivers/net/gve/gve_ethdev.c b/drivers/net/gve/gve_ethdev.c (rejected hunks) @@ -34,10 +34,39 @@ gve_dev_configure(__rte_unused struct rte_eth_dev *dev) return 0; } +static int +gve_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complete) +{ + struct gve_priv *priv = dev->data->dev_private; + struct rte_eth_link link; + int err; + + memset(&link, 0, sizeof(link)); + link.link_duplex = RTE_ETH_LINK_FULL_DUPLEX; + link.link_autoneg = RTE_ETH_LINK_AUTONEG; + + if (!dev->data->dev_started) { + link.link_status = RTE_ETH_LINK_DOWN; + link.link_speed = RTE_ETH_SPEED_NUM_NONE; + } else { + link.link_status = RTE_ETH_LINK_UP; + PMD_DRV_LOG(DEBUG, "Get link status from hw"); + err = gve_adminq_report_link_speed(priv); + if (err) { + PMD_DRV_LOG(ERR, "Failed to get link speed."); + priv->link_speed = RTE_ETH_SPEED_NUM_UNKNOWN; + } + link.link_speed = priv->link_speed; + } + + return rte_eth_linkstatus_set(dev, &link); +} + static int gve_dev_start(struct rte_eth_dev *dev) { dev->data->dev_started = 1; + gve_link_update(dev, 0); return 0; } @@ -72,6 +101,7 @@ static const struct eth_dev_ops gve_eth_dev_ops = { .dev_start = gve_dev_start, .dev_stop = gve_dev_stop, .dev_close = gve_dev_close, + .link_update = gve_link_update, }; static void Checking patch doc/guides/nics/features/gve.ini... error: while searching for: ; [Features] Link status = Y Linux = Y x86-32 = Y x86-64 = Y error: patch failed: doc/guides/nics/features/gve.ini:5 Checking patch doc/guides/nics/gve.rst... error: while searching for: DPDK release. Also, only GQI_QPL queue format is in use on GCP since GQI_RDA hasn't been released in production. error: patch failed: doc/guides/nics/gve.rst:69 Checking patch drivers/net/gve/gve_ethdev.c... error: while searching for: return err; } static const struct eth_dev_ops gve_eth_dev_ops = { .dev_configure = gve_dev_configure, .dev_start = gve_dev_start, .dev_stop = gve_dev_stop, .dev_close = gve_dev_close, .link_update = gve_link_update, }; static void error: patch failed: drivers/net/gve/gve_ethdev.c:96 Applying patch doc/guides/nics/features/gve.ini with 1 reject... Rejected hunk #1. Applying patch doc/guides/nics/gve.rst with 1 reject... Rejected hunk #1. Applying patch drivers/net/gve/gve_ethdev.c with 1 reject... Rejected hunk #1. diff a/doc/guides/nics/features/gve.ini b/doc/guides/nics/features/gve.ini (rejected hunks) @@ -5,6 +5,7 @@ ; [Features] Link status = Y +MTU update = Y Linux = Y x86-32 = Y x86-64 = Y diff a/doc/guides/nics/gve.rst b/doc/guides/nics/gve.rst (rejected hunks) @@ -69,3 +69,5 @@ Jumbo Frame is not supported in PMD for now. It'll be added in the future DPDK release. Also, only GQI_QPL queue format is in use on GCP since GQI_RDA hasn't been released in production. + +Currently, setting MTU with value larger than 1460 is not supported. diff a/drivers/net/gve/gve_ethdev.c b/drivers/net/gve/gve_ethdev.c (rejected hunks) @@ -96,12 +96,40 @@ gve_dev_close(struct rte_eth_dev *dev) return err; } +static int +gve_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) +{ + struct gve_priv *priv = dev->data->dev_private; + int err; + + if (mtu < RTE_ETHER_MIN_MTU || mtu > priv->max_mtu) { + PMD_DRV_LOG(ERR, "MIN MTU is %u, MAX MTU is %u", + RTE_ETHER_MIN_MTU, priv->max_mtu); + return -EINVAL; + } + + /* mtu setting is forbidden if port is start */ + if (dev->data->dev_started) { + PMD_DRV_LOG(ERR, "Port must be stopped before configuration"); + return -EBUSY; + } + + err = gve_adminq_set_mtu(priv, mtu); + if (err) { + PMD_DRV_LOG(ERR, "Failed to set mtu as %u err = %d", mtu, err); + return err; + } + + return 0; +} + static const struct eth_dev_ops gve_eth_dev_ops = { .dev_configure = gve_dev_configure, .dev_start = gve_dev_start, .dev_stop = gve_dev_stop, .dev_close = gve_dev_close, .link_update = gve_link_update, + .mtu_set = gve_dev_mtu_set, }; static void Checking patch doc/guides/nics/features/gve.ini... error: while searching for: ; Refer to default.ini for the full list of available PMD features. ; [Features] Link status = Y MTU update = Y Linux = Y x86-32 = Y x86-64 = Y error: patch failed: doc/guides/nics/features/gve.ini:4 Checking patch doc/guides/nics/gve.rst... error: while searching for: released in production. Currently, setting MTU with value larger than 1460 is not supported. error: patch failed: doc/guides/nics/gve.rst:71 Checking patch drivers/net/gve/gve_ethdev.c... error: while searching for: } static int gve_dev_configure(__rte_unused struct rte_eth_dev *dev) { return 0; } error: patch failed: drivers/net/gve/gve_ethdev.c:29 error: while searching for: return err; } static int gve_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) { error: patch failed: drivers/net/gve/gve_ethdev.c:96 error: while searching for: .dev_start = gve_dev_start, .dev_stop = gve_dev_stop, .dev_close = gve_dev_close, .link_update = gve_link_update, .mtu_set = gve_dev_mtu_set, }; error: patch failed: drivers/net/gve/gve_ethdev.c:128 Checking patch drivers/net/gve/gve_ethdev.h... error: while searching for: #define GVE_MIN_BUF_SIZE 1024 #define GVE_MAX_RX_PKTLEN 65535 /* A list of pages registered with the device during setup and used by a queue * as buffers */ error: patch failed: drivers/net/gve/gve_ethdev.h:18 Applying patch doc/guides/nics/features/gve.ini with 1 reject... Rejected hunk #1. Applying patch doc/guides/nics/gve.rst with 1 reject... Rejected hunk #1. Applying patch drivers/net/gve/gve_ethdev.c with 3 rejects... Rejected hunk #1. Rejected hunk #2. Rejected hunk #3. Applying patch drivers/net/gve/gve_ethdev.h with 1 reject... Rejected hunk #1. diff a/doc/guides/nics/features/gve.ini b/doc/guides/nics/features/gve.ini (rejected hunks) @@ -4,8 +4,10 @@ ; Refer to default.ini for the full list of available PMD features. ; [Features] +Speed capabilities = Y Link status = Y MTU update = Y +RSS hash = Y Linux = Y x86-32 = Y x86-64 = Y diff a/doc/guides/nics/gve.rst b/doc/guides/nics/gve.rst (rejected hunks) @@ -71,3 +71,8 @@ Also, only GQI_QPL queue format is in use on GCP since GQI_RDA hasn't been released in production. Currently, setting MTU with value larger than 1460 is not supported. + +Currently, only "RSS hash" is force enabled so that the backend hardware +device calculated hash values could be shared with applications. But for +RSS, there is no such API to config RSS hash function or RETA table. So, +limited RSS is supported only with default config/setting. diff a/drivers/net/gve/gve_ethdev.c b/drivers/net/gve/gve_ethdev.c (rejected hunks) @@ -29,8 +29,16 @@ gve_write_version(uint8_t *driver_version_register) } static int -gve_dev_configure(__rte_unused struct rte_eth_dev *dev) +gve_dev_configure(struct rte_eth_dev *dev) { + struct gve_priv *priv = dev->data->dev_private; + + if (dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG) + dev->data->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_RSS_HASH; + + if (dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO) + priv->enable_rsc = 1; + return 0; } @@ -96,6 +104,54 @@ gve_dev_close(struct rte_eth_dev *dev) return err; } +static int +gve_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) +{ + struct gve_priv *priv = dev->data->dev_private; + + dev_info->device = dev->device; + dev_info->max_mac_addrs = 1; + dev_info->max_rx_queues = priv->max_nb_rxq; + dev_info->max_tx_queues = priv->max_nb_txq; + dev_info->min_rx_bufsize = GVE_MIN_BUF_SIZE; + dev_info->max_rx_pktlen = GVE_MAX_RX_PKTLEN; + dev_info->max_mtu = GVE_MAX_MTU; + dev_info->min_mtu = GVE_MIN_MTU; + + dev_info->rx_offload_capa = 0; + dev_info->tx_offload_capa = 0; + + if (priv->queue_format == GVE_DQO_RDA_FORMAT) + dev_info->rx_offload_capa |= RTE_ETH_RX_OFFLOAD_TCP_LRO; + + dev_info->default_rxconf = (struct rte_eth_rxconf) { + .rx_free_thresh = GVE_DEFAULT_RX_FREE_THRESH, + .rx_drop_en = 0, + .offloads = 0, + }; + + dev_info->default_txconf = (struct rte_eth_txconf) { + .tx_free_thresh = GVE_DEFAULT_TX_FREE_THRESH, + .offloads = 0, + }; + + dev_info->default_rxportconf.ring_size = priv->rx_desc_cnt; + dev_info->rx_desc_lim = (struct rte_eth_desc_lim) { + .nb_max = priv->rx_desc_cnt, + .nb_min = priv->rx_desc_cnt, + .nb_align = 1, + }; + + dev_info->default_txportconf.ring_size = priv->tx_desc_cnt; + dev_info->tx_desc_lim = (struct rte_eth_desc_lim) { + .nb_max = priv->tx_desc_cnt, + .nb_min = priv->tx_desc_cnt, + .nb_align = 1, + }; + + return 0; +} + static int gve_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) { @@ -128,6 +184,7 @@ static const struct eth_dev_ops gve_eth_dev_ops = { .dev_start = gve_dev_start, .dev_stop = gve_dev_stop, .dev_close = gve_dev_close, + .dev_infos_get = gve_dev_info_get, .link_update = gve_link_update, .mtu_set = gve_dev_mtu_set, }; diff a/drivers/net/gve/gve_ethdev.h b/drivers/net/gve/gve_ethdev.h (rejected hunks) @@ -18,6 +18,9 @@ #define GVE_MIN_BUF_SIZE 1024 #define GVE_MAX_RX_PKTLEN 65535 +#define GVE_MAX_MTU RTE_ETHER_MTU +#define GVE_MIN_MTU RTE_ETHER_MIN_MTU + /* A list of pages registered with the device during setup and used by a queue * as buffers */ Checking patch drivers/net/gve/gve_ethdev.c... error: while searching for: writeb('\n', driver_version_register); } static int gve_dev_configure(struct rte_eth_dev *dev) { error: patch failed: drivers/net/gve/gve_ethdev.c:28 Hunk #2 succeeded at 141 (offset 37 lines). error: while searching for: static int gve_dev_start(struct rte_eth_dev *dev) { dev->data->dev_started = 1; gve_link_update(dev, 0); return 0; } static int gve_dev_stop(struct rte_eth_dev *dev) { dev->data->dev_link.link_status = RTE_ETH_LINK_DOWN; dev->data->dev_started = 0; return 0; error: patch failed: drivers/net/gve/gve_ethdev.c:73 error: while searching for: static int gve_dev_close(struct rte_eth_dev *dev) { int err = 0; if (dev->data->dev_started) { err = gve_dev_stop(dev); error: patch failed: drivers/net/gve/gve_ethdev.c:91 error: while searching for: PMD_DRV_LOG(ERR, "Failed to stop dev."); } dev->data->mac_addrs = NULL; return err; error: patch failed: drivers/net/gve/gve_ethdev.c:99 error: while searching for: .dev_stop = gve_dev_stop, .dev_close = gve_dev_close, .dev_infos_get = gve_dev_info_get, .link_update = gve_link_update, .mtu_set = gve_dev_mtu_set, }; error: patch failed: drivers/net/gve/gve_ethdev.c:185 error: while searching for: static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device) { int num_ntfy; int err; /* Set up the adminq */ error: patch failed: drivers/net/gve/gve_ethdev.c:322 error: while searching for: PMD_DRV_LOG(INFO, "Max TX queues %d, Max RX queues %d", priv->max_nb_txq, priv->max_nb_rxq); setup_device: err = gve_setup_device_resources(priv); if (!err) return 0; free_adminq: gve_adminq_free(priv); return err; error: patch failed: drivers/net/gve/gve_ethdev.c:373 Checking patch drivers/net/gve/gve_ethdev.h... error: while searching for: struct gve_tx_seg_desc seg; /* subsequent descs for a packet */ }; struct gve_tx_queue { volatile union gve_tx_desc *tx_desc_ring; const struct rte_memzone *mz; uint64_t tx_ring_phys_addr; uint16_t nb_tx_desc; /* Only valid for DQO_QPL queue format */ struct gve_queue_page_list *qpl; uint16_t port_id; uint16_t queue_id; error: patch failed: drivers/net/gve/gve_ethdev.h:37 error: while searching for: /* Only valid for DQO_RDA queue format */ struct gve_tx_queue *complq; }; struct gve_rx_queue { error: patch failed: drivers/net/gve/gve_ethdev.h:59 error: while searching for: const struct rte_memzone *mz; const struct rte_memzone *data_mz; uint64_t rx_ring_phys_addr; uint16_t nb_rx_desc; volatile rte_be32_t *ntfy_addr; /* only valid for GQI_QPL queue format */ error: patch failed: drivers/net/gve/gve_ethdev.h:67 error: while searching for: /* Only valid for DQO_RDA queue format */ struct gve_rx_queue *bufq; }; struct gve_priv { error: patch failed: drivers/net/gve/gve_ethdev.h:86 error: while searching for: &priv->state_flags); } #endif /* _GVE_ETHDEV_H_ */ error: patch failed: drivers/net/gve/gve_ethdev.h:225 Checking patch drivers/net/gve/gve_rx.c... error: drivers/net/gve/gve_rx.c: already exists in index Checking patch drivers/net/gve/gve_tx.c... error: drivers/net/gve/gve_tx.c: already exists in index Checking patch drivers/net/gve/meson.build... error: while searching for: sources = files( 'base/gve_adminq.c', 'gve_ethdev.c', ) includes += include_directories('base') error: patch failed: drivers/net/gve/meson.build:9 Applying patch drivers/net/gve/gve_ethdev.c with 7 rejects... Rejected hunk #1. Hunk #2 applied cleanly. Rejected hunk #3. Rejected hunk #4. Rejected hunk #5. Rejected hunk #6. Rejected hunk #7. Rejected hunk #8. Applying patch drivers/net/gve/gve_ethdev.h with 5 rejects... Rejected hunk #1. Rejected hunk #2. Rejected hunk #3. Rejected hunk #4. Rejected hunk #5. Applying patch drivers/net/gve/meson.build with 1 reject... Rejected hunk #1. diff a/drivers/net/gve/gve_ethdev.c b/drivers/net/gve/gve_ethdev.c (rejected hunks) @@ -28,6 +28,68 @@ gve_write_version(uint8_t *driver_version_register) writeb('\n', driver_version_register); } +static int +gve_alloc_queue_page_list(struct gve_priv *priv, uint32_t id, uint32_t pages) +{ + char z_name[RTE_MEMZONE_NAMESIZE]; + struct gve_queue_page_list *qpl; + const struct rte_memzone *mz; + dma_addr_t page_bus; + uint32_t i; + + if (priv->num_registered_pages + pages > + priv->max_registered_pages) { + PMD_DRV_LOG(ERR, "Pages %" PRIu64 " > max registered pages %" PRIu64, + priv->num_registered_pages + pages, + priv->max_registered_pages); + return -EINVAL; + } + qpl = &priv->qpl[id]; + snprintf(z_name, sizeof(z_name), "gve_%s_qpl%d", priv->pci_dev->device.name, id); + mz = rte_memzone_reserve_aligned(z_name, pages * PAGE_SIZE, + rte_socket_id(), + RTE_MEMZONE_IOVA_CONTIG, PAGE_SIZE); + if (mz == NULL) { + PMD_DRV_LOG(ERR, "Failed to alloc %s.", z_name); + return -ENOMEM; + } + qpl->page_buses = rte_zmalloc("qpl page buses", pages * sizeof(dma_addr_t), 0); + if (qpl->page_buses == NULL) { + PMD_DRV_LOG(ERR, "Failed to alloc qpl %u page buses", id); + return -ENOMEM; + } + page_bus = mz->iova; + for (i = 0; i < pages; i++) { + qpl->page_buses[i] = page_bus; + page_bus += PAGE_SIZE; + } + qpl->id = id; + qpl->mz = mz; + qpl->num_entries = pages; + + priv->num_registered_pages += pages; + + return 0; +} + +static void +gve_free_qpls(struct gve_priv *priv) +{ + uint16_t nb_txqs = priv->max_nb_txq; + uint16_t nb_rxqs = priv->max_nb_rxq; + uint32_t i; + + for (i = 0; i < nb_txqs + nb_rxqs; i++) { + if (priv->qpl[i].mz != NULL) + rte_memzone_free(priv->qpl[i].mz); + if (priv->qpl[i].page_buses != NULL) + rte_free(priv->qpl[i].page_buses); + } + + if (priv->qpl != NULL) + rte_free(priv->qpl); +} + static int gve_dev_configure(struct rte_eth_dev *dev) { @@ -73,16 +172,70 @@ gve_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complete) static int gve_dev_start(struct rte_eth_dev *dev) { + uint16_t num_queues = dev->data->nb_tx_queues; + struct gve_priv *priv = dev->data->dev_private; + struct gve_tx_queue *txq; + struct gve_rx_queue *rxq; + uint16_t i; + int err; + + priv->txqs = (struct gve_tx_queue **)dev->data->tx_queues; + err = gve_adminq_create_tx_queues(priv, num_queues); + if (err) { + PMD_DRV_LOG(ERR, "failed to create %u tx queues.", num_queues); + return err; + } + for (i = 0; i < num_queues; i++) { + txq = priv->txqs[i]; + txq->qtx_tail = + &priv->db_bar2[rte_be_to_cpu_32(txq->qres->db_index)]; + txq->qtx_head = + &priv->cnt_array[rte_be_to_cpu_32(txq->qres->counter_index)]; + + rte_write32(rte_cpu_to_be_32(GVE_IRQ_MASK), txq->ntfy_addr); + } + + num_queues = dev->data->nb_rx_queues; + priv->rxqs = (struct gve_rx_queue **)dev->data->rx_queues; + err = gve_adminq_create_rx_queues(priv, num_queues); + if (err) { + PMD_DRV_LOG(ERR, "failed to create %u rx queues.", num_queues); + goto err_tx; + } + for (i = 0; i < num_queues; i++) { + rxq = priv->rxqs[i]; + rxq->qrx_tail = + &priv->db_bar2[rte_be_to_cpu_32(rxq->qres->db_index)]; + + rte_write32(rte_cpu_to_be_32(GVE_IRQ_MASK), rxq->ntfy_addr); + + err = gve_refill_pages(rxq); + if (err) { + PMD_DRV_LOG(ERR, "Failed to refill for RX"); + goto err_rx; + } + } + dev->data->dev_started = 1; gve_link_update(dev, 0); return 0; + +err_rx: + gve_stop_rx_queues(dev); +err_tx: + gve_stop_tx_queues(dev); + return err; } static int gve_dev_stop(struct rte_eth_dev *dev) { dev->data->dev_link.link_status = RTE_ETH_LINK_DOWN; + + gve_stop_tx_queues(dev); + gve_stop_rx_queues(dev); + dev->data->dev_started = 0; return 0; @@ -91,7 +244,11 @@ gve_dev_stop(struct rte_eth_dev *dev) static int gve_dev_close(struct rte_eth_dev *dev) { + struct gve_priv *priv = dev->data->dev_private; + struct gve_tx_queue *txq; + struct gve_rx_queue *rxq; int err = 0; + uint16_t i; if (dev->data->dev_started) { err = gve_dev_stop(dev); @@ -99,6 +256,19 @@ gve_dev_close(struct rte_eth_dev *dev) PMD_DRV_LOG(ERR, "Failed to stop dev."); } + for (i = 0; i < dev->data->nb_tx_queues; i++) { + txq = dev->data->tx_queues[i]; + gve_tx_queue_release(txq); + } + + for (i = 0; i < dev->data->nb_rx_queues; i++) { + rxq = dev->data->rx_queues[i]; + gve_rx_queue_release(rxq); + } + + gve_free_qpls(priv); + rte_free(priv->adminq); + dev->data->mac_addrs = NULL; return err; @@ -185,6 +355,8 @@ static const struct eth_dev_ops gve_eth_dev_ops = { .dev_stop = gve_dev_stop, .dev_close = gve_dev_close, .dev_infos_get = gve_dev_info_get, + .rx_queue_setup = gve_rx_queue_setup, + .tx_queue_setup = gve_tx_queue_setup, .link_update = gve_link_update, .mtu_set = gve_dev_mtu_set, }; @@ -322,7 +494,9 @@ gve_setup_device_resources(struct gve_priv *priv) static int gve_init_priv(struct gve_priv *priv, bool skip_describe_device) { + uint16_t pages; int num_ntfy; + uint32_t i; int err; /* Set up the adminq */ @@ -373,10 +547,40 @@ gve_init_priv(struct gve_priv *priv, bool skip_describe_device) PMD_DRV_LOG(INFO, "Max TX queues %d, Max RX queues %d", priv->max_nb_txq, priv->max_nb_rxq); + /* In GQI_QPL queue format: + * Allocate queue page lists according to max queue number + * tx qpl id should start from 0 while rx qpl id should start + * from priv->max_nb_txq + */ + if (priv->queue_format == GVE_GQI_QPL_FORMAT) { + priv->qpl = rte_zmalloc("gve_qpl", + (priv->max_nb_txq + priv->max_nb_rxq) * + sizeof(struct gve_queue_page_list), 0); + if (priv->qpl == NULL) { + PMD_DRV_LOG(ERR, "Failed to alloc qpl."); + err = -ENOMEM; + goto free_adminq; + } + + for (i = 0; i < priv->max_nb_txq + priv->max_nb_rxq; i++) { + if (i < priv->max_nb_txq) + pages = priv->tx_pages_per_qpl; + else + pages = priv->rx_data_slot_cnt; + err = gve_alloc_queue_page_list(priv, i, pages); + if (err != 0) { + PMD_DRV_LOG(ERR, "Failed to alloc qpl %u.", i); + goto err_qpl; + } + } + } + setup_device: err = gve_setup_device_resources(priv); if (!err) return 0; +err_qpl: + gve_free_qpls(priv); free_adminq: gve_adminq_free(priv); return err; diff a/drivers/net/gve/gve_ethdev.h b/drivers/net/gve/gve_ethdev.h (rejected hunks) @@ -37,15 +37,35 @@ union gve_tx_desc { struct gve_tx_seg_desc seg; /* subsequent descs for a packet */ }; +struct gve_tx_iovec { + uint32_t iov_base; /* offset in fifo */ + uint32_t iov_len; +}; + struct gve_tx_queue { volatile union gve_tx_desc *tx_desc_ring; const struct rte_memzone *mz; uint64_t tx_ring_phys_addr; + struct rte_mbuf **sw_ring; + volatile rte_be32_t *qtx_tail; + volatile rte_be32_t *qtx_head; + uint32_t tx_tail; uint16_t nb_tx_desc; + uint16_t nb_free; + uint32_t next_to_clean; + uint16_t free_thresh; /* Only valid for DQO_QPL queue format */ + uint16_t sw_tail; + uint16_t sw_ntc; + uint16_t sw_nb_free; + uint32_t fifo_size; + uint32_t fifo_head; + uint32_t fifo_avail; + uint64_t fifo_base; struct gve_queue_page_list *qpl; + struct gve_tx_iovec *iov_ring; uint16_t port_id; uint16_t queue_id; @@ -59,6 +79,8 @@ struct gve_tx_queue { /* Only valid for DQO_RDA queue format */ struct gve_tx_queue *complq; + + uint8_t is_gqi_qpl; }; struct gve_rx_queue { @@ -67,9 +89,17 @@ struct gve_rx_queue { const struct rte_memzone *mz; const struct rte_memzone *data_mz; uint64_t rx_ring_phys_addr; + struct rte_mbuf **sw_ring; + struct rte_mempool *mpool; + uint16_t rx_tail; uint16_t nb_rx_desc; + uint16_t expected_seqno; /* the next expected seqno */ + uint16_t free_thresh; + uint32_t next_avail; + uint32_t nb_avail; + volatile rte_be32_t *qrx_tail; volatile rte_be32_t *ntfy_addr; /* only valid for GQI_QPL queue format */ @@ -86,6 +116,8 @@ struct gve_rx_queue { /* Only valid for DQO_RDA queue format */ struct gve_rx_queue *bufq; + + uint8_t is_gqi_qpl; }; struct gve_priv { @@ -225,4 +257,24 @@ gve_clear_device_rings_ok(struct gve_priv *priv) &priv->state_flags); } +int +gve_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id, uint16_t nb_desc, + unsigned int socket_id, const struct rte_eth_rxconf *conf, + struct rte_mempool *pool); +int +gve_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_id, uint16_t nb_desc, + unsigned int socket_id, const struct rte_eth_txconf *conf); + +void +gve_tx_queue_release(void *txq); + +void +gve_rx_queue_release(void *rxq); + +void +gve_stop_tx_queues(struct rte_eth_dev *dev); + +void +gve_stop_rx_queues(struct rte_eth_dev *dev); + #endif /* _GVE_ETHDEV_H_ */ diff a/drivers/net/gve/meson.build b/drivers/net/gve/meson.build (rejected hunks) @@ -9,6 +9,8 @@ endif sources = files( 'base/gve_adminq.c', + 'gve_rx.c', + 'gve_tx.c', 'gve_ethdev.c', ) includes += include_directories('base') Checking patch doc/guides/nics/features/gve.ini... error: while searching for: Speed capabilities = Y Link status = Y MTU update = Y RSS hash = Y Linux = Y x86-32 = Y x86-64 = Y error: patch failed: doc/guides/nics/features/gve.ini:7 Checking patch doc/guides/nics/gve.rst... error: while searching for: reception and transmission. Supported features of the GVE PMD are: - Link state information Currently, only GQI_QPL and GQI_RDA queue format are supported in PMD. Jumbo Frame is not supported in PMD for now. It'll be added in the future error: patch failed: doc/guides/nics/gve.rst:62 Checking patch drivers/net/gve/gve_ethdev.c... error: drivers/net/gve/gve_ethdev.c: does not match index Checking patch drivers/net/gve/gve_ethdev.h... error: while searching for: struct gve_tx_seg_desc seg; /* subsequent descs for a packet */ }; struct gve_tx_iovec { uint32_t iov_base; /* offset in fifo */ uint32_t iov_len; error: patch failed: drivers/net/gve/gve_ethdev.h:37 error: while searching for: void gve_stop_rx_queues(struct rte_eth_dev *dev); #endif /* _GVE_ETHDEV_H_ */ error: patch failed: drivers/net/gve/gve_ethdev.h:277 Checking patch drivers/net/gve/gve_rx.c... error: while searching for: #include "gve_ethdev.h" #include "base/gve_adminq.h" static inline void gve_reset_rxq(struct gve_rx_queue *rxq) { error: patch failed: drivers/net/gve/gve_rx.c:5 Checking patch drivers/net/gve/gve_tx.c... error: while searching for: #include "gve_ethdev.h" #include "base/gve_adminq.h" static inline void gve_reset_txq(struct gve_tx_queue *txq) { error: patch failed: drivers/net/gve/gve_tx.c:5 Applying patch doc/guides/nics/features/gve.ini with 1 reject... Rejected hunk #1. Applying patch doc/guides/nics/gve.rst with 1 reject... Rejected hunk #1. Applying patch drivers/net/gve/gve_ethdev.h with 2 rejects... Rejected hunk #1. Rejected hunk #2. Applying patch drivers/net/gve/gve_rx.c with 1 reject... Rejected hunk #1. Applying patch drivers/net/gve/gve_tx.c with 1 reject... Rejected hunk #1. diff a/doc/guides/nics/features/gve.ini b/doc/guides/nics/features/gve.ini (rejected hunks) @@ -7,7 +7,9 @@ Speed capabilities = Y Link status = Y MTU update = Y +TSO = Y RSS hash = Y +L4 checksum offload = Y Linux = Y x86-32 = Y x86-64 = Y diff a/doc/guides/nics/gve.rst b/doc/guides/nics/gve.rst (rejected hunks) @@ -62,7 +62,11 @@ In this release, the GVE PMD provides the basic functionality of packet reception and transmission. Supported features of the GVE PMD are: +- Multiple queues for TX and RX +- TSO offload - Link state information +- TX multi-segments (Scatter TX) +- Tx UDP/TCP/SCTP Checksum Currently, only GQI_QPL and GQI_RDA queue format are supported in PMD. Jumbo Frame is not supported in PMD for now. It'll be added in the future diff a/drivers/net/gve/gve_ethdev.h b/drivers/net/gve/gve_ethdev.h (rejected hunks) @@ -37,6 +37,18 @@ union gve_tx_desc { struct gve_tx_seg_desc seg; /* subsequent descs for a packet */ }; +/* Offload features */ +union gve_tx_offload { + uint64_t data; + struct { + uint64_t l2_len:7; /* L2 (MAC) Header Length. */ + uint64_t l3_len:9; /* L3 (IP) Header Length. */ + uint64_t l4_len:8; /* L4 Header Length. */ + uint64_t tso_segsz:16; /* TCP TSO segment size */ + /* uint64_t unused : 24; */ + }; +}; + struct gve_tx_iovec { uint32_t iov_base; /* offset in fifo */ uint32_t iov_len; @@ -277,4 +289,10 @@ gve_stop_tx_queues(struct rte_eth_dev *dev); void gve_stop_rx_queues(struct rte_eth_dev *dev); +uint16_t +gve_rx_burst(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts); + +uint16_t +gve_tx_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts); + #endif /* _GVE_ETHDEV_H_ */ diff a/drivers/net/gve/gve_rx.c b/drivers/net/gve/gve_rx.c (rejected hunks) @@ -5,6 +5,148 @@ #include "gve_ethdev.h" #include "base/gve_adminq.h" +static inline void +gve_rx_refill(struct gve_rx_queue *rxq) +{ + uint16_t mask = rxq->nb_rx_desc - 1; + uint16_t idx = rxq->next_avail & mask; + uint32_t next_avail = rxq->next_avail; + uint16_t nb_alloc, i; + struct rte_mbuf *nmb; + int diag; + + /* wrap around */ + nb_alloc = rxq->nb_rx_desc - idx; + if (nb_alloc <= rxq->nb_avail) { + diag = rte_pktmbuf_alloc_bulk(rxq->mpool, &rxq->sw_ring[idx], nb_alloc); + if (diag < 0) { + for (i = 0; i < nb_alloc; i++) { + nmb = rte_pktmbuf_alloc(rxq->mpool); + if (!nmb) + break; + rxq->sw_ring[idx + i] = nmb; + } + if (i != nb_alloc) + nb_alloc = i; + } + rxq->nb_avail -= nb_alloc; + next_avail += nb_alloc; + + /* queue page list mode doesn't need real refill. */ + if (rxq->is_gqi_qpl) { + idx += nb_alloc; + } else { + for (i = 0; i < nb_alloc; i++) { + nmb = rxq->sw_ring[idx]; + rxq->rx_data_ring[idx].addr = + rte_cpu_to_be_64(rte_mbuf_data_iova(nmb)); + idx++; + } + } + if (idx == rxq->nb_rx_desc) + idx = 0; + } + + if (rxq->nb_avail > 0) { + nb_alloc = rxq->nb_avail; + if (rxq->nb_rx_desc < idx + rxq->nb_avail) + nb_alloc = rxq->nb_rx_desc - idx; + diag = rte_pktmbuf_alloc_bulk(rxq->mpool, &rxq->sw_ring[idx], nb_alloc); + if (diag < 0) { + for (i = 0; i < nb_alloc; i++) { + nmb = rte_pktmbuf_alloc(rxq->mpool); + if (!nmb) + break; + rxq->sw_ring[idx + i] = nmb; + } + nb_alloc = i; + } + rxq->nb_avail -= nb_alloc; + next_avail += nb_alloc; + + if (!rxq->is_gqi_qpl) { + for (i = 0; i < nb_alloc; i++) { + nmb = rxq->sw_ring[idx]; + rxq->rx_data_ring[idx].addr = + rte_cpu_to_be_64(rte_mbuf_data_iova(nmb)); + idx++; + } + } + } + + if (next_avail != rxq->next_avail) { + rte_write32(rte_cpu_to_be_32(next_avail), rxq->qrx_tail); + rxq->next_avail = next_avail; + } +} + +uint16_t +gve_rx_burst(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) +{ + volatile struct gve_rx_desc *rxr, *rxd; + struct gve_rx_queue *rxq = rx_queue; + uint16_t rx_id = rxq->rx_tail; + struct rte_mbuf *rxe; + uint16_t nb_rx, len; + uint64_t addr; + uint16_t i; + + rxr = rxq->rx_desc_ring; + nb_rx = 0; + + for (i = 0; i < nb_pkts; i++) { + rxd = &rxr[rx_id]; + if (GVE_SEQNO(rxd->flags_seq) != rxq->expected_seqno) + break; + + if (rxd->flags_seq & GVE_RXF_ERR) + continue; + + len = rte_be_to_cpu_16(rxd->len) - GVE_RX_PAD; + rxe = rxq->sw_ring[rx_id]; + if (rxq->is_gqi_qpl) { + addr = (uint64_t)(rxq->qpl->mz->addr) + rx_id * PAGE_SIZE + GVE_RX_PAD; + rte_memcpy((void *)((size_t)rxe->buf_addr + rxe->data_off), + (void *)(size_t)addr, len); + } + rxe->pkt_len = len; + rxe->data_len = len; + rxe->port = rxq->port_id; + rxe->ol_flags = 0; + + if (rxd->flags_seq & GVE_RXF_TCP) + rxe->packet_type |= RTE_PTYPE_L4_TCP; + if (rxd->flags_seq & GVE_RXF_UDP) + rxe->packet_type |= RTE_PTYPE_L4_UDP; + if (rxd->flags_seq & GVE_RXF_IPV4) + rxe->packet_type |= RTE_PTYPE_L3_IPV4; + if (rxd->flags_seq & GVE_RXF_IPV6) + rxe->packet_type |= RTE_PTYPE_L3_IPV6; + + if (gve_needs_rss(rxd->flags_seq)) { + rxe->ol_flags |= RTE_MBUF_F_RX_RSS_HASH; + rxe->hash.rss = rte_be_to_cpu_32(rxd->rss_hash); + } + + rxq->expected_seqno = gve_next_seqno(rxq->expected_seqno); + + rx_id++; + if (rx_id == rxq->nb_rx_desc) + rx_id = 0; + + rx_pkts[nb_rx] = rxe; + nb_rx++; + } + + rxq->nb_avail += nb_rx; + rxq->rx_tail = rx_id; + + if (rxq->nb_avail > rxq->free_thresh) + gve_rx_refill(rxq); + + return nb_rx; +} + static inline void gve_reset_rxq(struct gve_rx_queue *rxq) { diff a/drivers/net/gve/gve_tx.c b/drivers/net/gve/gve_tx.c (rejected hunks) @@ -5,6 +5,461 @@ #include "gve_ethdev.h" #include "base/gve_adminq.h" +static inline void +gve_free_bulk_mbuf(struct rte_mbuf **txep, int num) +{ + struct rte_mbuf *m, *free[GVE_TX_MAX_FREE_SZ]; + int nb_free = 0; + int i, s; + + if (unlikely(num == 0)) + return; + + /* Find the 1st mbuf which needs to be free */ + for (s = 0; s < num; s++) { + if (txep[s] != NULL) { + m = rte_pktmbuf_prefree_seg(txep[s]); + if (m != NULL) + break; + } + } + + if (s == num) + return; + + free[0] = m; + nb_free = 1; + for (i = s + 1; i < num; i++) { + if (likely(txep[i] != NULL)) { + m = rte_pktmbuf_prefree_seg(txep[i]); + if (likely(m != NULL)) { + if (likely(m->pool == free[0]->pool)) { + free[nb_free++] = m; + } else { + rte_mempool_put_bulk(free[0]->pool, (void *)free, nb_free); + free[0] = m; + nb_free = 1; + } + } + txep[i] = NULL; + } + } + rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free); +} + +static inline void +gve_tx_clean(struct gve_tx_queue *txq) +{ + uint16_t mask = txq->nb_tx_desc - 1; + uint32_t start = txq->next_to_clean & mask; + uint32_t ntc, nb_clean, i; + struct gve_tx_iovec *iov; + + ntc = rte_be_to_cpu_32(rte_read32(txq->qtx_head)); + ntc = ntc & mask; + + if (ntc == start) + return; + + /* if wrap around, free twice. */ + if (ntc < start) { + nb_clean = txq->nb_tx_desc - start; + if (nb_clean > GVE_TX_MAX_FREE_SZ) + nb_clean = GVE_TX_MAX_FREE_SZ; + if (txq->is_gqi_qpl) { + for (i = start; i < start + nb_clean; i++) { + iov = &txq->iov_ring[i]; + txq->fifo_avail += iov->iov_len; + iov->iov_base = 0; + iov->iov_len = 0; + } + } else { + gve_free_bulk_mbuf(&txq->sw_ring[start], nb_clean); + } + txq->nb_free += nb_clean; + start += nb_clean; + if (start == txq->nb_tx_desc) + start = 0; + txq->next_to_clean += nb_clean; + } + + if (ntc > start) { + nb_clean = ntc - start; + if (nb_clean > GVE_TX_MAX_FREE_SZ) + nb_clean = GVE_TX_MAX_FREE_SZ; + if (txq->is_gqi_qpl) { + for (i = start; i < start + nb_clean; i++) { + iov = &txq->iov_ring[i]; + txq->fifo_avail += iov->iov_len; + iov->iov_base = 0; + iov->iov_len = 0; + } + } else { + gve_free_bulk_mbuf(&txq->sw_ring[start], nb_clean); + } + txq->nb_free += nb_clean; + txq->next_to_clean += nb_clean; + } +} + +static inline void +gve_tx_clean_swr_qpl(struct gve_tx_queue *txq) +{ + uint32_t start = txq->sw_ntc; + uint32_t ntc, nb_clean; + + ntc = txq->sw_tail; + + if (ntc == start) + return; + + /* if wrap around, free twice. */ + if (ntc < start) { + nb_clean = txq->nb_tx_desc - start; + if (nb_clean > GVE_TX_MAX_FREE_SZ) + nb_clean = GVE_TX_MAX_FREE_SZ; + gve_free_bulk_mbuf(&txq->sw_ring[start], nb_clean); + + txq->sw_nb_free += nb_clean; + start += nb_clean; + if (start == txq->nb_tx_desc) + start = 0; + txq->sw_ntc = start; + } + + if (ntc > start) { + nb_clean = ntc - start; + if (nb_clean > GVE_TX_MAX_FREE_SZ) + nb_clean = GVE_TX_MAX_FREE_SZ; + gve_free_bulk_mbuf(&txq->sw_ring[start], nb_clean); + txq->sw_nb_free += nb_clean; + start += nb_clean; + txq->sw_ntc = start; + } +} + +static inline void +gve_tx_fill_pkt_desc(volatile union gve_tx_desc *desc, struct rte_mbuf *mbuf, + uint8_t desc_cnt, uint16_t len, uint64_t addr) +{ + uint64_t csum_l4 = mbuf->ol_flags & RTE_MBUF_F_TX_L4_MASK; + uint8_t l4_csum_offset = 0; + uint8_t l4_hdr_offset = 0; + + if (mbuf->ol_flags & RTE_MBUF_F_TX_TCP_SEG) + csum_l4 |= RTE_MBUF_F_TX_TCP_CKSUM; + + switch (csum_l4) { + case RTE_MBUF_F_TX_TCP_CKSUM: + l4_csum_offset = offsetof(struct rte_tcp_hdr, cksum); + l4_hdr_offset = mbuf->l2_len + mbuf->l3_len; + break; + case RTE_MBUF_F_TX_UDP_CKSUM: + l4_csum_offset = offsetof(struct rte_udp_hdr, dgram_cksum); + l4_hdr_offset = mbuf->l2_len + mbuf->l3_len; + break; + case RTE_MBUF_F_TX_SCTP_CKSUM: + l4_csum_offset = offsetof(struct rte_sctp_hdr, cksum); + l4_hdr_offset = mbuf->l2_len + mbuf->l3_len; + break; + } + + if (mbuf->ol_flags & RTE_MBUF_F_TX_TCP_SEG) { + desc->pkt.type_flags = GVE_TXD_TSO | GVE_TXF_L4CSUM; + desc->pkt.l4_csum_offset = l4_csum_offset >> 1; + desc->pkt.l4_hdr_offset = l4_hdr_offset >> 1; + } else if (mbuf->ol_flags & RTE_MBUF_F_TX_L4_MASK) { + desc->pkt.type_flags = GVE_TXD_STD | GVE_TXF_L4CSUM; + desc->pkt.l4_csum_offset = l4_csum_offset >> 1; + desc->pkt.l4_hdr_offset = l4_hdr_offset >> 1; + } else { + desc->pkt.type_flags = GVE_TXD_STD; + desc->pkt.l4_csum_offset = 0; + desc->pkt.l4_hdr_offset = 0; + } + desc->pkt.desc_cnt = desc_cnt; + desc->pkt.len = rte_cpu_to_be_16(mbuf->pkt_len); + desc->pkt.seg_len = rte_cpu_to_be_16(len); + desc->pkt.seg_addr = rte_cpu_to_be_64(addr); +} + +static inline void +gve_tx_fill_seg_desc(volatile union gve_tx_desc *desc, uint64_t ol_flags, + union gve_tx_offload tx_offload, + uint16_t len, uint64_t addr) +{ + desc->seg.type_flags = GVE_TXD_SEG; + if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) { + if (ol_flags & RTE_MBUF_F_TX_IPV6) + desc->seg.type_flags |= GVE_TXSF_IPV6; + desc->seg.l3_offset = tx_offload.l2_len >> 1; + desc->seg.mss = rte_cpu_to_be_16(tx_offload.tso_segsz); + } + desc->seg.seg_len = rte_cpu_to_be_16(len); + desc->seg.seg_addr = rte_cpu_to_be_64(addr); +} + +static inline bool +is_fifo_avail(struct gve_tx_queue *txq, uint16_t len) +{ + if (txq->fifo_avail < len) + return false; + /* Don't split segment. */ + if (txq->fifo_head + len > txq->fifo_size && + txq->fifo_size - txq->fifo_head + len > txq->fifo_avail) + return false; + return true; +} +static inline uint64_t +gve_tx_alloc_from_fifo(struct gve_tx_queue *txq, uint16_t tx_id, uint16_t len) +{ + uint32_t head = txq->fifo_head; + uint32_t size = txq->fifo_size; + struct gve_tx_iovec *iov; + uint32_t aligned_head; + uint32_t iov_len = 0; + uint64_t fifo_addr; + + iov = &txq->iov_ring[tx_id]; + + /* Don't split segment */ + if (head + len > size) { + iov_len += (size - head); + head = 0; + } + + fifo_addr = head; + iov_len += len; + iov->iov_base = head; + + /* Re-align to a cacheline for next head */ + head += len; + aligned_head = RTE_ALIGN(head, RTE_CACHE_LINE_SIZE); + iov_len += (aligned_head - head); + iov->iov_len = iov_len; + + if (aligned_head == txq->fifo_size) + aligned_head = 0; + txq->fifo_head = aligned_head; + txq->fifo_avail -= iov_len; + + return fifo_addr; +} + +static inline uint16_t +gve_tx_burst_qpl(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) +{ + union gve_tx_offload tx_offload = {0}; + volatile union gve_tx_desc *txr, *txd; + struct gve_tx_queue *txq = tx_queue; + struct rte_mbuf **sw_ring = txq->sw_ring; + uint16_t mask = txq->nb_tx_desc - 1; + uint16_t tx_id = txq->tx_tail & mask; + uint64_t ol_flags, addr, fifo_addr; + uint32_t tx_tail = txq->tx_tail; + struct rte_mbuf *tx_pkt, *first; + uint16_t sw_id = txq->sw_tail; + uint16_t nb_used, i; + uint16_t nb_tx = 0; + uint32_t hlen; + + txr = txq->tx_desc_ring; + + if (txq->nb_free < txq->free_thresh || txq->fifo_avail == 0) + gve_tx_clean(txq); + + if (txq->sw_nb_free < txq->free_thresh) + gve_tx_clean_swr_qpl(txq); + + for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { + tx_pkt = *tx_pkts++; + ol_flags = tx_pkt->ol_flags; + + if (txq->sw_nb_free < tx_pkt->nb_segs) { + gve_tx_clean_swr_qpl(txq); + if (txq->sw_nb_free < tx_pkt->nb_segs) + goto end_of_tx; + } + + /* Even for multi-segs, use 1 qpl buf for data */ + nb_used = 1; + if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) + nb_used++; + + if (txq->nb_free < nb_used) + goto end_of_tx; + + tx_offload.l2_len = tx_pkt->l2_len; + tx_offload.l3_len = tx_pkt->l3_len; + tx_offload.l4_len = tx_pkt->l4_len; + tx_offload.tso_segsz = tx_pkt->tso_segsz; + + first = tx_pkt; + txd = &txr[tx_id]; + hlen = ol_flags & RTE_MBUF_F_TX_TCP_SEG ? + (uint32_t)(tx_offload.l2_len + tx_offload.l3_len + tx_offload.l4_len) : + tx_pkt->pkt_len; + + sw_ring[sw_id] = tx_pkt; + if (!is_fifo_avail(txq, hlen)) { + gve_tx_clean(txq); + if (!is_fifo_avail(txq, hlen)) + goto end_of_tx; + } + addr = (uint64_t)(tx_pkt->buf_addr) + tx_pkt->data_off; + fifo_addr = gve_tx_alloc_from_fifo(txq, tx_id, hlen); + + /* For TSO, check if there's enough fifo space for data first */ + if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) { + if (!is_fifo_avail(txq, tx_pkt->pkt_len - hlen)) { + gve_tx_clean(txq); + if (!is_fifo_avail(txq, tx_pkt->pkt_len - hlen)) + goto end_of_tx; + } + } + if (tx_pkt->nb_segs == 1 || ol_flags & RTE_MBUF_F_TX_TCP_SEG) + rte_memcpy((void *)(size_t)(fifo_addr + txq->fifo_base), + (void *)(size_t)addr, hlen); + else + rte_pktmbuf_read(tx_pkt, 0, hlen, + (void *)(size_t)(fifo_addr + txq->fifo_base)); + gve_tx_fill_pkt_desc(txd, tx_pkt, nb_used, hlen, fifo_addr); + + if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) { + tx_id = (tx_id + 1) & mask; + txd = &txr[tx_id]; + addr = (uint64_t)(tx_pkt->buf_addr) + tx_pkt->data_off + hlen; + fifo_addr = gve_tx_alloc_from_fifo(txq, tx_id, tx_pkt->pkt_len - hlen); + if (tx_pkt->nb_segs == 1) + rte_memcpy((void *)(size_t)(fifo_addr + txq->fifo_base), + (void *)(size_t)addr, + tx_pkt->pkt_len - hlen); + else + rte_pktmbuf_read(tx_pkt, hlen, tx_pkt->pkt_len - hlen, + (void *)(size_t)(fifo_addr + txq->fifo_base)); + + gve_tx_fill_seg_desc(txd, ol_flags, tx_offload, + tx_pkt->pkt_len - hlen, fifo_addr); + } + + /* record mbuf in sw_ring for free */ + for (i = 1; i < first->nb_segs; i++) { + sw_id = (sw_id + 1) & mask; + tx_pkt = tx_pkt->next; + sw_ring[sw_id] = tx_pkt; + } + + sw_id = (sw_id + 1) & mask; + tx_id = (tx_id + 1) & mask; + + txq->nb_free -= nb_used; + txq->sw_nb_free -= first->nb_segs; + tx_tail += nb_used; + } + +end_of_tx: + if (nb_tx) { + rte_write32(rte_cpu_to_be_32(tx_tail), txq->qtx_tail); + txq->tx_tail = tx_tail; + txq->sw_tail = sw_id; + } + + return nb_tx; +} + +static inline uint16_t +gve_tx_burst_ra(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) +{ + union gve_tx_offload tx_offload = {0}; + volatile union gve_tx_desc *txr, *txd; + struct gve_tx_queue *txq = tx_queue; + struct rte_mbuf **sw_ring = txq->sw_ring; + uint16_t mask = txq->nb_tx_desc - 1; + uint16_t tx_id = txq->tx_tail & mask; + uint32_t tx_tail = txq->tx_tail; + struct rte_mbuf *tx_pkt, *first; + uint16_t nb_used, hlen, i; + uint64_t ol_flags, addr; + uint16_t nb_tx = 0; + + txr = txq->tx_desc_ring; + + if (txq->nb_free < txq->free_thresh) + gve_tx_clean(txq); + + for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) { + tx_pkt = *tx_pkts++; + ol_flags = tx_pkt->ol_flags; + + nb_used = tx_pkt->nb_segs; + if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) + nb_used++; + + if (txq->nb_free < nb_used) + goto end_of_tx; + + tx_offload.l2_len = tx_pkt->l2_len; + tx_offload.l3_len = tx_pkt->l3_len; + tx_offload.l4_len = tx_pkt->l4_len; + tx_offload.tso_segsz = tx_pkt->tso_segsz; + + first = tx_pkt; + txd = &txr[tx_id]; + + hlen = ol_flags & RTE_MBUF_F_TX_TCP_SEG ? + (uint32_t)(tx_offload.l2_len + tx_offload.l3_len + tx_offload.l4_len) : + tx_pkt->pkt_len; + /* + * if tso, the driver needs to fill 2 descs for 1 mbuf + * so only put this mbuf into the 1st tx entry in sw ring + */ + sw_ring[tx_id] = tx_pkt; + addr = rte_mbuf_data_iova(tx_pkt); + gve_tx_fill_pkt_desc(txd, tx_pkt, nb_used, hlen, addr); + + if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) { + tx_id = (tx_id + 1) & mask; + txd = &txr[tx_id]; + addr = rte_mbuf_data_iova(tx_pkt) + hlen; + gve_tx_fill_seg_desc(txd, ol_flags, tx_offload, + tx_pkt->data_len - hlen, addr); + } + + for (i = 1; i < first->nb_segs; i++) { + tx_id = (tx_id + 1) & mask; + txd = &txr[tx_id]; + tx_pkt = tx_pkt->next; + sw_ring[tx_id] = tx_pkt; + addr = rte_mbuf_data_iova(tx_pkt); + gve_tx_fill_seg_desc(txd, ol_flags, tx_offload, + tx_pkt->data_len, addr); + } + tx_id = (tx_id + 1) & mask; + + txq->nb_free -= nb_used; + tx_tail += nb_used; + } + +end_of_tx: + if (nb_tx) { + rte_write32(rte_cpu_to_be_32(tx_tail), txq->qtx_tail); + txq->tx_tail = tx_tail; + } + + return nb_tx; +} + +uint16_t +gve_tx_burst(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) +{ + struct gve_tx_queue *txq = tx_queue; + + if (txq->is_gqi_qpl) + return gve_tx_burst_qpl(tx_queue, tx_pkts, nb_pkts); + + return gve_tx_burst_ra(tx_queue, tx_pkts, nb_pkts); +} + static inline void gve_reset_txq(struct gve_tx_queue *txq) { https://lab.dpdk.org/results/dashboard/patchsets/24180/ UNH-IOL DPDK Community Lab