RE: [PATCH v2 03/10] net/gve: support device initialization

["Guo, Junfeng" <junfeng.guo@intel.com>]

> -----Original Message-----
> From: Ferruh Yigit <ferruh.yigit@xilinx.com>
> Sent: Friday, September 2, 2022 01:22
> To: Guo, Junfeng <junfeng.guo@intel.com>; Zhang, Qi Z
> <qi.z.zhang@intel.com>; Wu, Jingjing <jingjing.wu@intel.com>
> Cc: dev@dpdk.org; Li, Xiaoyun <xiaoyun.li@intel.com>;
> awogbemila@google.com; Richardson, Bruce
> <bruce.richardson@intel.com>; Wang, Haiyue <haiyue.wang@intel.com>
> Subject: Re: [PATCH v2 03/10] net/gve: support device initialization
> 
> On 8/29/2022 9:41 AM, Junfeng Guo wrote:
> 
> >
> > Support device init and the fowllowing devops:
> >    - dev_configure
> >    - dev_start
> >    - dev_stop
> >    - dev_close
> >
> > Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
> > Signed-off-by: Xiaoyun Li <xiaoyun.li@intel.com>
> > Signed-off-by: Junfeng Guo <junfeng.guo@intel.com>
> > ---
> >   drivers/net/gve/gve.h        | 249 +++++++++++++++++++++++
> >   drivers/net/gve/gve_adminq.c |   1 +
> >   drivers/net/gve/gve_ethdev.c | 375
> +++++++++++++++++++++++++++++++++++
> >   drivers/net/gve/meson.build  |  13 ++
> >   drivers/net/gve/version.map  |   3 +
> >   drivers/net/meson.build      |   1 +
> >   6 files changed, 642 insertions(+)
> >   create mode 100644 drivers/net/gve/gve.h
> >   create mode 100644 drivers/net/gve/gve_ethdev.c
> >   create mode 100644 drivers/net/gve/meson.build
> >   create mode 100644 drivers/net/gve/version.map
> >
> > diff --git a/drivers/net/gve/gve.h b/drivers/net/gve/gve.h
> > new file mode 100644
> > index 0000000000..704c88983c
> > --- /dev/null
> > +++ b/drivers/net/gve/gve.h
> > @@ -0,0 +1,249 @@
> > +/* SPDX-License-Identifier: BSD-3-Clause
> > + * Copyright(C) 2022 Intel Corporation
> > + */
> > +
> > +#ifndef _GVE_H_
> > +#define _GVE_H_
> > +
> > +#include <ethdev_driver.h>
> > +#include <ethdev_pci.h>
> > +#include <rte_ether.h>
> > +
> > +#include "gve_desc.h"
> > +
> > +#ifndef GOOGLE_VENDOR_ID
> > +#define GOOGLE_VENDOR_ID       0x1ae0
> > +#endif
> > +
> > +#define GVE_DEV_ID             0x0042
> > +
> > +#define GVE_REG_BAR    0
> > +#define GVE_DB_BAR     2
> > +
> > +/* 1 for management, 1 for rx, 1 for tx */
> > +#define GVE_MIN_MSIX           3
> > +
> > +/* PTYPEs are always 10 bits. */
> > +#define GVE_NUM_PTYPES 1024
> > +
> > +/* A list of pages registered with the device during setup and used by a
> queue
> > + * as buffers
> > + */
> > +struct gve_queue_page_list {
> > +       uint32_t id; /* unique id */
> > +       uint32_t num_entries;
> > +       dma_addr_t *page_buses; /* the dma addrs of the pages */
> > +       const struct rte_memzone *mz;
> > +};
> > +
> > +/* A TX desc ring entry */
> > +union gve_tx_desc {
> > +       struct gve_tx_pkt_desc pkt; /* first desc for a packet */
> > +       struct gve_tx_seg_desc seg; /* subsequent descs for a packet */
> > +};
> > +
> > +struct gve_tx_queue {
> > +       volatile union gve_tx_desc *tx_desc_ring;
> > +       const struct rte_memzone *mz;
> > +       uint64_t tx_ring_phys_addr;
> > +
> > +       uint16_t nb_tx_desc;
> > +
> > +       /* Only valid for DQO_QPL queue format */
> > +       struct gve_queue_page_list *qpl;
> > +
> > +       uint16_t port_id;
> > +       uint16_t queue_id;
> > +
> > +       uint16_t ntfy_id;
> > +       volatile rte_be32_t *ntfy_addr;
> > +
> > +       struct gve_priv *hw;
> > +       const struct rte_memzone *qres_mz;
> > +       struct gve_queue_resources *qres;
> > +
> > +       /* Only valid for DQO_RDA queue format */
> > +       struct gve_tx_queue *complq;
> > +};
> > +
> > +struct gve_rx_queue {
> > +       volatile struct gve_rx_desc *rx_desc_ring;
> > +       volatile union gve_rx_data_slot *rx_data_ring;
> > +       const struct rte_memzone *mz;
> > +       const struct rte_memzone *data_mz;
> > +       uint64_t rx_ring_phys_addr;
> > +
> > +       uint16_t nb_rx_desc;
> > +
> > +       volatile rte_be32_t *ntfy_addr;
> > +
> > +       /* only valid for GQI_QPL queue format */
> > +       struct gve_queue_page_list *qpl;
> > +
> > +       struct gve_priv *hw;
> > +       const struct rte_memzone *qres_mz;
> > +       struct gve_queue_resources *qres;
> > +
> > +       uint16_t port_id;
> > +       uint16_t queue_id;
> > +       uint16_t ntfy_id;
> > +       uint16_t rx_buf_len;
> > +
> > +       /* Only valid for DQO_RDA queue format */
> > +       struct gve_rx_queue *bufq;
> > +};
> > +
> > +struct gve_irq_db {
> > +       rte_be32_t id;
> > +} ____cacheline_aligned;
> > +
> > +struct gve_ptype {
> > +       uint8_t l3_type;  /* `gve_l3_type` in gve_adminq.h */
> > +       uint8_t l4_type;  /* `gve_l4_type` in gve_adminq.h */
> > +};
> > +
> > +struct gve_ptype_lut {
> > +       struct gve_ptype ptypes[GVE_NUM_PTYPES];
> > +};
> > +
> > +enum gve_queue_format {
> > +       GVE_QUEUE_FORMAT_UNSPECIFIED = 0x0, /* default unspecified
> */
> > +       GVE_GQI_RDA_FORMAT           = 0x1, /* GQI Raw Addressing */
> > +       GVE_GQI_QPL_FORMAT           = 0x2, /* GQI Queue Page List */
> > +       GVE_DQO_RDA_FORMAT           = 0x3, /* DQO Raw Addressing */
> > +};
> > +
> 
> Shouldn't these queue format information be part of 'base' file? Both
> for licensing issues also to cover the case it is updated in the google
> repo.
> But if some dpdk related information is required, what do you think to
> split this file as one for base folder, other for dpdk?

Yes, current solution is to move these base-related code into another
gve.h file and rename this file as gve_ethdev.h to match the dpdk style. 
Will update the solution in the coming version.
Thanks!

> 
> > +struct gve_priv {
> > +       struct gve_irq_db *irq_dbs; /* array of num_ntfy_blks */
> > +       const struct rte_memzone *irq_dbs_mz;
> > +       uint32_t mgmt_msix_idx;
> > +       rte_be32_t *cnt_array; /* array of num_event_counters */
> > +       const struct rte_memzone *cnt_array_mz;
> > +
> > +       uint16_t num_event_counters;
> > +       uint16_t tx_desc_cnt; /* txq size */
> > +       uint16_t rx_desc_cnt; /* rxq size */
> > +       uint16_t tx_pages_per_qpl; /* tx buffer length */
> > +       uint16_t rx_data_slot_cnt; /* rx buffer length */
> > +
> > +       /* Only valid for DQO_RDA queue format */
> > +       uint16_t tx_compq_size; /* tx completion queue size */
> > +       uint16_t rx_bufq_size; /* rx buff queue size */
> > +
> > +       uint64_t max_registered_pages;
> > +       uint64_t num_registered_pages; /* num pages registered with NIC
> */
> > +       uint16_t default_num_queues; /* default num queues to set up */
> > +       enum gve_queue_format queue_format; /* see enum
> gve_queue_format */
> > +       uint8_t enable_lsc;
> > +
> > +       uint16_t max_nb_txq;
> > +       uint16_t max_nb_rxq;
> > +       uint32_t num_ntfy_blks; /* spilt between TX and RX so must be
> even */
> > +
> > +       struct gve_registers __iomem *reg_bar0; /* see gve_register.h */
> > +       rte_be32_t __iomem *db_bar2; /* "array" of doorbells */
> > +       struct rte_pci_device *pci_dev;
> > +
> > +       /* Admin queue - see gve_adminq.h*/
> > +       union gve_adminq_command *adminq;
> > +       struct gve_dma_mem adminq_dma_mem;
> > +       uint32_t adminq_mask; /* masks prod_cnt to adminq size */
> > +       uint32_t adminq_prod_cnt; /* free-running count of AQ cmds
> executed */
> > +       uint32_t adminq_cmd_fail; /* free-running count of AQ cmds failed
> */
> > +       uint32_t adminq_timeouts; /* free-running count of AQ cmds
> timeouts */
> > +       /* free-running count of per AQ cmd executed */
> > +       uint32_t adminq_describe_device_cnt;
> > +       uint32_t adminq_cfg_device_resources_cnt;
> > +       uint32_t adminq_register_page_list_cnt;
> > +       uint32_t adminq_unregister_page_list_cnt;
> > +       uint32_t adminq_create_tx_queue_cnt;
> > +       uint32_t adminq_create_rx_queue_cnt;
> > +       uint32_t adminq_destroy_tx_queue_cnt;
> > +       uint32_t adminq_destroy_rx_queue_cnt;
> > +       uint32_t adminq_dcfg_device_resources_cnt;
> > +       uint32_t adminq_set_driver_parameter_cnt;
> > +       uint32_t adminq_report_stats_cnt;
> > +       uint32_t adminq_report_link_speed_cnt;
> > +       uint32_t adminq_get_ptype_map_cnt;
> > +
> > +       volatile uint32_t state_flags;
> > +
> > +       /* Gvnic device link speed from hypervisor. */
> > +       uint64_t link_speed;
> > +
> > +       uint16_t max_mtu;
> > +       struct rte_ether_addr dev_addr; /* mac address */
> > +
> > +       struct gve_queue_page_list *qpl;
> > +
> > +       struct gve_tx_queue **txqs;
> > +       struct gve_rx_queue **rxqs;
> > +};
> > +
> 
> The device private data is provided fully here, as well as
> 'gve_rx_queue' and 'gve_tx_queue' structs etc, although most of the
> files not used at this stage and not clear why needed.
> 
> Instead of adding full structs, can you please add here bare minimum
> structs, whatever used at this point, and as they are used in .c file,
> keep adding them in structs too?
> This clarifies what is used/added for specific features, also lets us
> easily figure out unused fields / clutter in the header files.

Sure, will update in the coming version. Thanks!

> 
> > +enum gve_state_flags_bit {
> > +       GVE_PRIV_FLAGS_ADMIN_QUEUE_OK           = 1,
> > +       GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK      = 2,
> > +       GVE_PRIV_FLAGS_DEVICE_RINGS_OK          = 3,
> > +       GVE_PRIV_FLAGS_NAPI_ENABLED             = 4,
> > +};
> > +
> > +static inline bool gve_is_gqi(struct gve_priv *priv)
> > +{
> > +       return priv->queue_format == GVE_GQI_RDA_FORMAT ||
> > +               priv->queue_format == GVE_GQI_QPL_FORMAT;
> > +}
> > +
> > +static inline bool gve_get_admin_queue_ok(struct gve_priv *priv)
> > +{
> > +
> return !!rte_bit_relaxed_get32(GVE_PRIV_FLAGS_ADMIN_QUEUE_OK,
> > +                                      &priv->state_flags);
> > +}
> > +
> > +static inline void gve_set_admin_queue_ok(struct gve_priv *priv)
> > +{
> > +       rte_bit_relaxed_set32(GVE_PRIV_FLAGS_ADMIN_QUEUE_OK,
> > +                             &priv->state_flags);
> > +}
> > +
> > +static inline void gve_clear_admin_queue_ok(struct gve_priv *priv)
> > +{
> > +       rte_bit_relaxed_clear32(GVE_PRIV_FLAGS_ADMIN_QUEUE_OK,
> > +                               &priv->state_flags);
> > +}
> > +
> > +static inline bool gve_get_device_resources_ok(struct gve_priv *priv)
> > +{
> > +
> return !!rte_bit_relaxed_get32(GVE_PRIV_FLAGS_DEVICE_RESOURCES_O
> K,
> > +                                      &priv->state_flags);
> > +}
> > +
> > +static inline void gve_set_device_resources_ok(struct gve_priv *priv)
> > +{
> > +       rte_bit_relaxed_set32(GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK,
> > +                             &priv->state_flags);
> > +}
> > +
> > +static inline void gve_clear_device_resources_ok(struct gve_priv *priv)
> > +{
> > +
> rte_bit_relaxed_clear32(GVE_PRIV_FLAGS_DEVICE_RESOURCES_OK,
> > +                               &priv->state_flags);
> > +}
> > +
> > +static inline bool gve_get_device_rings_ok(struct gve_priv *priv)
> > +{
> > +
> return !!rte_bit_relaxed_get32(GVE_PRIV_FLAGS_DEVICE_RINGS_OK,
> > +                                      &priv->state_flags);
> > +}
> > +
> > +static inline void gve_set_device_rings_ok(struct gve_priv *priv)
> > +{
> > +       rte_bit_relaxed_set32(GVE_PRIV_FLAGS_DEVICE_RINGS_OK,
> > +                             &priv->state_flags);
> > +}
> > +
> > +static inline void gve_clear_device_rings_ok(struct gve_priv *priv)
> > +{
> > +       rte_bit_relaxed_clear32(GVE_PRIV_FLAGS_DEVICE_RINGS_OK,
> > +                               &priv->state_flags);
> > +}
> 
> If this is dpdk file, not base file, please follow dpdk coding
> convention, like return type should be in separate line.

Sure, will update in the coming version patchset. Thanks!

> 
> > +#endif /* _GVE_H_ */
> > diff --git a/drivers/net/gve/gve_adminq.c
> b/drivers/net/gve/gve_adminq.c
> > index 8a724f12c6..438ca2070e 100644
> > --- a/drivers/net/gve/gve_adminq.c
> > +++ b/drivers/net/gve/gve_adminq.c
> > @@ -5,6 +5,7 @@
> >    * Copyright(C) 2022 Intel Corporation
> >    */
> >
> > +#include "gve.h"
> >   #include "gve_adminq.h"
> >   #include "gve_register.h"
> >
> > diff --git a/drivers/net/gve/gve_ethdev.c
> b/drivers/net/gve/gve_ethdev.c
> > new file mode 100644
> > index 0000000000..f10f273f7d
> > --- /dev/null
> > +++ b/drivers/net/gve/gve_ethdev.c
> > @@ -0,0 +1,375 @@
> > +/* SPDX-License-Identifier: BSD-3-Clause
> > + * Copyright(C) 2022 Intel Corporation
> > + */
> > +#include <linux/pci_regs.h>
> > +
> > +#include "gve.h"
> > +#include "gve_adminq.h"
> > +#include "gve_register.h"
> > +
> > +#define GVE_VERSION            "1.3.0"
> > +#define GVE_VERSION_PREFIX     "GVE-"
> > +
> 
> Again, shouldn't these come from base file (google repo)?
> 
> Perhaps can be good to discuss what is the base file update
> strategy/plan for future?
> It can be easier if you can drop some external files with minimum change
> in dpdk files, that can be done by grouping external content in spefic
> files.
> Qi has lots of experince on this and I believe he can privide useful
> insight.

Yes, need to keep the base file at the base folder in the future.
Thanks!

> 
> > +const char gve_version_str[] = GVE_VERSION;
> > +static const char gve_version_prefix[] = GVE_VERSION_PREFIX;
> > +
> > +static void
> > +gve_write_version(uint8_t *driver_version_register)
> > +{
> > +       const char *c = gve_version_prefix;
> > +
> > +       while (*c) {
> > +               writeb(*c, driver_version_register);
> > +               c++;
> > +       }
> > +
> > +       c = gve_version_str;
> > +       while (*c) {
> > +               writeb(*c, driver_version_register);
> > +               c++;
> > +       }
> > +       writeb('\n', driver_version_register);
> > +}
> > +
> > +static int
> > +gve_dev_configure(__rte_unused struct rte_eth_dev *dev)
> > +{
> > +       return 0;
> > +}
> > +
> > +static int
> > +gve_dev_start(struct rte_eth_dev *dev)
> > +{
> > +       dev->data->dev_started = 1;
> > +
> > +       return 0;
> > +}
> > +
> > +static int
> > +gve_dev_stop(struct rte_eth_dev *dev)
> > +{
> > +       dev->data->dev_link.link_status = RTE_ETH_LINK_DOWN;
> > +       dev->data->dev_started = 0;
> > +
> > +       return 0;
> > +}
> > +
> > +static int
> > +gve_dev_close(struct rte_eth_dev *dev)
> > +{
> > +       int err = 0;
> > +
> > +       if (dev->data->dev_started) {
> > +               err = gve_dev_stop(dev);
> > +               if (err != 0)
> > +                       PMD_DRV_LOG(ERR, "Failed to stop dev.");
> > +       }
> > +
> 
> Just a reminder that in 'close' driver should free all the resources, if
> there is previously allocated memomory, it should be freed now.

Thanks! Will update in the coming version.