From: Jin Liu <jin.liu@corigine.com>
To: dev@dpdk.org
Cc: niklas.soderlund@corigine.com, Jin Liu <jin.liu@corigine.com>,
Diana Wang <na.wang@corigine.com>,
Peng Zhang <peng.zhang@corigine.com>,
Chaoyong He <chaoyong.he@corigine.com>
Subject: [PATCH v4 06/13] net/nfp: support NFP3800 card
Date: Thu, 23 Jun 2022 04:26:08 +0200 [thread overview]
Message-ID: <20220623022615.3628093-7-jin.liu@corigine.com> (raw)
In-Reply-To: <20220623022615.3628093-1-jin.liu@corigine.com>
Add support for a new type of NIC NFP3800 card, and update some
network card data acquisition interface functions.
Signed-off-by: Jin Liu <jin.liu@corigine.com>
Signed-off-by: Diana Wang <na.wang@corigine.com>
Signed-off-by: Peng Zhang <peng.zhang@corigine.com>
Signed-off-by: Chaoyong He <chaoyong.he@corigine.com>
Signed-off-by: Niklas Söderlund <niklas.soderlund@corigine.com>
---
doc/guides/nics/nfp.rst | 3 ++-
doc/guides/rel_notes/release_22_07.rst | 4 +++
drivers/net/nfp/nfp_common.h | 34 ++++++++++++++++++++++++--
drivers/net/nfp/nfp_ethdev.c | 28 ++++++++++++++++++---
drivers/net/nfp/nfp_ethdev_vf.c | 9 +++++--
drivers/net/nfp/nfpcore/nfp_cpp.h | 2 +-
drivers/net/nfp/nfpcore/nfp_nsp_eth.c | 20 +++++++++++----
7 files changed, 86 insertions(+), 14 deletions(-)
diff --git a/doc/guides/nics/nfp.rst b/doc/guides/nics/nfp.rst
index 30cdc69202..dcefac3ef6 100644
--- a/doc/guides/nics/nfp.rst
+++ b/doc/guides/nics/nfp.rst
@@ -12,7 +12,8 @@ up to 400-Gb/s.
This document explains how to use DPDK with the Netronome Poll Mode
Driver (PMD) supporting Netronome's Network Flow Processor 6xxx
-(NFP-6xxx) and Netronome's Flow Processor 4xxx (NFP-4xxx).
+(NFP-6xxx), Netronome's Network Flow Processor 4xxx (NFP-4xxx) and
+Netronome's Network Flow Processor 38xx (NFP-38xx).
NFP is a SRIOV capable device and the PMD supports the physical
function (PF) and the virtual functions (VFs).
diff --git a/doc/guides/rel_notes/release_22_07.rst b/doc/guides/rel_notes/release_22_07.rst
index 7d5e3fac77..d5d8c735b1 100644
--- a/doc/guides/rel_notes/release_22_07.rst
+++ b/doc/guides/rel_notes/release_22_07.rst
@@ -108,6 +108,10 @@ New Features
* Added support for MTU on Windows.
* Added matching and RSS on IPsec ESP.
+* **Updated Netronome nfp driver.**
+
+ * Added support for NFP3800 NIC.
+
* **Updated VMware vmxnet3 networking driver.**
* Added version 5 support.
diff --git a/drivers/net/nfp/nfp_common.h b/drivers/net/nfp/nfp_common.h
index 8e1b4fb6a4..19e96414e3 100644
--- a/drivers/net/nfp/nfp_common.h
+++ b/drivers/net/nfp/nfp_common.h
@@ -16,9 +16,11 @@
#define NFP_NET_PMD_VERSION "0.1"
#define PCI_VENDOR_ID_NETRONOME 0x19ee
+#define PCI_DEVICE_ID_NFP3800_PF_NIC 0x3800
+#define PCI_DEVICE_ID_NFP3800_VF_NIC 0x3803
#define PCI_DEVICE_ID_NFP4000_PF_NIC 0x4000
#define PCI_DEVICE_ID_NFP6000_PF_NIC 0x6000
-#define PCI_DEVICE_ID_NFP6000_VF_NIC 0x6003
+#define PCI_DEVICE_ID_NFP6000_VF_NIC 0x6003 /* Include NFP4000VF */
/* Forward declaration */
struct nfp_net_adapter;
@@ -41,8 +43,16 @@ struct nfp_net_adapter;
#define NFP_QCP_QUEUE_STS_HI 0x000c
#define NFP_QCP_QUEUE_STS_HI_WRITEPTR_mask (0x3ffff)
+#define NFP_PCIE_QCP_NFP3800_OFFSET 0x400000
+#define NFP_PCIE_QCP_NFP6000_OFFSET 0x80000
+#define NFP_PCIE_QUEUE_NFP3800_MASK 0x1ff
+#define NFP_PCIE_QUEUE_NFP6000_MASK 0xff
+#define NFP_PCIE_QCP_PF_OFFSET 0x0
+#define NFP_PCIE_QCP_VF_OFFSET 0x0
+
/* The offset of the queue controller queues in the PCIe Target */
-#define NFP_PCIE_QUEUE(_q) (0x80000 + (NFP_QCP_QUEUE_ADDR_SZ * ((_q) & 0xff)))
+#define NFP_PCIE_QUEUE(_offset, _q, _mask) \
+ ((_offset) + (NFP_QCP_QUEUE_ADDR_SZ * ((_q) & (_mask))))
/* Interrupt definitions */
#define NFP_NET_IRQ_LSC_IDX 0
@@ -342,6 +352,26 @@ nfp_qcp_read(uint8_t *q, enum nfp_qcp_ptr ptr)
return val & NFP_QCP_QUEUE_STS_HI_WRITEPTR_mask;
}
+static inline uint32_t
+nfp_pci_queue(struct rte_pci_device *pdev, uint16_t queue)
+{
+ switch (pdev->id.device_id) {
+ case PCI_DEVICE_ID_NFP4000_PF_NIC:
+ case PCI_DEVICE_ID_NFP6000_PF_NIC:
+ return NFP_PCIE_QUEUE(NFP_PCIE_QCP_PF_OFFSET, queue,
+ NFP_PCIE_QUEUE_NFP6000_MASK);
+ case PCI_DEVICE_ID_NFP3800_VF_NIC:
+ return NFP_PCIE_QUEUE(NFP_PCIE_QCP_VF_OFFSET, queue,
+ NFP_PCIE_QUEUE_NFP3800_MASK);
+ case PCI_DEVICE_ID_NFP6000_VF_NIC:
+ return NFP_PCIE_QUEUE(NFP_PCIE_QCP_VF_OFFSET, queue,
+ NFP_PCIE_QUEUE_NFP6000_MASK);
+ default:
+ return NFP_PCIE_QUEUE(NFP_PCIE_QCP_PF_OFFSET, queue,
+ NFP_PCIE_QUEUE_NFP3800_MASK);
+ }
+}
+
/* Prototypes for common NFP functions */
int nfp_net_reconfig(struct nfp_net_hw *hw, uint32_t ctrl, uint32_t update);
int nfp_net_configure(struct rte_eth_dev *dev);
diff --git a/drivers/net/nfp/nfp_ethdev.c b/drivers/net/nfp/nfp_ethdev.c
index ae6cb5943f..cb84dc3188 100644
--- a/drivers/net/nfp/nfp_ethdev.c
+++ b/drivers/net/nfp/nfp_ethdev.c
@@ -446,12 +446,13 @@ nfp_net_init(struct rte_eth_dev *eth_dev)
/* Work out where in the BAR the queues start. */
switch (pci_dev->id.device_id) {
+ case PCI_DEVICE_ID_NFP3800_PF_NIC:
case PCI_DEVICE_ID_NFP4000_PF_NIC:
case PCI_DEVICE_ID_NFP6000_PF_NIC:
start_q = nn_cfg_readl(hw, NFP_NET_CFG_START_TXQ);
- tx_bar_off = (uint64_t)start_q * NFP_QCP_QUEUE_ADDR_SZ;
+ tx_bar_off = nfp_pci_queue(pci_dev, start_q);
start_q = nn_cfg_readl(hw, NFP_NET_CFG_START_RXQ);
- rx_bar_off = (uint64_t)start_q * NFP_QCP_QUEUE_ADDR_SZ;
+ rx_bar_off = nfp_pci_queue(pci_dev, start_q);
break;
default:
PMD_DRV_LOG(ERR, "nfp_net: no device ID matching");
@@ -764,6 +765,7 @@ nfp_pf_init(struct rte_pci_device *pci_dev)
{
int err;
int ret = 0;
+ uint64_t addr;
int total_ports;
struct nfp_cpp *cpp;
struct nfp_pf_dev *pf_dev;
@@ -867,8 +869,24 @@ nfp_pf_init(struct rte_pci_device *pci_dev)
PMD_INIT_LOG(DEBUG, "ctrl bar: %p", pf_dev->ctrl_bar);
/* configure access to tx/rx vNIC BARs */
+ switch (pci_dev->id.device_id) {
+ case PCI_DEVICE_ID_NFP3800_PF_NIC:
+ addr = NFP_PCIE_QUEUE(NFP_PCIE_QCP_NFP3800_OFFSET,
+ 0, NFP_PCIE_QUEUE_NFP3800_MASK);
+ break;
+ case PCI_DEVICE_ID_NFP4000_PF_NIC:
+ case PCI_DEVICE_ID_NFP6000_PF_NIC:
+ addr = NFP_PCIE_QUEUE(NFP_PCIE_QCP_NFP6000_OFFSET,
+ 0, NFP_PCIE_QUEUE_NFP6000_MASK);
+ break;
+ default:
+ PMD_INIT_LOG(ERR, "nfp_net: no device ID matching");
+ err = -ENODEV;
+ goto ctrl_area_cleanup;
+ }
+
pf_dev->hw_queues = nfp_cpp_map_area(pf_dev->cpp, 0, 0,
- NFP_PCIE_QUEUE(0), NFP_QCP_QUEUE_AREA_SZ,
+ addr, NFP_QCP_QUEUE_AREA_SZ,
&pf_dev->hwqueues_area);
if (pf_dev->hw_queues == NULL) {
PMD_INIT_LOG(ERR, "nfp_rtsym_map fails for net.qc");
@@ -995,6 +1013,10 @@ nfp_pf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
}
static const struct rte_pci_id pci_id_nfp_pf_net_map[] = {
+ {
+ RTE_PCI_DEVICE(PCI_VENDOR_ID_NETRONOME,
+ PCI_DEVICE_ID_NFP3800_PF_NIC)
+ },
{
RTE_PCI_DEVICE(PCI_VENDOR_ID_NETRONOME,
PCI_DEVICE_ID_NFP4000_PF_NIC)
diff --git a/drivers/net/nfp/nfp_ethdev_vf.c b/drivers/net/nfp/nfp_ethdev_vf.c
index d0fa1df24d..c46ee0f913 100644
--- a/drivers/net/nfp/nfp_ethdev_vf.c
+++ b/drivers/net/nfp/nfp_ethdev_vf.c
@@ -327,11 +327,12 @@ nfp_netvf_init(struct rte_eth_dev *eth_dev)
/* Work out where in the BAR the queues start. */
switch (pci_dev->id.device_id) {
+ case PCI_DEVICE_ID_NFP3800_VF_NIC:
case PCI_DEVICE_ID_NFP6000_VF_NIC:
start_q = nn_cfg_readl(hw, NFP_NET_CFG_START_TXQ);
- tx_bar_off = (uint64_t)start_q * NFP_QCP_QUEUE_ADDR_SZ;
+ tx_bar_off = nfp_pci_queue(pci_dev, start_q);
start_q = nn_cfg_readl(hw, NFP_NET_CFG_START_RXQ);
- rx_bar_off = (uint64_t)start_q * NFP_QCP_QUEUE_ADDR_SZ;
+ rx_bar_off = nfp_pci_queue(pci_dev, start_q);
break;
default:
PMD_DRV_LOG(ERR, "nfp_net: no device ID matching");
@@ -456,6 +457,10 @@ nfp_netvf_init(struct rte_eth_dev *eth_dev)
}
static const struct rte_pci_id pci_id_nfp_vf_net_map[] = {
+ {
+ RTE_PCI_DEVICE(PCI_VENDOR_ID_NETRONOME,
+ PCI_DEVICE_ID_NFP3800_VF_NIC)
+ },
{
RTE_PCI_DEVICE(PCI_VENDOR_ID_NETRONOME,
PCI_DEVICE_ID_NFP6000_VF_NIC)
diff --git a/drivers/net/nfp/nfpcore/nfp_cpp.h b/drivers/net/nfp/nfpcore/nfp_cpp.h
index 720d3989e6..a04a68f546 100644
--- a/drivers/net/nfp/nfpcore/nfp_cpp.h
+++ b/drivers/net/nfp/nfpcore/nfp_cpp.h
@@ -214,7 +214,7 @@ void nfp_cpp_free(struct nfp_cpp *cpp);
* @return true if model is in the NFP6000 family, false otherwise.
*/
#define NFP_CPP_MODEL_IS_6000(model) \
- ((NFP_CPP_MODEL_CHIP_of(model) >= 0x4000) && \
+ ((NFP_CPP_MODEL_CHIP_of(model) >= 0x3800) && \
(NFP_CPP_MODEL_CHIP_of(model) < 0x7000))
/*
diff --git a/drivers/net/nfp/nfpcore/nfp_nsp_eth.c b/drivers/net/nfp/nfpcore/nfp_nsp_eth.c
index 67946891ab..f8f3c372ac 100644
--- a/drivers/net/nfp/nfpcore/nfp_nsp_eth.c
+++ b/drivers/net/nfp/nfpcore/nfp_nsp_eth.c
@@ -266,6 +266,7 @@ __nfp_eth_read_ports(struct nfp_nsp *nsp)
struct nfp_eth_table *table;
uint32_t table_sz;
int i, j, ret, cnt = 0;
+ const struct rte_ether_addr *mac;
entries = malloc(NSP_ETH_TABLE_SIZE);
if (!entries)
@@ -278,9 +279,15 @@ __nfp_eth_read_ports(struct nfp_nsp *nsp)
goto err;
}
- for (i = 0; i < NSP_ETH_MAX_COUNT; i++)
- if (entries[i].port & NSP_ETH_PORT_LANES_MASK)
+ /* The NFP3800 NIC support 8 ports, but only 2 ports are valid,
+ * the rest 6 ports mac are all 0, ensure we don't use these port
+ */
+ for (i = 0; i < NSP_ETH_MAX_COUNT; i++) {
+ mac = (const struct rte_ether_addr *)entries[i].mac_addr;
+ if ((entries[i].port & NSP_ETH_PORT_LANES_MASK) &&
+ (!rte_is_zero_ether_addr(mac)))
cnt++;
+ }
/* Some versions of flash will give us 0 instead of port count. For
* those that give a port count, verify it against the value calculated
@@ -299,10 +306,13 @@ __nfp_eth_read_ports(struct nfp_nsp *nsp)
memset(table, 0, table_sz);
table->count = cnt;
- for (i = 0, j = 0; i < NSP_ETH_MAX_COUNT; i++)
- if (entries[i].port & NSP_ETH_PORT_LANES_MASK)
+ for (i = 0, j = 0; i < NSP_ETH_MAX_COUNT; i++) {
+ mac = (const struct rte_ether_addr *)entries[i].mac_addr;
+ if ((entries[i].port & NSP_ETH_PORT_LANES_MASK) &&
+ (!rte_is_zero_ether_addr(mac)))
nfp_eth_port_translate(nsp, &entries[i], i,
- &table->ports[j++]);
+ &table->ports[j++]);
+ }
nfp_eth_calc_port_geometry(table);
for (i = 0; i < (int)table->count; i++)
--
2.27.0
next prev parent reply other threads:[~2022-06-23 2:27 UTC|newest]
Thread overview: 81+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-06-02 1:52 [PATCH 00/14] Add support of NFP3800 chip and firmware with NFDk Jin Liu
2022-06-02 1:52 ` [PATCH 01/14] net/nfp: change the coding style Jin Liu
2022-06-02 22:52 ` Ferruh Yigit
2022-06-14 8:50 ` Kevin Liu
2022-06-02 1:52 ` [PATCH 02/14] net/nfp: remove unnecessary forward function declaration Jin Liu
2022-06-02 1:52 ` [PATCH 03/14] net/nfp: remove pessimistic limit Jin Liu
2022-06-02 1:52 ` [PATCH 04/14] net/nfp: rename set MAC function Jin Liu
2022-06-02 1:52 ` [PATCH 05/14] net/nfp: rename function and struct Jin Liu
2022-06-02 1:52 ` [PATCH 06/14] net/nfp: support NFP3800 card Jin Liu
2022-06-02 22:52 ` Ferruh Yigit
2022-06-14 8:50 ` Kevin Liu
2022-06-02 1:52 ` [PATCH 07/14] net/nfp: support NFDK firmware Jin Liu
2022-06-02 22:53 ` Ferruh Yigit
2022-06-14 8:49 ` Kevin Liu
2022-06-14 9:21 ` Ferruh Yigit
2022-06-14 9:30 ` Kevin Liu
2022-06-02 1:52 ` [PATCH 08/14] net/nfp: structure adjustment Jin Liu
2022-06-02 22:54 ` Ferruh Yigit
2022-06-14 8:49 ` Kevin Liu
2022-06-02 1:52 ` [PATCH 09/14] net/nfp: nfdk netdev option and queue function Jin Liu
2022-06-02 1:53 ` [PATCH 10/14] net/nfp: add queue stop and close helper function Jin Liu
2022-06-02 1:53 ` [PATCH 11/14] net/nfp: nfdk stop and close function Jin Liu
2022-06-02 1:53 ` [PATCH 12/14] net/nfp: move macro from C file to head file Jin Liu
2022-06-02 1:53 ` [PATCH 13/14] net/nfp: nfdk packet xmit function Jin Liu
2022-06-02 1:53 ` [PATCH 14/14] net/nfp: modify RSS logic Jin Liu
2022-06-02 22:56 ` Ferruh Yigit
2022-06-14 8:50 ` Kevin Liu
2022-06-02 22:51 ` [PATCH 00/14] Add support of NFP3800 chip and firmware with NFDk Ferruh Yigit
2022-06-14 8:48 ` Kevin Liu
2022-06-16 2:39 ` [PATCH v2 00/15] " Jin Liu
2022-06-16 2:39 ` [PATCH v2 01/15] doc: update release note Jin Liu
2022-06-16 15:04 ` Ferruh Yigit
2022-06-16 2:39 ` [PATCH v2 02/15] doc: update nfp documentation Jin Liu
2022-06-16 15:04 ` Ferruh Yigit
2022-06-16 2:39 ` [PATCH v2 03/15] net/nfp: change the coding style Jin Liu
2022-06-16 2:39 ` [PATCH v2 04/15] net/nfp: remove unnecessary forward function declaration Jin Liu
2022-06-16 2:39 ` [PATCH v2 05/15] net/nfp: remove pessimistic limit Jin Liu
2022-06-16 2:39 ` [PATCH v2 06/15] net/nfp: rename set MAC function Jin Liu
2022-06-16 2:39 ` [PATCH v2 07/15] net/nfp: rename function and struct Jin Liu
2022-06-16 2:39 ` [PATCH v2 08/15] net/nfp: support NFP3800 card Jin Liu
2022-06-16 15:04 ` Ferruh Yigit
2022-06-16 2:39 ` [PATCH v2 09/15] net/nfp: support firmware with NFDk Jin Liu
2022-06-16 2:39 ` [PATCH v2 10/15] net/nfp: structure adjustment Jin Liu
2022-06-16 2:39 ` [PATCH v2 11/15] net/nfp: nfdk netdev option and queue function Jin Liu
2022-06-16 2:39 ` [PATCH v2 12/15] net/nfp: add queue stop and close helper function Jin Liu
2022-06-16 2:39 ` [PATCH v2 13/15] net/nfp: move macro from C file to head file Jin Liu
2022-06-16 2:39 ` [PATCH v2 14/15] net/nfp: nfdk packet xmit function Jin Liu
2022-06-16 2:39 ` [PATCH v2 15/15] net/nfp: modify RSS logic Jin Liu
2022-06-16 15:06 ` [PATCH v2 00/15] Add support of NFP3800 chip and firmware with NFDk Ferruh Yigit
2022-06-17 9:34 ` [PATCH v3 00/13] " Jin Liu
2022-06-17 9:34 ` [PATCH v3 01/13] net/nfp: change the coding style Jin Liu
2022-06-17 9:34 ` [PATCH v3 02/13] net/nfp: remove unnecessary forward function declaration Jin Liu
2022-06-17 9:34 ` [PATCH v3 03/13] net/nfp: remove pessimistic limit Jin Liu
2022-06-17 9:34 ` [PATCH v3 04/13] net/nfp: rename set MAC function Jin Liu
2022-06-17 9:34 ` [PATCH v3 05/13] net/nfp: rename function and struct Jin Liu
2022-06-17 9:34 ` [PATCH v3 06/13] net/nfp: support NFP3800 card Jin Liu
2022-06-17 9:34 ` [PATCH v3 07/13] net/nfp: support firmware with NFDk Jin Liu
2022-06-17 9:34 ` [PATCH v3 08/13] net/nfp: structure adjustment Jin Liu
2022-06-17 9:34 ` [PATCH v3 09/13] net/nfp: nfdk netdev option and queue function Jin Liu
2022-06-17 9:34 ` [PATCH v3 10/13] net/nfp: add queue stop and close helper function Jin Liu
2022-06-17 9:34 ` [PATCH v3 11/13] net/nfp: move macro from C file to head file Jin Liu
2022-06-17 9:34 ` [PATCH v3 12/13] net/nfp: nfdk packet xmit function Jin Liu
2022-06-22 16:03 ` Thomas Monjalon
2022-06-22 16:29 ` Ferruh Yigit
2022-06-17 9:34 ` [PATCH v3 13/13] net/nfp: modify RSS logic Jin Liu
2022-06-17 13:33 ` [PATCH v3 00/13] Add support of NFP3800 chip and firmware with NFDk Ferruh Yigit
2022-06-23 2:26 ` [PATCH v4 " Jin Liu
2022-06-23 2:26 ` [PATCH v4 01/13] net/nfp: change the coding style Jin Liu
2022-06-23 2:26 ` [PATCH v4 02/13] net/nfp: remove unnecessary forward function declaration Jin Liu
2022-06-23 2:26 ` [PATCH v4 03/13] net/nfp: remove pessimistic limit Jin Liu
2022-06-23 2:26 ` [PATCH v4 04/13] net/nfp: rename set MAC function Jin Liu
2022-06-23 2:26 ` [PATCH v4 05/13] net/nfp: rename function and struct Jin Liu
2022-06-23 2:26 ` Jin Liu [this message]
2022-06-23 2:26 ` [PATCH v4 07/13] net/nfp: support firmware with NFDk Jin Liu
2022-06-23 2:26 ` [PATCH v4 08/13] net/nfp: structure adjustment Jin Liu
2022-06-23 2:26 ` [PATCH v4 09/13] net/nfp: nfdk netdev option and queue function Jin Liu
2022-06-23 2:26 ` [PATCH v4 10/13] net/nfp: add queue stop and close helper function Jin Liu
2022-06-23 2:26 ` [PATCH v4 11/13] net/nfp: move macro from C file to head file Jin Liu
2022-06-23 2:26 ` [PATCH v4 12/13] net/nfp: nfdk packet xmit function Jin Liu
2022-06-23 2:26 ` [PATCH v4 13/13] net/nfp: modify RSS logic Jin Liu
2022-06-23 9:11 ` [PATCH v4 00/13] Add support of NFP3800 chip and firmware with NFDk Ferruh Yigit
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20220623022615.3628093-7-jin.liu@corigine.com \
--to=jin.liu@corigine.com \
--cc=chaoyong.he@corigine.com \
--cc=dev@dpdk.org \
--cc=na.wang@corigine.com \
--cc=niklas.soderlund@corigine.com \
--cc=peng.zhang@corigine.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).