[PATCH v3 16/22] net/ice: negotiate large VF and request more queues

[Kevin Liu <kevinx.liu@intel.com>]

From: Steve Yang <stevex.yang@intel.com>

Negotiate large VF capability with PF during VF initialization. If large
VF is supported and the number of queues larger than 16 is required, VF
requests additional queues from PF. Mark the state that large VF is
supported.

If the allocated queues number is larger than 16, the max RSS queue
region cannot be 16 anymore. Add the function to query max RSS queue
region from PF, use it in the RSS initialization and future filters
configuration.

Signed-off-by: Steve Yang <stevex.yang@intel.com>
Signed-off-by: Kevin Liu <kevinx.liu@intel.com>
---
 drivers/net/ice/ice_dcf.c        | 34 +++++++++++++++-
 drivers/net/ice/ice_dcf.h        |  4 ++
 drivers/net/ice/ice_dcf_ethdev.c | 69 +++++++++++++++++++++++++++++++-
 drivers/net/ice/ice_dcf_ethdev.h |  2 +
 4 files changed, 106 insertions(+), 3 deletions(-)

diff --git a/drivers/net/ice/ice_dcf.c b/drivers/net/ice/ice_dcf.c
index 6aeafa6681..7091658841 100644
--- a/drivers/net/ice/ice_dcf.c
+++ b/drivers/net/ice/ice_dcf.c
@@ -257,7 +257,8 @@ ice_dcf_get_vf_resource(struct ice_dcf_hw *hw)
 	       VIRTCHNL_VF_CAP_ADV_LINK_SPEED | VIRTCHNL_VF_CAP_DCF |
 	       VIRTCHNL_VF_OFFLOAD_VLAN_V2 |
 	       VF_BASE_MODE_OFFLOADS | VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC |
-	       VIRTCHNL_VF_OFFLOAD_QOS | VIRTCHNL_VF_OFFLOAD_REQ_QUEUES;
+	       VIRTCHNL_VF_OFFLOAD_QOS | VIRTCHNL_VF_OFFLOAD_REQ_QUEUES |
+		   VIRTCHNL_VF_LARGE_NUM_QPAIRS;
 
 	err = ice_dcf_send_cmd_req_no_irq(hw, VIRTCHNL_OP_GET_VF_RESOURCES,
 					  (uint8_t *)&caps, sizeof(caps));
@@ -1083,6 +1084,37 @@ ice_dcf_request_queues(struct ice_dcf_hw *hw, uint16_t num)
 	return -1;
 }
 
+int
+ice_dcf_get_max_rss_queue_region(struct ice_dcf_hw *hw)
+{
+	struct dcf_virtchnl_cmd args;
+	uint16_t qregion_width;
+	int err;
+
+	memset(&args, 0, sizeof(args));
+	args.v_op = VIRTCHNL_OP_GET_MAX_RSS_QREGION;
+	args.req_msg = NULL;
+	args.req_msglen = 0;
+	args.rsp_msgbuf = hw->arq_buf;
+	args.rsp_msglen = ICE_DCF_AQ_BUF_SZ;
+	args.rsp_buflen = ICE_DCF_AQ_BUF_SZ;
+
+	err = ice_dcf_execute_virtchnl_cmd(hw, &args);
+	if (err) {
+		PMD_DRV_LOG(ERR,
+			    "Failed to execute command of "
+			    "VIRTCHNL_OP_GET_MAX_RSS_QREGION");
+		return err;
+	}
+
+	qregion_width =	((struct virtchnl_max_rss_qregion *)
+				args.rsp_msgbuf)->qregion_width;
+	hw->max_rss_qregion = (uint16_t)(1 << qregion_width);
+
+	return 0;
+}
+
+
 int
 ice_dcf_config_irq_map(struct ice_dcf_hw *hw)
 {
diff --git a/drivers/net/ice/ice_dcf.h b/drivers/net/ice/ice_dcf.h
index 99498e2184..05ea91d2a5 100644
--- a/drivers/net/ice/ice_dcf.h
+++ b/drivers/net/ice/ice_dcf.h
@@ -105,6 +105,7 @@ struct ice_dcf_hw {
 
 	uint16_t msix_base;
 	uint16_t nb_msix;
+	uint16_t max_rss_qregion; /* max RSS queue region supported by PF */
 	uint16_t rxq_map[16];
 	struct virtchnl_eth_stats eth_stats_offset;
 	struct virtchnl_vlan_caps vlan_v2_caps;
@@ -114,6 +115,8 @@ struct ice_dcf_hw {
 	uint32_t link_speed;
 
 	bool resetting;
+	/* Indicate large VF support enabled or not */
+	bool lv_enabled;
 };
 
 int ice_dcf_execute_virtchnl_cmd(struct ice_dcf_hw *hw,
@@ -128,6 +131,7 @@ int ice_dcf_configure_rss_lut(struct ice_dcf_hw *hw);
 int ice_dcf_init_rss(struct ice_dcf_hw *hw);
 int ice_dcf_configure_queues(struct ice_dcf_hw *hw);
 int ice_dcf_request_queues(struct ice_dcf_hw *hw, uint16_t num);
+int ice_dcf_get_max_rss_queue_region(struct ice_dcf_hw *hw);
 int ice_dcf_config_irq_map(struct ice_dcf_hw *hw);
 int ice_dcf_switch_queue(struct ice_dcf_hw *hw, uint16_t qid, bool rx, bool on);
 int ice_dcf_disable_queues(struct ice_dcf_hw *hw);
diff --git a/drivers/net/ice/ice_dcf_ethdev.c b/drivers/net/ice/ice_dcf_ethdev.c
index d4bfa182a4..a43c5a320d 100644
--- a/drivers/net/ice/ice_dcf_ethdev.c
+++ b/drivers/net/ice/ice_dcf_ethdev.c
@@ -39,6 +39,8 @@ static int
 ice_dcf_dev_udp_tunnel_port_del(struct rte_eth_dev *dev,
 				struct rte_eth_udp_tunnel *udp_tunnel);
 
+static int ice_dcf_queues_req_reset(struct rte_eth_dev *dev, uint16_t num);
+
 static int
 ice_dcf_dev_init(struct rte_eth_dev *eth_dev);
 
@@ -663,6 +665,11 @@ ice_dcf_dev_configure(struct rte_eth_dev *dev)
 {
 	struct ice_dcf_adapter *dcf_ad = dev->data->dev_private;
 	struct ice_adapter *ad = &dcf_ad->parent;
+	struct ice_dcf_hw *hw = &dcf_ad->real_hw;
+	int ret;
+
+	uint16_t num_queue_pairs =
+		RTE_MAX(dev->data->nb_rx_queues, dev->data->nb_tx_queues);
 
 	ad->rx_bulk_alloc_allowed = true;
 	ad->tx_simple_allowed = true;
@@ -670,6 +677,47 @@ ice_dcf_dev_configure(struct rte_eth_dev *dev)
 	if (dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)
 		dev->data->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_RSS_HASH;
 
+		/* Large VF setting */
+	if (num_queue_pairs > ICE_DCF_MAX_NUM_QUEUES_DFLT) {
+		if (!(hw->vf_res->vf_cap_flags &
+				VIRTCHNL_VF_LARGE_NUM_QPAIRS)) {
+			PMD_DRV_LOG(ERR, "large VF is not supported");
+			return -1;
+		}
+
+		if (num_queue_pairs > ICE_DCF_MAX_NUM_QUEUES_LV) {
+			PMD_DRV_LOG(ERR,
+				"queue pairs number cannot be larger than %u",
+				ICE_DCF_MAX_NUM_QUEUES_LV);
+			return -1;
+		}
+
+		ret = ice_dcf_queues_req_reset(dev, num_queue_pairs);
+		if (ret)
+			return ret;
+
+		ret = ice_dcf_get_max_rss_queue_region(hw);
+		if (ret) {
+			PMD_INIT_LOG(ERR, "get max rss queue region failed");
+			return ret;
+		}
+
+		hw->lv_enabled = true;
+	} else {
+		/* Check if large VF is already enabled. If so, disable and
+		 * release redundant queue resource.
+		 */
+		if (hw->lv_enabled) {
+			ret = ice_dcf_queues_req_reset(dev, num_queue_pairs);
+			if (ret)
+				return ret;
+
+			hw->lv_enabled = false;
+		}
+		/* if large VF is not required, use default rss queue region */
+		hw->max_rss_qregion = ICE_DCF_MAX_NUM_QUEUES_DFLT;
+	}
+
 	return 0;
 }
 
@@ -681,8 +729,8 @@ ice_dcf_dev_info_get(struct rte_eth_dev *dev,
 	struct ice_dcf_hw *hw = &adapter->real_hw;
 
 	dev_info->max_mac_addrs = DCF_NUM_MACADDR_MAX;
-	dev_info->max_rx_queues = hw->vsi_res->num_queue_pairs;
-	dev_info->max_tx_queues = hw->vsi_res->num_queue_pairs;
+	dev_info->max_rx_queues = ICE_DCF_MAX_NUM_QUEUES_LV;
+	dev_info->max_tx_queues = ICE_DCF_MAX_NUM_QUEUES_LV;
 	dev_info->min_rx_bufsize = ICE_BUF_SIZE_MIN;
 	dev_info->max_rx_pktlen = ICE_FRAME_SIZE_MAX;
 	dev_info->hash_key_size = hw->vf_res->rss_key_size;
@@ -1829,6 +1877,23 @@ ice_dcf_dev_uninit(struct rte_eth_dev *eth_dev)
 	return 0;
 }
 
+static int ice_dcf_queues_req_reset(struct rte_eth_dev *dev, uint16_t num)
+{
+	struct ice_dcf_adapter *adapter = dev->data->dev_private;
+	struct ice_dcf_hw *hw = &adapter->real_hw;
+	int ret;
+
+	ret = ice_dcf_request_queues(hw, num);
+	if (ret) {
+		PMD_DRV_LOG(ERR, "request queues from PF failed");
+		return ret;
+	}
+	PMD_DRV_LOG(INFO, "change queue pairs from %u to %u",
+			hw->vsi_res->num_queue_pairs, num);
+
+	return ice_dcf_dev_reset(dev);
+}
+
 static int
 ice_dcf_cap_check_handler(__rte_unused const char *key,
 			  const char *value, __rte_unused void *opaque)
diff --git a/drivers/net/ice/ice_dcf_ethdev.h b/drivers/net/ice/ice_dcf_ethdev.h
index 27f6402786..4a08d32e0c 100644
--- a/drivers/net/ice/ice_dcf_ethdev.h
+++ b/drivers/net/ice/ice_dcf_ethdev.h
@@ -20,6 +20,8 @@
 #define ICE_DCF_ETH_OVERHEAD \
 	(RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN + ICE_DCF_VLAN_TAG_SIZE * 2)
 #define ICE_DCF_ETH_MAX_LEN (RTE_ETHER_MTU + ICE_DCF_ETH_OVERHEAD)
+#define ICE_DCF_MAX_NUM_QUEUES_DFLT 16
+#define ICE_DCF_MAX_NUM_QUEUES_LV   256
 
 struct ice_dcf_queue {
 	uint64_t dummy;
-- 
2.33.1