DPDK patches and discussions
 help / color / mirror / Atom feed
* [dpdk-dev] [PATCH v1 0/2] enable large VF configuration
@ 2020-09-09  7:20 Ting Xu
  2020-09-09  7:20 ` [dpdk-dev] [PATCH v1 1/2] net/iavf: add IAVF request queues Ting Xu
                   ` (9 more replies)
  0 siblings, 10 replies; 45+ messages in thread
From: Ting Xu @ 2020-09-09  7:20 UTC (permalink / raw)
  To: dev; +Cc: qi.z.zhang, jingjing.wu, Ting Xu

This patchset supports to configure up to 256 queue pairs per VF. If
large VF is supported, VF will request queues from PF as needed. New
virtual channel opcodes and structures are used to indicate 256 queues,
so VF is designed to handle the new function of configure VSI queues, IRQ
mapping and enable/disable queues. Enable VF to query the max RSS queue
regions and report to users.

Ting Xu (2):
  net/iavf: add IAVF request queues
  net/iavf: enable large VF configuration

 drivers/net/iavf/iavf.h        |  41 +++-
 drivers/net/iavf/iavf_ethdev.c |  93 +++++--
 drivers/net/iavf/iavf_rxtx.c   |  27 ++-
 drivers/net/iavf/iavf_vchnl.c  | 426 ++++++++++++++++++++++++++++++---
 4 files changed, 528 insertions(+), 59 deletions(-)

-- 
2.17.1


^ permalink raw reply	[flat|nested] 45+ messages in thread

* [dpdk-dev] [PATCH v1 1/2] net/iavf: add IAVF request queues
  2020-09-09  7:20 [dpdk-dev] [PATCH v1 0/2] enable large VF configuration Ting Xu
@ 2020-09-09  7:20 ` Ting Xu
  2020-09-09  7:20 ` [dpdk-dev] [PATCH v1 2/2] net/iavf: enable large VF configuration Ting Xu
                   ` (8 subsequent siblings)
  9 siblings, 0 replies; 45+ messages in thread
From: Ting Xu @ 2020-09-09  7:20 UTC (permalink / raw)
  To: dev; +Cc: qi.z.zhang, jingjing.wu, Ting Xu

Add support for IAVF request queues from PF. VF sends desired queue
pairs number to PF to ask for queue allocation. If the request succeeds,
VF should reset. If fails, PF will return available queue pairs number.

Signed-off-by: Ting Xu <ting.xu@intel.com>
---
 drivers/net/iavf/iavf.h        |  17 +++++
 drivers/net/iavf/iavf_ethdev.c |  12 +++-
 drivers/net/iavf/iavf_vchnl.c  | 117 +++++++++++++++++++++++++++++----
 3 files changed, 131 insertions(+), 15 deletions(-)

diff --git a/drivers/net/iavf/iavf.h b/drivers/net/iavf/iavf.h
index 9ad331ee9..1a42936a0 100644
--- a/drivers/net/iavf/iavf.h
+++ b/drivers/net/iavf/iavf.h
@@ -107,6 +107,21 @@ struct iavf_fdir_info {
 /* TODO: is that correct to assume the max number to be 16 ?*/
 #define IAVF_MAX_MSIX_VECTORS   16
 
+/* Event status from PF */
+enum pending_msg {
+	PFMSG_LINK_CHANGE = 0x1,
+	PFMSG_RESET_IMPENDING = 0x2,
+	PFMSG_DRIVER_CLOSE = 0x4,
+};
+
+/* Message type read in admin queue from PF */
+enum iavf_aq_result {
+	IAVF_MSG_ERR = -1, /* Meet error when accessing admin queue */
+	IAVF_MSG_NON,      /* Read nothing from admin queue */
+	IAVF_MSG_SYS,      /* Read system msg from admin queue */
+	IAVF_MSG_CMD,      /* Read async command result */
+};
+
 /* Structure to store private data specific for VF instance. */
 struct iavf_info {
 	uint16_t num_queue_pairs;
@@ -123,6 +138,7 @@ struct iavf_info {
 	volatile enum virtchnl_ops pend_cmd; /* pending command not finished */
 	uint32_t cmd_retval; /* return value of the cmd response from PF */
 	uint8_t *aq_resp; /* buffer to store the adminq response from PF */
+	uint16_t pend_msg; /* flags indicates events from pf not handled yet */
 
 	/* Event from pf */
 	bool dev_closed;
@@ -279,4 +295,5 @@ int iavf_add_del_rss_cfg(struct iavf_adapter *adapter,
 int iavf_add_del_mc_addr_list(struct iavf_adapter *adapter,
 			struct rte_ether_addr *mc_addrs,
 			uint32_t mc_addrs_num, bool add);
+int iavf_request_queues(struct rte_eth_dev *dev, uint16_t num);
 #endif /* _IAVF_ETHDEV_H_ */
diff --git a/drivers/net/iavf/iavf_ethdev.c b/drivers/net/iavf/iavf_ethdev.c
index 8e1d8a8d3..48e474f53 100644
--- a/drivers/net/iavf/iavf_ethdev.c
+++ b/drivers/net/iavf/iavf_ethdev.c
@@ -1196,7 +1196,7 @@ iavf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
 }
 
 static int
-iavf_check_vf_reset_done(struct iavf_hw *hw)
+iavf_check_vf_reset_done(struct iavf_hw *hw, struct iavf_info *vf)
 {
 	int i, reset;
 
@@ -1213,6 +1213,10 @@ iavf_check_vf_reset_done(struct iavf_hw *hw)
 	if (i >= IAVF_RESET_WAIT_CNT)
 		return -1;
 
+	/* VF is not in reset or reset is completed */
+	vf->vf_reset = false;
+	vf->pend_msg &= ~PFMSG_RESET_IMPENDING;
+
 	return 0;
 }
 
@@ -1231,7 +1235,7 @@ iavf_init_vf(struct rte_eth_dev *dev)
 		goto err;
 	}
 
-	err = iavf_check_vf_reset_done(hw);
+	err = iavf_check_vf_reset_done(hw, vf);
 	if (err) {
 		PMD_INIT_LOG(ERR, "VF is still resetting");
 		goto err;
@@ -1473,7 +1477,9 @@ iavf_dev_close(struct rte_eth_dev *dev)
 
 	iavf_dev_stop(dev);
 	iavf_flow_flush(dev, NULL);
-	iavf_flow_uninit(adapter);
+	/* if VF is in reset, adminq is disabled, skip the process via adminq */
+	if (!vf->vf_reset)
+		iavf_flow_uninit(adapter);
 	iavf_shutdown_adminq(hw);
 	/* disable uio intr before callback unregister */
 	rte_intr_disable(intr_handle);
diff --git a/drivers/net/iavf/iavf_vchnl.c b/drivers/net/iavf/iavf_vchnl.c
index 34c31a153..eab2f2a22 100644
--- a/drivers/net/iavf/iavf_vchnl.c
+++ b/drivers/net/iavf/iavf_vchnl.c
@@ -17,6 +17,7 @@
 #include <rte_eal.h>
 #include <rte_ether.h>
 #include <rte_ethdev_driver.h>
+#include <rte_ethdev_pci.h>
 #include <rte_dev.h>
 #include <rte_bus_pci.h>
 
@@ -27,13 +28,14 @@
 #define ASQ_DELAY_MS  10
 
 /* Read data in admin queue to get msg from pf driver */
-static enum iavf_status
+static enum iavf_aq_result
 iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,
 		     uint8_t *buf)
 {
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter);
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
 	struct iavf_arq_event_info event;
+	enum iavf_aq_result result = IAVF_MSG_NON;
 	enum virtchnl_ops opcode;
 	int ret;
 
@@ -43,7 +45,9 @@ iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,
 	/* Can't read any msg from adminQ */
 	if (ret) {
 		PMD_DRV_LOG(DEBUG, "Can't read msg from AQ");
-		return ret;
+		if (ret != IAVF_ERR_ADMIN_QUEUE_NO_WORK)
+			result = IAVF_MSG_ERR;
+		return result;
 	}
 
 	opcode = (enum virtchnl_ops)rte_le_to_cpu_32(event.desc.cookie_high);
@@ -53,16 +57,46 @@ iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,
 	PMD_DRV_LOG(DEBUG, "AQ from pf carries opcode %u, retval %d",
 		    opcode, vf->cmd_retval);
 
-	if (opcode != vf->pend_cmd) {
-		if (opcode != VIRTCHNL_OP_EVENT) {
-			PMD_DRV_LOG(WARNING,
-				    "command mismatch, expect %u, get %u",
-				    vf->pend_cmd, opcode);
+	if (opcode == VIRTCHNL_OP_EVENT) {
+		struct virtchnl_pf_event *vpe =
+			(struct virtchnl_pf_event *)event.msg_buf;
+
+		result = IAVF_MSG_SYS;
+		switch (vpe->event) {
+		case VIRTCHNL_EVENT_LINK_CHANGE:
+			vf->link_up =
+				vpe->event_data.link_event.link_status;
+			vf->link_speed =
+				vpe->event_data.link_event.link_speed;
+			vf->pend_msg |= PFMSG_LINK_CHANGE;
+			PMD_DRV_LOG(INFO, "Link status update:%s",
+					vf->link_up ? "up" : "down");
+			break;
+		case VIRTCHNL_EVENT_RESET_IMPENDING:
+			vf->vf_reset = true;
+			vf->pend_msg |= PFMSG_RESET_IMPENDING;
+			PMD_DRV_LOG(INFO, "vf is reseting");
+			break;
+		case VIRTCHNL_EVENT_PF_DRIVER_CLOSE:
+			vf->dev_closed = true;
+			vf->pend_msg |= PFMSG_DRIVER_CLOSE;
+			PMD_DRV_LOG(INFO, "PF driver closed");
+			break;
+		default:
+			PMD_DRV_LOG(ERR, "%s: Unknown event %d from pf",
+					__func__, vpe->event);
+		}
+	}  else {
+		/* async reply msg on command issued by vf previously */
+		result = IAVF_MSG_CMD;
+		if (opcode != vf->pend_cmd) {
+			PMD_DRV_LOG(WARNING, "command mismatch, expect %u, get %u",
+					vf->pend_cmd, opcode);
+			result = IAVF_MSG_ERR;
 		}
-		return IAVF_ERR_OPCODE_MISMATCH;
 	}
 
-	return IAVF_SUCCESS;
+	return result;
 }
 
 static int
@@ -70,6 +104,7 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter, struct iavf_cmd_info *args)
 {
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter);
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	enum iavf_aq_result result;
 	enum iavf_status ret;
 	int err = 0;
 	int i = 0;
@@ -95,9 +130,9 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter, struct iavf_cmd_info *args)
 	case VIRTCHNL_OP_GET_SUPPORTED_RXDIDS:
 		/* for init virtchnl ops, need to poll the response */
 		do {
-			ret = iavf_read_msg_from_pf(adapter, args->out_size,
+			result = iavf_read_msg_from_pf(adapter, args->out_size,
 						   args->out_buffer);
-			if (ret == IAVF_SUCCESS)
+			if (result == IAVF_MSG_CMD)
 				break;
 			rte_delay_ms(ASQ_DELAY_MS);
 		} while (i++ < MAX_TRY_TIMES);
@@ -109,7 +144,33 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter, struct iavf_cmd_info *args)
 		}
 		_clear_cmd(vf);
 		break;
-
+	case VIRTCHNL_OP_REQUEST_QUEUES:
+		/*
+		 * ignore async reply, only wait for system message,
+		 * vf_reset = true if get VIRTCHNL_EVENT_RESET_IMPENDING,
+		 * if not, means request queues failed.
+		 */
+		do {
+			result = iavf_read_msg_from_pf(adapter, args->out_size,
+						   args->out_buffer);
+			if (result == IAVF_MSG_SYS && vf->vf_reset) {
+				break;
+			} else if (result == IAVF_MSG_CMD ||
+				result == IAVF_MSG_ERR) {
+				err = -1;
+				break;
+			}
+			rte_delay_ms(ASQ_DELAY_MS);
+			/* If don't read msg or read sys event, continue */
+		} while (i++ < MAX_TRY_TIMES);
+		if (i >= MAX_TRY_TIMES ||
+		    vf->cmd_retval != VIRTCHNL_STATUS_SUCCESS) {
+			err = -1;
+			PMD_DRV_LOG(ERR, "No response or return failure (%d)"
+				    " for cmd %d", vf->cmd_retval, args->ops);
+		}
+		_clear_cmd(vf);
+		break;
 	default:
 		/* For other virtchnl ops in running time,
 		 * wait for the cmd done flag.
@@ -1168,3 +1229,35 @@ iavf_add_del_mc_addr_list(struct iavf_adapter *adapter,
 
 	return 0;
 }
+
+int
+iavf_request_queues(struct rte_eth_dev *dev, uint16_t num)
+{
+	struct iavf_adapter *ad =
+		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
+	struct iavf_info *vf =  IAVF_DEV_PRIVATE_TO_VF(ad);
+	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+	struct virtchnl_vf_res_request vfres;
+	struct iavf_cmd_info args;
+	int err;
+
+	vfres.num_queue_pairs = num;
+
+	args.ops = VIRTCHNL_OP_REQUEST_QUEUES;
+	args.in_args = (u8 *)&vfres;
+	args.in_args_size = sizeof(vfres);
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+
+	/*
+	 * disable interrupt to avoid the admin queue message to be read
+	 * before iavf_read_msg_from_pf.
+	 */
+	rte_intr_disable(&pci_dev->intr_handle);
+	err = iavf_execute_vf_cmd(ad, &args);
+	if (err)
+		PMD_DRV_LOG(ERR, "fail to execute command OP_REQUEST_QUEUES");
+	rte_intr_enable(&pci_dev->intr_handle);
+
+	return err;
+}
-- 
2.17.1


^ permalink raw reply	[flat|nested] 45+ messages in thread

* [dpdk-dev] [PATCH v1 2/2] net/iavf: enable large VF configuration
  2020-09-09  7:20 [dpdk-dev] [PATCH v1 0/2] enable large VF configuration Ting Xu
  2020-09-09  7:20 ` [dpdk-dev] [PATCH v1 1/2] net/iavf: add IAVF request queues Ting Xu
@ 2020-09-09  7:20 ` Ting Xu
  2020-09-25  5:59 ` [dpdk-dev] [PATCH v2 0/2] " Ting Xu
                   ` (7 subsequent siblings)
  9 siblings, 0 replies; 45+ messages in thread
From: Ting Xu @ 2020-09-09  7:20 UTC (permalink / raw)
  To: dev; +Cc: qi.z.zhang, jingjing.wu, Ting Xu

Add support to configure VSI queues, enable/disable queues and IRQ
mapping for large VF. The max VF queue pairs number is 256. Request
more queues from PF if allocated queues are not enough. Add support
to query max RSS queue regions number. Large VF offload should be
supported before configuring large VF.

Signed-off-by: Ting Xu <ting.xu@intel.com>
---
 drivers/net/iavf/iavf.h        |  24 ++-
 drivers/net/iavf/iavf_ethdev.c |  81 +++++++--
 drivers/net/iavf/iavf_rxtx.c   |  27 ++-
 drivers/net/iavf/iavf_vchnl.c  | 309 ++++++++++++++++++++++++++++++---
 4 files changed, 397 insertions(+), 44 deletions(-)

diff --git a/drivers/net/iavf/iavf.h b/drivers/net/iavf/iavf.h
index 1a42936a0..a16365d3b 100644
--- a/drivers/net/iavf/iavf.h
+++ b/drivers/net/iavf/iavf.h
@@ -19,7 +19,10 @@
 #define IAVF_FRAME_SIZE_MAX       9728
 #define IAVF_QUEUE_BASE_ADDR_UNIT 128
 
-#define IAVF_MAX_NUM_QUEUES       16
+#define IAVF_MAX_NUM_QUEUES_DFLT	 16
+#define IAVF_MAX_NUM_QUEUES_LV		 256
+#define IAVF_RXTX_QUEUE_CHUNKS_NUM	 2
+#define IAVF_CFG_Q_NUM_PER_BUF		 32
 
 #define IAVF_NUM_MACADDR_MAX      64
 
@@ -104,8 +107,10 @@ struct iavf_fdir_info {
 	struct iavf_fdir_conf conf;
 };
 
-/* TODO: is that correct to assume the max number to be 16 ?*/
-#define IAVF_MAX_MSIX_VECTORS   16
+struct iavf_qv_map {
+	uint16_t queue_id;
+	uint16_t vector_id;
+};
 
 /* Event status from PF */
 enum pending_msg {
@@ -157,14 +162,17 @@ struct iavf_info {
 	uint8_t *rss_key;
 	uint16_t nb_msix;   /* number of MSI-X interrupts on Rx */
 	uint16_t msix_base; /* msix vector base from */
-	/* queue bitmask for each vector */
-	uint16_t rxq_map[IAVF_MAX_MSIX_VECTORS];
+	uint32_t max_rss_qregion; /* max RSS queue region supported by PF */
+	/* queue vector mapping */
+	struct iavf_qv_map *qv_map;
 	struct iavf_flow_list flow_list;
 	rte_spinlock_t flow_ops_lock;
 	struct iavf_parser_list rss_parser_list;
 	struct iavf_parser_list dist_parser_list;
 
 	struct iavf_fdir_info fdir; /* flow director info */
+	/* indicate large VF support enabled or not */
+	bool lv_enabled;
 };
 
 #define IAVF_MAX_PKT_TYPE 1024
@@ -269,13 +277,18 @@ int iavf_enable_vlan_strip(struct iavf_adapter *adapter);
 int iavf_disable_vlan_strip(struct iavf_adapter *adapter);
 int iavf_switch_queue(struct iavf_adapter *adapter, uint16_t qid,
 		     bool rx, bool on);
+int iavf_switch_queue_lv(struct iavf_adapter *adapter, uint16_t qid,
+		     bool rx, bool on);
 int iavf_enable_queues(struct iavf_adapter *adapter);
+int iavf_enable_queues_lv(struct iavf_adapter *adapter);
 int iavf_disable_queues(struct iavf_adapter *adapter);
+int iavf_disable_queues_lv(struct iavf_adapter *adapter);
 int iavf_configure_rss_lut(struct iavf_adapter *adapter);
 int iavf_configure_rss_key(struct iavf_adapter *adapter);
 int iavf_configure_queues(struct iavf_adapter *adapter);
 int iavf_get_supported_rxdid(struct iavf_adapter *adapter);
 int iavf_config_irq_map(struct iavf_adapter *adapter);
+int iavf_config_irq_map_lv(struct iavf_adapter *adapter);
 void iavf_add_del_all_mac_addr(struct iavf_adapter *adapter, bool add);
 int iavf_dev_link_update(struct rte_eth_dev *dev,
 			__rte_unused int wait_to_complete);
@@ -296,4 +309,5 @@ int iavf_add_del_mc_addr_list(struct iavf_adapter *adapter,
 			struct rte_ether_addr *mc_addrs,
 			uint32_t mc_addrs_num, bool add);
 int iavf_request_queues(struct rte_eth_dev *dev, uint16_t num);
+int iavf_get_max_rss_queue_region(struct iavf_adapter *adapter);
 #endif /* _IAVF_ETHDEV_H_ */
diff --git a/drivers/net/iavf/iavf_ethdev.c b/drivers/net/iavf/iavf_ethdev.c
index 48e474f53..6765aac33 100644
--- a/drivers/net/iavf/iavf_ethdev.c
+++ b/drivers/net/iavf/iavf_ethdev.c
@@ -160,12 +160,12 @@ iavf_init_rss(struct iavf_adapter *adapter)
 {
 	struct iavf_info *vf =  IAVF_DEV_PRIVATE_TO_VF(adapter);
 	struct rte_eth_rss_conf *rss_conf;
-	uint8_t i, j, nb_q;
+	uint16_t i, j, nb_q;
 	int ret;
 
 	rss_conf = &adapter->eth_dev->data->dev_conf.rx_adv_conf.rss_conf;
 	nb_q = RTE_MIN(adapter->eth_dev->data->nb_rx_queues,
-		       IAVF_MAX_NUM_QUEUES);
+		       IAVF_MAX_NUM_QUEUES_LV);
 
 	if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) {
 		PMD_DRV_LOG(DEBUG, "RSS is not supported");
@@ -218,6 +218,9 @@ iavf_dev_configure(struct rte_eth_dev *dev)
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct iavf_info *vf =  IAVF_DEV_PRIVATE_TO_VF(ad);
 	struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
+	uint16_t num_queue_pairs = RTE_MAX(dev->data->nb_rx_queues,
+		dev->data->nb_tx_queues);
+	int ret = 0;
 
 	ad->rx_bulk_alloc_allowed = true;
 	/* Initialize to TRUE. If any of Rx queues doesn't meet the
@@ -229,6 +232,29 @@ iavf_dev_configure(struct rte_eth_dev *dev)
 	if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)
 		dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_RSS_HASH;
 
+	/* Large VF setting */
+	if (num_queue_pairs > IAVF_MAX_NUM_QUEUES_DFLT) {
+		if (!(vf->vf_res->vf_cap_flags &
+				VIRTCHNL_VF_OFFLOAD_LARGE_VF)) {
+			PMD_DRV_LOG(ERR, "large VF is not supported");
+			return -1;
+		}
+
+		ret = iavf_request_queues(dev, num_queue_pairs);
+		if (ret != 0) {
+			PMD_DRV_LOG(ERR, "request queues from PF failed");
+			return ret;
+		}
+		PMD_DRV_LOG(INFO, "change queue pairs from %u to %u",
+				vf->vsi_res->num_queue_pairs, num_queue_pairs);
+
+		ret = iavf_dev_reset(dev);
+		if (ret != 0)
+			return ret;
+
+		vf->lv_enabled = true;
+	}
+
 	/* Vlan stripping setting */
 	if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN) {
 		if (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
@@ -243,6 +269,7 @@ iavf_dev_configure(struct rte_eth_dev *dev)
 			return -1;
 		}
 	}
+
 	return 0;
 }
 
@@ -325,6 +352,7 @@ static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev,
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter);
+	struct iavf_qv_map *qv_map;
 	uint16_t interval, i;
 	int vec;
 
@@ -345,6 +373,14 @@ static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev,
 		}
 	}
 
+	qv_map = rte_zmalloc("qv_map",
+		dev->data->nb_rx_queues * sizeof(struct iavf_qv_map), 0);
+	if (!qv_map) {
+		PMD_DRV_LOG(ERR, "Failed to allocate %d queue-vector map",
+				dev->data->nb_rx_queues);
+		return -1;
+	}
+
 	if (!dev->data->dev_conf.intr_conf.rxq ||
 	    !rte_intr_dp_is_en(intr_handle)) {
 		/* Rx interrupt disabled, Map interrupt only for writeback */
@@ -375,16 +411,21 @@ static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev,
 		}
 		IAVF_WRITE_FLUSH(hw);
 		/* map all queues to the same interrupt */
-		for (i = 0; i < dev->data->nb_rx_queues; i++)
-			vf->rxq_map[vf->msix_base] |= 1 << i;
+		for (i = 0; i < dev->data->nb_rx_queues; i++) {
+			qv_map[i].queue_id = i;
+			qv_map[i].vector_id = vf->msix_base;
+		}
+		vf->qv_map = qv_map;
 	} else {
 		if (!rte_intr_allow_others(intr_handle)) {
 			vf->nb_msix = 1;
 			vf->msix_base = IAVF_MISC_VEC_ID;
 			for (i = 0; i < dev->data->nb_rx_queues; i++) {
-				vf->rxq_map[vf->msix_base] |= 1 << i;
+				qv_map[i].queue_id = i;
+				qv_map[i].vector_id = vf->msix_base;
 				intr_handle->intr_vec[i] = IAVF_MISC_VEC_ID;
 			}
+			vf->qv_map = qv_map;
 			PMD_DRV_LOG(DEBUG,
 				    "vector %u are mapping to all Rx queues",
 				    vf->msix_base);
@@ -397,21 +438,33 @@ static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev,
 			vf->msix_base = IAVF_RX_VEC_START;
 			vec = IAVF_RX_VEC_START;
 			for (i = 0; i < dev->data->nb_rx_queues; i++) {
-				vf->rxq_map[vec] |= 1 << i;
+				qv_map[i].queue_id = i;
+				qv_map[i].vector_id = vec;
 				intr_handle->intr_vec[i] = vec++;
 				if (vec >= vf->nb_msix)
 					vec = IAVF_RX_VEC_START;
 			}
+			vf->qv_map = qv_map;
 			PMD_DRV_LOG(DEBUG,
 				    "%u vectors are mapping to %u Rx queues",
 				    vf->nb_msix, dev->data->nb_rx_queues);
 		}
 	}
 
-	if (iavf_config_irq_map(adapter)) {
-		PMD_DRV_LOG(ERR, "config interrupt mapping failed");
-		return -1;
+	if (!vf->lv_enabled) {
+		if (iavf_config_irq_map(adapter)) {
+			PMD_DRV_LOG(ERR, "config interrupt mapping failed");
+			return -1;
+		}
+	} else {
+		if (iavf_config_irq_map_lv(adapter)) {
+			PMD_DRV_LOG(ERR, "config interrupt mapping "
+				"for large VF failed");
+			return -1;
+		}
 	}
+
+	rte_free(qv_map);
 	return 0;
 }
 
@@ -539,8 +592,8 @@ iavf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 {
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 
-	dev_info->max_rx_queues = vf->vsi_res->num_queue_pairs;
-	dev_info->max_tx_queues = vf->vsi_res->num_queue_pairs;
+	dev_info->max_rx_queues = IAVF_MAX_NUM_QUEUES_LV;
+	dev_info->max_tx_queues = IAVF_MAX_NUM_QUEUES_LV;
 	dev_info->min_rx_bufsize = IAVF_BUF_SIZE_MIN;
 	dev_info->max_rx_pktlen = IAVF_FRAME_SIZE_MAX;
 	dev_info->hash_key_size = vf->vf_res->rss_key_size;
@@ -1269,6 +1322,12 @@ iavf_init_vf(struct rte_eth_dev *dev)
 		PMD_INIT_LOG(ERR, "iavf_get_vf_config failed");
 		goto err_alloc;
 	}
+
+	if (iavf_get_max_rss_queue_region(adapter) != 0) {
+		PMD_INIT_LOG(ERR, "get max rss queue region failed");
+		goto err_alloc;
+	}
+
 	/* Allocate memort for RSS info */
 	if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) {
 		vf->rss_key = rte_zmalloc("rss_key",
diff --git a/drivers/net/iavf/iavf_rxtx.c b/drivers/net/iavf/iavf_rxtx.c
index 05a7dd898..86004ac90 100644
--- a/drivers/net/iavf/iavf_rxtx.c
+++ b/drivers/net/iavf/iavf_rxtx.c
@@ -532,6 +532,7 @@ iavf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 	struct iavf_adapter *adapter =
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	struct iavf_rx_queue *rxq;
 	int err = 0;
 
@@ -555,7 +556,11 @@ iavf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 	IAVF_WRITE_FLUSH(hw);
 
 	/* Ready to switch the queue on */
-	err = iavf_switch_queue(adapter, rx_queue_id, true, true);
+	if (!vf->lv_enabled)
+		err = iavf_switch_queue(adapter, rx_queue_id, true, true);
+	else
+		err = iavf_switch_queue_lv(adapter, rx_queue_id, true, true);
+
 	if (err)
 		PMD_DRV_LOG(ERR, "Failed to switch RX queue %u on",
 			    rx_queue_id);
@@ -572,6 +577,7 @@ iavf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 	struct iavf_adapter *adapter =
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	struct iavf_tx_queue *txq;
 	int err = 0;
 
@@ -587,7 +593,10 @@ iavf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 	IAVF_WRITE_FLUSH(hw);
 
 	/* Ready to switch the queue on */
-	err = iavf_switch_queue(adapter, tx_queue_id, false, true);
+	if (!vf->lv_enabled)
+		err = iavf_switch_queue(adapter, tx_queue_id, false, true);
+	else
+		err = iavf_switch_queue_lv(adapter, tx_queue_id, false, true);
 
 	if (err)
 		PMD_DRV_LOG(ERR, "Failed to switch TX queue %u on",
@@ -688,14 +697,22 @@ iavf_stop_queues(struct rte_eth_dev *dev)
 {
 	struct iavf_adapter *adapter =
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	struct iavf_rx_queue *rxq;
 	struct iavf_tx_queue *txq;
 	int ret, i;
 
 	/* Stop All queues */
-	ret = iavf_disable_queues(adapter);
-	if (ret)
-		PMD_DRV_LOG(WARNING, "Fail to stop queues");
+	if (!vf->lv_enabled) {
+		ret = iavf_disable_queues(adapter);
+		if (ret)
+			PMD_DRV_LOG(WARNING, "Fail to stop queues");
+	} else {
+		ret = iavf_disable_queues_lv(adapter);
+		if (ret)
+			PMD_DRV_LOG(WARNING, "Fail to stop queues for large VF");
+	}
+
 
 	for (i = 0; i < dev->data->nb_tx_queues; i++) {
 		txq = dev->data->tx_queues[i];
diff --git a/drivers/net/iavf/iavf_vchnl.c b/drivers/net/iavf/iavf_vchnl.c
index eab2f2a22..6090afab6 100644
--- a/drivers/net/iavf/iavf_vchnl.c
+++ b/drivers/net/iavf/iavf_vchnl.c
@@ -128,6 +128,7 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter, struct iavf_cmd_info *args)
 	case VIRTCHNL_OP_VERSION:
 	case VIRTCHNL_OP_GET_VF_RESOURCES:
 	case VIRTCHNL_OP_GET_SUPPORTED_RXDIDS:
+	case VIRTCHNL_OP_GET_MAX_RSS_QREGION:
 		/* for init virtchnl ops, need to poll the response */
 		do {
 			result = iavf_read_msg_from_pf(adapter, args->out_size,
@@ -448,7 +449,8 @@ iavf_get_vf_resource(struct iavf_adapter *adapter)
 	caps = IAVF_BASIC_OFFLOAD_CAPS | VIRTCHNL_VF_CAP_ADV_LINK_SPEED |
 		VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC |
 		VIRTCHNL_VF_OFFLOAD_FDIR_PF |
-		VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF;
+		VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF |
+		VIRTCHNL_VF_OFFLOAD_LARGE_VF;
 
 	args.in_args = (uint8_t *)&caps;
 	args.in_args_size = sizeof(caps);
@@ -598,6 +600,142 @@ iavf_switch_queue(struct iavf_adapter *adapter, uint16_t qid,
 	return err;
 }
 
+int
+iavf_enable_queues_lv(struct iavf_adapter *adapter)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct virtchnl_del_ena_dis_queues *queue_select;
+	struct virtchnl_queue_chunk *queue_chunk;
+	struct iavf_cmd_info args;
+	int err, len;
+
+	len = sizeof(struct virtchnl_del_ena_dis_queues) +
+		  sizeof(struct virtchnl_queue_chunk) *
+		  (IAVF_RXTX_QUEUE_CHUNKS_NUM - 1);
+	queue_select = rte_zmalloc("queue_select", len, 0);
+	if (!queue_select)
+		return -ENOMEM;
+
+	queue_chunk = queue_select->chunks.chunks;
+	queue_select->chunks.num_chunks = IAVF_RXTX_QUEUE_CHUNKS_NUM;
+	queue_select->vport_id = vf->vsi_res->vsi_id;
+
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].type = VIRTCHNL_QUEUE_TYPE_TX;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].start_queue_id = 0;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].num_queues =
+		adapter->eth_dev->data->nb_tx_queues;
+
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].type = VIRTCHNL_QUEUE_TYPE_RX;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].start_queue_id = 0;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].num_queues =
+		adapter->eth_dev->data->nb_rx_queues;
+
+	args.ops = VIRTCHNL_OP_ENABLE_QUEUES_V2;
+	args.in_args = (u8 *)queue_select;
+	args.in_args_size = len;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err) {
+		PMD_DRV_LOG(ERR,
+			    "Failed to execute command of OP_ENABLE_QUEUES_V2");
+		return err;
+	}
+	return 0;
+}
+
+int
+iavf_disable_queues_lv(struct iavf_adapter *adapter)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct virtchnl_del_ena_dis_queues *queue_select;
+	struct virtchnl_queue_chunk *queue_chunk;
+	struct iavf_cmd_info args;
+	int err, len;
+
+	len = sizeof(struct virtchnl_del_ena_dis_queues) +
+		  sizeof(struct virtchnl_queue_chunk) *
+		  (IAVF_RXTX_QUEUE_CHUNKS_NUM - 1);
+	queue_select = rte_zmalloc("queue_select", len, 0);
+	if (!queue_select)
+		return -ENOMEM;
+
+	queue_chunk = queue_select->chunks.chunks;
+	queue_select->chunks.num_chunks = IAVF_RXTX_QUEUE_CHUNKS_NUM;
+	queue_select->vport_id = vf->vsi_res->vsi_id;
+
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].type = VIRTCHNL_QUEUE_TYPE_TX;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].start_queue_id = 0;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].num_queues =
+		adapter->eth_dev->data->nb_tx_queues;
+
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].type = VIRTCHNL_QUEUE_TYPE_RX;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].start_queue_id = 0;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].num_queues =
+		adapter->eth_dev->data->nb_rx_queues;
+
+	args.ops = VIRTCHNL_OP_DISABLE_QUEUES_V2;
+	args.in_args = (u8 *)queue_select;
+	args.in_args_size = len;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err) {
+		PMD_DRV_LOG(ERR,
+			    "Failed to execute command of OP_DISABLE_QUEUES_V2");
+		return err;
+	}
+	return 0;
+}
+
+int
+iavf_switch_queue_lv(struct iavf_adapter *adapter, uint16_t qid,
+		 bool rx, bool on)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct virtchnl_del_ena_dis_queues *queue_select;
+	struct virtchnl_queue_chunk *queue_chunk;
+	struct iavf_cmd_info args;
+	int err, len;
+
+	len = sizeof(struct virtchnl_del_ena_dis_queues) +
+		  sizeof(struct virtchnl_queue_chunk) *
+		  (IAVF_RXTX_QUEUE_CHUNKS_NUM - 1);
+	queue_select = rte_zmalloc("queue_select", len, 0);
+	if (!queue_select)
+		return -ENOMEM;
+
+	queue_chunk = queue_select->chunks.chunks;
+	queue_select->chunks.num_chunks = IAVF_RXTX_QUEUE_CHUNKS_NUM;
+	queue_select->vport_id = vf->vsi_res->vsi_id;
+
+	if (rx) {
+		queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].type =
+			VIRTCHNL_QUEUE_TYPE_RX;
+		queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].start_queue_id = qid;
+		queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].num_queues = 1;
+	} else {
+		queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].type =
+			VIRTCHNL_QUEUE_TYPE_TX;
+		queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].start_queue_id = qid;
+		queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].num_queues = 1;
+	}
+
+	if (on)
+		args.ops = VIRTCHNL_OP_ENABLE_QUEUES_V2;
+	else
+		args.ops = VIRTCHNL_OP_DISABLE_QUEUES_V2;
+	args.in_args = (u8 *)queue_select;
+	args.in_args_size = len;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err)
+		PMD_DRV_LOG(ERR, "Failed to execute command of %s",
+			    on ? "OP_ENABLE_QUEUES_V2" : "OP_DISABLE_QUEUES_V2");
+	return err;
+}
+
 int
 iavf_configure_rss_lut(struct iavf_adapter *adapter)
 {
@@ -662,32 +800,24 @@ iavf_configure_rss_key(struct iavf_adapter *adapter)
 	return err;
 }
 
-int
-iavf_configure_queues(struct iavf_adapter *adapter)
+static int
+iavf_exec_queue_cfg(struct iavf_adapter *adapter,
+	struct virtchnl_vsi_queue_config_info *vc_config,
+	uint16_t count, uint16_t size)
 {
 	struct iavf_rx_queue **rxq =
 		(struct iavf_rx_queue **)adapter->eth_dev->data->rx_queues;
 	struct iavf_tx_queue **txq =
 		(struct iavf_tx_queue **)adapter->eth_dev->data->tx_queues;
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
-	struct virtchnl_vsi_queue_config_info *vc_config;
 	struct virtchnl_queue_pair_info *vc_qp;
 	struct iavf_cmd_info args;
-	uint16_t i, size;
-	int err;
-
-	size = sizeof(*vc_config) +
-	       sizeof(vc_config->qpair[0]) * vf->num_queue_pairs;
-	vc_config = rte_zmalloc("cfg_queue", size, 0);
-	if (!vc_config)
-		return -ENOMEM;
-
-	vc_config->vsi_id = vf->vsi_res->vsi_id;
-	vc_config->num_queue_pairs = vf->num_queue_pairs;
+	int err = 0;
+	uint16_t i;
 
-	for (i = 0, vc_qp = vc_config->qpair;
-	     i < vf->num_queue_pairs;
-	     i++, vc_qp++) {
+	for (i = count * IAVF_CFG_Q_NUM_PER_BUF, vc_qp = vc_config->qpair;
+	     i < count * IAVF_CFG_Q_NUM_PER_BUF + vc_config->num_queue_pairs;
+		 i++, vc_qp++) {
 		vc_qp->txq.vsi_id = vf->vsi_res->vsi_id;
 		vc_qp->txq.queue_id = i;
 		/* Virtchnnl configure queues by pairs */
@@ -741,8 +871,73 @@ iavf_configure_queues(struct iavf_adapter *adapter)
 	err = iavf_execute_vf_cmd(adapter, &args);
 	if (err)
 		PMD_DRV_LOG(ERR, "Failed to execute command of"
-			    " VIRTCHNL_OP_CONFIG_VSI_QUEUES");
+				" VIRTCHNL_OP_CONFIG_VSI_QUEUES");
+
+	return err;
+}
+
+/* Configure VSI queues. Max VF queue pairs number is 256, may
+ * send this command multiple times to configure all queues.
+ */
+int
+iavf_configure_queues(struct iavf_adapter *adapter)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct virtchnl_vsi_queue_config_info *vc_config;
+	uint16_t nb_cmd_full, nb_cmd_rest, nbq_rest;
+	uint16_t i, size;
+	int err = 0;
+
+	/* Compute how many times should the command to be sent */
+	nbq_rest = vf->num_queue_pairs % IAVF_CFG_Q_NUM_PER_BUF;
+	nb_cmd_rest = nbq_rest ? 1 : 0;
+	nb_cmd_full = vf->num_queue_pairs / IAVF_CFG_Q_NUM_PER_BUF;
+
+	/* send cmd only once for small queues number */
+	if (!nb_cmd_full) {
+		size = sizeof(*vc_config) +
+		       sizeof(vc_config->qpair[0]) * nbq_rest;
+		vc_config = rte_zmalloc("cfg_queue", size, 0);
+		if (!vc_config)
+			return -ENOMEM;
+
+		vc_config->vsi_id = vf->vsi_res->vsi_id;
+		vc_config->num_queue_pairs = nbq_rest;
+		err = iavf_exec_queue_cfg(adapter, vc_config, 0, size);
+		goto free;
+	}
+
+	size = sizeof(*vc_config) +
+		   sizeof(vc_config->qpair[0]) * IAVF_CFG_Q_NUM_PER_BUF;
+	vc_config = rte_zmalloc("cfg_queue", size, 0);
+	if (!vc_config)
+		return -ENOMEM;
+
+	for (i = 0; i < nb_cmd_full + nb_cmd_rest; i++) {
+		if (i < nb_cmd_full) {
+			vc_config->vsi_id = vf->vsi_res->vsi_id;
+			vc_config->num_queue_pairs = IAVF_CFG_Q_NUM_PER_BUF;
+		} else {
+			/* re-allocate virtchnl msg for less queues */
+			rte_free(vc_config);
+			size = sizeof(*vc_config) +
+			       sizeof(vc_config->qpair[0]) * nbq_rest;
+			vc_config = rte_zmalloc("cfg_queue", size, 0);
+			if (!vc_config)
+				return -ENOMEM;
+
+			vc_config->vsi_id = vf->vsi_res->vsi_id;
+			vc_config->num_queue_pairs = nbq_rest;
+		}
 
+		err = iavf_exec_queue_cfg(adapter, vc_config, i, size);
+		if (err)
+			goto free;
+
+		memset(vc_config, 0, size);
+	}
+
+free:
 	rte_free(vc_config);
 	return err;
 }
@@ -797,13 +992,14 @@ iavf_config_irq_map(struct iavf_adapter *adapter)
 
 	map_info = (struct virtchnl_irq_map_info *)cmd_buffer;
 	map_info->num_vectors = nb_msix;
-	for (i = 0; i < nb_msix; i++) {
-		vecmap = &map_info->vecmap[i];
+	for (i = 0; i < adapter->eth_dev->data->nb_rx_queues; i++) {
+		vecmap =
+		    &map_info->vecmap[vf->qv_map[i].vector_id - vf->msix_base];
 		vecmap->vsi_id = vf->vsi_res->vsi_id;
 		vecmap->rxitr_idx = IAVF_ITR_INDEX_DEFAULT;
-		vecmap->vector_id = vf->msix_base + i;
+		vecmap->vector_id = vf->qv_map[i].vector_id;
 		vecmap->txq_map = 0;
-		vecmap->rxq_map = vf->rxq_map[vf->msix_base + i];
+		vecmap->rxq_map |= 1 << vf->qv_map[i].queue_id;
 	}
 
 	args.ops = VIRTCHNL_OP_CONFIG_IRQ_MAP;
@@ -819,6 +1015,46 @@ iavf_config_irq_map(struct iavf_adapter *adapter)
 	return err;
 }
 
+int
+iavf_config_irq_map_lv(struct iavf_adapter *adapter)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct virtchnl_queue_vector_maps *map_info;
+	struct virtchnl_queue_vector *qv_maps;
+	struct iavf_cmd_info args;
+	int len, i, err;
+
+	len = sizeof(struct virtchnl_queue_vector_maps) +
+	      sizeof(struct virtchnl_queue_vector) *
+	      (adapter->eth_dev->data->nb_rx_queues - 1);
+
+	map_info = rte_zmalloc("map_info", len, 0);
+	if (!map_info)
+		return -ENOMEM;
+
+	map_info->vport_id = vf->vsi_res->vsi_id;
+	map_info->num_qv_maps = adapter->eth_dev->data->nb_rx_queues;
+	for (i = 0; i < map_info->num_qv_maps; i++) {
+		qv_maps = &map_info->qv_maps[i];
+		qv_maps->itr_idx = VIRTCHNL_ITR_IDX_0;
+		qv_maps->queue_type = VIRTCHNL_QUEUE_TYPE_RX;
+		qv_maps->queue_id = vf->qv_map[i].queue_id;
+		qv_maps->vector_id = vf->qv_map[i].vector_id;
+	}
+
+	args.ops = VIRTCHNL_OP_MAP_QUEUE_VECTOR;
+	args.in_args = (u8 *)map_info;
+	args.in_args_size = len;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err)
+		PMD_DRV_LOG(ERR, "fail to execute command OP_MAP_QUEUE_VECTOR");
+
+	rte_free(map_info);
+	return err;
+}
+
 void
 iavf_add_del_all_mac_addr(struct iavf_adapter *adapter, bool add)
 {
@@ -1261,3 +1497,30 @@ iavf_request_queues(struct rte_eth_dev *dev, uint16_t num)
 
 	return err;
 }
+
+int
+iavf_get_max_rss_queue_region(struct iavf_adapter *adapter)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct iavf_cmd_info args;
+	int ret;
+
+	args.ops = VIRTCHNL_OP_GET_MAX_RSS_QREGION;
+	args.in_args = NULL;
+	args.in_args_size = 0;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+
+	ret = iavf_execute_vf_cmd(adapter, &args);
+	if (ret) {
+		PMD_DRV_LOG(ERR,
+			    "Failed to execute command of "
+			    "VIRTCHNL_OP_GET_MAX_RSS_QREGION");
+		return ret;
+	}
+
+	vf->max_rss_qregion =
+	((struct virtchnl_max_rss_qregion *)args.out_buffer)->max_rss_qregion;
+
+	return 0;
+}
-- 
2.17.1


^ permalink raw reply	[flat|nested] 45+ messages in thread

* [dpdk-dev] [PATCH v2 0/2] enable large VF configuration
  2020-09-09  7:20 [dpdk-dev] [PATCH v1 0/2] enable large VF configuration Ting Xu
  2020-09-09  7:20 ` [dpdk-dev] [PATCH v1 1/2] net/iavf: add IAVF request queues Ting Xu
  2020-09-09  7:20 ` [dpdk-dev] [PATCH v1 2/2] net/iavf: enable large VF configuration Ting Xu
@ 2020-09-25  5:59 ` Ting Xu
  2020-09-25  5:59   ` [dpdk-dev] [PATCH v2 1/2] net/iavf: add IAVF request queues function Ting Xu
  2020-09-25  5:59   ` [dpdk-dev] [PATCH v2 2/2] net/iavf: enable large VF configuration Ting Xu
  2020-09-27 12:42 ` [dpdk-dev] [PATCH v3 0/2] " Ting Xu
                   ` (6 subsequent siblings)
  9 siblings, 2 replies; 45+ messages in thread
From: Ting Xu @ 2020-09-25  5:59 UTC (permalink / raw)
  To: dev; +Cc: qi.z.zhang, jingjing.wu, beilei.xing, Ting Xu

This patchset supports to configure up to 256 queue pairs per VF. If
large VF is supported after capability negotiation, VF will request
queues from PF as needed. New virtual channel opcodes and structures
are used to indicate 256 queues, so VF is designed to handle the new
function of configure VSI queues, IRQ mapping and enable/disable queues.
Also, enable VF to query the max RSS queue region.

Ting Xu (2):
  net/iavf: add IAVF request queues function
  net/iavf: enable large VF configuration

---
v1->v2:
Change the communication with kernel PF

---
 drivers/net/iavf/iavf.h        |  40 ++-
 drivers/net/iavf/iavf_ethdev.c |  91 ++++++-
 drivers/net/iavf/iavf_rxtx.c   |  27 +-
 drivers/net/iavf/iavf_vchnl.c  | 436 ++++++++++++++++++++++++++++++---
 4 files changed, 539 insertions(+), 55 deletions(-)

-- 
2.17.1


^ permalink raw reply	[flat|nested] 45+ messages in thread

* [dpdk-dev] [PATCH v2 1/2] net/iavf: add IAVF request queues function
  2020-09-25  5:59 ` [dpdk-dev] [PATCH v2 0/2] " Ting Xu
@ 2020-09-25  5:59   ` Ting Xu
  2020-09-25  5:59   ` [dpdk-dev] [PATCH v2 2/2] net/iavf: enable large VF configuration Ting Xu
  1 sibling, 0 replies; 45+ messages in thread
From: Ting Xu @ 2020-09-25  5:59 UTC (permalink / raw)
  To: dev; +Cc: qi.z.zhang, jingjing.wu, beilei.xing, Ting Xu

Add new virtchnl function to request additional queues from PF. Current
default queue pairs number when creating a VF is 16. In order to support
up to 256 queue pairs, enable this request queues function.
Since request queues command may return event message, modify function
iavf_read_msg_from_pf to identify event opcode and mark VF reset status.

Signed-off-by: Ting Xu <ting.xu@intel.com>
---
 drivers/net/iavf/iavf.h        |  17 +++++
 drivers/net/iavf/iavf_ethdev.c |  12 ++-
 drivers/net/iavf/iavf_vchnl.c  | 134 ++++++++++++++++++++++++++++++---
 3 files changed, 148 insertions(+), 15 deletions(-)

diff --git a/drivers/net/iavf/iavf.h b/drivers/net/iavf/iavf.h
index 3198d85b3..bdc75ff50 100644
--- a/drivers/net/iavf/iavf.h
+++ b/drivers/net/iavf/iavf.h
@@ -107,6 +107,21 @@ struct iavf_fdir_info {
 /* TODO: is that correct to assume the max number to be 16 ?*/
 #define IAVF_MAX_MSIX_VECTORS   16
 
+/* Event status from PF */
+enum pending_msg {
+	PFMSG_LINK_CHANGE = 0x1,
+	PFMSG_RESET_IMPENDING = 0x2,
+	PFMSG_DRIVER_CLOSE = 0x4,
+};
+
+/* Message type read in admin queue from PF */
+enum iavf_aq_result {
+	IAVF_MSG_ERR = -1, /* Meet error when accessing admin queue */
+	IAVF_MSG_NON,      /* Read nothing from admin queue */
+	IAVF_MSG_SYS,      /* Read system msg from admin queue */
+	IAVF_MSG_CMD,      /* Read async command result */
+};
+
 /* Structure to store private data specific for VF instance. */
 struct iavf_info {
 	uint16_t num_queue_pairs;
@@ -123,6 +138,7 @@ struct iavf_info {
 	volatile enum virtchnl_ops pend_cmd; /* pending command not finished */
 	uint32_t cmd_retval; /* return value of the cmd response from PF */
 	uint8_t *aq_resp; /* buffer to store the adminq response from PF */
+	uint16_t pend_msg; /* flags indicates events from pf not handled yet */
 
 	/* Event from pf */
 	bool dev_closed;
@@ -279,4 +295,5 @@ int iavf_add_del_rss_cfg(struct iavf_adapter *adapter,
 int iavf_add_del_mc_addr_list(struct iavf_adapter *adapter,
 			struct rte_ether_addr *mc_addrs,
 			uint32_t mc_addrs_num, bool add);
+int iavf_request_queues(struct rte_eth_dev *dev, uint16_t num);
 #endif /* _IAVF_ETHDEV_H_ */
diff --git a/drivers/net/iavf/iavf_ethdev.c b/drivers/net/iavf/iavf_ethdev.c
index 440da7d76..8fd9b44db 100644
--- a/drivers/net/iavf/iavf_ethdev.c
+++ b/drivers/net/iavf/iavf_ethdev.c
@@ -1193,7 +1193,7 @@ iavf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
 }
 
 static int
-iavf_check_vf_reset_done(struct iavf_hw *hw)
+iavf_check_vf_reset_done(struct iavf_hw *hw, struct iavf_info *vf)
 {
 	int i, reset;
 
@@ -1210,6 +1210,10 @@ iavf_check_vf_reset_done(struct iavf_hw *hw)
 	if (i >= IAVF_RESET_WAIT_CNT)
 		return -1;
 
+	/* VF is not in reset or reset is completed */
+	vf->vf_reset = false;
+	vf->pend_msg &= ~PFMSG_RESET_IMPENDING;
+
 	return 0;
 }
 
@@ -1228,7 +1232,7 @@ iavf_init_vf(struct rte_eth_dev *dev)
 		goto err;
 	}
 
-	err = iavf_check_vf_reset_done(hw);
+	err = iavf_check_vf_reset_done(hw, vf);
 	if (err) {
 		PMD_INIT_LOG(ERR, "VF is still resetting");
 		goto err;
@@ -1475,7 +1479,9 @@ iavf_dev_close(struct rte_eth_dev *dev)
 
 	iavf_dev_stop(dev);
 	iavf_flow_flush(dev, NULL);
-	iavf_flow_uninit(adapter);
+	/* if VF is in reset, adminq is disabled, skip the process via adminq */
+	if (!vf->vf_reset)
+		iavf_flow_uninit(adapter);
 	iavf_shutdown_adminq(hw);
 	/* disable uio intr before callback unregister */
 	rte_intr_disable(intr_handle);
diff --git a/drivers/net/iavf/iavf_vchnl.c b/drivers/net/iavf/iavf_vchnl.c
index 76f8e38d1..bd39f0db6 100644
--- a/drivers/net/iavf/iavf_vchnl.c
+++ b/drivers/net/iavf/iavf_vchnl.c
@@ -17,6 +17,7 @@
 #include <rte_eal.h>
 #include <rte_ether.h>
 #include <rte_ethdev_driver.h>
+#include <rte_ethdev_pci.h>
 #include <rte_dev.h>
 
 #include "iavf.h"
@@ -26,13 +27,14 @@
 #define ASQ_DELAY_MS  10
 
 /* Read data in admin queue to get msg from pf driver */
-static enum iavf_status
+static enum iavf_aq_result
 iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,
 		     uint8_t *buf)
 {
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter);
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
 	struct iavf_arq_event_info event;
+	enum iavf_aq_result result = IAVF_MSG_NON;
 	enum virtchnl_ops opcode;
 	int ret;
 
@@ -42,7 +44,9 @@ iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,
 	/* Can't read any msg from adminQ */
 	if (ret) {
 		PMD_DRV_LOG(DEBUG, "Can't read msg from AQ");
-		return ret;
+		if (ret != IAVF_ERR_ADMIN_QUEUE_NO_WORK)
+			result = IAVF_MSG_ERR;
+		return result;
 	}
 
 	opcode = (enum virtchnl_ops)rte_le_to_cpu_32(event.desc.cookie_high);
@@ -52,16 +56,46 @@ iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,
 	PMD_DRV_LOG(DEBUG, "AQ from pf carries opcode %u, retval %d",
 		    opcode, vf->cmd_retval);
 
-	if (opcode != vf->pend_cmd) {
-		if (opcode != VIRTCHNL_OP_EVENT) {
-			PMD_DRV_LOG(WARNING,
-				    "command mismatch, expect %u, get %u",
-				    vf->pend_cmd, opcode);
+	if (opcode == VIRTCHNL_OP_EVENT) {
+		struct virtchnl_pf_event *vpe =
+			(struct virtchnl_pf_event *)event.msg_buf;
+
+		result = IAVF_MSG_SYS;
+		switch (vpe->event) {
+		case VIRTCHNL_EVENT_LINK_CHANGE:
+			vf->link_up =
+				vpe->event_data.link_event.link_status;
+			vf->link_speed =
+				vpe->event_data.link_event.link_speed;
+			vf->pend_msg |= PFMSG_LINK_CHANGE;
+			PMD_DRV_LOG(INFO, "Link status update:%s",
+					vf->link_up ? "up" : "down");
+			break;
+		case VIRTCHNL_EVENT_RESET_IMPENDING:
+			vf->vf_reset = true;
+			vf->pend_msg |= PFMSG_RESET_IMPENDING;
+			PMD_DRV_LOG(INFO, "vf is reseting");
+			break;
+		case VIRTCHNL_EVENT_PF_DRIVER_CLOSE:
+			vf->dev_closed = true;
+			vf->pend_msg |= PFMSG_DRIVER_CLOSE;
+			PMD_DRV_LOG(INFO, "PF driver closed");
+			break;
+		default:
+			PMD_DRV_LOG(ERR, "%s: Unknown event %d from pf",
+					__func__, vpe->event);
+		}
+	}  else {
+		/* async reply msg on command issued by vf previously */
+		result = IAVF_MSG_CMD;
+		if (opcode != vf->pend_cmd) {
+			PMD_DRV_LOG(WARNING, "command mismatch, expect %u, get %u",
+					vf->pend_cmd, opcode);
+			result = IAVF_MSG_ERR;
 		}
-		return IAVF_ERR_OPCODE_MISMATCH;
 	}
 
-	return IAVF_SUCCESS;
+	return result;
 }
 
 static int
@@ -69,6 +103,7 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter, struct iavf_cmd_info *args)
 {
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter);
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	enum iavf_aq_result result;
 	enum iavf_status ret;
 	int err = 0;
 	int i = 0;
@@ -97,9 +132,9 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter, struct iavf_cmd_info *args)
 	case VIRTCHNL_OP_GET_SUPPORTED_RXDIDS:
 		/* for init virtchnl ops, need to poll the response */
 		do {
-			ret = iavf_read_msg_from_pf(adapter, args->out_size,
+			result = iavf_read_msg_from_pf(adapter, args->out_size,
 						   args->out_buffer);
-			if (ret == IAVF_SUCCESS)
+			if (result == IAVF_MSG_CMD)
 				break;
 			rte_delay_ms(ASQ_DELAY_MS);
 		} while (i++ < MAX_TRY_TIMES);
@@ -111,7 +146,33 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter, struct iavf_cmd_info *args)
 		}
 		_clear_cmd(vf);
 		break;
-
+	case VIRTCHNL_OP_REQUEST_QUEUES:
+		/*
+		 * ignore async reply, only wait for system message,
+		 * vf_reset = true if get VIRTCHNL_EVENT_RESET_IMPENDING,
+		 * if not, means request queues failed.
+		 */
+		do {
+			result = iavf_read_msg_from_pf(adapter, args->out_size,
+						   args->out_buffer);
+			if (result == IAVF_MSG_SYS && vf->vf_reset) {
+				break;
+			} else if (result == IAVF_MSG_CMD ||
+				result == IAVF_MSG_ERR) {
+				err = -1;
+				break;
+			}
+			rte_delay_ms(ASQ_DELAY_MS);
+			/* If don't read msg or read sys event, continue */
+		} while (i++ < MAX_TRY_TIMES);
+		if (i >= MAX_TRY_TIMES ||
+		    vf->cmd_retval != VIRTCHNL_STATUS_SUCCESS) {
+			err = -1;
+			PMD_DRV_LOG(ERR, "No response or return failure (%d)"
+				    " for cmd %d", vf->cmd_retval, args->ops);
+		}
+		_clear_cmd(vf);
+		break;
 	default:
 		/* For other virtchnl ops in running time,
 		 * wait for the cmd done flag.
@@ -1136,3 +1197,52 @@ iavf_add_del_mc_addr_list(struct iavf_adapter *adapter,
 
 	return 0;
 }
+
+int
+iavf_request_queues(struct rte_eth_dev *dev, uint16_t num)
+{
+	struct iavf_adapter *ad =
+		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
+	struct iavf_info *vf =  IAVF_DEV_PRIVATE_TO_VF(ad);
+	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+	struct virtchnl_vf_res_request vfres;
+	struct iavf_cmd_info args;
+	uint16_t num_queue_pairs;
+	int err;
+
+	if (num == 0)
+		return -1;
+	vfres.num_queue_pairs = num;
+
+	args.ops = VIRTCHNL_OP_REQUEST_QUEUES;
+	args.in_args = (u8 *)&vfres;
+	args.in_args_size = sizeof(vfres);
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+
+	/*
+	 * disable interrupt to avoid the admin queue message to be read
+	 * before iavf_read_msg_from_pf.
+	 */
+	rte_intr_disable(&pci_dev->intr_handle);
+	err = iavf_execute_vf_cmd(ad, &args);
+	rte_intr_enable(&pci_dev->intr_handle);
+	if (err) {
+		PMD_DRV_LOG(ERR, "fail to execute command OP_REQUEST_QUEUES");
+		return err;
+	}
+
+	/* request queues succeeded, vf is resetting */
+	if (vf->vf_reset) {
+		PMD_DRV_LOG(INFO, "vf reset");
+		return 0;
+	}
+
+	/* request additional queues failed, return available number */
+	num_queue_pairs =
+	  ((struct virtchnl_vf_res_request *)args.out_buffer)->num_queue_pairs;
+	PMD_DRV_LOG(ERR, "request queues failed, only %u queues "
+		"available", num_queue_pairs);
+
+	return -1;
+}
-- 
2.17.1


^ permalink raw reply	[flat|nested] 45+ messages in thread

* [dpdk-dev] [PATCH v2 2/2] net/iavf: enable large VF configuration
  2020-09-25  5:59 ` [dpdk-dev] [PATCH v2 0/2] " Ting Xu
  2020-09-25  5:59   ` [dpdk-dev] [PATCH v2 1/2] net/iavf: add IAVF request queues function Ting Xu
@ 2020-09-25  5:59   ` Ting Xu
  1 sibling, 0 replies; 45+ messages in thread
From: Ting Xu @ 2020-09-25  5:59 UTC (permalink / raw)
  To: dev; +Cc: qi.z.zhang, jingjing.wu, beilei.xing, Ting Xu

Add support to negotiate large VF capability, configure VSI queues,
enable/disable queues and IRQ mapping for large VF. Use new virtchnl
opcodes and structures to support max 256 queue pairs. Request
additional queues from PF first if current allocated queues are not
enough. Query max RSS queue region for future RSS configuration.

Signed-off-by: Ting Xu <ting.xu@intel.com>
---
 drivers/net/iavf/iavf.h        |  23 ++-
 drivers/net/iavf/iavf_ethdev.c |  79 +++++++--
 drivers/net/iavf/iavf_rxtx.c   |  27 ++-
 drivers/net/iavf/iavf_vchnl.c  | 302 ++++++++++++++++++++++++++++++---
 4 files changed, 391 insertions(+), 40 deletions(-)

diff --git a/drivers/net/iavf/iavf.h b/drivers/net/iavf/iavf.h
index bdc75ff50..f506a934a 100644
--- a/drivers/net/iavf/iavf.h
+++ b/drivers/net/iavf/iavf.h
@@ -19,7 +19,10 @@
 #define IAVF_FRAME_SIZE_MAX       9728
 #define IAVF_QUEUE_BASE_ADDR_UNIT 128
 
-#define IAVF_MAX_NUM_QUEUES       16
+#define IAVF_MAX_NUM_QUEUES_DFLT	 16
+#define IAVF_MAX_NUM_QUEUES_LV		 256
+#define IAVF_RXTX_QUEUE_CHUNKS_NUM	 2
+#define IAVF_CFG_Q_NUM_PER_BUF		 32
 
 #define IAVF_NUM_MACADDR_MAX      64
 
@@ -104,8 +107,10 @@ struct iavf_fdir_info {
 	struct iavf_fdir_conf conf;
 };
 
-/* TODO: is that correct to assume the max number to be 16 ?*/
-#define IAVF_MAX_MSIX_VECTORS   16
+struct iavf_qv_map {
+	uint16_t queue_id;
+	uint16_t vector_id;
+};
 
 /* Event status from PF */
 enum pending_msg {
@@ -157,14 +162,16 @@ struct iavf_info {
 	uint8_t *rss_key;
 	uint16_t nb_msix;   /* number of MSI-X interrupts on Rx */
 	uint16_t msix_base; /* msix vector base from */
-	/* queue bitmask for each vector */
-	uint16_t rxq_map[IAVF_MAX_MSIX_VECTORS];
+	uint16_t max_rss_qregion; /* max RSS queue region supported by PF */
+	struct iavf_qv_map *qv_map; /* queue vector mapping */
 	struct iavf_flow_list flow_list;
 	rte_spinlock_t flow_ops_lock;
 	struct iavf_parser_list rss_parser_list;
 	struct iavf_parser_list dist_parser_list;
 
 	struct iavf_fdir_info fdir; /* flow director info */
+	/* indicate large VF support enabled or not */
+	bool lv_enabled;
 };
 
 #define IAVF_MAX_PKT_TYPE 1024
@@ -269,13 +276,18 @@ int iavf_enable_vlan_strip(struct iavf_adapter *adapter);
 int iavf_disable_vlan_strip(struct iavf_adapter *adapter);
 int iavf_switch_queue(struct iavf_adapter *adapter, uint16_t qid,
 		     bool rx, bool on);
+int iavf_switch_queue_lv(struct iavf_adapter *adapter, uint16_t qid,
+		     bool rx, bool on);
 int iavf_enable_queues(struct iavf_adapter *adapter);
+int iavf_enable_queues_lv(struct iavf_adapter *adapter);
 int iavf_disable_queues(struct iavf_adapter *adapter);
+int iavf_disable_queues_lv(struct iavf_adapter *adapter);
 int iavf_configure_rss_lut(struct iavf_adapter *adapter);
 int iavf_configure_rss_key(struct iavf_adapter *adapter);
 int iavf_configure_queues(struct iavf_adapter *adapter);
 int iavf_get_supported_rxdid(struct iavf_adapter *adapter);
 int iavf_config_irq_map(struct iavf_adapter *adapter);
+int iavf_config_irq_map_lv(struct iavf_adapter *adapter);
 void iavf_add_del_all_mac_addr(struct iavf_adapter *adapter, bool add);
 int iavf_dev_link_update(struct rte_eth_dev *dev,
 			__rte_unused int wait_to_complete);
@@ -296,4 +308,5 @@ int iavf_add_del_mc_addr_list(struct iavf_adapter *adapter,
 			struct rte_ether_addr *mc_addrs,
 			uint32_t mc_addrs_num, bool add);
 int iavf_request_queues(struct rte_eth_dev *dev, uint16_t num);
+int iavf_get_max_rss_queue_region(struct iavf_adapter *adapter);
 #endif /* _IAVF_ETHDEV_H_ */
diff --git a/drivers/net/iavf/iavf_ethdev.c b/drivers/net/iavf/iavf_ethdev.c
index 8fd9b44db..bed2a43e8 100644
--- a/drivers/net/iavf/iavf_ethdev.c
+++ b/drivers/net/iavf/iavf_ethdev.c
@@ -162,7 +162,7 @@ iavf_init_rss(struct iavf_adapter *adapter)
 
 	rss_conf = &adapter->eth_dev->data->dev_conf.rx_adv_conf.rss_conf;
 	nb_q = RTE_MIN(adapter->eth_dev->data->nb_rx_queues,
-		       IAVF_MAX_NUM_QUEUES);
+		       IAVF_MAX_NUM_QUEUES_LV);
 
 	if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) {
 		PMD_DRV_LOG(DEBUG, "RSS is not supported");
@@ -215,6 +215,9 @@ iavf_dev_configure(struct rte_eth_dev *dev)
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct iavf_info *vf =  IAVF_DEV_PRIVATE_TO_VF(ad);
 	struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
+	uint16_t num_queue_pairs = RTE_MAX(dev->data->nb_rx_queues,
+		dev->data->nb_tx_queues);
+	int ret = 0;
 
 	ad->rx_bulk_alloc_allowed = true;
 	/* Initialize to TRUE. If any of Rx queues doesn't meet the
@@ -226,6 +229,29 @@ iavf_dev_configure(struct rte_eth_dev *dev)
 	if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)
 		dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_RSS_HASH;
 
+	/* Large VF setting */
+	if (num_queue_pairs > IAVF_MAX_NUM_QUEUES_DFLT) {
+		if (!(vf->vf_res->vf_cap_flags &
+				VIRTCHNL_VF_LARGE_NUM_QPAIRS)) {
+			PMD_DRV_LOG(ERR, "large VF is not supported");
+			return -1;
+		}
+
+		ret = iavf_request_queues(dev, num_queue_pairs);
+		if (ret != 0) {
+			PMD_DRV_LOG(ERR, "request queues from PF failed");
+			return ret;
+		}
+		PMD_DRV_LOG(INFO, "change queue pairs from %u to %u",
+				vf->vsi_res->num_queue_pairs, num_queue_pairs);
+
+		ret = iavf_dev_reset(dev);
+		if (ret != 0)
+			return ret;
+
+		vf->lv_enabled = true;
+	}
+
 	/* Vlan stripping setting */
 	if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN) {
 		if (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
@@ -240,6 +266,7 @@ iavf_dev_configure(struct rte_eth_dev *dev)
 			return -1;
 		}
 	}
+
 	return 0;
 }
 
@@ -322,6 +349,7 @@ static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev,
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter);
+	struct iavf_qv_map *qv_map;
 	uint16_t interval, i;
 	int vec;
 
@@ -342,6 +370,14 @@ static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev,
 		}
 	}
 
+	qv_map = rte_zmalloc("qv_map",
+		dev->data->nb_rx_queues * sizeof(struct iavf_qv_map), 0);
+	if (!qv_map) {
+		PMD_DRV_LOG(ERR, "Failed to allocate %d queue-vector map",
+				dev->data->nb_rx_queues);
+		return -1;
+	}
+
 	if (!dev->data->dev_conf.intr_conf.rxq ||
 	    !rte_intr_dp_is_en(intr_handle)) {
 		/* Rx interrupt disabled, Map interrupt only for writeback */
@@ -372,16 +408,21 @@ static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev,
 		}
 		IAVF_WRITE_FLUSH(hw);
 		/* map all queues to the same interrupt */
-		for (i = 0; i < dev->data->nb_rx_queues; i++)
-			vf->rxq_map[vf->msix_base] |= 1 << i;
+		for (i = 0; i < dev->data->nb_rx_queues; i++) {
+			qv_map[i].queue_id = i;
+			qv_map[i].vector_id = vf->msix_base;
+		}
+		vf->qv_map = qv_map;
 	} else {
 		if (!rte_intr_allow_others(intr_handle)) {
 			vf->nb_msix = 1;
 			vf->msix_base = IAVF_MISC_VEC_ID;
 			for (i = 0; i < dev->data->nb_rx_queues; i++) {
-				vf->rxq_map[vf->msix_base] |= 1 << i;
+				qv_map[i].queue_id = i;
+				qv_map[i].vector_id = vf->msix_base;
 				intr_handle->intr_vec[i] = IAVF_MISC_VEC_ID;
 			}
+			vf->qv_map = qv_map;
 			PMD_DRV_LOG(DEBUG,
 				    "vector %u are mapping to all Rx queues",
 				    vf->msix_base);
@@ -394,21 +435,32 @@ static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev,
 			vf->msix_base = IAVF_RX_VEC_START;
 			vec = IAVF_RX_VEC_START;
 			for (i = 0; i < dev->data->nb_rx_queues; i++) {
-				vf->rxq_map[vec] |= 1 << i;
+				qv_map[i].queue_id = i;
+				qv_map[i].vector_id = vec;
 				intr_handle->intr_vec[i] = vec++;
 				if (vec >= vf->nb_msix)
 					vec = IAVF_RX_VEC_START;
 			}
+			vf->qv_map = qv_map;
 			PMD_DRV_LOG(DEBUG,
 				    "%u vectors are mapping to %u Rx queues",
 				    vf->nb_msix, dev->data->nb_rx_queues);
 		}
 	}
 
-	if (iavf_config_irq_map(adapter)) {
-		PMD_DRV_LOG(ERR, "config interrupt mapping failed");
-		return -1;
+	if (!vf->lv_enabled) {
+		if (iavf_config_irq_map(adapter)) {
+			PMD_DRV_LOG(ERR, "config interrupt mapping failed");
+			return -1;
+		}
+	} else {
+		if (iavf_config_irq_map_lv(adapter)) {
+			PMD_DRV_LOG(ERR, "config interrupt mapping "
+				"for large VF failed");
+			return -1;
+		}
 	}
+
 	return 0;
 }
 
@@ -472,6 +524,7 @@ iavf_dev_start(struct rte_eth_dev *dev)
 		PMD_DRV_LOG(ERR, "configure irq failed");
 		goto err_queue;
 	}
+
 	/* re-enable intr again, because efd assign may change */
 	if (dev->data->dev_conf.intr_conf.rxq != 0) {
 		rte_intr_disable(intr_handle);
@@ -536,8 +589,8 @@ iavf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 {
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 
-	dev_info->max_rx_queues = vf->vsi_res->num_queue_pairs;
-	dev_info->max_tx_queues = vf->vsi_res->num_queue_pairs;
+	dev_info->max_rx_queues = IAVF_MAX_NUM_QUEUES_LV;
+	dev_info->max_tx_queues = IAVF_MAX_NUM_QUEUES_LV;
 	dev_info->min_rx_bufsize = IAVF_BUF_SIZE_MIN;
 	dev_info->max_rx_pktlen = IAVF_FRAME_SIZE_MAX;
 	dev_info->hash_key_size = vf->vf_res->rss_key_size;
@@ -1266,6 +1319,12 @@ iavf_init_vf(struct rte_eth_dev *dev)
 		PMD_INIT_LOG(ERR, "iavf_get_vf_config failed");
 		goto err_alloc;
 	}
+
+	if (iavf_get_max_rss_queue_region(adapter) != 0) {
+		PMD_INIT_LOG(ERR, "get max rss queue region failed");
+		goto err_alloc;
+	}
+
 	/* Allocate memort for RSS info */
 	if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) {
 		vf->rss_key = rte_zmalloc("rss_key",
diff --git a/drivers/net/iavf/iavf_rxtx.c b/drivers/net/iavf/iavf_rxtx.c
index 05a7dd898..86004ac90 100644
--- a/drivers/net/iavf/iavf_rxtx.c
+++ b/drivers/net/iavf/iavf_rxtx.c
@@ -532,6 +532,7 @@ iavf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 	struct iavf_adapter *adapter =
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	struct iavf_rx_queue *rxq;
 	int err = 0;
 
@@ -555,7 +556,11 @@ iavf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 	IAVF_WRITE_FLUSH(hw);
 
 	/* Ready to switch the queue on */
-	err = iavf_switch_queue(adapter, rx_queue_id, true, true);
+	if (!vf->lv_enabled)
+		err = iavf_switch_queue(adapter, rx_queue_id, true, true);
+	else
+		err = iavf_switch_queue_lv(adapter, rx_queue_id, true, true);
+
 	if (err)
 		PMD_DRV_LOG(ERR, "Failed to switch RX queue %u on",
 			    rx_queue_id);
@@ -572,6 +577,7 @@ iavf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 	struct iavf_adapter *adapter =
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	struct iavf_tx_queue *txq;
 	int err = 0;
 
@@ -587,7 +593,10 @@ iavf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 	IAVF_WRITE_FLUSH(hw);
 
 	/* Ready to switch the queue on */
-	err = iavf_switch_queue(adapter, tx_queue_id, false, true);
+	if (!vf->lv_enabled)
+		err = iavf_switch_queue(adapter, tx_queue_id, false, true);
+	else
+		err = iavf_switch_queue_lv(adapter, tx_queue_id, false, true);
 
 	if (err)
 		PMD_DRV_LOG(ERR, "Failed to switch TX queue %u on",
@@ -688,14 +697,22 @@ iavf_stop_queues(struct rte_eth_dev *dev)
 {
 	struct iavf_adapter *adapter =
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	struct iavf_rx_queue *rxq;
 	struct iavf_tx_queue *txq;
 	int ret, i;
 
 	/* Stop All queues */
-	ret = iavf_disable_queues(adapter);
-	if (ret)
-		PMD_DRV_LOG(WARNING, "Fail to stop queues");
+	if (!vf->lv_enabled) {
+		ret = iavf_disable_queues(adapter);
+		if (ret)
+			PMD_DRV_LOG(WARNING, "Fail to stop queues");
+	} else {
+		ret = iavf_disable_queues_lv(adapter);
+		if (ret)
+			PMD_DRV_LOG(WARNING, "Fail to stop queues for large VF");
+	}
+
 
 	for (i = 0; i < dev->data->nb_tx_queues; i++) {
 		txq = dev->data->tx_queues[i];
diff --git a/drivers/net/iavf/iavf_vchnl.c b/drivers/net/iavf/iavf_vchnl.c
index bd39f0db6..8a85a81f8 100644
--- a/drivers/net/iavf/iavf_vchnl.c
+++ b/drivers/net/iavf/iavf_vchnl.c
@@ -130,6 +130,7 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter, struct iavf_cmd_info *args)
 	case VIRTCHNL_OP_VERSION:
 	case VIRTCHNL_OP_GET_VF_RESOURCES:
 	case VIRTCHNL_OP_GET_SUPPORTED_RXDIDS:
+	case VIRTCHNL_OP_GET_MAX_RSS_QREGION:
 		/* for init virtchnl ops, need to poll the response */
 		do {
 			result = iavf_read_msg_from_pf(adapter, args->out_size,
@@ -449,7 +450,8 @@ iavf_get_vf_resource(struct iavf_adapter *adapter)
 	caps = IAVF_BASIC_OFFLOAD_CAPS | VIRTCHNL_VF_CAP_ADV_LINK_SPEED |
 		VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC |
 		VIRTCHNL_VF_OFFLOAD_FDIR_PF |
-		VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF;
+		VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF |
+		VIRTCHNL_VF_LARGE_NUM_QPAIRS;
 
 	args.in_args = (uint8_t *)&caps;
 	args.in_args_size = sizeof(caps);
@@ -599,6 +601,138 @@ iavf_switch_queue(struct iavf_adapter *adapter, uint16_t qid,
 	return err;
 }
 
+int
+iavf_enable_queues_lv(struct iavf_adapter *adapter)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct virtchnl_del_ena_dis_queues *queue_select;
+	struct virtchnl_queue_chunk *queue_chunk;
+	struct iavf_cmd_info args;
+	int err, len;
+
+	len = sizeof(struct virtchnl_del_ena_dis_queues) +
+		  sizeof(struct virtchnl_queue_chunk) *
+		  (IAVF_RXTX_QUEUE_CHUNKS_NUM - 1);
+	queue_select = rte_zmalloc("queue_select", len, 0);
+	if (!queue_select)
+		return -ENOMEM;
+
+	queue_chunk = queue_select->chunks.chunks;
+	queue_select->chunks.num_chunks = IAVF_RXTX_QUEUE_CHUNKS_NUM;
+	queue_select->vport_id = vf->vsi_res->vsi_id;
+
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].type = VIRTCHNL_QUEUE_TYPE_TX;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].start_queue_id = 0;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].num_queues =
+		adapter->eth_dev->data->nb_tx_queues;
+
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].type = VIRTCHNL_QUEUE_TYPE_RX;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].start_queue_id = 0;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].num_queues =
+		adapter->eth_dev->data->nb_rx_queues;
+
+	args.ops = VIRTCHNL_OP_ENABLE_QUEUES_V2;
+	args.in_args = (u8 *)queue_select;
+	args.in_args_size = len;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err) {
+		PMD_DRV_LOG(ERR,
+			    "Failed to execute command of OP_ENABLE_QUEUES_V2");
+		return err;
+	}
+	return 0;
+}
+
+int
+iavf_disable_queues_lv(struct iavf_adapter *adapter)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct virtchnl_del_ena_dis_queues *queue_select;
+	struct virtchnl_queue_chunk *queue_chunk;
+	struct iavf_cmd_info args;
+	int err, len;
+
+	len = sizeof(struct virtchnl_del_ena_dis_queues) +
+		  sizeof(struct virtchnl_queue_chunk) *
+		  (IAVF_RXTX_QUEUE_CHUNKS_NUM - 1);
+	queue_select = rte_zmalloc("queue_select", len, 0);
+	if (!queue_select)
+		return -ENOMEM;
+
+	queue_chunk = queue_select->chunks.chunks;
+	queue_select->chunks.num_chunks = IAVF_RXTX_QUEUE_CHUNKS_NUM;
+	queue_select->vport_id = vf->vsi_res->vsi_id;
+
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].type = VIRTCHNL_QUEUE_TYPE_TX;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].start_queue_id = 0;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].num_queues =
+		adapter->eth_dev->data->nb_tx_queues;
+
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].type = VIRTCHNL_QUEUE_TYPE_RX;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].start_queue_id = 0;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].num_queues =
+		adapter->eth_dev->data->nb_rx_queues;
+
+	args.ops = VIRTCHNL_OP_DISABLE_QUEUES_V2;
+	args.in_args = (u8 *)queue_select;
+	args.in_args_size = len;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err) {
+		PMD_DRV_LOG(ERR,
+			    "Failed to execute command of OP_DISABLE_QUEUES_V2");
+		return err;
+	}
+	return 0;
+}
+
+int
+iavf_switch_queue_lv(struct iavf_adapter *adapter, uint16_t qid,
+		 bool rx, bool on)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct virtchnl_del_ena_dis_queues *queue_select;
+	struct virtchnl_queue_chunk *queue_chunk;
+	struct iavf_cmd_info args;
+	int err, len;
+
+	len = sizeof(struct virtchnl_del_ena_dis_queues);
+	queue_select = rte_zmalloc("queue_select", len, 0);
+	if (!queue_select)
+		return -ENOMEM;
+
+	queue_chunk = queue_select->chunks.chunks;
+	queue_select->chunks.num_chunks = 1;
+	queue_select->vport_id = vf->vsi_res->vsi_id;
+
+	if (rx) {
+		queue_chunk->type = VIRTCHNL_QUEUE_TYPE_RX;
+		queue_chunk->start_queue_id = qid;
+		queue_chunk->num_queues = 1;
+	} else {
+		queue_chunk->type = VIRTCHNL_QUEUE_TYPE_TX;
+		queue_chunk->start_queue_id = qid;
+		queue_chunk->num_queues = 1;
+	}
+
+	if (on)
+		args.ops = VIRTCHNL_OP_ENABLE_QUEUES_V2;
+	else
+		args.ops = VIRTCHNL_OP_DISABLE_QUEUES_V2;
+	args.in_args = (u8 *)queue_select;
+	args.in_args_size = len;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err)
+		PMD_DRV_LOG(ERR, "Failed to execute command of %s",
+			    on ? "OP_ENABLE_QUEUES_V2" : "OP_DISABLE_QUEUES_V2");
+	return err;
+}
+
 int
 iavf_configure_rss_lut(struct iavf_adapter *adapter)
 {
@@ -663,32 +797,26 @@ iavf_configure_rss_key(struct iavf_adapter *adapter)
 	return err;
 }
 
-int
-iavf_configure_queues(struct iavf_adapter *adapter)
+static int
+iavf_exec_queue_cfg(struct iavf_adapter *adapter,
+	struct virtchnl_vsi_queue_config_info *vc_config, uint16_t count)
 {
 	struct iavf_rx_queue **rxq =
 		(struct iavf_rx_queue **)adapter->eth_dev->data->rx_queues;
 	struct iavf_tx_queue **txq =
 		(struct iavf_tx_queue **)adapter->eth_dev->data->tx_queues;
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
-	struct virtchnl_vsi_queue_config_info *vc_config;
 	struct virtchnl_queue_pair_info *vc_qp;
 	struct iavf_cmd_info args;
 	uint16_t i, size;
-	int err;
+	int err = 0;
 
 	size = sizeof(*vc_config) +
-	       sizeof(vc_config->qpair[0]) * vf->num_queue_pairs;
-	vc_config = rte_zmalloc("cfg_queue", size, 0);
-	if (!vc_config)
-		return -ENOMEM;
+		sizeof(vc_config->qpair[0]) * vc_config->num_queue_pairs;
 
-	vc_config->vsi_id = vf->vsi_res->vsi_id;
-	vc_config->num_queue_pairs = vf->num_queue_pairs;
-
-	for (i = 0, vc_qp = vc_config->qpair;
-	     i < vf->num_queue_pairs;
-	     i++, vc_qp++) {
+	for (i = count * IAVF_CFG_Q_NUM_PER_BUF, vc_qp = vc_config->qpair;
+	     i < count * IAVF_CFG_Q_NUM_PER_BUF + vc_config->num_queue_pairs;
+		 i++, vc_qp++) {
 		vc_qp->txq.vsi_id = vf->vsi_res->vsi_id;
 		vc_qp->txq.queue_id = i;
 		/* Virtchnnl configure queues by pairs */
@@ -742,8 +870,71 @@ iavf_configure_queues(struct iavf_adapter *adapter)
 	err = iavf_execute_vf_cmd(adapter, &args);
 	if (err)
 		PMD_DRV_LOG(ERR, "Failed to execute command of"
-			    " VIRTCHNL_OP_CONFIG_VSI_QUEUES");
+				" VIRTCHNL_OP_CONFIG_VSI_QUEUES");
+
+	return err;
+}
+
+/* Configure VSI queues. Max VF queue pairs number is 256, may
+ * send this command multiple times to configure all queues.
+ */
+int
+iavf_configure_queues(struct iavf_adapter *adapter)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct virtchnl_vsi_queue_config_info *vc_config = NULL;
+	uint16_t i, size_full, size_inc;
+	uint16_t nb_cmd_full, nbq_inc;
+	int err = 0;
+
+	/* Compute how many times should the command to be sent,
+	 * including the commands with full buffer and incomplete
+	 * buffer.
+	 */
+	nbq_inc = vf->num_queue_pairs % IAVF_CFG_Q_NUM_PER_BUF;
+	nb_cmd_full = vf->num_queue_pairs / IAVF_CFG_Q_NUM_PER_BUF;
+
+	size_full = sizeof(*vc_config) +
+		sizeof(vc_config->qpair[0]) * IAVF_CFG_Q_NUM_PER_BUF;
+	size_inc = sizeof(*vc_config) +
+		sizeof(vc_config->qpair[0]) * nbq_inc;
+
+	if (!nb_cmd_full) {
+		vc_config = rte_zmalloc("cfg_queue", size_inc, 0);
+		if (!vc_config)
+			return -ENOMEM;
+
+		vc_config->vsi_id = vf->vsi_res->vsi_id;
+		vc_config->num_queue_pairs = nbq_inc;
+		err = iavf_exec_queue_cfg(adapter, vc_config, 0);
+		goto free;
+	}
+
+	vc_config = rte_zmalloc("cfg_queue", size_full, 0);
+	if (!vc_config)
+		return -ENOMEM;
 
+	vc_config->vsi_id = vf->vsi_res->vsi_id;
+	vc_config->num_queue_pairs = IAVF_CFG_Q_NUM_PER_BUF;
+
+	for (i = 0; i < nb_cmd_full + (nbq_inc ? 1 : 0); i++) {
+		if (i >= nb_cmd_full) {
+			/* re-allocate virtchnl msg for less queues */
+			rte_free(vc_config);
+			vc_config = rte_zmalloc("cfg_queue", size_inc, 0);
+			if (!vc_config)
+				return -ENOMEM;
+
+			vc_config->vsi_id = vf->vsi_res->vsi_id;
+			vc_config->num_queue_pairs = nbq_inc;
+		}
+
+		err = iavf_exec_queue_cfg(adapter, vc_config, i);
+		if (err)
+			break;
+	}
+
+free:
 	rte_free(vc_config);
 	return err;
 }
@@ -765,13 +956,14 @@ iavf_config_irq_map(struct iavf_adapter *adapter)
 		return -ENOMEM;
 
 	map_info->num_vectors = vf->nb_msix;
-	for (i = 0; i < vf->nb_msix; i++) {
-		vecmap = &map_info->vecmap[i];
+	for (i = 0; i < adapter->eth_dev->data->nb_rx_queues; i++) {
+		vecmap =
+		    &map_info->vecmap[vf->qv_map[i].vector_id - vf->msix_base];
 		vecmap->vsi_id = vf->vsi_res->vsi_id;
 		vecmap->rxitr_idx = IAVF_ITR_INDEX_DEFAULT;
-		vecmap->vector_id = vf->msix_base + i;
+		vecmap->vector_id = vf->qv_map[i].vector_id;
 		vecmap->txq_map = 0;
-		vecmap->rxq_map = vf->rxq_map[vf->msix_base + i];
+		vecmap->rxq_map |= 1 << vf->qv_map[i].queue_id;
 	}
 
 	args.ops = VIRTCHNL_OP_CONFIG_IRQ_MAP;
@@ -787,6 +979,46 @@ iavf_config_irq_map(struct iavf_adapter *adapter)
 	return err;
 }
 
+int
+iavf_config_irq_map_lv(struct iavf_adapter *adapter)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct virtchnl_queue_vector_maps *map_info;
+	struct virtchnl_queue_vector *qv_maps;
+	struct iavf_cmd_info args;
+	int len, i, err;
+
+	len = sizeof(struct virtchnl_queue_vector_maps) +
+	      sizeof(struct virtchnl_queue_vector) *
+	      (adapter->eth_dev->data->nb_rx_queues - 1);
+
+	map_info = rte_zmalloc("map_info", len, 0);
+	if (!map_info)
+		return -ENOMEM;
+
+	map_info->vport_id = vf->vsi_res->vsi_id;
+	map_info->num_qv_maps = adapter->eth_dev->data->nb_rx_queues;
+	for (i = 0; i < map_info->num_qv_maps; i++) {
+		qv_maps = &map_info->qv_maps[i];
+		qv_maps->itr_idx = VIRTCHNL_ITR_IDX_0;
+		qv_maps->queue_type = VIRTCHNL_QUEUE_TYPE_RX;
+		qv_maps->queue_id = vf->qv_map[i].queue_id;
+		qv_maps->vector_id = vf->qv_map[i].vector_id;
+	}
+
+	args.ops = VIRTCHNL_OP_MAP_QUEUE_VECTOR;
+	args.in_args = (u8 *)map_info;
+	args.in_args_size = len;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err)
+		PMD_DRV_LOG(ERR, "fail to execute command OP_MAP_QUEUE_VECTOR");
+
+	rte_free(map_info);
+	return err;
+}
+
 void
 iavf_add_del_all_mac_addr(struct iavf_adapter *adapter, bool add)
 {
@@ -1246,3 +1478,33 @@ iavf_request_queues(struct rte_eth_dev *dev, uint16_t num)
 
 	return -1;
 }
+
+int
+iavf_get_max_rss_queue_region(struct iavf_adapter *adapter)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct iavf_cmd_info args;
+	uint16_t qregion_width;
+	int err;
+
+	args.ops = VIRTCHNL_OP_GET_MAX_RSS_QREGION;
+	args.in_args = NULL;
+	args.in_args_size = 0;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err) {
+		PMD_DRV_LOG(ERR,
+			    "Failed to execute command of "
+			    "VIRTCHNL_OP_GET_MAX_RSS_QREGION");
+		return err;
+	}
+
+	qregion_width =
+	((struct virtchnl_max_rss_qregion *)args.out_buffer)->qregion_width;
+
+	vf->max_rss_qregion = (uint16_t)(1 << qregion_width);
+
+	return 0;
+}
-- 
2.17.1


^ permalink raw reply	[flat|nested] 45+ messages in thread

* [dpdk-dev] [PATCH v3 0/2] enable large VF configuration
  2020-09-09  7:20 [dpdk-dev] [PATCH v1 0/2] enable large VF configuration Ting Xu
                   ` (2 preceding siblings ...)
  2020-09-25  5:59 ` [dpdk-dev] [PATCH v2 0/2] " Ting Xu
@ 2020-09-27 12:42 ` Ting Xu
  2020-09-27 12:42   ` [dpdk-dev] [PATCH v3 1/2] net/iavf: add IAVF request queues function Ting Xu
  2020-09-27 12:42   ` [dpdk-dev] [PATCH v3 2/2] net/iavf: enable large VF configuration Ting Xu
  2020-10-15  5:21 ` [dpdk-dev] [PATCH v4 0/2] " Ting Xu
                   ` (5 subsequent siblings)
  9 siblings, 2 replies; 45+ messages in thread
From: Ting Xu @ 2020-09-27 12:42 UTC (permalink / raw)
  To: dev; +Cc: qi.z.zhang, jingjing.wu, beilei.xing, Ting Xu

This patchset supports to configure up to 256 queue pairs per VF. If
large VF is supported after capability negotiation, VF will request
queues from PF as needed. New virtual channel opcodes and structures
are used to indicate 256 queues, so VF is designed to handle the new
function of configure VSI queues, IRQ mapping and enable/disable queues.
Also, enable VF to query the max RSS queue region.

Ting Xu (2):
  net/iavf: add IAVF request queues function
  net/iavf: enable large VF configuration

---
v2->v3:
Fix coding style issue. Modify based on comments

v1->v2:
Change the communication with kernel PF

---
 drivers/net/iavf/iavf.h        |  40 ++-
 drivers/net/iavf/iavf_ethdev.c | 102 +++++++-
 drivers/net/iavf/iavf_rxtx.c   |  27 +-
 drivers/net/iavf/iavf_vchnl.c  | 444 ++++++++++++++++++++++++++++++---
 4 files changed, 558 insertions(+), 55 deletions(-)

-- 
2.17.1


^ permalink raw reply	[flat|nested] 45+ messages in thread

* [dpdk-dev] [PATCH v3 1/2] net/iavf: add IAVF request queues function
  2020-09-27 12:42 ` [dpdk-dev] [PATCH v3 0/2] " Ting Xu
@ 2020-09-27 12:42   ` Ting Xu
  2020-09-27 12:42   ` [dpdk-dev] [PATCH v3 2/2] net/iavf: enable large VF configuration Ting Xu
  1 sibling, 0 replies; 45+ messages in thread
From: Ting Xu @ 2020-09-27 12:42 UTC (permalink / raw)
  To: dev; +Cc: qi.z.zhang, jingjing.wu, beilei.xing, Ting Xu

Add new virtchnl function to request additional queues from PF. Current
default queue pairs number when creating a VF is 16. In order to support
up to 256 queue pairs, enable this request queues function.
Since request queues command may return event message, modify function
iavf_read_msg_from_pf to identify event opcode and mark VF reset status.

Signed-off-by: Ting Xu <ting.xu@intel.com>
---
 drivers/net/iavf/iavf.h        |  17 ++++
 drivers/net/iavf/iavf_ethdev.c |  12 ++-
 drivers/net/iavf/iavf_vchnl.c  | 145 ++++++++++++++++++++++++++++++---
 3 files changed, 158 insertions(+), 16 deletions(-)

diff --git a/drivers/net/iavf/iavf.h b/drivers/net/iavf/iavf.h
index d56611608..1c40f9bdf 100644
--- a/drivers/net/iavf/iavf.h
+++ b/drivers/net/iavf/iavf.h
@@ -107,6 +107,21 @@ struct iavf_fdir_info {
 /* TODO: is that correct to assume the max number to be 16 ?*/
 #define IAVF_MAX_MSIX_VECTORS   16
 
+/* Event status from PF */
+enum pending_msg {
+	PFMSG_LINK_CHANGE = 0x1,
+	PFMSG_RESET_IMPENDING = 0x2,
+	PFMSG_DRIVER_CLOSE = 0x4,
+};
+
+/* Message type read in admin queue from PF */
+enum iavf_aq_result {
+	IAVF_MSG_ERR = -1, /* Meet error when accessing admin queue */
+	IAVF_MSG_NON,      /* Read nothing from admin queue */
+	IAVF_MSG_SYS,      /* Read system msg from admin queue */
+	IAVF_MSG_CMD,      /* Read async command result */
+};
+
 /* Structure to store private data specific for VF instance. */
 struct iavf_info {
 	uint16_t num_queue_pairs;
@@ -123,6 +138,7 @@ struct iavf_info {
 	volatile enum virtchnl_ops pend_cmd; /* pending command not finished */
 	uint32_t cmd_retval; /* return value of the cmd response from PF */
 	uint8_t *aq_resp; /* buffer to store the adminq response from PF */
+	uint16_t pend_msg; /* flags indicates events from pf not handled yet */
 
 	/* Event from pf */
 	bool dev_closed;
@@ -301,4 +317,5 @@ int iavf_add_del_rss_cfg(struct iavf_adapter *adapter,
 int iavf_add_del_mc_addr_list(struct iavf_adapter *adapter,
 			struct rte_ether_addr *mc_addrs,
 			uint32_t mc_addrs_num, bool add);
+int iavf_request_queues(struct rte_eth_dev *dev, uint16_t num);
 #endif /* _IAVF_ETHDEV_H_ */
diff --git a/drivers/net/iavf/iavf_ethdev.c b/drivers/net/iavf/iavf_ethdev.c
index a88d53ab0..33745a7b2 100644
--- a/drivers/net/iavf/iavf_ethdev.c
+++ b/drivers/net/iavf/iavf_ethdev.c
@@ -1236,7 +1236,7 @@ iavf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
 }
 
 static int
-iavf_check_vf_reset_done(struct iavf_hw *hw)
+iavf_check_vf_reset_done(struct iavf_hw *hw, struct iavf_info *vf)
 {
 	int i, reset;
 
@@ -1253,6 +1253,10 @@ iavf_check_vf_reset_done(struct iavf_hw *hw)
 	if (i >= IAVF_RESET_WAIT_CNT)
 		return -1;
 
+	/* VF is not in reset or reset is completed */
+	vf->vf_reset = false;
+	vf->pend_msg &= ~PFMSG_RESET_IMPENDING;
+
 	return 0;
 }
 
@@ -1620,7 +1624,7 @@ iavf_init_vf(struct rte_eth_dev *dev)
 		goto err;
 	}
 
-	err = iavf_check_vf_reset_done(hw);
+	err = iavf_check_vf_reset_done(hw, vf);
 	if (err) {
 		PMD_INIT_LOG(ERR, "VF is still resetting");
 		goto err;
@@ -1869,7 +1873,9 @@ iavf_dev_close(struct rte_eth_dev *dev)
 
 	iavf_dev_stop(dev);
 	iavf_flow_flush(dev, NULL);
-	iavf_flow_uninit(adapter);
+	/* if VF is in reset, adminq is disabled, skip the process via adminq */
+	if (!vf->vf_reset)
+		iavf_flow_uninit(adapter);
 	iavf_shutdown_adminq(hw);
 	/* disable uio intr before callback unregister */
 	rte_intr_disable(intr_handle);
diff --git a/drivers/net/iavf/iavf_vchnl.c b/drivers/net/iavf/iavf_vchnl.c
index 7981dfa30..b1149ef4d 100644
--- a/drivers/net/iavf/iavf_vchnl.c
+++ b/drivers/net/iavf/iavf_vchnl.c
@@ -17,6 +17,7 @@
 #include <rte_eal.h>
 #include <rte_ether.h>
 #include <rte_ethdev_driver.h>
+#include <rte_ethdev_pci.h>
 #include <rte_dev.h>
 
 #include "iavf.h"
@@ -26,13 +27,14 @@
 #define ASQ_DELAY_MS  10
 
 /* Read data in admin queue to get msg from pf driver */
-static enum iavf_status
+static enum iavf_aq_result
 iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,
 		     uint8_t *buf)
 {
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter);
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
 	struct iavf_arq_event_info event;
+	enum iavf_aq_result result = IAVF_MSG_NON;
 	enum virtchnl_ops opcode;
 	int ret;
 
@@ -42,7 +44,9 @@ iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,
 	/* Can't read any msg from adminQ */
 	if (ret) {
 		PMD_DRV_LOG(DEBUG, "Can't read msg from AQ");
-		return ret;
+		if (ret != IAVF_ERR_ADMIN_QUEUE_NO_WORK)
+			result = IAVF_MSG_ERR;
+		return result;
 	}
 
 	opcode = (enum virtchnl_ops)rte_le_to_cpu_32(event.desc.cookie_high);
@@ -52,16 +56,46 @@ iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,
 	PMD_DRV_LOG(DEBUG, "AQ from pf carries opcode %u, retval %d",
 		    opcode, vf->cmd_retval);
 
-	if (opcode != vf->pend_cmd) {
-		if (opcode != VIRTCHNL_OP_EVENT) {
-			PMD_DRV_LOG(WARNING,
-				    "command mismatch, expect %u, get %u",
-				    vf->pend_cmd, opcode);
+	if (opcode == VIRTCHNL_OP_EVENT) {
+		struct virtchnl_pf_event *vpe =
+			(struct virtchnl_pf_event *)event.msg_buf;
+
+		result = IAVF_MSG_SYS;
+		switch (vpe->event) {
+		case VIRTCHNL_EVENT_LINK_CHANGE:
+			vf->link_up =
+				vpe->event_data.link_event.link_status;
+			vf->link_speed =
+				vpe->event_data.link_event.link_speed;
+			vf->pend_msg |= PFMSG_LINK_CHANGE;
+			PMD_DRV_LOG(INFO, "Link status update:%s",
+					vf->link_up ? "up" : "down");
+			break;
+		case VIRTCHNL_EVENT_RESET_IMPENDING:
+			vf->vf_reset = true;
+			vf->pend_msg |= PFMSG_RESET_IMPENDING;
+			PMD_DRV_LOG(INFO, "vf is resetting");
+			break;
+		case VIRTCHNL_EVENT_PF_DRIVER_CLOSE:
+			vf->dev_closed = true;
+			vf->pend_msg |= PFMSG_DRIVER_CLOSE;
+			PMD_DRV_LOG(INFO, "PF driver closed");
+			break;
+		default:
+			PMD_DRV_LOG(ERR, "%s: Unknown event %d from pf",
+					__func__, vpe->event);
+		}
+	}  else {
+		/* async reply msg on command issued by vf previously */
+		result = IAVF_MSG_CMD;
+		if (opcode != vf->pend_cmd) {
+			PMD_DRV_LOG(WARNING, "command mismatch, expect %u, get %u",
+					vf->pend_cmd, opcode);
+			result = IAVF_MSG_ERR;
 		}
-		return IAVF_ERR_OPCODE_MISMATCH;
 	}
 
-	return IAVF_SUCCESS;
+	return result;
 }
 
 static int
@@ -69,6 +103,7 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter, struct iavf_cmd_info *args)
 {
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter);
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	enum iavf_aq_result result;
 	enum iavf_status ret;
 	int err = 0;
 	int i = 0;
@@ -97,9 +132,9 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter, struct iavf_cmd_info *args)
 	case VIRTCHNL_OP_GET_SUPPORTED_RXDIDS:
 		/* for init virtchnl ops, need to poll the response */
 		do {
-			ret = iavf_read_msg_from_pf(adapter, args->out_size,
+			result = iavf_read_msg_from_pf(adapter, args->out_size,
 						   args->out_buffer);
-			if (ret == IAVF_SUCCESS)
+			if (result == IAVF_MSG_CMD)
 				break;
 			rte_delay_ms(ASQ_DELAY_MS);
 		} while (i++ < MAX_TRY_TIMES);
@@ -111,7 +146,33 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter, struct iavf_cmd_info *args)
 		}
 		_clear_cmd(vf);
 		break;
-
+	case VIRTCHNL_OP_REQUEST_QUEUES:
+		/*
+		 * ignore async reply, only wait for system message,
+		 * vf_reset = true if get VIRTCHNL_EVENT_RESET_IMPENDING,
+		 * if not, means request queues failed.
+		 */
+		do {
+			result = iavf_read_msg_from_pf(adapter, args->out_size,
+						   args->out_buffer);
+			if (result == IAVF_MSG_SYS && vf->vf_reset) {
+				break;
+			} else if (result == IAVF_MSG_CMD ||
+				result == IAVF_MSG_ERR) {
+				err = -1;
+				break;
+			}
+			rte_delay_ms(ASQ_DELAY_MS);
+			/* If don't read msg or read sys event, continue */
+		} while (i++ < MAX_TRY_TIMES);
+		if (i >= MAX_TRY_TIMES ||
+		    vf->cmd_retval != VIRTCHNL_STATUS_SUCCESS) {
+			err = -1;
+			PMD_DRV_LOG(ERR, "No response or return failure (%d)"
+				    " for cmd %d", vf->cmd_retval, args->ops);
+		}
+		_clear_cmd(vf);
+		break;
 	default:
 		/* For other virtchnl ops in running time,
 		 * wait for the cmd done flag.
@@ -388,7 +449,8 @@ iavf_get_vf_resource(struct iavf_adapter *adapter)
 	caps = IAVF_BASIC_OFFLOAD_CAPS | VIRTCHNL_VF_CAP_ADV_LINK_SPEED |
 		VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC |
 		VIRTCHNL_VF_OFFLOAD_FDIR_PF |
-		VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF;
+		VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF |
+		VIRTCHNL_VF_OFFLOAD_REQ_QUEUES;
 
 	args.in_args = (uint8_t *)&caps;
 	args.in_args_size = sizeof(caps);
@@ -1138,3 +1200,60 @@ iavf_add_del_mc_addr_list(struct iavf_adapter *adapter,
 
 	return 0;
 }
+
+int
+iavf_request_queues(struct rte_eth_dev *dev, uint16_t num)
+{
+	struct iavf_adapter *ad =
+		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
+	struct iavf_info *vf =  IAVF_DEV_PRIVATE_TO_VF(ad);
+	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+	struct virtchnl_vf_res_request vfres;
+	struct iavf_cmd_info args;
+	uint16_t num_queue_pairs;
+	int err;
+
+	if (vf->vf_res->vf_cap_flags &
+		VIRTCHNL_VF_OFFLOAD_REQ_QUEUES) {
+		PMD_DRV_LOG(ERR, "request queues not supported");
+		return -1;
+	}
+
+	if (num == 0) {
+		PMD_DRV_LOG(ERR, "queue number cannot be zero");
+		return -1;
+	}
+	vfres.num_queue_pairs = num;
+
+	args.ops = VIRTCHNL_OP_REQUEST_QUEUES;
+	args.in_args = (u8 *)&vfres;
+	args.in_args_size = sizeof(vfres);
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+
+	/*
+	 * disable interrupt to avoid the admin queue message to be read
+	 * before iavf_read_msg_from_pf.
+	 */
+	rte_intr_disable(&pci_dev->intr_handle);
+	err = iavf_execute_vf_cmd(ad, &args);
+	rte_intr_enable(&pci_dev->intr_handle);
+	if (err) {
+		PMD_DRV_LOG(ERR, "fail to execute command OP_REQUEST_QUEUES");
+		return err;
+	}
+
+	/* request queues succeeded, vf is resetting */
+	if (vf->vf_reset) {
+		PMD_DRV_LOG(INFO, "vf is resetting");
+		return 0;
+	}
+
+	/* request additional queues failed, return available number */
+	num_queue_pairs =
+	  ((struct virtchnl_vf_res_request *)args.out_buffer)->num_queue_pairs;
+	PMD_DRV_LOG(ERR, "request queues failed, only %u queues "
+		"available", num_queue_pairs);
+
+	return -1;
+}
-- 
2.17.1


^ permalink raw reply	[flat|nested] 45+ messages in thread

* [dpdk-dev] [PATCH v3 2/2] net/iavf: enable large VF configuration
  2020-09-27 12:42 ` [dpdk-dev] [PATCH v3 0/2] " Ting Xu
  2020-09-27 12:42   ` [dpdk-dev] [PATCH v3 1/2] net/iavf: add IAVF request queues function Ting Xu
@ 2020-09-27 12:42   ` Ting Xu
  1 sibling, 0 replies; 45+ messages in thread
From: Ting Xu @ 2020-09-27 12:42 UTC (permalink / raw)
  To: dev; +Cc: qi.z.zhang, jingjing.wu, beilei.xing, Ting Xu

Add support to negotiate large VF capability, configure VSI queues,
enable/disable queues and IRQ mapping for large VF. Use new virtchnl
opcodes and structures to support max 256 queue pairs. Request
additional queues from PF first if current allocated queues are not
enough. Query max RSS queue region for future RSS configuration.

Signed-off-by: Ting Xu <ting.xu@intel.com>
---
 drivers/net/iavf/iavf.h        |  23 ++-
 drivers/net/iavf/iavf_ethdev.c |  90 ++++++++--
 drivers/net/iavf/iavf_rxtx.c   |  27 ++-
 drivers/net/iavf/iavf_vchnl.c  | 301 ++++++++++++++++++++++++++++++---
 4 files changed, 401 insertions(+), 40 deletions(-)

diff --git a/drivers/net/iavf/iavf.h b/drivers/net/iavf/iavf.h
index 1c40f9bdf..b2e896598 100644
--- a/drivers/net/iavf/iavf.h
+++ b/drivers/net/iavf/iavf.h
@@ -19,7 +19,10 @@
 #define IAVF_FRAME_SIZE_MAX       9728
 #define IAVF_QUEUE_BASE_ADDR_UNIT 128
 
-#define IAVF_MAX_NUM_QUEUES       16
+#define IAVF_MAX_NUM_QUEUES_DFLT	 16
+#define IAVF_MAX_NUM_QUEUES_LV		 256
+#define IAVF_RXTX_QUEUE_CHUNKS_NUM	 2
+#define IAVF_CFG_Q_NUM_PER_BUF		 32
 
 #define IAVF_NUM_MACADDR_MAX      64
 
@@ -104,8 +107,10 @@ struct iavf_fdir_info {
 	struct iavf_fdir_conf conf;
 };
 
-/* TODO: is that correct to assume the max number to be 16 ?*/
-#define IAVF_MAX_MSIX_VECTORS   16
+struct iavf_qv_map {
+	uint16_t queue_id;
+	uint16_t vector_id;
+};
 
 /* Event status from PF */
 enum pending_msg {
@@ -157,14 +162,16 @@ struct iavf_info {
 	uint8_t *rss_key;
 	uint16_t nb_msix;   /* number of MSI-X interrupts on Rx */
 	uint16_t msix_base; /* msix vector base from */
-	/* queue bitmask for each vector */
-	uint16_t rxq_map[IAVF_MAX_MSIX_VECTORS];
+	uint16_t max_rss_qregion; /* max RSS queue region supported by PF */
+	struct iavf_qv_map *qv_map; /* queue vector mapping */
 	struct iavf_flow_list flow_list;
 	rte_spinlock_t flow_ops_lock;
 	struct iavf_parser_list rss_parser_list;
 	struct iavf_parser_list dist_parser_list;
 
 	struct iavf_fdir_info fdir; /* flow director info */
+	/* indicate large VF support enabled or not */
+	bool lv_enabled;
 };
 
 #define IAVF_MAX_PKT_TYPE 1024
@@ -291,13 +298,18 @@ int iavf_enable_vlan_strip(struct iavf_adapter *adapter);
 int iavf_disable_vlan_strip(struct iavf_adapter *adapter);
 int iavf_switch_queue(struct iavf_adapter *adapter, uint16_t qid,
 		     bool rx, bool on);
+int iavf_switch_queue_lv(struct iavf_adapter *adapter, uint16_t qid,
+		     bool rx, bool on);
 int iavf_enable_queues(struct iavf_adapter *adapter);
+int iavf_enable_queues_lv(struct iavf_adapter *adapter);
 int iavf_disable_queues(struct iavf_adapter *adapter);
+int iavf_disable_queues_lv(struct iavf_adapter *adapter);
 int iavf_configure_rss_lut(struct iavf_adapter *adapter);
 int iavf_configure_rss_key(struct iavf_adapter *adapter);
 int iavf_configure_queues(struct iavf_adapter *adapter);
 int iavf_get_supported_rxdid(struct iavf_adapter *adapter);
 int iavf_config_irq_map(struct iavf_adapter *adapter);
+int iavf_config_irq_map_lv(struct iavf_adapter *adapter);
 void iavf_add_del_all_mac_addr(struct iavf_adapter *adapter, bool add);
 int iavf_dev_link_update(struct rte_eth_dev *dev,
 			__rte_unused int wait_to_complete);
@@ -318,4 +330,5 @@ int iavf_add_del_mc_addr_list(struct iavf_adapter *adapter,
 			struct rte_ether_addr *mc_addrs,
 			uint32_t mc_addrs_num, bool add);
 int iavf_request_queues(struct rte_eth_dev *dev, uint16_t num);
+int iavf_get_max_rss_queue_region(struct iavf_adapter *adapter);
 #endif /* _IAVF_ETHDEV_H_ */
diff --git a/drivers/net/iavf/iavf_ethdev.c b/drivers/net/iavf/iavf_ethdev.c
index 33745a7b2..fdb2294a8 100644
--- a/drivers/net/iavf/iavf_ethdev.c
+++ b/drivers/net/iavf/iavf_ethdev.c
@@ -205,7 +205,7 @@ iavf_init_rss(struct iavf_adapter *adapter)
 
 	rss_conf = &adapter->eth_dev->data->dev_conf.rx_adv_conf.rss_conf;
 	nb_q = RTE_MIN(adapter->eth_dev->data->nb_rx_queues,
-		       IAVF_MAX_NUM_QUEUES);
+		       vf->max_rss_qregion);
 
 	if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) {
 		PMD_DRV_LOG(DEBUG, "RSS is not supported");
@@ -258,6 +258,9 @@ iavf_dev_configure(struct rte_eth_dev *dev)
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct iavf_info *vf =  IAVF_DEV_PRIVATE_TO_VF(ad);
 	struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
+	uint16_t num_queue_pairs = RTE_MAX(dev->data->nb_rx_queues,
+		dev->data->nb_tx_queues);
+	int ret = 0;
 
 	ad->rx_bulk_alloc_allowed = true;
 	/* Initialize to TRUE. If any of Rx queues doesn't meet the
@@ -269,6 +272,45 @@ iavf_dev_configure(struct rte_eth_dev *dev)
 	if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)
 		dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_RSS_HASH;
 
+	/* Large VF setting */
+	if (num_queue_pairs > IAVF_MAX_NUM_QUEUES_DFLT) {
+		if (!(vf->vf_res->vf_cap_flags &
+				VIRTCHNL_VF_LARGE_NUM_QPAIRS)) {
+			PMD_DRV_LOG(ERR, "large VF is not supported");
+			return -1;
+		}
+
+		if (num_queue_pairs > IAVF_MAX_NUM_QUEUES_LV) {
+			PMD_DRV_LOG(ERR, "queue pairs number cannot be larger "
+				"than %u", IAVF_MAX_NUM_QUEUES_LV);
+			return -1;
+		}
+
+		ret = iavf_request_queues(dev, num_queue_pairs);
+		if (ret != 0) {
+			PMD_DRV_LOG(ERR, "request queues from PF failed");
+			return ret;
+		}
+		PMD_DRV_LOG(INFO, "change queue pairs from %u to %u",
+				vf->vsi_res->num_queue_pairs, num_queue_pairs);
+
+		ret = iavf_dev_reset(dev);
+		if (ret != 0)
+			return ret;
+
+		vf->lv_enabled = true;
+	}
+
+	/* Set max RSS queue region */
+	if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_LARGE_NUM_QPAIRS) {
+		if (iavf_get_max_rss_queue_region(ad) != 0) {
+			PMD_INIT_LOG(ERR, "get max rss queue region failed");
+			return -1;
+		}
+	} else {
+		vf->max_rss_qregion = IAVF_MAX_NUM_QUEUES_DFLT;
+	}
+
 	/* Vlan stripping setting */
 	if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN) {
 		if (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
@@ -283,6 +325,7 @@ iavf_dev_configure(struct rte_eth_dev *dev)
 			return -1;
 		}
 	}
+
 	return 0;
 }
 
@@ -365,6 +408,7 @@ static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev,
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter);
+	struct iavf_qv_map *qv_map;
 	uint16_t interval, i;
 	int vec;
 
@@ -385,6 +429,14 @@ static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev,
 		}
 	}
 
+	qv_map = rte_zmalloc("qv_map",
+		dev->data->nb_rx_queues * sizeof(struct iavf_qv_map), 0);
+	if (!qv_map) {
+		PMD_DRV_LOG(ERR, "Failed to allocate %d queue-vector map",
+				dev->data->nb_rx_queues);
+		return -1;
+	}
+
 	if (!dev->data->dev_conf.intr_conf.rxq ||
 	    !rte_intr_dp_is_en(intr_handle)) {
 		/* Rx interrupt disabled, Map interrupt only for writeback */
@@ -415,16 +467,21 @@ static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev,
 		}
 		IAVF_WRITE_FLUSH(hw);
 		/* map all queues to the same interrupt */
-		for (i = 0; i < dev->data->nb_rx_queues; i++)
-			vf->rxq_map[vf->msix_base] |= 1 << i;
+		for (i = 0; i < dev->data->nb_rx_queues; i++) {
+			qv_map[i].queue_id = i;
+			qv_map[i].vector_id = vf->msix_base;
+		}
+		vf->qv_map = qv_map;
 	} else {
 		if (!rte_intr_allow_others(intr_handle)) {
 			vf->nb_msix = 1;
 			vf->msix_base = IAVF_MISC_VEC_ID;
 			for (i = 0; i < dev->data->nb_rx_queues; i++) {
-				vf->rxq_map[vf->msix_base] |= 1 << i;
+				qv_map[i].queue_id = i;
+				qv_map[i].vector_id = vf->msix_base;
 				intr_handle->intr_vec[i] = IAVF_MISC_VEC_ID;
 			}
+			vf->qv_map = qv_map;
 			PMD_DRV_LOG(DEBUG,
 				    "vector %u are mapping to all Rx queues",
 				    vf->msix_base);
@@ -437,21 +494,32 @@ static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev,
 			vf->msix_base = IAVF_RX_VEC_START;
 			vec = IAVF_RX_VEC_START;
 			for (i = 0; i < dev->data->nb_rx_queues; i++) {
-				vf->rxq_map[vec] |= 1 << i;
+				qv_map[i].queue_id = i;
+				qv_map[i].vector_id = vec;
 				intr_handle->intr_vec[i] = vec++;
 				if (vec >= vf->nb_msix)
 					vec = IAVF_RX_VEC_START;
 			}
+			vf->qv_map = qv_map;
 			PMD_DRV_LOG(DEBUG,
 				    "%u vectors are mapping to %u Rx queues",
 				    vf->nb_msix, dev->data->nb_rx_queues);
 		}
 	}
 
-	if (iavf_config_irq_map(adapter)) {
-		PMD_DRV_LOG(ERR, "config interrupt mapping failed");
-		return -1;
+	if (!vf->lv_enabled) {
+		if (iavf_config_irq_map(adapter)) {
+			PMD_DRV_LOG(ERR, "config interrupt mapping failed");
+			return -1;
+		}
+	} else {
+		if (iavf_config_irq_map_lv(adapter)) {
+			PMD_DRV_LOG(ERR, "config interrupt mapping "
+				"for large VF failed");
+			return -1;
+		}
 	}
+
 	return 0;
 }
 
@@ -515,6 +583,7 @@ iavf_dev_start(struct rte_eth_dev *dev)
 		PMD_DRV_LOG(ERR, "configure irq failed");
 		goto err_queue;
 	}
+
 	/* re-enable intr again, because efd assign may change */
 	if (dev->data->dev_conf.intr_conf.rxq != 0) {
 		rte_intr_disable(intr_handle);
@@ -579,8 +648,8 @@ iavf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 {
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 
-	dev_info->max_rx_queues = vf->vsi_res->num_queue_pairs;
-	dev_info->max_tx_queues = vf->vsi_res->num_queue_pairs;
+	dev_info->max_rx_queues = IAVF_MAX_NUM_QUEUES_LV;
+	dev_info->max_tx_queues = IAVF_MAX_NUM_QUEUES_LV;
 	dev_info->min_rx_bufsize = IAVF_BUF_SIZE_MIN;
 	dev_info->max_rx_pktlen = IAVF_FRAME_SIZE_MAX;
 	dev_info->hash_key_size = vf->vf_res->rss_key_size;
@@ -1658,6 +1727,7 @@ iavf_init_vf(struct rte_eth_dev *dev)
 		PMD_INIT_LOG(ERR, "iavf_get_vf_config failed");
 		goto err_alloc;
 	}
+
 	/* Allocate memort for RSS info */
 	if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) {
 		vf->rss_key = rte_zmalloc("rss_key",
diff --git a/drivers/net/iavf/iavf_rxtx.c b/drivers/net/iavf/iavf_rxtx.c
index 7b81bf8ad..67300db5b 100644
--- a/drivers/net/iavf/iavf_rxtx.c
+++ b/drivers/net/iavf/iavf_rxtx.c
@@ -720,6 +720,7 @@ iavf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 	struct iavf_adapter *adapter =
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	struct iavf_rx_queue *rxq;
 	int err = 0;
 
@@ -743,7 +744,11 @@ iavf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 	IAVF_WRITE_FLUSH(hw);
 
 	/* Ready to switch the queue on */
-	err = iavf_switch_queue(adapter, rx_queue_id, true, true);
+	if (!vf->lv_enabled)
+		err = iavf_switch_queue(adapter, rx_queue_id, true, true);
+	else
+		err = iavf_switch_queue_lv(adapter, rx_queue_id, true, true);
+
 	if (err)
 		PMD_DRV_LOG(ERR, "Failed to switch RX queue %u on",
 			    rx_queue_id);
@@ -760,6 +765,7 @@ iavf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 	struct iavf_adapter *adapter =
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	struct iavf_tx_queue *txq;
 	int err = 0;
 
@@ -775,7 +781,10 @@ iavf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 	IAVF_WRITE_FLUSH(hw);
 
 	/* Ready to switch the queue on */
-	err = iavf_switch_queue(adapter, tx_queue_id, false, true);
+	if (!vf->lv_enabled)
+		err = iavf_switch_queue(adapter, tx_queue_id, false, true);
+	else
+		err = iavf_switch_queue_lv(adapter, tx_queue_id, false, true);
 
 	if (err)
 		PMD_DRV_LOG(ERR, "Failed to switch TX queue %u on",
@@ -876,14 +885,22 @@ iavf_stop_queues(struct rte_eth_dev *dev)
 {
 	struct iavf_adapter *adapter =
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	struct iavf_rx_queue *rxq;
 	struct iavf_tx_queue *txq;
 	int ret, i;
 
 	/* Stop All queues */
-	ret = iavf_disable_queues(adapter);
-	if (ret)
-		PMD_DRV_LOG(WARNING, "Fail to stop queues");
+	if (!vf->lv_enabled) {
+		ret = iavf_disable_queues(adapter);
+		if (ret)
+			PMD_DRV_LOG(WARNING, "Fail to stop queues");
+	} else {
+		ret = iavf_disable_queues_lv(adapter);
+		if (ret)
+			PMD_DRV_LOG(WARNING, "Fail to stop queues for large VF");
+	}
+
 
 	for (i = 0; i < dev->data->nb_tx_queues; i++) {
 		txq = dev->data->tx_queues[i];
diff --git a/drivers/net/iavf/iavf_vchnl.c b/drivers/net/iavf/iavf_vchnl.c
index b1149ef4d..2e7d6e28d 100644
--- a/drivers/net/iavf/iavf_vchnl.c
+++ b/drivers/net/iavf/iavf_vchnl.c
@@ -450,7 +450,8 @@ iavf_get_vf_resource(struct iavf_adapter *adapter)
 		VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC |
 		VIRTCHNL_VF_OFFLOAD_FDIR_PF |
 		VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF |
-		VIRTCHNL_VF_OFFLOAD_REQ_QUEUES;
+		VIRTCHNL_VF_OFFLOAD_REQ_QUEUES |
+		VIRTCHNL_VF_LARGE_NUM_QPAIRS;
 
 	args.in_args = (uint8_t *)&caps;
 	args.in_args_size = sizeof(caps);
@@ -600,6 +601,138 @@ iavf_switch_queue(struct iavf_adapter *adapter, uint16_t qid,
 	return err;
 }
 
+int
+iavf_enable_queues_lv(struct iavf_adapter *adapter)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct virtchnl_del_ena_dis_queues *queue_select;
+	struct virtchnl_queue_chunk *queue_chunk;
+	struct iavf_cmd_info args;
+	int err, len;
+
+	len = sizeof(struct virtchnl_del_ena_dis_queues) +
+		  sizeof(struct virtchnl_queue_chunk) *
+		  (IAVF_RXTX_QUEUE_CHUNKS_NUM - 1);
+	queue_select = rte_zmalloc("queue_select", len, 0);
+	if (!queue_select)
+		return -ENOMEM;
+
+	queue_chunk = queue_select->chunks.chunks;
+	queue_select->chunks.num_chunks = IAVF_RXTX_QUEUE_CHUNKS_NUM;
+	queue_select->vport_id = vf->vsi_res->vsi_id;
+
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].type = VIRTCHNL_QUEUE_TYPE_TX;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].start_queue_id = 0;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].num_queues =
+		adapter->eth_dev->data->nb_tx_queues;
+
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].type = VIRTCHNL_QUEUE_TYPE_RX;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].start_queue_id = 0;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].num_queues =
+		adapter->eth_dev->data->nb_rx_queues;
+
+	args.ops = VIRTCHNL_OP_ENABLE_QUEUES_V2;
+	args.in_args = (u8 *)queue_select;
+	args.in_args_size = len;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err) {
+		PMD_DRV_LOG(ERR,
+			    "Failed to execute command of OP_ENABLE_QUEUES_V2");
+		return err;
+	}
+	return 0;
+}
+
+int
+iavf_disable_queues_lv(struct iavf_adapter *adapter)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct virtchnl_del_ena_dis_queues *queue_select;
+	struct virtchnl_queue_chunk *queue_chunk;
+	struct iavf_cmd_info args;
+	int err, len;
+
+	len = sizeof(struct virtchnl_del_ena_dis_queues) +
+		  sizeof(struct virtchnl_queue_chunk) *
+		  (IAVF_RXTX_QUEUE_CHUNKS_NUM - 1);
+	queue_select = rte_zmalloc("queue_select", len, 0);
+	if (!queue_select)
+		return -ENOMEM;
+
+	queue_chunk = queue_select->chunks.chunks;
+	queue_select->chunks.num_chunks = IAVF_RXTX_QUEUE_CHUNKS_NUM;
+	queue_select->vport_id = vf->vsi_res->vsi_id;
+
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].type = VIRTCHNL_QUEUE_TYPE_TX;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].start_queue_id = 0;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].num_queues =
+		adapter->eth_dev->data->nb_tx_queues;
+
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].type = VIRTCHNL_QUEUE_TYPE_RX;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].start_queue_id = 0;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].num_queues =
+		adapter->eth_dev->data->nb_rx_queues;
+
+	args.ops = VIRTCHNL_OP_DISABLE_QUEUES_V2;
+	args.in_args = (u8 *)queue_select;
+	args.in_args_size = len;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err) {
+		PMD_DRV_LOG(ERR,
+			    "Failed to execute command of OP_DISABLE_QUEUES_V2");
+		return err;
+	}
+	return 0;
+}
+
+int
+iavf_switch_queue_lv(struct iavf_adapter *adapter, uint16_t qid,
+		 bool rx, bool on)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct virtchnl_del_ena_dis_queues *queue_select;
+	struct virtchnl_queue_chunk *queue_chunk;
+	struct iavf_cmd_info args;
+	int err, len;
+
+	len = sizeof(struct virtchnl_del_ena_dis_queues);
+	queue_select = rte_zmalloc("queue_select", len, 0);
+	if (!queue_select)
+		return -ENOMEM;
+
+	queue_chunk = queue_select->chunks.chunks;
+	queue_select->chunks.num_chunks = 1;
+	queue_select->vport_id = vf->vsi_res->vsi_id;
+
+	if (rx) {
+		queue_chunk->type = VIRTCHNL_QUEUE_TYPE_RX;
+		queue_chunk->start_queue_id = qid;
+		queue_chunk->num_queues = 1;
+	} else {
+		queue_chunk->type = VIRTCHNL_QUEUE_TYPE_TX;
+		queue_chunk->start_queue_id = qid;
+		queue_chunk->num_queues = 1;
+	}
+
+	if (on)
+		args.ops = VIRTCHNL_OP_ENABLE_QUEUES_V2;
+	else
+		args.ops = VIRTCHNL_OP_DISABLE_QUEUES_V2;
+	args.in_args = (u8 *)queue_select;
+	args.in_args_size = len;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err)
+		PMD_DRV_LOG(ERR, "Failed to execute command of %s",
+			    on ? "OP_ENABLE_QUEUES_V2" : "OP_DISABLE_QUEUES_V2");
+	return err;
+}
+
 int
 iavf_configure_rss_lut(struct iavf_adapter *adapter)
 {
@@ -664,32 +797,26 @@ iavf_configure_rss_key(struct iavf_adapter *adapter)
 	return err;
 }
 
-int
-iavf_configure_queues(struct iavf_adapter *adapter)
+static int
+iavf_exec_queue_cfg(struct iavf_adapter *adapter,
+	struct virtchnl_vsi_queue_config_info *vc_config, uint16_t count)
 {
 	struct iavf_rx_queue **rxq =
 		(struct iavf_rx_queue **)adapter->eth_dev->data->rx_queues;
 	struct iavf_tx_queue **txq =
 		(struct iavf_tx_queue **)adapter->eth_dev->data->tx_queues;
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
-	struct virtchnl_vsi_queue_config_info *vc_config;
 	struct virtchnl_queue_pair_info *vc_qp;
 	struct iavf_cmd_info args;
 	uint16_t i, size;
-	int err;
+	int err = 0;
 
 	size = sizeof(*vc_config) +
-	       sizeof(vc_config->qpair[0]) * vf->num_queue_pairs;
-	vc_config = rte_zmalloc("cfg_queue", size, 0);
-	if (!vc_config)
-		return -ENOMEM;
+		sizeof(vc_config->qpair[0]) * vc_config->num_queue_pairs;
 
-	vc_config->vsi_id = vf->vsi_res->vsi_id;
-	vc_config->num_queue_pairs = vf->num_queue_pairs;
-
-	for (i = 0, vc_qp = vc_config->qpair;
-	     i < vf->num_queue_pairs;
-	     i++, vc_qp++) {
+	for (i = count * IAVF_CFG_Q_NUM_PER_BUF, vc_qp = vc_config->qpair;
+	     i < count * IAVF_CFG_Q_NUM_PER_BUF + vc_config->num_queue_pairs;
+		 i++, vc_qp++) {
 		vc_qp->txq.vsi_id = vf->vsi_res->vsi_id;
 		vc_qp->txq.queue_id = i;
 		/* Virtchnnl configure queues by pairs */
@@ -745,8 +872,71 @@ iavf_configure_queues(struct iavf_adapter *adapter)
 	err = iavf_execute_vf_cmd(adapter, &args);
 	if (err)
 		PMD_DRV_LOG(ERR, "Failed to execute command of"
-			    " VIRTCHNL_OP_CONFIG_VSI_QUEUES");
+				" VIRTCHNL_OP_CONFIG_VSI_QUEUES");
+
+	return err;
+}
+
+/* Configure VSI queues. Max VF queue pairs number is 256, may
+ * send this command multiple times to configure all queues.
+ */
+int
+iavf_configure_queues(struct iavf_adapter *adapter)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct virtchnl_vsi_queue_config_info *vc_config = NULL;
+	uint16_t i, size_full, size_inc;
+	uint16_t nb_cmd_full, nbq_inc;
+	int err = 0;
+
+	/* Compute how many times should the command to be sent,
+	 * including the commands with full buffer and incomplete
+	 * buffer.
+	 */
+	nbq_inc = vf->num_queue_pairs % IAVF_CFG_Q_NUM_PER_BUF;
+	nb_cmd_full = vf->num_queue_pairs / IAVF_CFG_Q_NUM_PER_BUF;
+
+	size_full = sizeof(*vc_config) +
+		sizeof(vc_config->qpair[0]) * IAVF_CFG_Q_NUM_PER_BUF;
+	size_inc = sizeof(*vc_config) +
+		sizeof(vc_config->qpair[0]) * nbq_inc;
+
+	if (!nb_cmd_full) {
+		vc_config = rte_zmalloc("cfg_queue", size_inc, 0);
+		if (!vc_config)
+			return -ENOMEM;
+
+		vc_config->vsi_id = vf->vsi_res->vsi_id;
+		vc_config->num_queue_pairs = nbq_inc;
+		err = iavf_exec_queue_cfg(adapter, vc_config, 0);
+		goto free;
+	}
+
+	vc_config = rte_zmalloc("cfg_queue", size_full, 0);
+	if (!vc_config)
+		return -ENOMEM;
 
+	vc_config->vsi_id = vf->vsi_res->vsi_id;
+	vc_config->num_queue_pairs = IAVF_CFG_Q_NUM_PER_BUF;
+
+	for (i = 0; i < nb_cmd_full + (nbq_inc ? 1 : 0); i++) {
+		if (i >= nb_cmd_full) {
+			/* re-allocate virtchnl msg for less queues */
+			rte_free(vc_config);
+			vc_config = rte_zmalloc("cfg_queue", size_inc, 0);
+			if (!vc_config)
+				return -ENOMEM;
+
+			vc_config->vsi_id = vf->vsi_res->vsi_id;
+			vc_config->num_queue_pairs = nbq_inc;
+		}
+
+		err = iavf_exec_queue_cfg(adapter, vc_config, i);
+		if (err)
+			break;
+	}
+
+free:
 	rte_free(vc_config);
 	return err;
 }
@@ -768,13 +958,14 @@ iavf_config_irq_map(struct iavf_adapter *adapter)
 		return -ENOMEM;
 
 	map_info->num_vectors = vf->nb_msix;
-	for (i = 0; i < vf->nb_msix; i++) {
-		vecmap = &map_info->vecmap[i];
+	for (i = 0; i < adapter->eth_dev->data->nb_rx_queues; i++) {
+		vecmap =
+		    &map_info->vecmap[vf->qv_map[i].vector_id - vf->msix_base];
 		vecmap->vsi_id = vf->vsi_res->vsi_id;
 		vecmap->rxitr_idx = IAVF_ITR_INDEX_DEFAULT;
-		vecmap->vector_id = vf->msix_base + i;
+		vecmap->vector_id = vf->qv_map[i].vector_id;
 		vecmap->txq_map = 0;
-		vecmap->rxq_map = vf->rxq_map[vf->msix_base + i];
+		vecmap->rxq_map |= 1 << vf->qv_map[i].queue_id;
 	}
 
 	args.ops = VIRTCHNL_OP_CONFIG_IRQ_MAP;
@@ -790,6 +981,46 @@ iavf_config_irq_map(struct iavf_adapter *adapter)
 	return err;
 }
 
+int
+iavf_config_irq_map_lv(struct iavf_adapter *adapter)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct virtchnl_queue_vector_maps *map_info;
+	struct virtchnl_queue_vector *qv_maps;
+	struct iavf_cmd_info args;
+	int len, i, err;
+
+	len = sizeof(struct virtchnl_queue_vector_maps) +
+	      sizeof(struct virtchnl_queue_vector) *
+	      (adapter->eth_dev->data->nb_rx_queues - 1);
+
+	map_info = rte_zmalloc("map_info", len, 0);
+	if (!map_info)
+		return -ENOMEM;
+
+	map_info->vport_id = vf->vsi_res->vsi_id;
+	map_info->num_qv_maps = adapter->eth_dev->data->nb_rx_queues;
+	for (i = 0; i < map_info->num_qv_maps; i++) {
+		qv_maps = &map_info->qv_maps[i];
+		qv_maps->itr_idx = VIRTCHNL_ITR_IDX_0;
+		qv_maps->queue_type = VIRTCHNL_QUEUE_TYPE_RX;
+		qv_maps->queue_id = vf->qv_map[i].queue_id;
+		qv_maps->vector_id = vf->qv_map[i].vector_id;
+	}
+
+	args.ops = VIRTCHNL_OP_MAP_QUEUE_VECTOR;
+	args.in_args = (u8 *)map_info;
+	args.in_args_size = len;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err)
+		PMD_DRV_LOG(ERR, "fail to execute command OP_MAP_QUEUE_VECTOR");
+
+	rte_free(map_info);
+	return err;
+}
+
 void
 iavf_add_del_all_mac_addr(struct iavf_adapter *adapter, bool add)
 {
@@ -1257,3 +1488,33 @@ iavf_request_queues(struct rte_eth_dev *dev, uint16_t num)
 
 	return -1;
 }
+
+int
+iavf_get_max_rss_queue_region(struct iavf_adapter *adapter)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct iavf_cmd_info args;
+	uint16_t qregion_width;
+	int err;
+
+	args.ops = VIRTCHNL_OP_GET_MAX_RSS_QREGION;
+	args.in_args = NULL;
+	args.in_args_size = 0;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err) {
+		PMD_DRV_LOG(ERR,
+			    "Failed to execute command of "
+			    "VIRTCHNL_OP_GET_MAX_RSS_QREGION");
+		return err;
+	}
+
+	qregion_width =
+	((struct virtchnl_max_rss_qregion *)args.out_buffer)->qregion_width;
+
+	vf->max_rss_qregion = (uint16_t)(1 << qregion_width);
+
+	return 0;
+}
-- 
2.17.1


^ permalink raw reply	[flat|nested] 45+ messages in thread

* [dpdk-dev] [PATCH v4 0/2] enable large VF configuration
  2020-09-09  7:20 [dpdk-dev] [PATCH v1 0/2] enable large VF configuration Ting Xu
                   ` (3 preceding siblings ...)
  2020-09-27 12:42 ` [dpdk-dev] [PATCH v3 0/2] " Ting Xu
@ 2020-10-15  5:21 ` Ting Xu
  2020-10-15  5:21   ` [dpdk-dev] [PATCH v4 1/2] net/iavf: add IAVF request queues function Ting Xu
  2020-10-15  5:21   ` [dpdk-dev] [PATCH v4 2/2] net/iavf: enable large VF configuration Ting Xu
  2020-10-16  1:21 ` [dpdk-dev] [PATCH v5 0/2] " Ting Xu
                   ` (4 subsequent siblings)
  9 siblings, 2 replies; 45+ messages in thread
From: Ting Xu @ 2020-10-15  5:21 UTC (permalink / raw)
  To: dev; +Cc: qi.z.zhang, beilei.xing, jingjing.wu, Ting Xu

This patchset supports to configure up to 256 queue pairs per VF. If
large VF is supported after capability negotiation, VF will request
queues from PF as needed. New virtual channel opcodes and structures
are used to indicate 256 queues, so VF is designed to handle the new
function of configure VSI queues, IRQ mapping and enable/disable queues.
Also, enable VF to query the max RSS queue region for RSS and filter
configuration.

Ting Xu (2):
  net/iavf: add IAVF request queues function
  net/iavf: enable large VF configuration

---
v3->v4:
Optimize handling messages from PF

v2->v3:
Fix coding style issue.
Move get max RSS queue region after VF reset.
Add request queues capability negotiation.

v1->v2:
Change the communication with kernel PF.

---
 drivers/net/iavf/iavf.h        |  32 +-
 drivers/net/iavf/iavf_ethdev.c | 126 +++++++-
 drivers/net/iavf/iavf_rxtx.c   |  27 +-
 drivers/net/iavf/iavf_vchnl.c  | 525 ++++++++++++++++++++++++++++-----
 4 files changed, 617 insertions(+), 93 deletions(-)

-- 
2.17.1


^ permalink raw reply	[flat|nested] 45+ messages in thread

* [dpdk-dev] [PATCH v4 1/2] net/iavf: add IAVF request queues function
  2020-10-15  5:21 ` [dpdk-dev] [PATCH v4 0/2] " Ting Xu
@ 2020-10-15  5:21   ` Ting Xu
  2020-10-15  5:21   ` [dpdk-dev] [PATCH v4 2/2] net/iavf: enable large VF configuration Ting Xu
  1 sibling, 0 replies; 45+ messages in thread
From: Ting Xu @ 2020-10-15  5:21 UTC (permalink / raw)
  To: dev; +Cc: qi.z.zhang, beilei.xing, jingjing.wu, Ting Xu

Add new virtchnl function to request additional queues from PF. Current
default queue pairs number when creating a VF is 16. In order to support
up to 256 queue pairs, enable this request queues function.
Since request queues command may return event message, modify function
iavf_read_msg_from_pf to identify event opcode and mark VF reset status.

Signed-off-by: Ting Xu <ting.xu@intel.com>
---
 drivers/net/iavf/iavf.h        |   9 ++
 drivers/net/iavf/iavf_ethdev.c |  11 +-
 drivers/net/iavf/iavf_vchnl.c  | 226 +++++++++++++++++++++++++--------
 3 files changed, 192 insertions(+), 54 deletions(-)

diff --git a/drivers/net/iavf/iavf.h b/drivers/net/iavf/iavf.h
index d56611608..93c165c62 100644
--- a/drivers/net/iavf/iavf.h
+++ b/drivers/net/iavf/iavf.h
@@ -107,6 +107,14 @@ struct iavf_fdir_info {
 /* TODO: is that correct to assume the max number to be 16 ?*/
 #define IAVF_MAX_MSIX_VECTORS   16
 
+/* Message type read in admin queue from PF */
+enum iavf_aq_result {
+	IAVF_MSG_ERR = -1, /* Meet error when accessing admin queue */
+	IAVF_MSG_NON,      /* Read nothing from admin queue */
+	IAVF_MSG_SYS,      /* Read system msg from admin queue */
+	IAVF_MSG_CMD,      /* Read async command result */
+};
+
 /* Structure to store private data specific for VF instance. */
 struct iavf_info {
 	uint16_t num_queue_pairs;
@@ -301,4 +309,5 @@ int iavf_add_del_rss_cfg(struct iavf_adapter *adapter,
 int iavf_add_del_mc_addr_list(struct iavf_adapter *adapter,
 			struct rte_ether_addr *mc_addrs,
 			uint32_t mc_addrs_num, bool add);
+int iavf_request_queues(struct iavf_adapter *adapter, uint16_t num);
 #endif /* _IAVF_ETHDEV_H_ */
diff --git a/drivers/net/iavf/iavf_ethdev.c b/drivers/net/iavf/iavf_ethdev.c
index 8b1cf8f1c..a4a28b885 100644
--- a/drivers/net/iavf/iavf_ethdev.c
+++ b/drivers/net/iavf/iavf_ethdev.c
@@ -1282,7 +1282,7 @@ iavf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
 }
 
 static int
-iavf_check_vf_reset_done(struct iavf_hw *hw)
+iavf_check_vf_reset_done(struct iavf_hw *hw, struct iavf_info *vf)
 {
 	int i, reset;
 
@@ -1299,6 +1299,9 @@ iavf_check_vf_reset_done(struct iavf_hw *hw)
 	if (i >= IAVF_RESET_WAIT_CNT)
 		return -1;
 
+	/* VF is not in reset or reset is completed */
+	vf->vf_reset = false;
+
 	return 0;
 }
 
@@ -1666,7 +1669,7 @@ iavf_init_vf(struct rte_eth_dev *dev)
 		goto err;
 	}
 
-	err = iavf_check_vf_reset_done(hw);
+	err = iavf_check_vf_reset_done(hw, vf);
 	if (err) {
 		PMD_INIT_LOG(ERR, "VF is still resetting");
 		goto err;
@@ -1911,7 +1914,9 @@ iavf_dev_close(struct rte_eth_dev *dev)
 
 	iavf_dev_stop(dev);
 	iavf_flow_flush(dev, NULL);
-	iavf_flow_uninit(adapter);
+	/* if VF is in reset, adminq is disabled, skip the process via adminq */
+	if (!vf->vf_reset)
+		iavf_flow_uninit(adapter);
 
 	/*
 	 * disable promiscuous mode before reset vf
diff --git a/drivers/net/iavf/iavf_vchnl.c b/drivers/net/iavf/iavf_vchnl.c
index 5e7142893..11a1ff608 100644
--- a/drivers/net/iavf/iavf_vchnl.c
+++ b/drivers/net/iavf/iavf_vchnl.c
@@ -17,6 +17,7 @@
 #include <rte_eal.h>
 #include <rte_ether.h>
 #include <rte_ethdev_driver.h>
+#include <rte_ethdev_pci.h>
 #include <rte_dev.h>
 
 #include "iavf.h"
@@ -25,14 +26,54 @@
 #define MAX_TRY_TIMES 200
 #define ASQ_DELAY_MS  10
 
+static uint32_t
+iavf_convert_link_speed(enum virtchnl_link_speed virt_link_speed)
+{
+	uint32_t speed;
+
+	switch (virt_link_speed) {
+	case VIRTCHNL_LINK_SPEED_100MB:
+		speed = 100;
+		break;
+	case VIRTCHNL_LINK_SPEED_1GB:
+		speed = 1000;
+		break;
+	case VIRTCHNL_LINK_SPEED_10GB:
+		speed = 10000;
+		break;
+	case VIRTCHNL_LINK_SPEED_40GB:
+		speed = 40000;
+		break;
+	case VIRTCHNL_LINK_SPEED_20GB:
+		speed = 20000;
+		break;
+	case VIRTCHNL_LINK_SPEED_25GB:
+		speed = 25000;
+		break;
+	case VIRTCHNL_LINK_SPEED_2_5GB:
+		speed = 2500;
+		break;
+	case VIRTCHNL_LINK_SPEED_5GB:
+		speed = 5000;
+		break;
+	default:
+		speed = 0;
+		break;
+	}
+
+	return speed;
+}
+
 /* Read data in admin queue to get msg from pf driver */
-static enum iavf_status
+static enum iavf_aq_result
 iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,
 		     uint8_t *buf)
 {
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter);
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct rte_eth_dev *dev = adapter->eth_dev;
 	struct iavf_arq_event_info event;
+	enum iavf_aq_result result = IAVF_MSG_NON;
 	enum virtchnl_ops opcode;
 	int ret;
 
@@ -42,7 +83,9 @@ iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,
 	/* Can't read any msg from adminQ */
 	if (ret) {
 		PMD_DRV_LOG(DEBUG, "Can't read msg from AQ");
-		return ret;
+		if (ret != IAVF_ERR_ADMIN_QUEUE_NO_WORK)
+			result = IAVF_MSG_ERR;
+		return result;
 	}
 
 	opcode = (enum virtchnl_ops)rte_le_to_cpu_32(event.desc.cookie_high);
@@ -52,16 +95,51 @@ iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,
 	PMD_DRV_LOG(DEBUG, "AQ from pf carries opcode %u, retval %d",
 		    opcode, vf->cmd_retval);
 
-	if (opcode != vf->pend_cmd) {
-		if (opcode != VIRTCHNL_OP_EVENT) {
-			PMD_DRV_LOG(WARNING,
-				    "command mismatch, expect %u, get %u",
-				    vf->pend_cmd, opcode);
+	if (opcode == VIRTCHNL_OP_EVENT) {
+		struct virtchnl_pf_event *vpe =
+			(struct virtchnl_pf_event *)event.msg_buf;
+
+		result = IAVF_MSG_SYS;
+		switch (vpe->event) {
+		case VIRTCHNL_EVENT_LINK_CHANGE:
+			vf->link_up =
+				vpe->event_data.link_event.link_status;
+			if (vf->vf_res->vf_cap_flags &
+				VIRTCHNL_VF_CAP_ADV_LINK_SPEED) {
+				vf->link_speed =
+				    vpe->event_data.link_event_adv.link_speed;
+			} else {
+				enum virtchnl_link_speed speed;
+				speed = vpe->event_data.link_event.link_speed;
+				vf->link_speed = iavf_convert_link_speed(speed);
+			}
+			iavf_dev_link_update(dev, 0);
+			PMD_DRV_LOG(INFO, "Link status update:%s",
+					vf->link_up ? "up" : "down");
+			break;
+		case VIRTCHNL_EVENT_RESET_IMPENDING:
+			vf->vf_reset = true;
+			PMD_DRV_LOG(INFO, "VF is resetting");
+			break;
+		case VIRTCHNL_EVENT_PF_DRIVER_CLOSE:
+			vf->dev_closed = true;
+			PMD_DRV_LOG(INFO, "PF driver closed");
+			break;
+		default:
+			PMD_DRV_LOG(ERR, "%s: Unknown event %d from pf",
+					__func__, vpe->event);
+		}
+	}  else {
+		/* async reply msg on command issued by vf previously */
+		result = IAVF_MSG_CMD;
+		if (opcode != vf->pend_cmd) {
+			PMD_DRV_LOG(WARNING, "command mismatch, expect %u, get %u",
+					vf->pend_cmd, opcode);
+			result = IAVF_MSG_ERR;
 		}
-		return IAVF_ERR_OPCODE_MISMATCH;
 	}
 
-	return IAVF_SUCCESS;
+	return result;
 }
 
 static int
@@ -69,6 +147,7 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter, struct iavf_cmd_info *args)
 {
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter);
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	enum iavf_aq_result result;
 	enum iavf_status ret;
 	int err = 0;
 	int i = 0;
@@ -97,9 +176,9 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter, struct iavf_cmd_info *args)
 	case VIRTCHNL_OP_GET_SUPPORTED_RXDIDS:
 		/* for init virtchnl ops, need to poll the response */
 		do {
-			ret = iavf_read_msg_from_pf(adapter, args->out_size,
+			result = iavf_read_msg_from_pf(adapter, args->out_size,
 						   args->out_buffer);
-			if (ret == IAVF_SUCCESS)
+			if (result == IAVF_MSG_CMD)
 				break;
 			rte_delay_ms(ASQ_DELAY_MS);
 		} while (i++ < MAX_TRY_TIMES);
@@ -111,7 +190,33 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter, struct iavf_cmd_info *args)
 		}
 		_clear_cmd(vf);
 		break;
-
+	case VIRTCHNL_OP_REQUEST_QUEUES:
+		/*
+		 * ignore async reply, only wait for system message,
+		 * vf_reset = true if get VIRTCHNL_EVENT_RESET_IMPENDING,
+		 * if not, means request queues failed.
+		 */
+		do {
+			result = iavf_read_msg_from_pf(adapter, args->out_size,
+						   args->out_buffer);
+			if (result == IAVF_MSG_SYS && vf->vf_reset) {
+				break;
+			} else if (result == IAVF_MSG_CMD ||
+				result == IAVF_MSG_ERR) {
+				err = -1;
+				break;
+			}
+			rte_delay_ms(ASQ_DELAY_MS);
+			/* If don't read msg or read sys event, continue */
+		} while (i++ < MAX_TRY_TIMES);
+		if (i >= MAX_TRY_TIMES ||
+		    vf->cmd_retval != VIRTCHNL_STATUS_SUCCESS) {
+			err = -1;
+			PMD_DRV_LOG(ERR, "No response or return failure (%d)"
+				    " for cmd %d", vf->cmd_retval, args->ops);
+		}
+		_clear_cmd(vf);
+		break;
 	default:
 		/* For other virtchnl ops in running time,
 		 * wait for the cmd done flag.
@@ -136,44 +241,6 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter, struct iavf_cmd_info *args)
 	return err;
 }
 
-static uint32_t
-iavf_convert_link_speed(enum virtchnl_link_speed virt_link_speed)
-{
-	uint32_t speed;
-
-	switch (virt_link_speed) {
-	case VIRTCHNL_LINK_SPEED_100MB:
-		speed = 100;
-		break;
-	case VIRTCHNL_LINK_SPEED_1GB:
-		speed = 1000;
-		break;
-	case VIRTCHNL_LINK_SPEED_10GB:
-		speed = 10000;
-		break;
-	case VIRTCHNL_LINK_SPEED_40GB:
-		speed = 40000;
-		break;
-	case VIRTCHNL_LINK_SPEED_20GB:
-		speed = 20000;
-		break;
-	case VIRTCHNL_LINK_SPEED_25GB:
-		speed = 25000;
-		break;
-	case VIRTCHNL_LINK_SPEED_2_5GB:
-		speed = 2500;
-		break;
-	case VIRTCHNL_LINK_SPEED_5GB:
-		speed = 5000;
-		break;
-	default:
-		speed = 0;
-		break;
-	}
-
-	return speed;
-}
-
 static void
 iavf_handle_pf_event_msg(struct rte_eth_dev *dev, uint8_t *msg,
 			uint16_t msglen)
@@ -389,7 +456,8 @@ iavf_get_vf_resource(struct iavf_adapter *adapter)
 	caps = IAVF_BASIC_OFFLOAD_CAPS | VIRTCHNL_VF_CAP_ADV_LINK_SPEED |
 		VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC |
 		VIRTCHNL_VF_OFFLOAD_FDIR_PF |
-		VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF;
+		VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF |
+		VIRTCHNL_VF_OFFLOAD_REQ_QUEUES;
 
 	args.in_args = (uint8_t *)&caps;
 	args.in_args_size = sizeof(caps);
@@ -1148,3 +1216,59 @@ iavf_add_del_mc_addr_list(struct iavf_adapter *adapter,
 
 	return 0;
 }
+
+int
+iavf_request_queues(struct iavf_adapter *adapter, uint16_t num)
+{
+	struct rte_eth_dev *dev = adapter->eth_dev;
+	struct iavf_info *vf =  IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+	struct virtchnl_vf_res_request vfres;
+	struct iavf_cmd_info args;
+	uint16_t num_queue_pairs;
+	int err;
+
+	if (!(vf->vf_res->vf_cap_flags &
+		VIRTCHNL_VF_OFFLOAD_REQ_QUEUES)) {
+		PMD_DRV_LOG(ERR, "request queues not supported");
+		return -1;
+	}
+
+	if (num == 0) {
+		PMD_DRV_LOG(ERR, "queue number cannot be zero");
+		return -1;
+	}
+	vfres.num_queue_pairs = num;
+
+	args.ops = VIRTCHNL_OP_REQUEST_QUEUES;
+	args.in_args = (u8 *)&vfres;
+	args.in_args_size = sizeof(vfres);
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+
+	/*
+	 * disable interrupt to avoid the admin queue message to be read
+	 * before iavf_read_msg_from_pf.
+	 */
+	rte_intr_disable(&pci_dev->intr_handle);
+	err = iavf_execute_vf_cmd(adapter, &args);
+	rte_intr_enable(&pci_dev->intr_handle);
+	if (err) {
+		PMD_DRV_LOG(ERR, "fail to execute command OP_REQUEST_QUEUES");
+		return err;
+	}
+
+	/* request queues succeeded, vf is resetting */
+	if (vf->vf_reset) {
+		PMD_DRV_LOG(INFO, "vf is resetting");
+		return 0;
+	}
+
+	/* request additional queues failed, return available number */
+	num_queue_pairs =
+	  ((struct virtchnl_vf_res_request *)args.out_buffer)->num_queue_pairs;
+	PMD_DRV_LOG(ERR, "request queues failed, only %u queues "
+		"available", num_queue_pairs);
+
+	return -1;
+}
-- 
2.17.1


^ permalink raw reply	[flat|nested] 45+ messages in thread

* [dpdk-dev] [PATCH v4 2/2] net/iavf: enable large VF configuration
  2020-10-15  5:21 ` [dpdk-dev] [PATCH v4 0/2] " Ting Xu
  2020-10-15  5:21   ` [dpdk-dev] [PATCH v4 1/2] net/iavf: add IAVF request queues function Ting Xu
@ 2020-10-15  5:21   ` Ting Xu
  1 sibling, 0 replies; 45+ messages in thread
From: Ting Xu @ 2020-10-15  5:21 UTC (permalink / raw)
  To: dev; +Cc: qi.z.zhang, beilei.xing, jingjing.wu, Ting Xu

Add support to negotiate large VF capability, including configure VSI
queues, enable/disable queues and IRQ mapping for large VF. Use new
virtchnl opcodes and structures to help indicate up to 256 queue pairs.
Need to request additional queues from PF first if the current allocated
queues are not enough. Add the step to query the current max RSS queue
region for RSS configuration since the max RSS queue region could be
smaller than the allocated queue pairs.

Signed-off-by: Ting Xu <ting.xu@intel.com>
---
 drivers/net/iavf/iavf.h        |  23 ++-
 drivers/net/iavf/iavf_ethdev.c | 115 +++++++++++--
 drivers/net/iavf/iavf_rxtx.c   |  27 ++-
 drivers/net/iavf/iavf_vchnl.c  | 301 ++++++++++++++++++++++++++++++---
 4 files changed, 426 insertions(+), 40 deletions(-)

diff --git a/drivers/net/iavf/iavf.h b/drivers/net/iavf/iavf.h
index 93c165c62..fa07f1545 100644
--- a/drivers/net/iavf/iavf.h
+++ b/drivers/net/iavf/iavf.h
@@ -19,7 +19,10 @@
 #define IAVF_FRAME_SIZE_MAX       9728
 #define IAVF_QUEUE_BASE_ADDR_UNIT 128
 
-#define IAVF_MAX_NUM_QUEUES       16
+#define IAVF_MAX_NUM_QUEUES_DFLT	 16
+#define IAVF_MAX_NUM_QUEUES_LV		 256
+#define IAVF_RXTX_QUEUE_CHUNKS_NUM	 2
+#define IAVF_CFG_Q_NUM_PER_BUF		 32
 
 #define IAVF_NUM_MACADDR_MAX      64
 
@@ -104,8 +107,10 @@ struct iavf_fdir_info {
 	struct iavf_fdir_conf conf;
 };
 
-/* TODO: is that correct to assume the max number to be 16 ?*/
-#define IAVF_MAX_MSIX_VECTORS   16
+struct iavf_qv_map {
+	uint16_t queue_id;
+	uint16_t vector_id;
+};
 
 /* Message type read in admin queue from PF */
 enum iavf_aq_result {
@@ -149,14 +154,16 @@ struct iavf_info {
 	uint8_t *rss_key;
 	uint16_t nb_msix;   /* number of MSI-X interrupts on Rx */
 	uint16_t msix_base; /* msix vector base from */
-	/* queue bitmask for each vector */
-	uint16_t rxq_map[IAVF_MAX_MSIX_VECTORS];
+	uint16_t max_rss_qregion; /* max RSS queue region supported by PF */
+	struct iavf_qv_map *qv_map; /* queue vector mapping */
 	struct iavf_flow_list flow_list;
 	rte_spinlock_t flow_ops_lock;
 	struct iavf_parser_list rss_parser_list;
 	struct iavf_parser_list dist_parser_list;
 
 	struct iavf_fdir_info fdir; /* flow director info */
+	/* indicate large VF support enabled or not */
+	bool lv_enabled;
 };
 
 #define IAVF_MAX_PKT_TYPE 1024
@@ -283,13 +290,18 @@ int iavf_enable_vlan_strip(struct iavf_adapter *adapter);
 int iavf_disable_vlan_strip(struct iavf_adapter *adapter);
 int iavf_switch_queue(struct iavf_adapter *adapter, uint16_t qid,
 		     bool rx, bool on);
+int iavf_switch_queue_lv(struct iavf_adapter *adapter, uint16_t qid,
+		     bool rx, bool on);
 int iavf_enable_queues(struct iavf_adapter *adapter);
+int iavf_enable_queues_lv(struct iavf_adapter *adapter);
 int iavf_disable_queues(struct iavf_adapter *adapter);
+int iavf_disable_queues_lv(struct iavf_adapter *adapter);
 int iavf_configure_rss_lut(struct iavf_adapter *adapter);
 int iavf_configure_rss_key(struct iavf_adapter *adapter);
 int iavf_configure_queues(struct iavf_adapter *adapter);
 int iavf_get_supported_rxdid(struct iavf_adapter *adapter);
 int iavf_config_irq_map(struct iavf_adapter *adapter);
+int iavf_config_irq_map_lv(struct iavf_adapter *adapter);
 void iavf_add_del_all_mac_addr(struct iavf_adapter *adapter, bool add);
 int iavf_dev_link_update(struct rte_eth_dev *dev,
 			__rte_unused int wait_to_complete);
@@ -310,4 +322,5 @@ int iavf_add_del_mc_addr_list(struct iavf_adapter *adapter,
 			struct rte_ether_addr *mc_addrs,
 			uint32_t mc_addrs_num, bool add);
 int iavf_request_queues(struct iavf_adapter *adapter, uint16_t num);
+int iavf_get_max_rss_queue_region(struct iavf_adapter *adapter);
 #endif /* _IAVF_ETHDEV_H_ */
diff --git a/drivers/net/iavf/iavf_ethdev.c b/drivers/net/iavf/iavf_ethdev.c
index a4a28b885..3dd52ef2b 100644
--- a/drivers/net/iavf/iavf_ethdev.c
+++ b/drivers/net/iavf/iavf_ethdev.c
@@ -238,7 +238,7 @@ iavf_init_rss(struct iavf_adapter *adapter)
 
 	rss_conf = &adapter->eth_dev->data->dev_conf.rx_adv_conf.rss_conf;
 	nb_q = RTE_MIN(adapter->eth_dev->data->nb_rx_queues,
-		       IAVF_MAX_NUM_QUEUES);
+		       vf->max_rss_qregion);
 
 	if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) {
 		PMD_DRV_LOG(DEBUG, "RSS is not supported");
@@ -284,6 +284,31 @@ iavf_init_rss(struct iavf_adapter *adapter)
 	return 0;
 }
 
+static int
+iavf_queues_req_reset(struct rte_eth_dev *dev, uint16_t num)
+{
+	struct iavf_adapter *ad =
+		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
+	struct iavf_info *vf =  IAVF_DEV_PRIVATE_TO_VF(ad);
+	int ret = 0;
+
+	ret = iavf_request_queues(ad, num);
+	if (ret) {
+		PMD_DRV_LOG(ERR, "request queues from PF failed");
+		return ret;
+	}
+	PMD_DRV_LOG(INFO, "change queue pairs from %u to %u",
+			vf->vsi_res->num_queue_pairs, num);
+
+	ret = iavf_dev_reset(dev);
+	if (ret) {
+		PMD_DRV_LOG(ERR, "vf reset failed");
+		return ret;
+	}
+
+	return 0;
+}
+
 static int
 iavf_dev_configure(struct rte_eth_dev *dev)
 {
@@ -292,6 +317,8 @@ iavf_dev_configure(struct rte_eth_dev *dev)
 	struct iavf_info *vf =  IAVF_DEV_PRIVATE_TO_VF(ad);
 	struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 	uint32_t frame_size = dev->data->mtu + IAVF_ETH_OVERHEAD;
+	uint16_t num_queue_pairs = RTE_MAX(dev->data->nb_rx_queues,
+		dev->data->nb_tx_queues);
 	int ret;
 
 	ad->rx_bulk_alloc_allowed = true;
@@ -314,6 +341,46 @@ iavf_dev_configure(struct rte_eth_dev *dev)
 			return ret;
 	}
 
+	/* Large VF setting */
+	if (num_queue_pairs > IAVF_MAX_NUM_QUEUES_DFLT) {
+		if (!(vf->vf_res->vf_cap_flags &
+				VIRTCHNL_VF_LARGE_NUM_QPAIRS)) {
+			PMD_DRV_LOG(ERR, "large VF is not supported");
+			return -1;
+		}
+
+		if (num_queue_pairs > IAVF_MAX_NUM_QUEUES_LV) {
+			PMD_DRV_LOG(ERR, "queue pairs number cannot be larger "
+				"than %u", IAVF_MAX_NUM_QUEUES_LV);
+			return -1;
+		}
+
+		ret = iavf_queues_req_reset(dev, num_queue_pairs);
+		if (ret)
+			return ret;
+
+		ret = iavf_get_max_rss_queue_region(ad);
+		if (ret) {
+			PMD_INIT_LOG(ERR, "get max rss queue region failed");
+			return ret;
+		}
+
+		vf->lv_enabled = true;
+	} else {
+		/* Check if large VF is already enabled. If so, disable and
+		 * release redundant queue resource.
+		 */
+		if (vf->lv_enabled) {
+			ret = iavf_queues_req_reset(dev, num_queue_pairs);
+			if (ret)
+				return ret;
+
+			vf->lv_enabled = false;
+		}
+		/* if large VF is not required, use default rss queue region */
+		vf->max_rss_qregion = IAVF_MAX_NUM_QUEUES_DFLT;
+	}
+
 	/* Vlan stripping setting */
 	if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN) {
 		if (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
@@ -328,6 +395,7 @@ iavf_dev_configure(struct rte_eth_dev *dev)
 			return -1;
 		}
 	}
+
 	return 0;
 }
 
@@ -410,6 +478,7 @@ static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev,
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter);
+	struct iavf_qv_map *qv_map;
 	uint16_t interval, i;
 	int vec;
 
@@ -430,6 +499,14 @@ static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev,
 		}
 	}
 
+	qv_map = rte_zmalloc("qv_map",
+		dev->data->nb_rx_queues * sizeof(struct iavf_qv_map), 0);
+	if (!qv_map) {
+		PMD_DRV_LOG(ERR, "Failed to allocate %d queue-vector map",
+				dev->data->nb_rx_queues);
+		return -1;
+	}
+
 	if (!dev->data->dev_conf.intr_conf.rxq ||
 	    !rte_intr_dp_is_en(intr_handle)) {
 		/* Rx interrupt disabled, Map interrupt only for writeback */
@@ -460,16 +537,21 @@ static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev,
 		}
 		IAVF_WRITE_FLUSH(hw);
 		/* map all queues to the same interrupt */
-		for (i = 0; i < dev->data->nb_rx_queues; i++)
-			vf->rxq_map[vf->msix_base] |= 1 << i;
+		for (i = 0; i < dev->data->nb_rx_queues; i++) {
+			qv_map[i].queue_id = i;
+			qv_map[i].vector_id = vf->msix_base;
+		}
+		vf->qv_map = qv_map;
 	} else {
 		if (!rte_intr_allow_others(intr_handle)) {
 			vf->nb_msix = 1;
 			vf->msix_base = IAVF_MISC_VEC_ID;
 			for (i = 0; i < dev->data->nb_rx_queues; i++) {
-				vf->rxq_map[vf->msix_base] |= 1 << i;
+				qv_map[i].queue_id = i;
+				qv_map[i].vector_id = vf->msix_base;
 				intr_handle->intr_vec[i] = IAVF_MISC_VEC_ID;
 			}
+			vf->qv_map = qv_map;
 			PMD_DRV_LOG(DEBUG,
 				    "vector %u are mapping to all Rx queues",
 				    vf->msix_base);
@@ -482,21 +564,32 @@ static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev,
 			vf->msix_base = IAVF_RX_VEC_START;
 			vec = IAVF_RX_VEC_START;
 			for (i = 0; i < dev->data->nb_rx_queues; i++) {
-				vf->rxq_map[vec] |= 1 << i;
+				qv_map[i].queue_id = i;
+				qv_map[i].vector_id = vec;
 				intr_handle->intr_vec[i] = vec++;
 				if (vec >= vf->nb_msix)
 					vec = IAVF_RX_VEC_START;
 			}
+			vf->qv_map = qv_map;
 			PMD_DRV_LOG(DEBUG,
 				    "%u vectors are mapping to %u Rx queues",
 				    vf->nb_msix, dev->data->nb_rx_queues);
 		}
 	}
 
-	if (iavf_config_irq_map(adapter)) {
-		PMD_DRV_LOG(ERR, "config interrupt mapping failed");
-		return -1;
+	if (!vf->lv_enabled) {
+		if (iavf_config_irq_map(adapter)) {
+			PMD_DRV_LOG(ERR, "config interrupt mapping failed");
+			return -1;
+		}
+	} else {
+		if (iavf_config_irq_map_lv(adapter)) {
+			PMD_DRV_LOG(ERR, "config interrupt mapping "
+				"for large VF failed");
+			return -1;
+		}
 	}
+
 	return 0;
 }
 
@@ -560,6 +653,7 @@ iavf_dev_start(struct rte_eth_dev *dev)
 		PMD_DRV_LOG(ERR, "configure irq failed");
 		goto err_queue;
 	}
+
 	/* re-enable intr again, because efd assign may change */
 	if (dev->data->dev_conf.intr_conf.rxq != 0) {
 		rte_intr_disable(intr_handle);
@@ -624,8 +718,8 @@ iavf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 {
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 
-	dev_info->max_rx_queues = vf->vsi_res->num_queue_pairs;
-	dev_info->max_tx_queues = vf->vsi_res->num_queue_pairs;
+	dev_info->max_rx_queues = IAVF_MAX_NUM_QUEUES_LV;
+	dev_info->max_tx_queues = IAVF_MAX_NUM_QUEUES_LV;
 	dev_info->min_rx_bufsize = IAVF_BUF_SIZE_MIN;
 	dev_info->max_rx_pktlen = IAVF_FRAME_SIZE_MAX;
 	dev_info->max_mtu = dev_info->max_rx_pktlen - IAVF_ETH_OVERHEAD;
@@ -1703,6 +1797,7 @@ iavf_init_vf(struct rte_eth_dev *dev)
 		PMD_INIT_LOG(ERR, "iavf_get_vf_config failed");
 		goto err_alloc;
 	}
+
 	/* Allocate memort for RSS info */
 	if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) {
 		vf->rss_key = rte_zmalloc("rss_key",
diff --git a/drivers/net/iavf/iavf_rxtx.c b/drivers/net/iavf/iavf_rxtx.c
index ca0f5d072..987eb3c4e 100644
--- a/drivers/net/iavf/iavf_rxtx.c
+++ b/drivers/net/iavf/iavf_rxtx.c
@@ -720,6 +720,7 @@ iavf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 	struct iavf_adapter *adapter =
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	struct iavf_rx_queue *rxq;
 	int err = 0;
 
@@ -743,7 +744,11 @@ iavf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 	IAVF_WRITE_FLUSH(hw);
 
 	/* Ready to switch the queue on */
-	err = iavf_switch_queue(adapter, rx_queue_id, true, true);
+	if (!vf->lv_enabled)
+		err = iavf_switch_queue(adapter, rx_queue_id, true, true);
+	else
+		err = iavf_switch_queue_lv(adapter, rx_queue_id, true, true);
+
 	if (err)
 		PMD_DRV_LOG(ERR, "Failed to switch RX queue %u on",
 			    rx_queue_id);
@@ -760,6 +765,7 @@ iavf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 	struct iavf_adapter *adapter =
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	struct iavf_tx_queue *txq;
 	int err = 0;
 
@@ -775,7 +781,10 @@ iavf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 	IAVF_WRITE_FLUSH(hw);
 
 	/* Ready to switch the queue on */
-	err = iavf_switch_queue(adapter, tx_queue_id, false, true);
+	if (!vf->lv_enabled)
+		err = iavf_switch_queue(adapter, tx_queue_id, false, true);
+	else
+		err = iavf_switch_queue_lv(adapter, tx_queue_id, false, true);
 
 	if (err)
 		PMD_DRV_LOG(ERR, "Failed to switch TX queue %u on",
@@ -876,14 +885,22 @@ iavf_stop_queues(struct rte_eth_dev *dev)
 {
 	struct iavf_adapter *adapter =
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	struct iavf_rx_queue *rxq;
 	struct iavf_tx_queue *txq;
 	int ret, i;
 
 	/* Stop All queues */
-	ret = iavf_disable_queues(adapter);
-	if (ret)
-		PMD_DRV_LOG(WARNING, "Fail to stop queues");
+	if (!vf->lv_enabled) {
+		ret = iavf_disable_queues(adapter);
+		if (ret)
+			PMD_DRV_LOG(WARNING, "Fail to stop queues");
+	} else {
+		ret = iavf_disable_queues_lv(adapter);
+		if (ret)
+			PMD_DRV_LOG(WARNING, "Fail to stop queues for large VF");
+	}
+
 
 	for (i = 0; i < dev->data->nb_tx_queues; i++) {
 		txq = dev->data->tx_queues[i];
diff --git a/drivers/net/iavf/iavf_vchnl.c b/drivers/net/iavf/iavf_vchnl.c
index 11a1ff608..742a9d489 100644
--- a/drivers/net/iavf/iavf_vchnl.c
+++ b/drivers/net/iavf/iavf_vchnl.c
@@ -457,7 +457,8 @@ iavf_get_vf_resource(struct iavf_adapter *adapter)
 		VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC |
 		VIRTCHNL_VF_OFFLOAD_FDIR_PF |
 		VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF |
-		VIRTCHNL_VF_OFFLOAD_REQ_QUEUES;
+		VIRTCHNL_VF_OFFLOAD_REQ_QUEUES |
+		VIRTCHNL_VF_LARGE_NUM_QPAIRS;
 
 	args.in_args = (uint8_t *)&caps;
 	args.in_args_size = sizeof(caps);
@@ -607,6 +608,138 @@ iavf_switch_queue(struct iavf_adapter *adapter, uint16_t qid,
 	return err;
 }
 
+int
+iavf_enable_queues_lv(struct iavf_adapter *adapter)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct virtchnl_del_ena_dis_queues *queue_select;
+	struct virtchnl_queue_chunk *queue_chunk;
+	struct iavf_cmd_info args;
+	int err, len;
+
+	len = sizeof(struct virtchnl_del_ena_dis_queues) +
+		  sizeof(struct virtchnl_queue_chunk) *
+		  (IAVF_RXTX_QUEUE_CHUNKS_NUM - 1);
+	queue_select = rte_zmalloc("queue_select", len, 0);
+	if (!queue_select)
+		return -ENOMEM;
+
+	queue_chunk = queue_select->chunks.chunks;
+	queue_select->chunks.num_chunks = IAVF_RXTX_QUEUE_CHUNKS_NUM;
+	queue_select->vport_id = vf->vsi_res->vsi_id;
+
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].type = VIRTCHNL_QUEUE_TYPE_TX;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].start_queue_id = 0;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].num_queues =
+		adapter->eth_dev->data->nb_tx_queues;
+
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].type = VIRTCHNL_QUEUE_TYPE_RX;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].start_queue_id = 0;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].num_queues =
+		adapter->eth_dev->data->nb_rx_queues;
+
+	args.ops = VIRTCHNL_OP_ENABLE_QUEUES_V2;
+	args.in_args = (u8 *)queue_select;
+	args.in_args_size = len;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err) {
+		PMD_DRV_LOG(ERR,
+			    "Failed to execute command of OP_ENABLE_QUEUES_V2");
+		return err;
+	}
+	return 0;
+}
+
+int
+iavf_disable_queues_lv(struct iavf_adapter *adapter)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct virtchnl_del_ena_dis_queues *queue_select;
+	struct virtchnl_queue_chunk *queue_chunk;
+	struct iavf_cmd_info args;
+	int err, len;
+
+	len = sizeof(struct virtchnl_del_ena_dis_queues) +
+		  sizeof(struct virtchnl_queue_chunk) *
+		  (IAVF_RXTX_QUEUE_CHUNKS_NUM - 1);
+	queue_select = rte_zmalloc("queue_select", len, 0);
+	if (!queue_select)
+		return -ENOMEM;
+
+	queue_chunk = queue_select->chunks.chunks;
+	queue_select->chunks.num_chunks = IAVF_RXTX_QUEUE_CHUNKS_NUM;
+	queue_select->vport_id = vf->vsi_res->vsi_id;
+
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].type = VIRTCHNL_QUEUE_TYPE_TX;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].start_queue_id = 0;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].num_queues =
+		adapter->eth_dev->data->nb_tx_queues;
+
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].type = VIRTCHNL_QUEUE_TYPE_RX;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].start_queue_id = 0;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].num_queues =
+		adapter->eth_dev->data->nb_rx_queues;
+
+	args.ops = VIRTCHNL_OP_DISABLE_QUEUES_V2;
+	args.in_args = (u8 *)queue_select;
+	args.in_args_size = len;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err) {
+		PMD_DRV_LOG(ERR,
+			    "Failed to execute command of OP_DISABLE_QUEUES_V2");
+		return err;
+	}
+	return 0;
+}
+
+int
+iavf_switch_queue_lv(struct iavf_adapter *adapter, uint16_t qid,
+		 bool rx, bool on)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct virtchnl_del_ena_dis_queues *queue_select;
+	struct virtchnl_queue_chunk *queue_chunk;
+	struct iavf_cmd_info args;
+	int err, len;
+
+	len = sizeof(struct virtchnl_del_ena_dis_queues);
+	queue_select = rte_zmalloc("queue_select", len, 0);
+	if (!queue_select)
+		return -ENOMEM;
+
+	queue_chunk = queue_select->chunks.chunks;
+	queue_select->chunks.num_chunks = 1;
+	queue_select->vport_id = vf->vsi_res->vsi_id;
+
+	if (rx) {
+		queue_chunk->type = VIRTCHNL_QUEUE_TYPE_RX;
+		queue_chunk->start_queue_id = qid;
+		queue_chunk->num_queues = 1;
+	} else {
+		queue_chunk->type = VIRTCHNL_QUEUE_TYPE_TX;
+		queue_chunk->start_queue_id = qid;
+		queue_chunk->num_queues = 1;
+	}
+
+	if (on)
+		args.ops = VIRTCHNL_OP_ENABLE_QUEUES_V2;
+	else
+		args.ops = VIRTCHNL_OP_DISABLE_QUEUES_V2;
+	args.in_args = (u8 *)queue_select;
+	args.in_args_size = len;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err)
+		PMD_DRV_LOG(ERR, "Failed to execute command of %s",
+			    on ? "OP_ENABLE_QUEUES_V2" : "OP_DISABLE_QUEUES_V2");
+	return err;
+}
+
 int
 iavf_configure_rss_lut(struct iavf_adapter *adapter)
 {
@@ -671,32 +804,26 @@ iavf_configure_rss_key(struct iavf_adapter *adapter)
 	return err;
 }
 
-int
-iavf_configure_queues(struct iavf_adapter *adapter)
+static int
+iavf_exec_queue_cfg(struct iavf_adapter *adapter,
+	struct virtchnl_vsi_queue_config_info *vc_config, uint16_t count)
 {
 	struct iavf_rx_queue **rxq =
 		(struct iavf_rx_queue **)adapter->eth_dev->data->rx_queues;
 	struct iavf_tx_queue **txq =
 		(struct iavf_tx_queue **)adapter->eth_dev->data->tx_queues;
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
-	struct virtchnl_vsi_queue_config_info *vc_config;
 	struct virtchnl_queue_pair_info *vc_qp;
 	struct iavf_cmd_info args;
 	uint16_t i, size;
-	int err;
+	int err = 0;
 
 	size = sizeof(*vc_config) +
-	       sizeof(vc_config->qpair[0]) * vf->num_queue_pairs;
-	vc_config = rte_zmalloc("cfg_queue", size, 0);
-	if (!vc_config)
-		return -ENOMEM;
+		sizeof(vc_config->qpair[0]) * vc_config->num_queue_pairs;
 
-	vc_config->vsi_id = vf->vsi_res->vsi_id;
-	vc_config->num_queue_pairs = vf->num_queue_pairs;
-
-	for (i = 0, vc_qp = vc_config->qpair;
-	     i < vf->num_queue_pairs;
-	     i++, vc_qp++) {
+	for (i = count * IAVF_CFG_Q_NUM_PER_BUF, vc_qp = vc_config->qpair;
+	     i < count * IAVF_CFG_Q_NUM_PER_BUF + vc_config->num_queue_pairs;
+		 i++, vc_qp++) {
 		vc_qp->txq.vsi_id = vf->vsi_res->vsi_id;
 		vc_qp->txq.queue_id = i;
 		/* Virtchnnl configure queues by pairs */
@@ -752,8 +879,71 @@ iavf_configure_queues(struct iavf_adapter *adapter)
 	err = iavf_execute_vf_cmd(adapter, &args);
 	if (err)
 		PMD_DRV_LOG(ERR, "Failed to execute command of"
-			    " VIRTCHNL_OP_CONFIG_VSI_QUEUES");
+				" VIRTCHNL_OP_CONFIG_VSI_QUEUES");
+
+	return err;
+}
+
+/* Configure VSI queues. Max VF queue pairs number is 256, may
+ * send this command multiple times to configure all queues.
+ */
+int
+iavf_configure_queues(struct iavf_adapter *adapter)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct virtchnl_vsi_queue_config_info *vc_config = NULL;
+	uint16_t i, size_full, size_inc;
+	uint16_t nb_cmd_full, nbq_inc;
+	int err = 0;
+
+	/* Compute how many times should the command to be sent,
+	 * including the commands with full buffer and incomplete
+	 * buffer.
+	 */
+	nbq_inc = vf->num_queue_pairs % IAVF_CFG_Q_NUM_PER_BUF;
+	nb_cmd_full = vf->num_queue_pairs / IAVF_CFG_Q_NUM_PER_BUF;
+
+	size_full = sizeof(*vc_config) +
+		sizeof(vc_config->qpair[0]) * IAVF_CFG_Q_NUM_PER_BUF;
+	size_inc = sizeof(*vc_config) +
+		sizeof(vc_config->qpair[0]) * nbq_inc;
+
+	if (!nb_cmd_full) {
+		vc_config = rte_zmalloc("cfg_queue", size_inc, 0);
+		if (!vc_config)
+			return -ENOMEM;
+
+		vc_config->vsi_id = vf->vsi_res->vsi_id;
+		vc_config->num_queue_pairs = nbq_inc;
+		err = iavf_exec_queue_cfg(adapter, vc_config, 0);
+		goto free;
+	}
+
+	vc_config = rte_zmalloc("cfg_queue", size_full, 0);
+	if (!vc_config)
+		return -ENOMEM;
 
+	vc_config->vsi_id = vf->vsi_res->vsi_id;
+	vc_config->num_queue_pairs = IAVF_CFG_Q_NUM_PER_BUF;
+
+	for (i = 0; i < nb_cmd_full + (nbq_inc ? 1 : 0); i++) {
+		if (i >= nb_cmd_full) {
+			/* re-allocate virtchnl msg for less queues */
+			rte_free(vc_config);
+			vc_config = rte_zmalloc("cfg_queue", size_inc, 0);
+			if (!vc_config)
+				return -ENOMEM;
+
+			vc_config->vsi_id = vf->vsi_res->vsi_id;
+			vc_config->num_queue_pairs = nbq_inc;
+		}
+
+		err = iavf_exec_queue_cfg(adapter, vc_config, i);
+		if (err)
+			break;
+	}
+
+free:
 	rte_free(vc_config);
 	return err;
 }
@@ -775,13 +965,14 @@ iavf_config_irq_map(struct iavf_adapter *adapter)
 		return -ENOMEM;
 
 	map_info->num_vectors = vf->nb_msix;
-	for (i = 0; i < vf->nb_msix; i++) {
-		vecmap = &map_info->vecmap[i];
+	for (i = 0; i < adapter->eth_dev->data->nb_rx_queues; i++) {
+		vecmap =
+		    &map_info->vecmap[vf->qv_map[i].vector_id - vf->msix_base];
 		vecmap->vsi_id = vf->vsi_res->vsi_id;
 		vecmap->rxitr_idx = IAVF_ITR_INDEX_DEFAULT;
-		vecmap->vector_id = vf->msix_base + i;
+		vecmap->vector_id = vf->qv_map[i].vector_id;
 		vecmap->txq_map = 0;
-		vecmap->rxq_map = vf->rxq_map[vf->msix_base + i];
+		vecmap->rxq_map |= 1 << vf->qv_map[i].queue_id;
 	}
 
 	args.ops = VIRTCHNL_OP_CONFIG_IRQ_MAP;
@@ -797,6 +988,46 @@ iavf_config_irq_map(struct iavf_adapter *adapter)
 	return err;
 }
 
+int
+iavf_config_irq_map_lv(struct iavf_adapter *adapter)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct virtchnl_queue_vector_maps *map_info;
+	struct virtchnl_queue_vector *qv_maps;
+	struct iavf_cmd_info args;
+	int len, i, err;
+
+	len = sizeof(struct virtchnl_queue_vector_maps) +
+	      sizeof(struct virtchnl_queue_vector) *
+	      (adapter->eth_dev->data->nb_rx_queues - 1);
+
+	map_info = rte_zmalloc("map_info", len, 0);
+	if (!map_info)
+		return -ENOMEM;
+
+	map_info->vport_id = vf->vsi_res->vsi_id;
+	map_info->num_qv_maps = adapter->eth_dev->data->nb_rx_queues;
+	for (i = 0; i < map_info->num_qv_maps; i++) {
+		qv_maps = &map_info->qv_maps[i];
+		qv_maps->itr_idx = VIRTCHNL_ITR_IDX_0;
+		qv_maps->queue_type = VIRTCHNL_QUEUE_TYPE_RX;
+		qv_maps->queue_id = vf->qv_map[i].queue_id;
+		qv_maps->vector_id = vf->qv_map[i].vector_id;
+	}
+
+	args.ops = VIRTCHNL_OP_MAP_QUEUE_VECTOR;
+	args.in_args = (u8 *)map_info;
+	args.in_args_size = len;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err)
+		PMD_DRV_LOG(ERR, "fail to execute command OP_MAP_QUEUE_VECTOR");
+
+	rte_free(map_info);
+	return err;
+}
+
 void
 iavf_add_del_all_mac_addr(struct iavf_adapter *adapter, bool add)
 {
@@ -1272,3 +1503,33 @@ iavf_request_queues(struct iavf_adapter *adapter, uint16_t num)
 
 	return -1;
 }
+
+int
+iavf_get_max_rss_queue_region(struct iavf_adapter *adapter)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct iavf_cmd_info args;
+	uint16_t qregion_width;
+	int err;
+
+	args.ops = VIRTCHNL_OP_GET_MAX_RSS_QREGION;
+	args.in_args = NULL;
+	args.in_args_size = 0;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err) {
+		PMD_DRV_LOG(ERR,
+			    "Failed to execute command of "
+			    "VIRTCHNL_OP_GET_MAX_RSS_QREGION");
+		return err;
+	}
+
+	qregion_width =
+	((struct virtchnl_max_rss_qregion *)args.out_buffer)->qregion_width;
+
+	vf->max_rss_qregion = (uint16_t)(1 << qregion_width);
+
+	return 0;
+}
-- 
2.17.1


^ permalink raw reply	[flat|nested] 45+ messages in thread

* [dpdk-dev] [PATCH v5 0/2] enable large VF configuration
  2020-09-09  7:20 [dpdk-dev] [PATCH v1 0/2] enable large VF configuration Ting Xu
                   ` (4 preceding siblings ...)
  2020-10-15  5:21 ` [dpdk-dev] [PATCH v4 0/2] " Ting Xu
@ 2020-10-16  1:21 ` Ting Xu
  2020-10-16  1:21   ` [dpdk-dev] [PATCH v5 1/2] net/iavf: add IAVF request queues function Ting Xu
  2020-10-16  1:21   ` [dpdk-dev] [PATCH v5 2/2] net/iavf: enable large VF configuration Ting Xu
  2020-10-16  1:34 ` [dpdk-dev] [PATCH v5 0/2] " Ting Xu
                   ` (3 subsequent siblings)
  9 siblings, 2 replies; 45+ messages in thread
From: Ting Xu @ 2020-10-16  1:21 UTC (permalink / raw)
  To: dev; +Cc: qi.z.zhang, beilei.xing, jingjing.wu, Ting Xu

This patchset supports to configure up to 256 queue pairs per VF. If
large VF is supported after capability negotiation, VF will request
queues from PF as needed. New virtual channel opcodes and structures
are used to indicate 256 queues, so VF is designed to handle the new
function of configure VSI queues, IRQ mapping and enable/disable queues.
Also, enable VF to query the max RSS queue region for RSS and filter
configuration.

Ting Xu (2):
  net/iavf: add IAVF request queues function
  net/iavf: enable large VF configuration

---
v4->v5:
Solve queue mapping buffer limitation issue.
Optimize VSI queue configuration

v3->v4:
Optimize handling messages from PF.

v2->v3:
Fix coding style issue.
Move get max RSS queue region after VF reset.
Add request queues capability negotiation.

v1->v2:
Change the communication with kernel PF.

---
 drivers/net/iavf/iavf.h        |  36 ++-
 drivers/net/iavf/iavf_ethdev.c | 156 ++++++++++--
 drivers/net/iavf/iavf_rxtx.c   |  27 +-
 drivers/net/iavf/iavf_vchnl.c  | 448 ++++++++++++++++++++++++++++-----
 4 files changed, 582 insertions(+), 85 deletions(-)

-- 
2.17.1


^ permalink raw reply	[flat|nested] 45+ messages in thread

* [dpdk-dev] [PATCH v5 1/2] net/iavf: add IAVF request queues function
  2020-10-16  1:21 ` [dpdk-dev] [PATCH v5 0/2] " Ting Xu
@ 2020-10-16  1:21   ` Ting Xu
  2020-10-16  1:21   ` [dpdk-dev] [PATCH v5 2/2] net/iavf: enable large VF configuration Ting Xu
  1 sibling, 0 replies; 45+ messages in thread
From: Ting Xu @ 2020-10-16  1:21 UTC (permalink / raw)
  To: dev; +Cc: qi.z.zhang, beilei.xing, jingjing.wu, Ting Xu

Add new virtchnl function to request additional queues from PF. Current
default queue pairs number when creating a VF is 16. In order to support
up to 256 queue pairs, enable this request queues function.
Since request queues command may return event message, modify function
iavf_read_msg_from_pf to identify event opcode and mark VF reset status.

Signed-off-by: Ting Xu <ting.xu@intel.com>
---
 drivers/net/iavf/iavf.h        |   9 ++
 drivers/net/iavf/iavf_ethdev.c |  11 +-
 drivers/net/iavf/iavf_vchnl.c  | 226 +++++++++++++++++++++++++--------
 3 files changed, 192 insertions(+), 54 deletions(-)

diff --git a/drivers/net/iavf/iavf.h b/drivers/net/iavf/iavf.h
index d56611608..93c165c62 100644
--- a/drivers/net/iavf/iavf.h
+++ b/drivers/net/iavf/iavf.h
@@ -107,6 +107,14 @@ struct iavf_fdir_info {
 /* TODO: is that correct to assume the max number to be 16 ?*/
 #define IAVF_MAX_MSIX_VECTORS   16
 
+/* Message type read in admin queue from PF */
+enum iavf_aq_result {
+	IAVF_MSG_ERR = -1, /* Meet error when accessing admin queue */
+	IAVF_MSG_NON,      /* Read nothing from admin queue */
+	IAVF_MSG_SYS,      /* Read system msg from admin queue */
+	IAVF_MSG_CMD,      /* Read async command result */
+};
+
 /* Structure to store private data specific for VF instance. */
 struct iavf_info {
 	uint16_t num_queue_pairs;
@@ -301,4 +309,5 @@ int iavf_add_del_rss_cfg(struct iavf_adapter *adapter,
 int iavf_add_del_mc_addr_list(struct iavf_adapter *adapter,
 			struct rte_ether_addr *mc_addrs,
 			uint32_t mc_addrs_num, bool add);
+int iavf_request_queues(struct iavf_adapter *adapter, uint16_t num);
 #endif /* _IAVF_ETHDEV_H_ */
diff --git a/drivers/net/iavf/iavf_ethdev.c b/drivers/net/iavf/iavf_ethdev.c
index 8b1cf8f1c..a4a28b885 100644
--- a/drivers/net/iavf/iavf_ethdev.c
+++ b/drivers/net/iavf/iavf_ethdev.c
@@ -1282,7 +1282,7 @@ iavf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
 }
 
 static int
-iavf_check_vf_reset_done(struct iavf_hw *hw)
+iavf_check_vf_reset_done(struct iavf_hw *hw, struct iavf_info *vf)
 {
 	int i, reset;
 
@@ -1299,6 +1299,9 @@ iavf_check_vf_reset_done(struct iavf_hw *hw)
 	if (i >= IAVF_RESET_WAIT_CNT)
 		return -1;
 
+	/* VF is not in reset or reset is completed */
+	vf->vf_reset = false;
+
 	return 0;
 }
 
@@ -1666,7 +1669,7 @@ iavf_init_vf(struct rte_eth_dev *dev)
 		goto err;
 	}
 
-	err = iavf_check_vf_reset_done(hw);
+	err = iavf_check_vf_reset_done(hw, vf);
 	if (err) {
 		PMD_INIT_LOG(ERR, "VF is still resetting");
 		goto err;
@@ -1911,7 +1914,9 @@ iavf_dev_close(struct rte_eth_dev *dev)
 
 	iavf_dev_stop(dev);
 	iavf_flow_flush(dev, NULL);
-	iavf_flow_uninit(adapter);
+	/* if VF is in reset, adminq is disabled, skip the process via adminq */
+	if (!vf->vf_reset)
+		iavf_flow_uninit(adapter);
 
 	/*
 	 * disable promiscuous mode before reset vf
diff --git a/drivers/net/iavf/iavf_vchnl.c b/drivers/net/iavf/iavf_vchnl.c
index 5e7142893..11a1ff608 100644
--- a/drivers/net/iavf/iavf_vchnl.c
+++ b/drivers/net/iavf/iavf_vchnl.c
@@ -17,6 +17,7 @@
 #include <rte_eal.h>
 #include <rte_ether.h>
 #include <rte_ethdev_driver.h>
+#include <rte_ethdev_pci.h>
 #include <rte_dev.h>
 
 #include "iavf.h"
@@ -25,14 +26,54 @@
 #define MAX_TRY_TIMES 200
 #define ASQ_DELAY_MS  10
 
+static uint32_t
+iavf_convert_link_speed(enum virtchnl_link_speed virt_link_speed)
+{
+	uint32_t speed;
+
+	switch (virt_link_speed) {
+	case VIRTCHNL_LINK_SPEED_100MB:
+		speed = 100;
+		break;
+	case VIRTCHNL_LINK_SPEED_1GB:
+		speed = 1000;
+		break;
+	case VIRTCHNL_LINK_SPEED_10GB:
+		speed = 10000;
+		break;
+	case VIRTCHNL_LINK_SPEED_40GB:
+		speed = 40000;
+		break;
+	case VIRTCHNL_LINK_SPEED_20GB:
+		speed = 20000;
+		break;
+	case VIRTCHNL_LINK_SPEED_25GB:
+		speed = 25000;
+		break;
+	case VIRTCHNL_LINK_SPEED_2_5GB:
+		speed = 2500;
+		break;
+	case VIRTCHNL_LINK_SPEED_5GB:
+		speed = 5000;
+		break;
+	default:
+		speed = 0;
+		break;
+	}
+
+	return speed;
+}
+
 /* Read data in admin queue to get msg from pf driver */
-static enum iavf_status
+static enum iavf_aq_result
 iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,
 		     uint8_t *buf)
 {
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter);
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct rte_eth_dev *dev = adapter->eth_dev;
 	struct iavf_arq_event_info event;
+	enum iavf_aq_result result = IAVF_MSG_NON;
 	enum virtchnl_ops opcode;
 	int ret;
 
@@ -42,7 +83,9 @@ iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,
 	/* Can't read any msg from adminQ */
 	if (ret) {
 		PMD_DRV_LOG(DEBUG, "Can't read msg from AQ");
-		return ret;
+		if (ret != IAVF_ERR_ADMIN_QUEUE_NO_WORK)
+			result = IAVF_MSG_ERR;
+		return result;
 	}
 
 	opcode = (enum virtchnl_ops)rte_le_to_cpu_32(event.desc.cookie_high);
@@ -52,16 +95,51 @@ iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,
 	PMD_DRV_LOG(DEBUG, "AQ from pf carries opcode %u, retval %d",
 		    opcode, vf->cmd_retval);
 
-	if (opcode != vf->pend_cmd) {
-		if (opcode != VIRTCHNL_OP_EVENT) {
-			PMD_DRV_LOG(WARNING,
-				    "command mismatch, expect %u, get %u",
-				    vf->pend_cmd, opcode);
+	if (opcode == VIRTCHNL_OP_EVENT) {
+		struct virtchnl_pf_event *vpe =
+			(struct virtchnl_pf_event *)event.msg_buf;
+
+		result = IAVF_MSG_SYS;
+		switch (vpe->event) {
+		case VIRTCHNL_EVENT_LINK_CHANGE:
+			vf->link_up =
+				vpe->event_data.link_event.link_status;
+			if (vf->vf_res->vf_cap_flags &
+				VIRTCHNL_VF_CAP_ADV_LINK_SPEED) {
+				vf->link_speed =
+				    vpe->event_data.link_event_adv.link_speed;
+			} else {
+				enum virtchnl_link_speed speed;
+				speed = vpe->event_data.link_event.link_speed;
+				vf->link_speed = iavf_convert_link_speed(speed);
+			}
+			iavf_dev_link_update(dev, 0);
+			PMD_DRV_LOG(INFO, "Link status update:%s",
+					vf->link_up ? "up" : "down");
+			break;
+		case VIRTCHNL_EVENT_RESET_IMPENDING:
+			vf->vf_reset = true;
+			PMD_DRV_LOG(INFO, "VF is resetting");
+			break;
+		case VIRTCHNL_EVENT_PF_DRIVER_CLOSE:
+			vf->dev_closed = true;
+			PMD_DRV_LOG(INFO, "PF driver closed");
+			break;
+		default:
+			PMD_DRV_LOG(ERR, "%s: Unknown event %d from pf",
+					__func__, vpe->event);
+		}
+	}  else {
+		/* async reply msg on command issued by vf previously */
+		result = IAVF_MSG_CMD;
+		if (opcode != vf->pend_cmd) {
+			PMD_DRV_LOG(WARNING, "command mismatch, expect %u, get %u",
+					vf->pend_cmd, opcode);
+			result = IAVF_MSG_ERR;
 		}
-		return IAVF_ERR_OPCODE_MISMATCH;
 	}
 
-	return IAVF_SUCCESS;
+	return result;
 }
 
 static int
@@ -69,6 +147,7 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter, struct iavf_cmd_info *args)
 {
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter);
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	enum iavf_aq_result result;
 	enum iavf_status ret;
 	int err = 0;
 	int i = 0;
@@ -97,9 +176,9 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter, struct iavf_cmd_info *args)
 	case VIRTCHNL_OP_GET_SUPPORTED_RXDIDS:
 		/* for init virtchnl ops, need to poll the response */
 		do {
-			ret = iavf_read_msg_from_pf(adapter, args->out_size,
+			result = iavf_read_msg_from_pf(adapter, args->out_size,
 						   args->out_buffer);
-			if (ret == IAVF_SUCCESS)
+			if (result == IAVF_MSG_CMD)
 				break;
 			rte_delay_ms(ASQ_DELAY_MS);
 		} while (i++ < MAX_TRY_TIMES);
@@ -111,7 +190,33 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter, struct iavf_cmd_info *args)
 		}
 		_clear_cmd(vf);
 		break;
-
+	case VIRTCHNL_OP_REQUEST_QUEUES:
+		/*
+		 * ignore async reply, only wait for system message,
+		 * vf_reset = true if get VIRTCHNL_EVENT_RESET_IMPENDING,
+		 * if not, means request queues failed.
+		 */
+		do {
+			result = iavf_read_msg_from_pf(adapter, args->out_size,
+						   args->out_buffer);
+			if (result == IAVF_MSG_SYS && vf->vf_reset) {
+				break;
+			} else if (result == IAVF_MSG_CMD ||
+				result == IAVF_MSG_ERR) {
+				err = -1;
+				break;
+			}
+			rte_delay_ms(ASQ_DELAY_MS);
+			/* If don't read msg or read sys event, continue */
+		} while (i++ < MAX_TRY_TIMES);
+		if (i >= MAX_TRY_TIMES ||
+		    vf->cmd_retval != VIRTCHNL_STATUS_SUCCESS) {
+			err = -1;
+			PMD_DRV_LOG(ERR, "No response or return failure (%d)"
+				    " for cmd %d", vf->cmd_retval, args->ops);
+		}
+		_clear_cmd(vf);
+		break;
 	default:
 		/* For other virtchnl ops in running time,
 		 * wait for the cmd done flag.
@@ -136,44 +241,6 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter, struct iavf_cmd_info *args)
 	return err;
 }
 
-static uint32_t
-iavf_convert_link_speed(enum virtchnl_link_speed virt_link_speed)
-{
-	uint32_t speed;
-
-	switch (virt_link_speed) {
-	case VIRTCHNL_LINK_SPEED_100MB:
-		speed = 100;
-		break;
-	case VIRTCHNL_LINK_SPEED_1GB:
-		speed = 1000;
-		break;
-	case VIRTCHNL_LINK_SPEED_10GB:
-		speed = 10000;
-		break;
-	case VIRTCHNL_LINK_SPEED_40GB:
-		speed = 40000;
-		break;
-	case VIRTCHNL_LINK_SPEED_20GB:
-		speed = 20000;
-		break;
-	case VIRTCHNL_LINK_SPEED_25GB:
-		speed = 25000;
-		break;
-	case VIRTCHNL_LINK_SPEED_2_5GB:
-		speed = 2500;
-		break;
-	case VIRTCHNL_LINK_SPEED_5GB:
-		speed = 5000;
-		break;
-	default:
-		speed = 0;
-		break;
-	}
-
-	return speed;
-}
-
 static void
 iavf_handle_pf_event_msg(struct rte_eth_dev *dev, uint8_t *msg,
 			uint16_t msglen)
@@ -389,7 +456,8 @@ iavf_get_vf_resource(struct iavf_adapter *adapter)
 	caps = IAVF_BASIC_OFFLOAD_CAPS | VIRTCHNL_VF_CAP_ADV_LINK_SPEED |
 		VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC |
 		VIRTCHNL_VF_OFFLOAD_FDIR_PF |
-		VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF;
+		VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF |
+		VIRTCHNL_VF_OFFLOAD_REQ_QUEUES;
 
 	args.in_args = (uint8_t *)&caps;
 	args.in_args_size = sizeof(caps);
@@ -1148,3 +1216,59 @@ iavf_add_del_mc_addr_list(struct iavf_adapter *adapter,
 
 	return 0;
 }
+
+int
+iavf_request_queues(struct iavf_adapter *adapter, uint16_t num)
+{
+	struct rte_eth_dev *dev = adapter->eth_dev;
+	struct iavf_info *vf =  IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+	struct virtchnl_vf_res_request vfres;
+	struct iavf_cmd_info args;
+	uint16_t num_queue_pairs;
+	int err;
+
+	if (!(vf->vf_res->vf_cap_flags &
+		VIRTCHNL_VF_OFFLOAD_REQ_QUEUES)) {
+		PMD_DRV_LOG(ERR, "request queues not supported");
+		return -1;
+	}
+
+	if (num == 0) {
+		PMD_DRV_LOG(ERR, "queue number cannot be zero");
+		return -1;
+	}
+	vfres.num_queue_pairs = num;
+
+	args.ops = VIRTCHNL_OP_REQUEST_QUEUES;
+	args.in_args = (u8 *)&vfres;
+	args.in_args_size = sizeof(vfres);
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+
+	/*
+	 * disable interrupt to avoid the admin queue message to be read
+	 * before iavf_read_msg_from_pf.
+	 */
+	rte_intr_disable(&pci_dev->intr_handle);
+	err = iavf_execute_vf_cmd(adapter, &args);
+	rte_intr_enable(&pci_dev->intr_handle);
+	if (err) {
+		PMD_DRV_LOG(ERR, "fail to execute command OP_REQUEST_QUEUES");
+		return err;
+	}
+
+	/* request queues succeeded, vf is resetting */
+	if (vf->vf_reset) {
+		PMD_DRV_LOG(INFO, "vf is resetting");
+		return 0;
+	}
+
+	/* request additional queues failed, return available number */
+	num_queue_pairs =
+	  ((struct virtchnl_vf_res_request *)args.out_buffer)->num_queue_pairs;
+	PMD_DRV_LOG(ERR, "request queues failed, only %u queues "
+		"available", num_queue_pairs);
+
+	return -1;
+}
-- 
2.17.1


^ permalink raw reply	[flat|nested] 45+ messages in thread

* [dpdk-dev] [PATCH v5 2/2] net/iavf: enable large VF configuration
  2020-10-16  1:21 ` [dpdk-dev] [PATCH v5 0/2] " Ting Xu
  2020-10-16  1:21   ` [dpdk-dev] [PATCH v5 1/2] net/iavf: add IAVF request queues function Ting Xu
@ 2020-10-16  1:21   ` Ting Xu
  1 sibling, 0 replies; 45+ messages in thread
From: Ting Xu @ 2020-10-16  1:21 UTC (permalink / raw)
  To: dev; +Cc: qi.z.zhang, beilei.xing, jingjing.wu, Ting Xu

Add support to negotiate large VF capability, including configure VSI
queues, enable/disable queues and IRQ mapping for large VF. Use new
virtchnl opcodes and structures to help indicate up to 256 queue pairs.
Need to request additional queues from PF first if the current allocated
queues are not enough. Add the step to query the current max RSS queue
region for RSS configuration since the max RSS queue region could be
smaller than the allocated queue pairs.

Signed-off-by: Ting Xu <ting.xu@intel.com>
---
 drivers/net/iavf/iavf.h        |  27 +++-
 drivers/net/iavf/iavf_ethdev.c | 145 +++++++++++++++++++--
 drivers/net/iavf/iavf_rxtx.c   |  27 +++-
 drivers/net/iavf/iavf_vchnl.c  | 224 +++++++++++++++++++++++++++++++--
 4 files changed, 391 insertions(+), 32 deletions(-)

diff --git a/drivers/net/iavf/iavf.h b/drivers/net/iavf/iavf.h
index 93c165c62..3de096879 100644
--- a/drivers/net/iavf/iavf.h
+++ b/drivers/net/iavf/iavf.h
@@ -19,7 +19,11 @@
 #define IAVF_FRAME_SIZE_MAX       9728
 #define IAVF_QUEUE_BASE_ADDR_UNIT 128
 
-#define IAVF_MAX_NUM_QUEUES       16
+#define IAVF_MAX_NUM_QUEUES_DFLT	 16
+#define IAVF_MAX_NUM_QUEUES_LV		 256
+#define IAVF_RXTX_QUEUE_CHUNKS_NUM	 2
+#define IAVF_CFG_Q_NUM_PER_BUF		 32
+#define IAVF_IRQ_MAP_NUM_PER_BUF	 128
 
 #define IAVF_NUM_MACADDR_MAX      64
 
@@ -104,8 +108,10 @@ struct iavf_fdir_info {
 	struct iavf_fdir_conf conf;
 };
 
-/* TODO: is that correct to assume the max number to be 16 ?*/
-#define IAVF_MAX_MSIX_VECTORS   16
+struct iavf_qv_map {
+	uint16_t queue_id;
+	uint16_t vector_id;
+};
 
 /* Message type read in admin queue from PF */
 enum iavf_aq_result {
@@ -149,14 +155,16 @@ struct iavf_info {
 	uint8_t *rss_key;
 	uint16_t nb_msix;   /* number of MSI-X interrupts on Rx */
 	uint16_t msix_base; /* msix vector base from */
-	/* queue bitmask for each vector */
-	uint16_t rxq_map[IAVF_MAX_MSIX_VECTORS];
+	uint16_t max_rss_qregion; /* max RSS queue region supported by PF */
+	struct iavf_qv_map *qv_map; /* queue vector mapping */
 	struct iavf_flow_list flow_list;
 	rte_spinlock_t flow_ops_lock;
 	struct iavf_parser_list rss_parser_list;
 	struct iavf_parser_list dist_parser_list;
 
 	struct iavf_fdir_info fdir; /* flow director info */
+	/* indicate large VF support enabled or not */
+	bool lv_enabled;
 };
 
 #define IAVF_MAX_PKT_TYPE 1024
@@ -283,13 +291,19 @@ int iavf_enable_vlan_strip(struct iavf_adapter *adapter);
 int iavf_disable_vlan_strip(struct iavf_adapter *adapter);
 int iavf_switch_queue(struct iavf_adapter *adapter, uint16_t qid,
 		     bool rx, bool on);
+int iavf_switch_queue_lv(struct iavf_adapter *adapter, uint16_t qid,
+		     bool rx, bool on);
 int iavf_enable_queues(struct iavf_adapter *adapter);
+int iavf_enable_queues_lv(struct iavf_adapter *adapter);
 int iavf_disable_queues(struct iavf_adapter *adapter);
+int iavf_disable_queues_lv(struct iavf_adapter *adapter);
 int iavf_configure_rss_lut(struct iavf_adapter *adapter);
 int iavf_configure_rss_key(struct iavf_adapter *adapter);
-int iavf_configure_queues(struct iavf_adapter *adapter);
+int iavf_configure_queues(struct iavf_adapter *adapter,
+			uint16_t num_queue_pairs, uint16_t index);
 int iavf_get_supported_rxdid(struct iavf_adapter *adapter);
 int iavf_config_irq_map(struct iavf_adapter *adapter);
+int iavf_config_irq_map_lv(struct iavf_adapter *adapter, uint16_t num);
 void iavf_add_del_all_mac_addr(struct iavf_adapter *adapter, bool add);
 int iavf_dev_link_update(struct rte_eth_dev *dev,
 			__rte_unused int wait_to_complete);
@@ -310,4 +324,5 @@ int iavf_add_del_mc_addr_list(struct iavf_adapter *adapter,
 			struct rte_ether_addr *mc_addrs,
 			uint32_t mc_addrs_num, bool add);
 int iavf_request_queues(struct iavf_adapter *adapter, uint16_t num);
+int iavf_get_max_rss_queue_region(struct iavf_adapter *adapter);
 #endif /* _IAVF_ETHDEV_H_ */
diff --git a/drivers/net/iavf/iavf_ethdev.c b/drivers/net/iavf/iavf_ethdev.c
index a4a28b885..c33021178 100644
--- a/drivers/net/iavf/iavf_ethdev.c
+++ b/drivers/net/iavf/iavf_ethdev.c
@@ -238,7 +238,7 @@ iavf_init_rss(struct iavf_adapter *adapter)
 
 	rss_conf = &adapter->eth_dev->data->dev_conf.rx_adv_conf.rss_conf;
 	nb_q = RTE_MIN(adapter->eth_dev->data->nb_rx_queues,
-		       IAVF_MAX_NUM_QUEUES);
+		       vf->max_rss_qregion);
 
 	if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) {
 		PMD_DRV_LOG(DEBUG, "RSS is not supported");
@@ -284,6 +284,31 @@ iavf_init_rss(struct iavf_adapter *adapter)
 	return 0;
 }
 
+static int
+iavf_queues_req_reset(struct rte_eth_dev *dev, uint16_t num)
+{
+	struct iavf_adapter *ad =
+		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
+	struct iavf_info *vf =  IAVF_DEV_PRIVATE_TO_VF(ad);
+	int ret = 0;
+
+	ret = iavf_request_queues(ad, num);
+	if (ret) {
+		PMD_DRV_LOG(ERR, "request queues from PF failed");
+		return ret;
+	}
+	PMD_DRV_LOG(INFO, "change queue pairs from %u to %u",
+			vf->vsi_res->num_queue_pairs, num);
+
+	ret = iavf_dev_reset(dev);
+	if (ret) {
+		PMD_DRV_LOG(ERR, "vf reset failed");
+		return ret;
+	}
+
+	return 0;
+}
+
 static int
 iavf_dev_configure(struct rte_eth_dev *dev)
 {
@@ -292,6 +317,8 @@ iavf_dev_configure(struct rte_eth_dev *dev)
 	struct iavf_info *vf =  IAVF_DEV_PRIVATE_TO_VF(ad);
 	struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 	uint32_t frame_size = dev->data->mtu + IAVF_ETH_OVERHEAD;
+	uint16_t num_queue_pairs = RTE_MAX(dev->data->nb_rx_queues,
+		dev->data->nb_tx_queues);
 	int ret;
 
 	ad->rx_bulk_alloc_allowed = true;
@@ -314,6 +341,46 @@ iavf_dev_configure(struct rte_eth_dev *dev)
 			return ret;
 	}
 
+	/* Large VF setting */
+	if (num_queue_pairs > IAVF_MAX_NUM_QUEUES_DFLT) {
+		if (!(vf->vf_res->vf_cap_flags &
+				VIRTCHNL_VF_LARGE_NUM_QPAIRS)) {
+			PMD_DRV_LOG(ERR, "large VF is not supported");
+			return -1;
+		}
+
+		if (num_queue_pairs > IAVF_MAX_NUM_QUEUES_LV) {
+			PMD_DRV_LOG(ERR, "queue pairs number cannot be larger "
+				"than %u", IAVF_MAX_NUM_QUEUES_LV);
+			return -1;
+		}
+
+		ret = iavf_queues_req_reset(dev, num_queue_pairs);
+		if (ret)
+			return ret;
+
+		ret = iavf_get_max_rss_queue_region(ad);
+		if (ret) {
+			PMD_INIT_LOG(ERR, "get max rss queue region failed");
+			return ret;
+		}
+
+		vf->lv_enabled = true;
+	} else {
+		/* Check if large VF is already enabled. If so, disable and
+		 * release redundant queue resource.
+		 */
+		if (vf->lv_enabled) {
+			ret = iavf_queues_req_reset(dev, num_queue_pairs);
+			if (ret)
+				return ret;
+
+			vf->lv_enabled = false;
+		}
+		/* if large VF is not required, use default rss queue region */
+		vf->max_rss_qregion = IAVF_MAX_NUM_QUEUES_DFLT;
+	}
+
 	/* Vlan stripping setting */
 	if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN) {
 		if (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
@@ -328,6 +395,7 @@ iavf_dev_configure(struct rte_eth_dev *dev)
 			return -1;
 		}
 	}
+
 	return 0;
 }
 
@@ -410,6 +478,7 @@ static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev,
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter);
+	struct iavf_qv_map *qv_map;
 	uint16_t interval, i;
 	int vec;
 
@@ -430,6 +499,14 @@ static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev,
 		}
 	}
 
+	qv_map = rte_zmalloc("qv_map",
+		dev->data->nb_rx_queues * sizeof(struct iavf_qv_map), 0);
+	if (!qv_map) {
+		PMD_DRV_LOG(ERR, "Failed to allocate %d queue-vector map",
+				dev->data->nb_rx_queues);
+		return -1;
+	}
+
 	if (!dev->data->dev_conf.intr_conf.rxq ||
 	    !rte_intr_dp_is_en(intr_handle)) {
 		/* Rx interrupt disabled, Map interrupt only for writeback */
@@ -460,16 +537,21 @@ static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev,
 		}
 		IAVF_WRITE_FLUSH(hw);
 		/* map all queues to the same interrupt */
-		for (i = 0; i < dev->data->nb_rx_queues; i++)
-			vf->rxq_map[vf->msix_base] |= 1 << i;
+		for (i = 0; i < dev->data->nb_rx_queues; i++) {
+			qv_map[i].queue_id = i;
+			qv_map[i].vector_id = vf->msix_base;
+		}
+		vf->qv_map = qv_map;
 	} else {
 		if (!rte_intr_allow_others(intr_handle)) {
 			vf->nb_msix = 1;
 			vf->msix_base = IAVF_MISC_VEC_ID;
 			for (i = 0; i < dev->data->nb_rx_queues; i++) {
-				vf->rxq_map[vf->msix_base] |= 1 << i;
+				qv_map[i].queue_id = i;
+				qv_map[i].vector_id = vf->msix_base;
 				intr_handle->intr_vec[i] = IAVF_MISC_VEC_ID;
 			}
+			vf->qv_map = qv_map;
 			PMD_DRV_LOG(DEBUG,
 				    "vector %u are mapping to all Rx queues",
 				    vf->msix_base);
@@ -482,21 +564,44 @@ static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev,
 			vf->msix_base = IAVF_RX_VEC_START;
 			vec = IAVF_RX_VEC_START;
 			for (i = 0; i < dev->data->nb_rx_queues; i++) {
-				vf->rxq_map[vec] |= 1 << i;
+				qv_map[i].queue_id = i;
+				qv_map[i].vector_id = vec;
 				intr_handle->intr_vec[i] = vec++;
 				if (vec >= vf->nb_msix)
 					vec = IAVF_RX_VEC_START;
 			}
+			vf->qv_map = qv_map;
 			PMD_DRV_LOG(DEBUG,
 				    "%u vectors are mapping to %u Rx queues",
 				    vf->nb_msix, dev->data->nb_rx_queues);
 		}
 	}
 
-	if (iavf_config_irq_map(adapter)) {
-		PMD_DRV_LOG(ERR, "config interrupt mapping failed");
-		return -1;
+	if (!vf->lv_enabled) {
+		if (iavf_config_irq_map(adapter)) {
+			PMD_DRV_LOG(ERR, "config interrupt mapping failed");
+			return -1;
+		}
+	} else {
+		uint16_t num_qv_maps = dev->data->nb_rx_queues;
+
+		while (num_qv_maps > IAVF_IRQ_MAP_NUM_PER_BUF) {
+			if (iavf_config_irq_map_lv(adapter,
+					IAVF_IRQ_MAP_NUM_PER_BUF)) {
+				PMD_DRV_LOG(ERR, "config interrupt mapping "
+					"for large VF failed");
+				return -1;
+			}
+			num_qv_maps -= IAVF_IRQ_MAP_NUM_PER_BUF;
+		}
+
+		if (iavf_config_irq_map_lv(adapter, num_qv_maps)) {
+			PMD_DRV_LOG(ERR, "config interrupt mapping "
+				"for large VF failed");
+			return -1;
+		}
 	}
+
 	return 0;
 }
 
@@ -537,6 +642,8 @@ iavf_dev_start(struct rte_eth_dev *dev)
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	struct rte_intr_handle *intr_handle = dev->intr_handle;
+	uint16_t num_queue_pairs;
+	uint16_t index = 0;
 
 	PMD_INIT_FUNC_TRACE();
 
@@ -545,13 +652,27 @@ iavf_dev_start(struct rte_eth_dev *dev)
 	vf->max_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
 	vf->num_queue_pairs = RTE_MAX(dev->data->nb_rx_queues,
 				      dev->data->nb_tx_queues);
+	num_queue_pairs = vf->num_queue_pairs;
 
 	if (iavf_init_queues(dev) != 0) {
 		PMD_DRV_LOG(ERR, "failed to do Queue init");
 		return -1;
 	}
 
-	if (iavf_configure_queues(adapter) != 0) {
+	/* If needed, send configure queues msg multiple times to make the
+	 * adminq buffer length smaller than the 4K limitation.
+	 */
+	while (num_queue_pairs > IAVF_CFG_Q_NUM_PER_BUF) {
+		if (iavf_configure_queues(adapter,
+				IAVF_CFG_Q_NUM_PER_BUF, index) != 0) {
+			PMD_DRV_LOG(ERR, "configure queues failed");
+			goto err_queue;
+		}
+		num_queue_pairs -= IAVF_CFG_Q_NUM_PER_BUF;
+		index += IAVF_CFG_Q_NUM_PER_BUF;
+	}
+
+	if (iavf_configure_queues(adapter, num_queue_pairs, index) != 0) {
 		PMD_DRV_LOG(ERR, "configure queues failed");
 		goto err_queue;
 	}
@@ -560,6 +681,7 @@ iavf_dev_start(struct rte_eth_dev *dev)
 		PMD_DRV_LOG(ERR, "configure irq failed");
 		goto err_queue;
 	}
+
 	/* re-enable intr again, because efd assign may change */
 	if (dev->data->dev_conf.intr_conf.rxq != 0) {
 		rte_intr_disable(intr_handle);
@@ -624,8 +746,8 @@ iavf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 {
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 
-	dev_info->max_rx_queues = vf->vsi_res->num_queue_pairs;
-	dev_info->max_tx_queues = vf->vsi_res->num_queue_pairs;
+	dev_info->max_rx_queues = IAVF_MAX_NUM_QUEUES_LV;
+	dev_info->max_tx_queues = IAVF_MAX_NUM_QUEUES_LV;
 	dev_info->min_rx_bufsize = IAVF_BUF_SIZE_MIN;
 	dev_info->max_rx_pktlen = IAVF_FRAME_SIZE_MAX;
 	dev_info->max_mtu = dev_info->max_rx_pktlen - IAVF_ETH_OVERHEAD;
@@ -1703,6 +1825,7 @@ iavf_init_vf(struct rte_eth_dev *dev)
 		PMD_INIT_LOG(ERR, "iavf_get_vf_config failed");
 		goto err_alloc;
 	}
+
 	/* Allocate memort for RSS info */
 	if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) {
 		vf->rss_key = rte_zmalloc("rss_key",
diff --git a/drivers/net/iavf/iavf_rxtx.c b/drivers/net/iavf/iavf_rxtx.c
index ca0f5d072..987eb3c4e 100644
--- a/drivers/net/iavf/iavf_rxtx.c
+++ b/drivers/net/iavf/iavf_rxtx.c
@@ -720,6 +720,7 @@ iavf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 	struct iavf_adapter *adapter =
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	struct iavf_rx_queue *rxq;
 	int err = 0;
 
@@ -743,7 +744,11 @@ iavf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 	IAVF_WRITE_FLUSH(hw);
 
 	/* Ready to switch the queue on */
-	err = iavf_switch_queue(adapter, rx_queue_id, true, true);
+	if (!vf->lv_enabled)
+		err = iavf_switch_queue(adapter, rx_queue_id, true, true);
+	else
+		err = iavf_switch_queue_lv(adapter, rx_queue_id, true, true);
+
 	if (err)
 		PMD_DRV_LOG(ERR, "Failed to switch RX queue %u on",
 			    rx_queue_id);
@@ -760,6 +765,7 @@ iavf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 	struct iavf_adapter *adapter =
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	struct iavf_tx_queue *txq;
 	int err = 0;
 
@@ -775,7 +781,10 @@ iavf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 	IAVF_WRITE_FLUSH(hw);
 
 	/* Ready to switch the queue on */
-	err = iavf_switch_queue(adapter, tx_queue_id, false, true);
+	if (!vf->lv_enabled)
+		err = iavf_switch_queue(adapter, tx_queue_id, false, true);
+	else
+		err = iavf_switch_queue_lv(adapter, tx_queue_id, false, true);
 
 	if (err)
 		PMD_DRV_LOG(ERR, "Failed to switch TX queue %u on",
@@ -876,14 +885,22 @@ iavf_stop_queues(struct rte_eth_dev *dev)
 {
 	struct iavf_adapter *adapter =
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	struct iavf_rx_queue *rxq;
 	struct iavf_tx_queue *txq;
 	int ret, i;
 
 	/* Stop All queues */
-	ret = iavf_disable_queues(adapter);
-	if (ret)
-		PMD_DRV_LOG(WARNING, "Fail to stop queues");
+	if (!vf->lv_enabled) {
+		ret = iavf_disable_queues(adapter);
+		if (ret)
+			PMD_DRV_LOG(WARNING, "Fail to stop queues");
+	} else {
+		ret = iavf_disable_queues_lv(adapter);
+		if (ret)
+			PMD_DRV_LOG(WARNING, "Fail to stop queues for large VF");
+	}
+
 
 	for (i = 0; i < dev->data->nb_tx_queues; i++) {
 		txq = dev->data->tx_queues[i];
diff --git a/drivers/net/iavf/iavf_vchnl.c b/drivers/net/iavf/iavf_vchnl.c
index 11a1ff608..0ad353b4f 100644
--- a/drivers/net/iavf/iavf_vchnl.c
+++ b/drivers/net/iavf/iavf_vchnl.c
@@ -457,7 +457,8 @@ iavf_get_vf_resource(struct iavf_adapter *adapter)
 		VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC |
 		VIRTCHNL_VF_OFFLOAD_FDIR_PF |
 		VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF |
-		VIRTCHNL_VF_OFFLOAD_REQ_QUEUES;
+		VIRTCHNL_VF_OFFLOAD_REQ_QUEUES |
+		VIRTCHNL_VF_LARGE_NUM_QPAIRS;
 
 	args.in_args = (uint8_t *)&caps;
 	args.in_args_size = sizeof(caps);
@@ -607,6 +608,138 @@ iavf_switch_queue(struct iavf_adapter *adapter, uint16_t qid,
 	return err;
 }
 
+int
+iavf_enable_queues_lv(struct iavf_adapter *adapter)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct virtchnl_del_ena_dis_queues *queue_select;
+	struct virtchnl_queue_chunk *queue_chunk;
+	struct iavf_cmd_info args;
+	int err, len;
+
+	len = sizeof(struct virtchnl_del_ena_dis_queues) +
+		  sizeof(struct virtchnl_queue_chunk) *
+		  (IAVF_RXTX_QUEUE_CHUNKS_NUM - 1);
+	queue_select = rte_zmalloc("queue_select", len, 0);
+	if (!queue_select)
+		return -ENOMEM;
+
+	queue_chunk = queue_select->chunks.chunks;
+	queue_select->chunks.num_chunks = IAVF_RXTX_QUEUE_CHUNKS_NUM;
+	queue_select->vport_id = vf->vsi_res->vsi_id;
+
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].type = VIRTCHNL_QUEUE_TYPE_TX;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].start_queue_id = 0;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].num_queues =
+		adapter->eth_dev->data->nb_tx_queues;
+
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].type = VIRTCHNL_QUEUE_TYPE_RX;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].start_queue_id = 0;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].num_queues =
+		adapter->eth_dev->data->nb_rx_queues;
+
+	args.ops = VIRTCHNL_OP_ENABLE_QUEUES_V2;
+	args.in_args = (u8 *)queue_select;
+	args.in_args_size = len;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err) {
+		PMD_DRV_LOG(ERR,
+			    "Failed to execute command of OP_ENABLE_QUEUES_V2");
+		return err;
+	}
+	return 0;
+}
+
+int
+iavf_disable_queues_lv(struct iavf_adapter *adapter)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct virtchnl_del_ena_dis_queues *queue_select;
+	struct virtchnl_queue_chunk *queue_chunk;
+	struct iavf_cmd_info args;
+	int err, len;
+
+	len = sizeof(struct virtchnl_del_ena_dis_queues) +
+		  sizeof(struct virtchnl_queue_chunk) *
+		  (IAVF_RXTX_QUEUE_CHUNKS_NUM - 1);
+	queue_select = rte_zmalloc("queue_select", len, 0);
+	if (!queue_select)
+		return -ENOMEM;
+
+	queue_chunk = queue_select->chunks.chunks;
+	queue_select->chunks.num_chunks = IAVF_RXTX_QUEUE_CHUNKS_NUM;
+	queue_select->vport_id = vf->vsi_res->vsi_id;
+
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].type = VIRTCHNL_QUEUE_TYPE_TX;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].start_queue_id = 0;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].num_queues =
+		adapter->eth_dev->data->nb_tx_queues;
+
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].type = VIRTCHNL_QUEUE_TYPE_RX;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].start_queue_id = 0;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].num_queues =
+		adapter->eth_dev->data->nb_rx_queues;
+
+	args.ops = VIRTCHNL_OP_DISABLE_QUEUES_V2;
+	args.in_args = (u8 *)queue_select;
+	args.in_args_size = len;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err) {
+		PMD_DRV_LOG(ERR,
+			    "Failed to execute command of OP_DISABLE_QUEUES_V2");
+		return err;
+	}
+	return 0;
+}
+
+int
+iavf_switch_queue_lv(struct iavf_adapter *adapter, uint16_t qid,
+		 bool rx, bool on)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct virtchnl_del_ena_dis_queues *queue_select;
+	struct virtchnl_queue_chunk *queue_chunk;
+	struct iavf_cmd_info args;
+	int err, len;
+
+	len = sizeof(struct virtchnl_del_ena_dis_queues);
+	queue_select = rte_zmalloc("queue_select", len, 0);
+	if (!queue_select)
+		return -ENOMEM;
+
+	queue_chunk = queue_select->chunks.chunks;
+	queue_select->chunks.num_chunks = 1;
+	queue_select->vport_id = vf->vsi_res->vsi_id;
+
+	if (rx) {
+		queue_chunk->type = VIRTCHNL_QUEUE_TYPE_RX;
+		queue_chunk->start_queue_id = qid;
+		queue_chunk->num_queues = 1;
+	} else {
+		queue_chunk->type = VIRTCHNL_QUEUE_TYPE_TX;
+		queue_chunk->start_queue_id = qid;
+		queue_chunk->num_queues = 1;
+	}
+
+	if (on)
+		args.ops = VIRTCHNL_OP_ENABLE_QUEUES_V2;
+	else
+		args.ops = VIRTCHNL_OP_DISABLE_QUEUES_V2;
+	args.in_args = (u8 *)queue_select;
+	args.in_args_size = len;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err)
+		PMD_DRV_LOG(ERR, "Failed to execute command of %s",
+			    on ? "OP_ENABLE_QUEUES_V2" : "OP_DISABLE_QUEUES_V2");
+	return err;
+}
+
 int
 iavf_configure_rss_lut(struct iavf_adapter *adapter)
 {
@@ -672,7 +805,8 @@ iavf_configure_rss_key(struct iavf_adapter *adapter)
 }
 
 int
-iavf_configure_queues(struct iavf_adapter *adapter)
+iavf_configure_queues(struct iavf_adapter *adapter,
+		uint16_t num_queue_pairs, uint16_t index)
 {
 	struct iavf_rx_queue **rxq =
 		(struct iavf_rx_queue **)adapter->eth_dev->data->rx_queues;
@@ -686,16 +820,16 @@ iavf_configure_queues(struct iavf_adapter *adapter)
 	int err;
 
 	size = sizeof(*vc_config) +
-	       sizeof(vc_config->qpair[0]) * vf->num_queue_pairs;
+	       sizeof(vc_config->qpair[0]) * num_queue_pairs;
 	vc_config = rte_zmalloc("cfg_queue", size, 0);
 	if (!vc_config)
 		return -ENOMEM;
 
 	vc_config->vsi_id = vf->vsi_res->vsi_id;
-	vc_config->num_queue_pairs = vf->num_queue_pairs;
+	vc_config->num_queue_pairs = num_queue_pairs;
 
-	for (i = 0, vc_qp = vc_config->qpair;
-	     i < vf->num_queue_pairs;
+	for (i = index, vc_qp = vc_config->qpair;
+	     i < index + num_queue_pairs;
 	     i++, vc_qp++) {
 		vc_qp->txq.vsi_id = vf->vsi_res->vsi_id;
 		vc_qp->txq.queue_id = i;
@@ -775,13 +909,14 @@ iavf_config_irq_map(struct iavf_adapter *adapter)
 		return -ENOMEM;
 
 	map_info->num_vectors = vf->nb_msix;
-	for (i = 0; i < vf->nb_msix; i++) {
-		vecmap = &map_info->vecmap[i];
+	for (i = 0; i < adapter->eth_dev->data->nb_rx_queues; i++) {
+		vecmap =
+		    &map_info->vecmap[vf->qv_map[i].vector_id - vf->msix_base];
 		vecmap->vsi_id = vf->vsi_res->vsi_id;
 		vecmap->rxitr_idx = IAVF_ITR_INDEX_DEFAULT;
-		vecmap->vector_id = vf->msix_base + i;
+		vecmap->vector_id = vf->qv_map[i].vector_id;
 		vecmap->txq_map = 0;
-		vecmap->rxq_map = vf->rxq_map[vf->msix_base + i];
+		vecmap->rxq_map |= 1 << vf->qv_map[i].queue_id;
 	}
 
 	args.ops = VIRTCHNL_OP_CONFIG_IRQ_MAP;
@@ -797,6 +932,45 @@ iavf_config_irq_map(struct iavf_adapter *adapter)
 	return err;
 }
 
+int
+iavf_config_irq_map_lv(struct iavf_adapter *adapter, uint16_t num)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct virtchnl_queue_vector_maps *map_info;
+	struct virtchnl_queue_vector *qv_maps;
+	struct iavf_cmd_info args;
+	int len, i, err;
+
+	len = sizeof(struct virtchnl_queue_vector_maps) +
+	      sizeof(struct virtchnl_queue_vector) * (num - 1);
+
+	map_info = rte_zmalloc("map_info", len, 0);
+	if (!map_info)
+		return -ENOMEM;
+
+	map_info->vport_id = vf->vsi_res->vsi_id;
+	map_info->num_qv_maps = num;
+	for (i = 0; i < map_info->num_qv_maps; i++) {
+		qv_maps = &map_info->qv_maps[i];
+		qv_maps->itr_idx = VIRTCHNL_ITR_IDX_0;
+		qv_maps->queue_type = VIRTCHNL_QUEUE_TYPE_RX;
+		qv_maps->queue_id = vf->qv_map[i].queue_id;
+		qv_maps->vector_id = vf->qv_map[i].vector_id;
+	}
+
+	args.ops = VIRTCHNL_OP_MAP_QUEUE_VECTOR;
+	args.in_args = (u8 *)map_info;
+	args.in_args_size = len;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err)
+		PMD_DRV_LOG(ERR, "fail to execute command OP_MAP_QUEUE_VECTOR");
+
+	rte_free(map_info);
+	return err;
+}
+
 void
 iavf_add_del_all_mac_addr(struct iavf_adapter *adapter, bool add)
 {
@@ -1272,3 +1446,33 @@ iavf_request_queues(struct iavf_adapter *adapter, uint16_t num)
 
 	return -1;
 }
+
+int
+iavf_get_max_rss_queue_region(struct iavf_adapter *adapter)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct iavf_cmd_info args;
+	uint16_t qregion_width;
+	int err;
+
+	args.ops = VIRTCHNL_OP_GET_MAX_RSS_QREGION;
+	args.in_args = NULL;
+	args.in_args_size = 0;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err) {
+		PMD_DRV_LOG(ERR,
+			    "Failed to execute command of "
+			    "VIRTCHNL_OP_GET_MAX_RSS_QREGION");
+		return err;
+	}
+
+	qregion_width =
+	((struct virtchnl_max_rss_qregion *)args.out_buffer)->qregion_width;
+
+	vf->max_rss_qregion = (uint16_t)(1 << qregion_width);
+
+	return 0;
+}
-- 
2.17.1


^ permalink raw reply	[flat|nested] 45+ messages in thread

* [dpdk-dev] [PATCH v5 0/2] enable large VF configuration
  2020-09-09  7:20 [dpdk-dev] [PATCH v1 0/2] enable large VF configuration Ting Xu
                   ` (5 preceding siblings ...)
  2020-10-16  1:21 ` [dpdk-dev] [PATCH v5 0/2] " Ting Xu
@ 2020-10-16  1:34 ` Ting Xu
  2020-10-16  1:34   ` [dpdk-dev] [PATCH v5 1/2] net/iavf: add IAVF request queues function Ting Xu
  2020-10-16  1:34   ` [dpdk-dev] [PATCH v5 2/2] net/iavf: enable large VF configuration Ting Xu
  2020-10-16  1:43 ` [dpdk-dev] [PATCH v6 0/2] " Ting Xu
                   ` (2 subsequent siblings)
  9 siblings, 2 replies; 45+ messages in thread
From: Ting Xu @ 2020-10-16  1:34 UTC (permalink / raw)
  To: dev; +Cc: qi.z.zhang, beilei.xing, jingjing.wu, Ting Xu

This patchset supports to configure up to 256 queue pairs per VF. If
large VF is supported after capability negotiation, VF will request
queues from PF as needed. New virtual channel opcodes and structures
are used to indicate 256 queues, so VF is designed to handle the new
function of configure VSI queues, IRQ mapping and enable/disable queues.
Also, enable VF to query the max RSS queue region for RSS and filter
configuration.

Ting Xu (2):
  net/iavf: add IAVF request queues function
  net/iavf: enable large VF configuration

---
v4->v5:
Solve queue mapping buffer limitation issue.
Optimize VSI queue configuration

v3->v4:
Optimize handling messages from PF.

v2->v3:
Fix coding style issue.
Move get max RSS queue region after VF reset.
Add request queues capability negotiation.

v1->v2:
Change the communication with kernel PF.

---
 drivers/net/iavf/iavf.h        |  37 ++-
 drivers/net/iavf/iavf_ethdev.c | 158 +++++++++++-
 drivers/net/iavf/iavf_rxtx.c   |  27 +-
 drivers/net/iavf/iavf_vchnl.c  | 450 ++++++++++++++++++++++++++++-----
 4 files changed, 587 insertions(+), 85 deletions(-)

-- 
2.17.1


^ permalink raw reply	[flat|nested] 45+ messages in thread

* [dpdk-dev] [PATCH v5 1/2] net/iavf: add IAVF request queues function
  2020-10-16  1:34 ` [dpdk-dev] [PATCH v5 0/2] " Ting Xu
@ 2020-10-16  1:34   ` Ting Xu
  2020-10-16  1:34   ` [dpdk-dev] [PATCH v5 2/2] net/iavf: enable large VF configuration Ting Xu
  1 sibling, 0 replies; 45+ messages in thread
From: Ting Xu @ 2020-10-16  1:34 UTC (permalink / raw)
  To: dev; +Cc: qi.z.zhang, beilei.xing, jingjing.wu, Ting Xu

Add new virtchnl function to request additional queues from PF. Current
default queue pairs number when creating a VF is 16. In order to support
up to 256 queue pairs, enable this request queues function.
Since request queues command may return event message, modify function
iavf_read_msg_from_pf to identify event opcode and mark VF reset status.

Signed-off-by: Ting Xu <ting.xu@intel.com>
---
 drivers/net/iavf/iavf.h        |   9 ++
 drivers/net/iavf/iavf_ethdev.c |  11 +-
 drivers/net/iavf/iavf_vchnl.c  | 226 +++++++++++++++++++++++++--------
 3 files changed, 192 insertions(+), 54 deletions(-)

diff --git a/drivers/net/iavf/iavf.h b/drivers/net/iavf/iavf.h
index d56611608..93c165c62 100644
--- a/drivers/net/iavf/iavf.h
+++ b/drivers/net/iavf/iavf.h
@@ -107,6 +107,14 @@ struct iavf_fdir_info {
 /* TODO: is that correct to assume the max number to be 16 ?*/
 #define IAVF_MAX_MSIX_VECTORS   16
 
+/* Message type read in admin queue from PF */
+enum iavf_aq_result {
+	IAVF_MSG_ERR = -1, /* Meet error when accessing admin queue */
+	IAVF_MSG_NON,      /* Read nothing from admin queue */
+	IAVF_MSG_SYS,      /* Read system msg from admin queue */
+	IAVF_MSG_CMD,      /* Read async command result */
+};
+
 /* Structure to store private data specific for VF instance. */
 struct iavf_info {
 	uint16_t num_queue_pairs;
@@ -301,4 +309,5 @@ int iavf_add_del_rss_cfg(struct iavf_adapter *adapter,
 int iavf_add_del_mc_addr_list(struct iavf_adapter *adapter,
 			struct rte_ether_addr *mc_addrs,
 			uint32_t mc_addrs_num, bool add);
+int iavf_request_queues(struct iavf_adapter *adapter, uint16_t num);
 #endif /* _IAVF_ETHDEV_H_ */
diff --git a/drivers/net/iavf/iavf_ethdev.c b/drivers/net/iavf/iavf_ethdev.c
index 8b1cf8f1c..a4a28b885 100644
--- a/drivers/net/iavf/iavf_ethdev.c
+++ b/drivers/net/iavf/iavf_ethdev.c
@@ -1282,7 +1282,7 @@ iavf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
 }
 
 static int
-iavf_check_vf_reset_done(struct iavf_hw *hw)
+iavf_check_vf_reset_done(struct iavf_hw *hw, struct iavf_info *vf)
 {
 	int i, reset;
 
@@ -1299,6 +1299,9 @@ iavf_check_vf_reset_done(struct iavf_hw *hw)
 	if (i >= IAVF_RESET_WAIT_CNT)
 		return -1;
 
+	/* VF is not in reset or reset is completed */
+	vf->vf_reset = false;
+
 	return 0;
 }
 
@@ -1666,7 +1669,7 @@ iavf_init_vf(struct rte_eth_dev *dev)
 		goto err;
 	}
 
-	err = iavf_check_vf_reset_done(hw);
+	err = iavf_check_vf_reset_done(hw, vf);
 	if (err) {
 		PMD_INIT_LOG(ERR, "VF is still resetting");
 		goto err;
@@ -1911,7 +1914,9 @@ iavf_dev_close(struct rte_eth_dev *dev)
 
 	iavf_dev_stop(dev);
 	iavf_flow_flush(dev, NULL);
-	iavf_flow_uninit(adapter);
+	/* if VF is in reset, adminq is disabled, skip the process via adminq */
+	if (!vf->vf_reset)
+		iavf_flow_uninit(adapter);
 
 	/*
 	 * disable promiscuous mode before reset vf
diff --git a/drivers/net/iavf/iavf_vchnl.c b/drivers/net/iavf/iavf_vchnl.c
index 5e7142893..11a1ff608 100644
--- a/drivers/net/iavf/iavf_vchnl.c
+++ b/drivers/net/iavf/iavf_vchnl.c
@@ -17,6 +17,7 @@
 #include <rte_eal.h>
 #include <rte_ether.h>
 #include <rte_ethdev_driver.h>
+#include <rte_ethdev_pci.h>
 #include <rte_dev.h>
 
 #include "iavf.h"
@@ -25,14 +26,54 @@
 #define MAX_TRY_TIMES 200
 #define ASQ_DELAY_MS  10
 
+static uint32_t
+iavf_convert_link_speed(enum virtchnl_link_speed virt_link_speed)
+{
+	uint32_t speed;
+
+	switch (virt_link_speed) {
+	case VIRTCHNL_LINK_SPEED_100MB:
+		speed = 100;
+		break;
+	case VIRTCHNL_LINK_SPEED_1GB:
+		speed = 1000;
+		break;
+	case VIRTCHNL_LINK_SPEED_10GB:
+		speed = 10000;
+		break;
+	case VIRTCHNL_LINK_SPEED_40GB:
+		speed = 40000;
+		break;
+	case VIRTCHNL_LINK_SPEED_20GB:
+		speed = 20000;
+		break;
+	case VIRTCHNL_LINK_SPEED_25GB:
+		speed = 25000;
+		break;
+	case VIRTCHNL_LINK_SPEED_2_5GB:
+		speed = 2500;
+		break;
+	case VIRTCHNL_LINK_SPEED_5GB:
+		speed = 5000;
+		break;
+	default:
+		speed = 0;
+		break;
+	}
+
+	return speed;
+}
+
 /* Read data in admin queue to get msg from pf driver */
-static enum iavf_status
+static enum iavf_aq_result
 iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,
 		     uint8_t *buf)
 {
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter);
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct rte_eth_dev *dev = adapter->eth_dev;
 	struct iavf_arq_event_info event;
+	enum iavf_aq_result result = IAVF_MSG_NON;
 	enum virtchnl_ops opcode;
 	int ret;
 
@@ -42,7 +83,9 @@ iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,
 	/* Can't read any msg from adminQ */
 	if (ret) {
 		PMD_DRV_LOG(DEBUG, "Can't read msg from AQ");
-		return ret;
+		if (ret != IAVF_ERR_ADMIN_QUEUE_NO_WORK)
+			result = IAVF_MSG_ERR;
+		return result;
 	}
 
 	opcode = (enum virtchnl_ops)rte_le_to_cpu_32(event.desc.cookie_high);
@@ -52,16 +95,51 @@ iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,
 	PMD_DRV_LOG(DEBUG, "AQ from pf carries opcode %u, retval %d",
 		    opcode, vf->cmd_retval);
 
-	if (opcode != vf->pend_cmd) {
-		if (opcode != VIRTCHNL_OP_EVENT) {
-			PMD_DRV_LOG(WARNING,
-				    "command mismatch, expect %u, get %u",
-				    vf->pend_cmd, opcode);
+	if (opcode == VIRTCHNL_OP_EVENT) {
+		struct virtchnl_pf_event *vpe =
+			(struct virtchnl_pf_event *)event.msg_buf;
+
+		result = IAVF_MSG_SYS;
+		switch (vpe->event) {
+		case VIRTCHNL_EVENT_LINK_CHANGE:
+			vf->link_up =
+				vpe->event_data.link_event.link_status;
+			if (vf->vf_res->vf_cap_flags &
+				VIRTCHNL_VF_CAP_ADV_LINK_SPEED) {
+				vf->link_speed =
+				    vpe->event_data.link_event_adv.link_speed;
+			} else {
+				enum virtchnl_link_speed speed;
+				speed = vpe->event_data.link_event.link_speed;
+				vf->link_speed = iavf_convert_link_speed(speed);
+			}
+			iavf_dev_link_update(dev, 0);
+			PMD_DRV_LOG(INFO, "Link status update:%s",
+					vf->link_up ? "up" : "down");
+			break;
+		case VIRTCHNL_EVENT_RESET_IMPENDING:
+			vf->vf_reset = true;
+			PMD_DRV_LOG(INFO, "VF is resetting");
+			break;
+		case VIRTCHNL_EVENT_PF_DRIVER_CLOSE:
+			vf->dev_closed = true;
+			PMD_DRV_LOG(INFO, "PF driver closed");
+			break;
+		default:
+			PMD_DRV_LOG(ERR, "%s: Unknown event %d from pf",
+					__func__, vpe->event);
+		}
+	}  else {
+		/* async reply msg on command issued by vf previously */
+		result = IAVF_MSG_CMD;
+		if (opcode != vf->pend_cmd) {
+			PMD_DRV_LOG(WARNING, "command mismatch, expect %u, get %u",
+					vf->pend_cmd, opcode);
+			result = IAVF_MSG_ERR;
 		}
-		return IAVF_ERR_OPCODE_MISMATCH;
 	}
 
-	return IAVF_SUCCESS;
+	return result;
 }
 
 static int
@@ -69,6 +147,7 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter, struct iavf_cmd_info *args)
 {
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter);
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	enum iavf_aq_result result;
 	enum iavf_status ret;
 	int err = 0;
 	int i = 0;
@@ -97,9 +176,9 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter, struct iavf_cmd_info *args)
 	case VIRTCHNL_OP_GET_SUPPORTED_RXDIDS:
 		/* for init virtchnl ops, need to poll the response */
 		do {
-			ret = iavf_read_msg_from_pf(adapter, args->out_size,
+			result = iavf_read_msg_from_pf(adapter, args->out_size,
 						   args->out_buffer);
-			if (ret == IAVF_SUCCESS)
+			if (result == IAVF_MSG_CMD)
 				break;
 			rte_delay_ms(ASQ_DELAY_MS);
 		} while (i++ < MAX_TRY_TIMES);
@@ -111,7 +190,33 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter, struct iavf_cmd_info *args)
 		}
 		_clear_cmd(vf);
 		break;
-
+	case VIRTCHNL_OP_REQUEST_QUEUES:
+		/*
+		 * ignore async reply, only wait for system message,
+		 * vf_reset = true if get VIRTCHNL_EVENT_RESET_IMPENDING,
+		 * if not, means request queues failed.
+		 */
+		do {
+			result = iavf_read_msg_from_pf(adapter, args->out_size,
+						   args->out_buffer);
+			if (result == IAVF_MSG_SYS && vf->vf_reset) {
+				break;
+			} else if (result == IAVF_MSG_CMD ||
+				result == IAVF_MSG_ERR) {
+				err = -1;
+				break;
+			}
+			rte_delay_ms(ASQ_DELAY_MS);
+			/* If don't read msg or read sys event, continue */
+		} while (i++ < MAX_TRY_TIMES);
+		if (i >= MAX_TRY_TIMES ||
+		    vf->cmd_retval != VIRTCHNL_STATUS_SUCCESS) {
+			err = -1;
+			PMD_DRV_LOG(ERR, "No response or return failure (%d)"
+				    " for cmd %d", vf->cmd_retval, args->ops);
+		}
+		_clear_cmd(vf);
+		break;
 	default:
 		/* For other virtchnl ops in running time,
 		 * wait for the cmd done flag.
@@ -136,44 +241,6 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter, struct iavf_cmd_info *args)
 	return err;
 }
 
-static uint32_t
-iavf_convert_link_speed(enum virtchnl_link_speed virt_link_speed)
-{
-	uint32_t speed;
-
-	switch (virt_link_speed) {
-	case VIRTCHNL_LINK_SPEED_100MB:
-		speed = 100;
-		break;
-	case VIRTCHNL_LINK_SPEED_1GB:
-		speed = 1000;
-		break;
-	case VIRTCHNL_LINK_SPEED_10GB:
-		speed = 10000;
-		break;
-	case VIRTCHNL_LINK_SPEED_40GB:
-		speed = 40000;
-		break;
-	case VIRTCHNL_LINK_SPEED_20GB:
-		speed = 20000;
-		break;
-	case VIRTCHNL_LINK_SPEED_25GB:
-		speed = 25000;
-		break;
-	case VIRTCHNL_LINK_SPEED_2_5GB:
-		speed = 2500;
-		break;
-	case VIRTCHNL_LINK_SPEED_5GB:
-		speed = 5000;
-		break;
-	default:
-		speed = 0;
-		break;
-	}
-
-	return speed;
-}
-
 static void
 iavf_handle_pf_event_msg(struct rte_eth_dev *dev, uint8_t *msg,
 			uint16_t msglen)
@@ -389,7 +456,8 @@ iavf_get_vf_resource(struct iavf_adapter *adapter)
 	caps = IAVF_BASIC_OFFLOAD_CAPS | VIRTCHNL_VF_CAP_ADV_LINK_SPEED |
 		VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC |
 		VIRTCHNL_VF_OFFLOAD_FDIR_PF |
-		VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF;
+		VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF |
+		VIRTCHNL_VF_OFFLOAD_REQ_QUEUES;
 
 	args.in_args = (uint8_t *)&caps;
 	args.in_args_size = sizeof(caps);
@@ -1148,3 +1216,59 @@ iavf_add_del_mc_addr_list(struct iavf_adapter *adapter,
 
 	return 0;
 }
+
+int
+iavf_request_queues(struct iavf_adapter *adapter, uint16_t num)
+{
+	struct rte_eth_dev *dev = adapter->eth_dev;
+	struct iavf_info *vf =  IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+	struct virtchnl_vf_res_request vfres;
+	struct iavf_cmd_info args;
+	uint16_t num_queue_pairs;
+	int err;
+
+	if (!(vf->vf_res->vf_cap_flags &
+		VIRTCHNL_VF_OFFLOAD_REQ_QUEUES)) {
+		PMD_DRV_LOG(ERR, "request queues not supported");
+		return -1;
+	}
+
+	if (num == 0) {
+		PMD_DRV_LOG(ERR, "queue number cannot be zero");
+		return -1;
+	}
+	vfres.num_queue_pairs = num;
+
+	args.ops = VIRTCHNL_OP_REQUEST_QUEUES;
+	args.in_args = (u8 *)&vfres;
+	args.in_args_size = sizeof(vfres);
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+
+	/*
+	 * disable interrupt to avoid the admin queue message to be read
+	 * before iavf_read_msg_from_pf.
+	 */
+	rte_intr_disable(&pci_dev->intr_handle);
+	err = iavf_execute_vf_cmd(adapter, &args);
+	rte_intr_enable(&pci_dev->intr_handle);
+	if (err) {
+		PMD_DRV_LOG(ERR, "fail to execute command OP_REQUEST_QUEUES");
+		return err;
+	}
+
+	/* request queues succeeded, vf is resetting */
+	if (vf->vf_reset) {
+		PMD_DRV_LOG(INFO, "vf is resetting");
+		return 0;
+	}
+
+	/* request additional queues failed, return available number */
+	num_queue_pairs =
+	  ((struct virtchnl_vf_res_request *)args.out_buffer)->num_queue_pairs;
+	PMD_DRV_LOG(ERR, "request queues failed, only %u queues "
+		"available", num_queue_pairs);
+
+	return -1;
+}
-- 
2.17.1


^ permalink raw reply	[flat|nested] 45+ messages in thread

* [dpdk-dev] [PATCH v5 2/2] net/iavf: enable large VF configuration
  2020-10-16  1:34 ` [dpdk-dev] [PATCH v5 0/2] " Ting Xu
  2020-10-16  1:34   ` [dpdk-dev] [PATCH v5 1/2] net/iavf: add IAVF request queues function Ting Xu
@ 2020-10-16  1:34   ` Ting Xu
  1 sibling, 0 replies; 45+ messages in thread
From: Ting Xu @ 2020-10-16  1:34 UTC (permalink / raw)
  To: dev; +Cc: qi.z.zhang, beilei.xing, jingjing.wu, Ting Xu

Add support to negotiate large VF capability, including configure VSI
queues, enable/disable queues and IRQ mapping for large VF. Use new
virtchnl opcodes and structures to help indicate up to 256 queue pairs.
Need to request additional queues from PF first if the current allocated
queues are not enough. Add the step to query the current max RSS queue
region for RSS configuration since the max RSS queue region could be
smaller than the allocated queue pairs.

Signed-off-by: Ting Xu <ting.xu@intel.com>
---
 drivers/net/iavf/iavf.h        |  28 +++-
 drivers/net/iavf/iavf_ethdev.c | 147 +++++++++++++++++++--
 drivers/net/iavf/iavf_rxtx.c   |  27 +++-
 drivers/net/iavf/iavf_vchnl.c  | 226 +++++++++++++++++++++++++++++++--
 4 files changed, 396 insertions(+), 32 deletions(-)

diff --git a/drivers/net/iavf/iavf.h b/drivers/net/iavf/iavf.h
index 93c165c62..f4ef92225 100644
--- a/drivers/net/iavf/iavf.h
+++ b/drivers/net/iavf/iavf.h
@@ -19,7 +19,11 @@
 #define IAVF_FRAME_SIZE_MAX       9728
 #define IAVF_QUEUE_BASE_ADDR_UNIT 128
 
-#define IAVF_MAX_NUM_QUEUES       16
+#define IAVF_MAX_NUM_QUEUES_DFLT	 16
+#define IAVF_MAX_NUM_QUEUES_LV		 256
+#define IAVF_RXTX_QUEUE_CHUNKS_NUM	 2
+#define IAVF_CFG_Q_NUM_PER_BUF		 32
+#define IAVF_IRQ_MAP_NUM_PER_BUF	 128
 
 #define IAVF_NUM_MACADDR_MAX      64
 
@@ -104,8 +108,10 @@ struct iavf_fdir_info {
 	struct iavf_fdir_conf conf;
 };
 
-/* TODO: is that correct to assume the max number to be 16 ?*/
-#define IAVF_MAX_MSIX_VECTORS   16
+struct iavf_qv_map {
+	uint16_t queue_id;
+	uint16_t vector_id;
+};
 
 /* Message type read in admin queue from PF */
 enum iavf_aq_result {
@@ -149,14 +155,16 @@ struct iavf_info {
 	uint8_t *rss_key;
 	uint16_t nb_msix;   /* number of MSI-X interrupts on Rx */
 	uint16_t msix_base; /* msix vector base from */
-	/* queue bitmask for each vector */
-	uint16_t rxq_map[IAVF_MAX_MSIX_VECTORS];
+	uint16_t max_rss_qregion; /* max RSS queue region supported by PF */
+	struct iavf_qv_map *qv_map; /* queue vector mapping */
 	struct iavf_flow_list flow_list;
 	rte_spinlock_t flow_ops_lock;
 	struct iavf_parser_list rss_parser_list;
 	struct iavf_parser_list dist_parser_list;
 
 	struct iavf_fdir_info fdir; /* flow director info */
+	/* indicate large VF support enabled or not */
+	bool lv_enabled;
 };
 
 #define IAVF_MAX_PKT_TYPE 1024
@@ -283,13 +291,20 @@ int iavf_enable_vlan_strip(struct iavf_adapter *adapter);
 int iavf_disable_vlan_strip(struct iavf_adapter *adapter);
 int iavf_switch_queue(struct iavf_adapter *adapter, uint16_t qid,
 		     bool rx, bool on);
+int iavf_switch_queue_lv(struct iavf_adapter *adapter, uint16_t qid,
+		     bool rx, bool on);
 int iavf_enable_queues(struct iavf_adapter *adapter);
+int iavf_enable_queues_lv(struct iavf_adapter *adapter);
 int iavf_disable_queues(struct iavf_adapter *adapter);
+int iavf_disable_queues_lv(struct iavf_adapter *adapter);
 int iavf_configure_rss_lut(struct iavf_adapter *adapter);
 int iavf_configure_rss_key(struct iavf_adapter *adapter);
-int iavf_configure_queues(struct iavf_adapter *adapter);
+int iavf_configure_queues(struct iavf_adapter *adapter,
+			uint16_t num_queue_pairs, uint16_t index);
 int iavf_get_supported_rxdid(struct iavf_adapter *adapter);
 int iavf_config_irq_map(struct iavf_adapter *adapter);
+int iavf_config_irq_map_lv(struct iavf_adapter *adapter, uint16_t num,
+			uint16_t index);
 void iavf_add_del_all_mac_addr(struct iavf_adapter *adapter, bool add);
 int iavf_dev_link_update(struct rte_eth_dev *dev,
 			__rte_unused int wait_to_complete);
@@ -310,4 +325,5 @@ int iavf_add_del_mc_addr_list(struct iavf_adapter *adapter,
 			struct rte_ether_addr *mc_addrs,
 			uint32_t mc_addrs_num, bool add);
 int iavf_request_queues(struct iavf_adapter *adapter, uint16_t num);
+int iavf_get_max_rss_queue_region(struct iavf_adapter *adapter);
 #endif /* _IAVF_ETHDEV_H_ */
diff --git a/drivers/net/iavf/iavf_ethdev.c b/drivers/net/iavf/iavf_ethdev.c
index a4a28b885..f9c5ef1c7 100644
--- a/drivers/net/iavf/iavf_ethdev.c
+++ b/drivers/net/iavf/iavf_ethdev.c
@@ -238,7 +238,7 @@ iavf_init_rss(struct iavf_adapter *adapter)
 
 	rss_conf = &adapter->eth_dev->data->dev_conf.rx_adv_conf.rss_conf;
 	nb_q = RTE_MIN(adapter->eth_dev->data->nb_rx_queues,
-		       IAVF_MAX_NUM_QUEUES);
+		       vf->max_rss_qregion);
 
 	if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) {
 		PMD_DRV_LOG(DEBUG, "RSS is not supported");
@@ -284,6 +284,31 @@ iavf_init_rss(struct iavf_adapter *adapter)
 	return 0;
 }
 
+static int
+iavf_queues_req_reset(struct rte_eth_dev *dev, uint16_t num)
+{
+	struct iavf_adapter *ad =
+		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
+	struct iavf_info *vf =  IAVF_DEV_PRIVATE_TO_VF(ad);
+	int ret = 0;
+
+	ret = iavf_request_queues(ad, num);
+	if (ret) {
+		PMD_DRV_LOG(ERR, "request queues from PF failed");
+		return ret;
+	}
+	PMD_DRV_LOG(INFO, "change queue pairs from %u to %u",
+			vf->vsi_res->num_queue_pairs, num);
+
+	ret = iavf_dev_reset(dev);
+	if (ret) {
+		PMD_DRV_LOG(ERR, "vf reset failed");
+		return ret;
+	}
+
+	return 0;
+}
+
 static int
 iavf_dev_configure(struct rte_eth_dev *dev)
 {
@@ -292,6 +317,8 @@ iavf_dev_configure(struct rte_eth_dev *dev)
 	struct iavf_info *vf =  IAVF_DEV_PRIVATE_TO_VF(ad);
 	struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 	uint32_t frame_size = dev->data->mtu + IAVF_ETH_OVERHEAD;
+	uint16_t num_queue_pairs = RTE_MAX(dev->data->nb_rx_queues,
+		dev->data->nb_tx_queues);
 	int ret;
 
 	ad->rx_bulk_alloc_allowed = true;
@@ -314,6 +341,46 @@ iavf_dev_configure(struct rte_eth_dev *dev)
 			return ret;
 	}
 
+	/* Large VF setting */
+	if (num_queue_pairs > IAVF_MAX_NUM_QUEUES_DFLT) {
+		if (!(vf->vf_res->vf_cap_flags &
+				VIRTCHNL_VF_LARGE_NUM_QPAIRS)) {
+			PMD_DRV_LOG(ERR, "large VF is not supported");
+			return -1;
+		}
+
+		if (num_queue_pairs > IAVF_MAX_NUM_QUEUES_LV) {
+			PMD_DRV_LOG(ERR, "queue pairs number cannot be larger "
+				"than %u", IAVF_MAX_NUM_QUEUES_LV);
+			return -1;
+		}
+
+		ret = iavf_queues_req_reset(dev, num_queue_pairs);
+		if (ret)
+			return ret;
+
+		ret = iavf_get_max_rss_queue_region(ad);
+		if (ret) {
+			PMD_INIT_LOG(ERR, "get max rss queue region failed");
+			return ret;
+		}
+
+		vf->lv_enabled = true;
+	} else {
+		/* Check if large VF is already enabled. If so, disable and
+		 * release redundant queue resource.
+		 */
+		if (vf->lv_enabled) {
+			ret = iavf_queues_req_reset(dev, num_queue_pairs);
+			if (ret)
+				return ret;
+
+			vf->lv_enabled = false;
+		}
+		/* if large VF is not required, use default rss queue region */
+		vf->max_rss_qregion = IAVF_MAX_NUM_QUEUES_DFLT;
+	}
+
 	/* Vlan stripping setting */
 	if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN) {
 		if (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
@@ -328,6 +395,7 @@ iavf_dev_configure(struct rte_eth_dev *dev)
 			return -1;
 		}
 	}
+
 	return 0;
 }
 
@@ -410,6 +478,7 @@ static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev,
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter);
+	struct iavf_qv_map *qv_map;
 	uint16_t interval, i;
 	int vec;
 
@@ -430,6 +499,14 @@ static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev,
 		}
 	}
 
+	qv_map = rte_zmalloc("qv_map",
+		dev->data->nb_rx_queues * sizeof(struct iavf_qv_map), 0);
+	if (!qv_map) {
+		PMD_DRV_LOG(ERR, "Failed to allocate %d queue-vector map",
+				dev->data->nb_rx_queues);
+		return -1;
+	}
+
 	if (!dev->data->dev_conf.intr_conf.rxq ||
 	    !rte_intr_dp_is_en(intr_handle)) {
 		/* Rx interrupt disabled, Map interrupt only for writeback */
@@ -460,16 +537,21 @@ static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev,
 		}
 		IAVF_WRITE_FLUSH(hw);
 		/* map all queues to the same interrupt */
-		for (i = 0; i < dev->data->nb_rx_queues; i++)
-			vf->rxq_map[vf->msix_base] |= 1 << i;
+		for (i = 0; i < dev->data->nb_rx_queues; i++) {
+			qv_map[i].queue_id = i;
+			qv_map[i].vector_id = vf->msix_base;
+		}
+		vf->qv_map = qv_map;
 	} else {
 		if (!rte_intr_allow_others(intr_handle)) {
 			vf->nb_msix = 1;
 			vf->msix_base = IAVF_MISC_VEC_ID;
 			for (i = 0; i < dev->data->nb_rx_queues; i++) {
-				vf->rxq_map[vf->msix_base] |= 1 << i;
+				qv_map[i].queue_id = i;
+				qv_map[i].vector_id = vf->msix_base;
 				intr_handle->intr_vec[i] = IAVF_MISC_VEC_ID;
 			}
+			vf->qv_map = qv_map;
 			PMD_DRV_LOG(DEBUG,
 				    "vector %u are mapping to all Rx queues",
 				    vf->msix_base);
@@ -482,21 +564,46 @@ static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev,
 			vf->msix_base = IAVF_RX_VEC_START;
 			vec = IAVF_RX_VEC_START;
 			for (i = 0; i < dev->data->nb_rx_queues; i++) {
-				vf->rxq_map[vec] |= 1 << i;
+				qv_map[i].queue_id = i;
+				qv_map[i].vector_id = vec;
 				intr_handle->intr_vec[i] = vec++;
 				if (vec >= vf->nb_msix)
 					vec = IAVF_RX_VEC_START;
 			}
+			vf->qv_map = qv_map;
 			PMD_DRV_LOG(DEBUG,
 				    "%u vectors are mapping to %u Rx queues",
 				    vf->nb_msix, dev->data->nb_rx_queues);
 		}
 	}
 
-	if (iavf_config_irq_map(adapter)) {
-		PMD_DRV_LOG(ERR, "config interrupt mapping failed");
-		return -1;
+	if (!vf->lv_enabled) {
+		if (iavf_config_irq_map(adapter)) {
+			PMD_DRV_LOG(ERR, "config interrupt mapping failed");
+			return -1;
+		}
+	} else {
+		uint16_t num_qv_maps = dev->data->nb_rx_queues;
+		uint16_t index = 0;
+
+		while (num_qv_maps > IAVF_IRQ_MAP_NUM_PER_BUF) {
+			if (iavf_config_irq_map_lv(adapter,
+					IAVF_IRQ_MAP_NUM_PER_BUF, index)) {
+				PMD_DRV_LOG(ERR, "config interrupt mapping "
+					"for large VF failed");
+				return -1;
+			}
+			num_qv_maps -= IAVF_IRQ_MAP_NUM_PER_BUF;
+			index += IAVF_IRQ_MAP_NUM_PER_BUF;
+		}
+
+		if (iavf_config_irq_map_lv(adapter, num_qv_maps, index)) {
+			PMD_DRV_LOG(ERR, "config interrupt mapping "
+				"for large VF failed");
+			return -1;
+		}
 	}
+
 	return 0;
 }
 
@@ -537,6 +644,8 @@ iavf_dev_start(struct rte_eth_dev *dev)
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	struct rte_intr_handle *intr_handle = dev->intr_handle;
+	uint16_t num_queue_pairs;
+	uint16_t index = 0;
 
 	PMD_INIT_FUNC_TRACE();
 
@@ -545,13 +654,27 @@ iavf_dev_start(struct rte_eth_dev *dev)
 	vf->max_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
 	vf->num_queue_pairs = RTE_MAX(dev->data->nb_rx_queues,
 				      dev->data->nb_tx_queues);
+	num_queue_pairs = vf->num_queue_pairs;
 
 	if (iavf_init_queues(dev) != 0) {
 		PMD_DRV_LOG(ERR, "failed to do Queue init");
 		return -1;
 	}
 
-	if (iavf_configure_queues(adapter) != 0) {
+	/* If needed, send configure queues msg multiple times to make the
+	 * adminq buffer length smaller than the 4K limitation.
+	 */
+	while (num_queue_pairs > IAVF_CFG_Q_NUM_PER_BUF) {
+		if (iavf_configure_queues(adapter,
+				IAVF_CFG_Q_NUM_PER_BUF, index) != 0) {
+			PMD_DRV_LOG(ERR, "configure queues failed");
+			goto err_queue;
+		}
+		num_queue_pairs -= IAVF_CFG_Q_NUM_PER_BUF;
+		index += IAVF_CFG_Q_NUM_PER_BUF;
+	}
+
+	if (iavf_configure_queues(adapter, num_queue_pairs, index) != 0) {
 		PMD_DRV_LOG(ERR, "configure queues failed");
 		goto err_queue;
 	}
@@ -560,6 +683,7 @@ iavf_dev_start(struct rte_eth_dev *dev)
 		PMD_DRV_LOG(ERR, "configure irq failed");
 		goto err_queue;
 	}
+
 	/* re-enable intr again, because efd assign may change */
 	if (dev->data->dev_conf.intr_conf.rxq != 0) {
 		rte_intr_disable(intr_handle);
@@ -624,8 +748,8 @@ iavf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 {
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 
-	dev_info->max_rx_queues = vf->vsi_res->num_queue_pairs;
-	dev_info->max_tx_queues = vf->vsi_res->num_queue_pairs;
+	dev_info->max_rx_queues = IAVF_MAX_NUM_QUEUES_LV;
+	dev_info->max_tx_queues = IAVF_MAX_NUM_QUEUES_LV;
 	dev_info->min_rx_bufsize = IAVF_BUF_SIZE_MIN;
 	dev_info->max_rx_pktlen = IAVF_FRAME_SIZE_MAX;
 	dev_info->max_mtu = dev_info->max_rx_pktlen - IAVF_ETH_OVERHEAD;
@@ -1703,6 +1827,7 @@ iavf_init_vf(struct rte_eth_dev *dev)
 		PMD_INIT_LOG(ERR, "iavf_get_vf_config failed");
 		goto err_alloc;
 	}
+
 	/* Allocate memort for RSS info */
 	if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) {
 		vf->rss_key = rte_zmalloc("rss_key",
diff --git a/drivers/net/iavf/iavf_rxtx.c b/drivers/net/iavf/iavf_rxtx.c
index ca0f5d072..987eb3c4e 100644
--- a/drivers/net/iavf/iavf_rxtx.c
+++ b/drivers/net/iavf/iavf_rxtx.c
@@ -720,6 +720,7 @@ iavf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 	struct iavf_adapter *adapter =
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	struct iavf_rx_queue *rxq;
 	int err = 0;
 
@@ -743,7 +744,11 @@ iavf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 	IAVF_WRITE_FLUSH(hw);
 
 	/* Ready to switch the queue on */
-	err = iavf_switch_queue(adapter, rx_queue_id, true, true);
+	if (!vf->lv_enabled)
+		err = iavf_switch_queue(adapter, rx_queue_id, true, true);
+	else
+		err = iavf_switch_queue_lv(adapter, rx_queue_id, true, true);
+
 	if (err)
 		PMD_DRV_LOG(ERR, "Failed to switch RX queue %u on",
 			    rx_queue_id);
@@ -760,6 +765,7 @@ iavf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 	struct iavf_adapter *adapter =
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	struct iavf_tx_queue *txq;
 	int err = 0;
 
@@ -775,7 +781,10 @@ iavf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 	IAVF_WRITE_FLUSH(hw);
 
 	/* Ready to switch the queue on */
-	err = iavf_switch_queue(adapter, tx_queue_id, false, true);
+	if (!vf->lv_enabled)
+		err = iavf_switch_queue(adapter, tx_queue_id, false, true);
+	else
+		err = iavf_switch_queue_lv(adapter, tx_queue_id, false, true);
 
 	if (err)
 		PMD_DRV_LOG(ERR, "Failed to switch TX queue %u on",
@@ -876,14 +885,22 @@ iavf_stop_queues(struct rte_eth_dev *dev)
 {
 	struct iavf_adapter *adapter =
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	struct iavf_rx_queue *rxq;
 	struct iavf_tx_queue *txq;
 	int ret, i;
 
 	/* Stop All queues */
-	ret = iavf_disable_queues(adapter);
-	if (ret)
-		PMD_DRV_LOG(WARNING, "Fail to stop queues");
+	if (!vf->lv_enabled) {
+		ret = iavf_disable_queues(adapter);
+		if (ret)
+			PMD_DRV_LOG(WARNING, "Fail to stop queues");
+	} else {
+		ret = iavf_disable_queues_lv(adapter);
+		if (ret)
+			PMD_DRV_LOG(WARNING, "Fail to stop queues for large VF");
+	}
+
 
 	for (i = 0; i < dev->data->nb_tx_queues; i++) {
 		txq = dev->data->tx_queues[i];
diff --git a/drivers/net/iavf/iavf_vchnl.c b/drivers/net/iavf/iavf_vchnl.c
index 11a1ff608..b1b0aa000 100644
--- a/drivers/net/iavf/iavf_vchnl.c
+++ b/drivers/net/iavf/iavf_vchnl.c
@@ -457,7 +457,8 @@ iavf_get_vf_resource(struct iavf_adapter *adapter)
 		VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC |
 		VIRTCHNL_VF_OFFLOAD_FDIR_PF |
 		VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF |
-		VIRTCHNL_VF_OFFLOAD_REQ_QUEUES;
+		VIRTCHNL_VF_OFFLOAD_REQ_QUEUES |
+		VIRTCHNL_VF_LARGE_NUM_QPAIRS;
 
 	args.in_args = (uint8_t *)&caps;
 	args.in_args_size = sizeof(caps);
@@ -607,6 +608,138 @@ iavf_switch_queue(struct iavf_adapter *adapter, uint16_t qid,
 	return err;
 }
 
+int
+iavf_enable_queues_lv(struct iavf_adapter *adapter)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct virtchnl_del_ena_dis_queues *queue_select;
+	struct virtchnl_queue_chunk *queue_chunk;
+	struct iavf_cmd_info args;
+	int err, len;
+
+	len = sizeof(struct virtchnl_del_ena_dis_queues) +
+		  sizeof(struct virtchnl_queue_chunk) *
+		  (IAVF_RXTX_QUEUE_CHUNKS_NUM - 1);
+	queue_select = rte_zmalloc("queue_select", len, 0);
+	if (!queue_select)
+		return -ENOMEM;
+
+	queue_chunk = queue_select->chunks.chunks;
+	queue_select->chunks.num_chunks = IAVF_RXTX_QUEUE_CHUNKS_NUM;
+	queue_select->vport_id = vf->vsi_res->vsi_id;
+
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].type = VIRTCHNL_QUEUE_TYPE_TX;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].start_queue_id = 0;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].num_queues =
+		adapter->eth_dev->data->nb_tx_queues;
+
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].type = VIRTCHNL_QUEUE_TYPE_RX;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].start_queue_id = 0;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].num_queues =
+		adapter->eth_dev->data->nb_rx_queues;
+
+	args.ops = VIRTCHNL_OP_ENABLE_QUEUES_V2;
+	args.in_args = (u8 *)queue_select;
+	args.in_args_size = len;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err) {
+		PMD_DRV_LOG(ERR,
+			    "Failed to execute command of OP_ENABLE_QUEUES_V2");
+		return err;
+	}
+	return 0;
+}
+
+int
+iavf_disable_queues_lv(struct iavf_adapter *adapter)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct virtchnl_del_ena_dis_queues *queue_select;
+	struct virtchnl_queue_chunk *queue_chunk;
+	struct iavf_cmd_info args;
+	int err, len;
+
+	len = sizeof(struct virtchnl_del_ena_dis_queues) +
+		  sizeof(struct virtchnl_queue_chunk) *
+		  (IAVF_RXTX_QUEUE_CHUNKS_NUM - 1);
+	queue_select = rte_zmalloc("queue_select", len, 0);
+	if (!queue_select)
+		return -ENOMEM;
+
+	queue_chunk = queue_select->chunks.chunks;
+	queue_select->chunks.num_chunks = IAVF_RXTX_QUEUE_CHUNKS_NUM;
+	queue_select->vport_id = vf->vsi_res->vsi_id;
+
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].type = VIRTCHNL_QUEUE_TYPE_TX;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].start_queue_id = 0;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].num_queues =
+		adapter->eth_dev->data->nb_tx_queues;
+
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].type = VIRTCHNL_QUEUE_TYPE_RX;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].start_queue_id = 0;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].num_queues =
+		adapter->eth_dev->data->nb_rx_queues;
+
+	args.ops = VIRTCHNL_OP_DISABLE_QUEUES_V2;
+	args.in_args = (u8 *)queue_select;
+	args.in_args_size = len;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err) {
+		PMD_DRV_LOG(ERR,
+			    "Failed to execute command of OP_DISABLE_QUEUES_V2");
+		return err;
+	}
+	return 0;
+}
+
+int
+iavf_switch_queue_lv(struct iavf_adapter *adapter, uint16_t qid,
+		 bool rx, bool on)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct virtchnl_del_ena_dis_queues *queue_select;
+	struct virtchnl_queue_chunk *queue_chunk;
+	struct iavf_cmd_info args;
+	int err, len;
+
+	len = sizeof(struct virtchnl_del_ena_dis_queues);
+	queue_select = rte_zmalloc("queue_select", len, 0);
+	if (!queue_select)
+		return -ENOMEM;
+
+	queue_chunk = queue_select->chunks.chunks;
+	queue_select->chunks.num_chunks = 1;
+	queue_select->vport_id = vf->vsi_res->vsi_id;
+
+	if (rx) {
+		queue_chunk->type = VIRTCHNL_QUEUE_TYPE_RX;
+		queue_chunk->start_queue_id = qid;
+		queue_chunk->num_queues = 1;
+	} else {
+		queue_chunk->type = VIRTCHNL_QUEUE_TYPE_TX;
+		queue_chunk->start_queue_id = qid;
+		queue_chunk->num_queues = 1;
+	}
+
+	if (on)
+		args.ops = VIRTCHNL_OP_ENABLE_QUEUES_V2;
+	else
+		args.ops = VIRTCHNL_OP_DISABLE_QUEUES_V2;
+	args.in_args = (u8 *)queue_select;
+	args.in_args_size = len;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err)
+		PMD_DRV_LOG(ERR, "Failed to execute command of %s",
+			    on ? "OP_ENABLE_QUEUES_V2" : "OP_DISABLE_QUEUES_V2");
+	return err;
+}
+
 int
 iavf_configure_rss_lut(struct iavf_adapter *adapter)
 {
@@ -672,7 +805,8 @@ iavf_configure_rss_key(struct iavf_adapter *adapter)
 }
 
 int
-iavf_configure_queues(struct iavf_adapter *adapter)
+iavf_configure_queues(struct iavf_adapter *adapter,
+		uint16_t num_queue_pairs, uint16_t index)
 {
 	struct iavf_rx_queue **rxq =
 		(struct iavf_rx_queue **)adapter->eth_dev->data->rx_queues;
@@ -686,16 +820,16 @@ iavf_configure_queues(struct iavf_adapter *adapter)
 	int err;
 
 	size = sizeof(*vc_config) +
-	       sizeof(vc_config->qpair[0]) * vf->num_queue_pairs;
+	       sizeof(vc_config->qpair[0]) * num_queue_pairs;
 	vc_config = rte_zmalloc("cfg_queue", size, 0);
 	if (!vc_config)
 		return -ENOMEM;
 
 	vc_config->vsi_id = vf->vsi_res->vsi_id;
-	vc_config->num_queue_pairs = vf->num_queue_pairs;
+	vc_config->num_queue_pairs = num_queue_pairs;
 
-	for (i = 0, vc_qp = vc_config->qpair;
-	     i < vf->num_queue_pairs;
+	for (i = index, vc_qp = vc_config->qpair;
+	     i < index + num_queue_pairs;
 	     i++, vc_qp++) {
 		vc_qp->txq.vsi_id = vf->vsi_res->vsi_id;
 		vc_qp->txq.queue_id = i;
@@ -775,13 +909,14 @@ iavf_config_irq_map(struct iavf_adapter *adapter)
 		return -ENOMEM;
 
 	map_info->num_vectors = vf->nb_msix;
-	for (i = 0; i < vf->nb_msix; i++) {
-		vecmap = &map_info->vecmap[i];
+	for (i = 0; i < adapter->eth_dev->data->nb_rx_queues; i++) {
+		vecmap =
+		    &map_info->vecmap[vf->qv_map[i].vector_id - vf->msix_base];
 		vecmap->vsi_id = vf->vsi_res->vsi_id;
 		vecmap->rxitr_idx = IAVF_ITR_INDEX_DEFAULT;
-		vecmap->vector_id = vf->msix_base + i;
+		vecmap->vector_id = vf->qv_map[i].vector_id;
 		vecmap->txq_map = 0;
-		vecmap->rxq_map = vf->rxq_map[vf->msix_base + i];
+		vecmap->rxq_map |= 1 << vf->qv_map[i].queue_id;
 	}
 
 	args.ops = VIRTCHNL_OP_CONFIG_IRQ_MAP;
@@ -797,6 +932,47 @@ iavf_config_irq_map(struct iavf_adapter *adapter)
 	return err;
 }
 
+int
+iavf_config_irq_map_lv(struct iavf_adapter *adapter, uint16_t num,
+		uint16_t index)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct virtchnl_queue_vector_maps *map_info;
+	struct virtchnl_queue_vector *qv_maps;
+	struct iavf_cmd_info args;
+	int len, i, err;
+	int count = 0;
+
+	len = sizeof(struct virtchnl_queue_vector_maps) +
+	      sizeof(struct virtchnl_queue_vector) * (num - 1);
+
+	map_info = rte_zmalloc("map_info", len, 0);
+	if (!map_info)
+		return -ENOMEM;
+
+	map_info->vport_id = vf->vsi_res->vsi_id;
+	map_info->num_qv_maps = num;
+	for (i = index; i < index + map_info->num_qv_maps; i++) {
+		qv_maps = &map_info->qv_maps[count++];
+		qv_maps->itr_idx = VIRTCHNL_ITR_IDX_0;
+		qv_maps->queue_type = VIRTCHNL_QUEUE_TYPE_RX;
+		qv_maps->queue_id = vf->qv_map[i].queue_id;
+		qv_maps->vector_id = vf->qv_map[i].vector_id;
+	}
+
+	args.ops = VIRTCHNL_OP_MAP_QUEUE_VECTOR;
+	args.in_args = (u8 *)map_info;
+	args.in_args_size = len;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err)
+		PMD_DRV_LOG(ERR, "fail to execute command OP_MAP_QUEUE_VECTOR");
+
+	rte_free(map_info);
+	return err;
+}
+
 void
 iavf_add_del_all_mac_addr(struct iavf_adapter *adapter, bool add)
 {
@@ -1272,3 +1448,33 @@ iavf_request_queues(struct iavf_adapter *adapter, uint16_t num)
 
 	return -1;
 }
+
+int
+iavf_get_max_rss_queue_region(struct iavf_adapter *adapter)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct iavf_cmd_info args;
+	uint16_t qregion_width;
+	int err;
+
+	args.ops = VIRTCHNL_OP_GET_MAX_RSS_QREGION;
+	args.in_args = NULL;
+	args.in_args_size = 0;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err) {
+		PMD_DRV_LOG(ERR,
+			    "Failed to execute command of "
+			    "VIRTCHNL_OP_GET_MAX_RSS_QREGION");
+		return err;
+	}
+
+	qregion_width =
+	((struct virtchnl_max_rss_qregion *)args.out_buffer)->qregion_width;
+
+	vf->max_rss_qregion = (uint16_t)(1 << qregion_width);
+
+	return 0;
+}
-- 
2.17.1


^ permalink raw reply	[flat|nested] 45+ messages in thread

* [dpdk-dev] [PATCH v6 0/2] enable large VF configuration
  2020-09-09  7:20 [dpdk-dev] [PATCH v1 0/2] enable large VF configuration Ting Xu
                   ` (6 preceding siblings ...)
  2020-10-16  1:34 ` [dpdk-dev] [PATCH v5 0/2] " Ting Xu
@ 2020-10-16  1:43 ` Ting Xu
  2020-10-16  1:43   ` [dpdk-dev] [PATCH v6 1/2] net/iavf: add IAVF request queues function Ting Xu
  2020-10-16  1:43   ` [dpdk-dev] [PATCH v6 2/2] net/iavf: enable large VF configuration Ting Xu
  2020-10-18 10:34 ` [dpdk-dev] [PATCH v7 0/6] " Ting Xu
  2020-10-22  6:48 ` [dpdk-dev] [PATCH v8 0/6] enable large VF configuration Ting Xu
  9 siblings, 2 replies; 45+ messages in thread
From: Ting Xu @ 2020-10-16  1:43 UTC (permalink / raw)
  To: dev; +Cc: qi.z.zhang, beilei.xing, jingjing.wu, Ting Xu

This patchset supports to configure up to 256 queue pairs per VF. If
large VF is supported after capability negotiation, VF will request
queues from PF as needed. New virtual channel opcodes and structures
are used to indicate 256 queues, so VF is designed to handle the new
function of configure VSI queues, IRQ mapping and enable/disable queues.
Also, enable VF to query the max RSS queue region for RSS and filter
configuration.

Ting Xu (2):
  net/iavf: add IAVF request queues function
  net/iavf: enable large VF configuration

---
v4->v6:
Solve queue mapping buffer limitation issue.
Optimize VSI queue configuration

v3->v4:
Optimize handling messages from PF.

v2->v3:
Fix coding style issue.
Move get max RSS queue region after VF reset.
Add request queues capability negotiation.

v1->v2:
Change the communication with kernel PF.

---
 drivers/net/iavf/iavf.h        |  37 ++-
 drivers/net/iavf/iavf_ethdev.c | 158 +++++++++++-
 drivers/net/iavf/iavf_rxtx.c   |  27 +-
 drivers/net/iavf/iavf_vchnl.c  | 450 ++++++++++++++++++++++++++++-----
 4 files changed, 587 insertions(+), 85 deletions(-)

-- 
2.17.1


^ permalink raw reply	[flat|nested] 45+ messages in thread

* [dpdk-dev] [PATCH v6 1/2] net/iavf: add IAVF request queues function
  2020-10-16  1:43 ` [dpdk-dev] [PATCH v6 0/2] " Ting Xu
@ 2020-10-16  1:43   ` Ting Xu
  2020-10-16  8:41     ` Xing, Beilei
  2020-10-16  1:43   ` [dpdk-dev] [PATCH v6 2/2] net/iavf: enable large VF configuration Ting Xu
  1 sibling, 1 reply; 45+ messages in thread
From: Ting Xu @ 2020-10-16  1:43 UTC (permalink / raw)
  To: dev; +Cc: qi.z.zhang, beilei.xing, jingjing.wu, Ting Xu

Add new virtchnl function to request additional queues from PF. Current
default queue pairs number when creating a VF is 16. In order to support
up to 256 queue pairs, enable this request queues function.
Since request queues command may return event message, modify function
iavf_read_msg_from_pf to identify event opcode and mark VF reset status.

Signed-off-by: Ting Xu <ting.xu@intel.com>
---
 drivers/net/iavf/iavf.h        |   9 ++
 drivers/net/iavf/iavf_ethdev.c |  11 +-
 drivers/net/iavf/iavf_vchnl.c  | 226 +++++++++++++++++++++++++--------
 3 files changed, 192 insertions(+), 54 deletions(-)

diff --git a/drivers/net/iavf/iavf.h b/drivers/net/iavf/iavf.h
index d56611608..93c165c62 100644
--- a/drivers/net/iavf/iavf.h
+++ b/drivers/net/iavf/iavf.h
@@ -107,6 +107,14 @@ struct iavf_fdir_info {
 /* TODO: is that correct to assume the max number to be 16 ?*/
 #define IAVF_MAX_MSIX_VECTORS   16
 
+/* Message type read in admin queue from PF */
+enum iavf_aq_result {
+	IAVF_MSG_ERR = -1, /* Meet error when accessing admin queue */
+	IAVF_MSG_NON,      /* Read nothing from admin queue */
+	IAVF_MSG_SYS,      /* Read system msg from admin queue */
+	IAVF_MSG_CMD,      /* Read async command result */
+};
+
 /* Structure to store private data specific for VF instance. */
 struct iavf_info {
 	uint16_t num_queue_pairs;
@@ -301,4 +309,5 @@ int iavf_add_del_rss_cfg(struct iavf_adapter *adapter,
 int iavf_add_del_mc_addr_list(struct iavf_adapter *adapter,
 			struct rte_ether_addr *mc_addrs,
 			uint32_t mc_addrs_num, bool add);
+int iavf_request_queues(struct iavf_adapter *adapter, uint16_t num);
 #endif /* _IAVF_ETHDEV_H_ */
diff --git a/drivers/net/iavf/iavf_ethdev.c b/drivers/net/iavf/iavf_ethdev.c
index 8b1cf8f1c..a4a28b885 100644
--- a/drivers/net/iavf/iavf_ethdev.c
+++ b/drivers/net/iavf/iavf_ethdev.c
@@ -1282,7 +1282,7 @@ iavf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
 }
 
 static int
-iavf_check_vf_reset_done(struct iavf_hw *hw)
+iavf_check_vf_reset_done(struct iavf_hw *hw, struct iavf_info *vf)
 {
 	int i, reset;
 
@@ -1299,6 +1299,9 @@ iavf_check_vf_reset_done(struct iavf_hw *hw)
 	if (i >= IAVF_RESET_WAIT_CNT)
 		return -1;
 
+	/* VF is not in reset or reset is completed */
+	vf->vf_reset = false;
+
 	return 0;
 }
 
@@ -1666,7 +1669,7 @@ iavf_init_vf(struct rte_eth_dev *dev)
 		goto err;
 	}
 
-	err = iavf_check_vf_reset_done(hw);
+	err = iavf_check_vf_reset_done(hw, vf);
 	if (err) {
 		PMD_INIT_LOG(ERR, "VF is still resetting");
 		goto err;
@@ -1911,7 +1914,9 @@ iavf_dev_close(struct rte_eth_dev *dev)
 
 	iavf_dev_stop(dev);
 	iavf_flow_flush(dev, NULL);
-	iavf_flow_uninit(adapter);
+	/* if VF is in reset, adminq is disabled, skip the process via adminq */
+	if (!vf->vf_reset)
+		iavf_flow_uninit(adapter);
 
 	/*
 	 * disable promiscuous mode before reset vf
diff --git a/drivers/net/iavf/iavf_vchnl.c b/drivers/net/iavf/iavf_vchnl.c
index 5e7142893..11a1ff608 100644
--- a/drivers/net/iavf/iavf_vchnl.c
+++ b/drivers/net/iavf/iavf_vchnl.c
@@ -17,6 +17,7 @@
 #include <rte_eal.h>
 #include <rte_ether.h>
 #include <rte_ethdev_driver.h>
+#include <rte_ethdev_pci.h>
 #include <rte_dev.h>
 
 #include "iavf.h"
@@ -25,14 +26,54 @@
 #define MAX_TRY_TIMES 200
 #define ASQ_DELAY_MS  10
 
+static uint32_t
+iavf_convert_link_speed(enum virtchnl_link_speed virt_link_speed)
+{
+	uint32_t speed;
+
+	switch (virt_link_speed) {
+	case VIRTCHNL_LINK_SPEED_100MB:
+		speed = 100;
+		break;
+	case VIRTCHNL_LINK_SPEED_1GB:
+		speed = 1000;
+		break;
+	case VIRTCHNL_LINK_SPEED_10GB:
+		speed = 10000;
+		break;
+	case VIRTCHNL_LINK_SPEED_40GB:
+		speed = 40000;
+		break;
+	case VIRTCHNL_LINK_SPEED_20GB:
+		speed = 20000;
+		break;
+	case VIRTCHNL_LINK_SPEED_25GB:
+		speed = 25000;
+		break;
+	case VIRTCHNL_LINK_SPEED_2_5GB:
+		speed = 2500;
+		break;
+	case VIRTCHNL_LINK_SPEED_5GB:
+		speed = 5000;
+		break;
+	default:
+		speed = 0;
+		break;
+	}
+
+	return speed;
+}
+
 /* Read data in admin queue to get msg from pf driver */
-static enum iavf_status
+static enum iavf_aq_result
 iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,
 		     uint8_t *buf)
 {
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter);
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct rte_eth_dev *dev = adapter->eth_dev;
 	struct iavf_arq_event_info event;
+	enum iavf_aq_result result = IAVF_MSG_NON;
 	enum virtchnl_ops opcode;
 	int ret;
 
@@ -42,7 +83,9 @@ iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,
 	/* Can't read any msg from adminQ */
 	if (ret) {
 		PMD_DRV_LOG(DEBUG, "Can't read msg from AQ");
-		return ret;
+		if (ret != IAVF_ERR_ADMIN_QUEUE_NO_WORK)
+			result = IAVF_MSG_ERR;
+		return result;
 	}
 
 	opcode = (enum virtchnl_ops)rte_le_to_cpu_32(event.desc.cookie_high);
@@ -52,16 +95,51 @@ iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,
 	PMD_DRV_LOG(DEBUG, "AQ from pf carries opcode %u, retval %d",
 		    opcode, vf->cmd_retval);
 
-	if (opcode != vf->pend_cmd) {
-		if (opcode != VIRTCHNL_OP_EVENT) {
-			PMD_DRV_LOG(WARNING,
-				    "command mismatch, expect %u, get %u",
-				    vf->pend_cmd, opcode);
+	if (opcode == VIRTCHNL_OP_EVENT) {
+		struct virtchnl_pf_event *vpe =
+			(struct virtchnl_pf_event *)event.msg_buf;
+
+		result = IAVF_MSG_SYS;
+		switch (vpe->event) {
+		case VIRTCHNL_EVENT_LINK_CHANGE:
+			vf->link_up =
+				vpe->event_data.link_event.link_status;
+			if (vf->vf_res->vf_cap_flags &
+				VIRTCHNL_VF_CAP_ADV_LINK_SPEED) {
+				vf->link_speed =
+				    vpe->event_data.link_event_adv.link_speed;
+			} else {
+				enum virtchnl_link_speed speed;
+				speed = vpe->event_data.link_event.link_speed;
+				vf->link_speed = iavf_convert_link_speed(speed);
+			}
+			iavf_dev_link_update(dev, 0);
+			PMD_DRV_LOG(INFO, "Link status update:%s",
+					vf->link_up ? "up" : "down");
+			break;
+		case VIRTCHNL_EVENT_RESET_IMPENDING:
+			vf->vf_reset = true;
+			PMD_DRV_LOG(INFO, "VF is resetting");
+			break;
+		case VIRTCHNL_EVENT_PF_DRIVER_CLOSE:
+			vf->dev_closed = true;
+			PMD_DRV_LOG(INFO, "PF driver closed");
+			break;
+		default:
+			PMD_DRV_LOG(ERR, "%s: Unknown event %d from pf",
+					__func__, vpe->event);
+		}
+	}  else {
+		/* async reply msg on command issued by vf previously */
+		result = IAVF_MSG_CMD;
+		if (opcode != vf->pend_cmd) {
+			PMD_DRV_LOG(WARNING, "command mismatch, expect %u, get %u",
+					vf->pend_cmd, opcode);
+			result = IAVF_MSG_ERR;
 		}
-		return IAVF_ERR_OPCODE_MISMATCH;
 	}
 
-	return IAVF_SUCCESS;
+	return result;
 }
 
 static int
@@ -69,6 +147,7 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter, struct iavf_cmd_info *args)
 {
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter);
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	enum iavf_aq_result result;
 	enum iavf_status ret;
 	int err = 0;
 	int i = 0;
@@ -97,9 +176,9 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter, struct iavf_cmd_info *args)
 	case VIRTCHNL_OP_GET_SUPPORTED_RXDIDS:
 		/* for init virtchnl ops, need to poll the response */
 		do {
-			ret = iavf_read_msg_from_pf(adapter, args->out_size,
+			result = iavf_read_msg_from_pf(adapter, args->out_size,
 						   args->out_buffer);
-			if (ret == IAVF_SUCCESS)
+			if (result == IAVF_MSG_CMD)
 				break;
 			rte_delay_ms(ASQ_DELAY_MS);
 		} while (i++ < MAX_TRY_TIMES);
@@ -111,7 +190,33 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter, struct iavf_cmd_info *args)
 		}
 		_clear_cmd(vf);
 		break;
-
+	case VIRTCHNL_OP_REQUEST_QUEUES:
+		/*
+		 * ignore async reply, only wait for system message,
+		 * vf_reset = true if get VIRTCHNL_EVENT_RESET_IMPENDING,
+		 * if not, means request queues failed.
+		 */
+		do {
+			result = iavf_read_msg_from_pf(adapter, args->out_size,
+						   args->out_buffer);
+			if (result == IAVF_MSG_SYS && vf->vf_reset) {
+				break;
+			} else if (result == IAVF_MSG_CMD ||
+				result == IAVF_MSG_ERR) {
+				err = -1;
+				break;
+			}
+			rte_delay_ms(ASQ_DELAY_MS);
+			/* If don't read msg or read sys event, continue */
+		} while (i++ < MAX_TRY_TIMES);
+		if (i >= MAX_TRY_TIMES ||
+		    vf->cmd_retval != VIRTCHNL_STATUS_SUCCESS) {
+			err = -1;
+			PMD_DRV_LOG(ERR, "No response or return failure (%d)"
+				    " for cmd %d", vf->cmd_retval, args->ops);
+		}
+		_clear_cmd(vf);
+		break;
 	default:
 		/* For other virtchnl ops in running time,
 		 * wait for the cmd done flag.
@@ -136,44 +241,6 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter, struct iavf_cmd_info *args)
 	return err;
 }
 
-static uint32_t
-iavf_convert_link_speed(enum virtchnl_link_speed virt_link_speed)
-{
-	uint32_t speed;
-
-	switch (virt_link_speed) {
-	case VIRTCHNL_LINK_SPEED_100MB:
-		speed = 100;
-		break;
-	case VIRTCHNL_LINK_SPEED_1GB:
-		speed = 1000;
-		break;
-	case VIRTCHNL_LINK_SPEED_10GB:
-		speed = 10000;
-		break;
-	case VIRTCHNL_LINK_SPEED_40GB:
-		speed = 40000;
-		break;
-	case VIRTCHNL_LINK_SPEED_20GB:
-		speed = 20000;
-		break;
-	case VIRTCHNL_LINK_SPEED_25GB:
-		speed = 25000;
-		break;
-	case VIRTCHNL_LINK_SPEED_2_5GB:
-		speed = 2500;
-		break;
-	case VIRTCHNL_LINK_SPEED_5GB:
-		speed = 5000;
-		break;
-	default:
-		speed = 0;
-		break;
-	}
-
-	return speed;
-}
-
 static void
 iavf_handle_pf_event_msg(struct rte_eth_dev *dev, uint8_t *msg,
 			uint16_t msglen)
@@ -389,7 +456,8 @@ iavf_get_vf_resource(struct iavf_adapter *adapter)
 	caps = IAVF_BASIC_OFFLOAD_CAPS | VIRTCHNL_VF_CAP_ADV_LINK_SPEED |
 		VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC |
 		VIRTCHNL_VF_OFFLOAD_FDIR_PF |
-		VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF;
+		VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF |
+		VIRTCHNL_VF_OFFLOAD_REQ_QUEUES;
 
 	args.in_args = (uint8_t *)&caps;
 	args.in_args_size = sizeof(caps);
@@ -1148,3 +1216,59 @@ iavf_add_del_mc_addr_list(struct iavf_adapter *adapter,
 
 	return 0;
 }
+
+int
+iavf_request_queues(struct iavf_adapter *adapter, uint16_t num)
+{
+	struct rte_eth_dev *dev = adapter->eth_dev;
+	struct iavf_info *vf =  IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+	struct virtchnl_vf_res_request vfres;
+	struct iavf_cmd_info args;
+	uint16_t num_queue_pairs;
+	int err;
+
+	if (!(vf->vf_res->vf_cap_flags &
+		VIRTCHNL_VF_OFFLOAD_REQ_QUEUES)) {
+		PMD_DRV_LOG(ERR, "request queues not supported");
+		return -1;
+	}
+
+	if (num == 0) {
+		PMD_DRV_LOG(ERR, "queue number cannot be zero");
+		return -1;
+	}
+	vfres.num_queue_pairs = num;
+
+	args.ops = VIRTCHNL_OP_REQUEST_QUEUES;
+	args.in_args = (u8 *)&vfres;
+	args.in_args_size = sizeof(vfres);
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+
+	/*
+	 * disable interrupt to avoid the admin queue message to be read
+	 * before iavf_read_msg_from_pf.
+	 */
+	rte_intr_disable(&pci_dev->intr_handle);
+	err = iavf_execute_vf_cmd(adapter, &args);
+	rte_intr_enable(&pci_dev->intr_handle);
+	if (err) {
+		PMD_DRV_LOG(ERR, "fail to execute command OP_REQUEST_QUEUES");
+		return err;
+	}
+
+	/* request queues succeeded, vf is resetting */
+	if (vf->vf_reset) {
+		PMD_DRV_LOG(INFO, "vf is resetting");
+		return 0;
+	}
+
+	/* request additional queues failed, return available number */
+	num_queue_pairs =
+	  ((struct virtchnl_vf_res_request *)args.out_buffer)->num_queue_pairs;
+	PMD_DRV_LOG(ERR, "request queues failed, only %u queues "
+		"available", num_queue_pairs);
+
+	return -1;
+}
-- 
2.17.1


^ permalink raw reply	[flat|nested] 45+ messages in thread

* [dpdk-dev] [PATCH v6 2/2] net/iavf: enable large VF configuration
  2020-10-16  1:43 ` [dpdk-dev] [PATCH v6 0/2] " Ting Xu
  2020-10-16  1:43   ` [dpdk-dev] [PATCH v6 1/2] net/iavf: add IAVF request queues function Ting Xu
@ 2020-10-16  1:43   ` Ting Xu
  1 sibling, 0 replies; 45+ messages in thread
From: Ting Xu @ 2020-10-16  1:43 UTC (permalink / raw)
  To: dev; +Cc: qi.z.zhang, beilei.xing, jingjing.wu, Ting Xu

Add support to negotiate large VF capability, including configure VSI
queues, enable/disable queues and IRQ mapping for large VF. Use new
virtchnl opcodes and structures to help indicate up to 256 queue pairs.
Need to request additional queues from PF first if the current allocated
queues are not enough. Add the step to query the current max RSS queue
region for RSS configuration since the max RSS queue region could be
smaller than the allocated queue pairs.

Signed-off-by: Ting Xu <ting.xu@intel.com>
---
 drivers/net/iavf/iavf.h        |  28 +++-
 drivers/net/iavf/iavf_ethdev.c | 147 +++++++++++++++++++--
 drivers/net/iavf/iavf_rxtx.c   |  27 +++-
 drivers/net/iavf/iavf_vchnl.c  | 226 +++++++++++++++++++++++++++++++--
 4 files changed, 396 insertions(+), 32 deletions(-)

diff --git a/drivers/net/iavf/iavf.h b/drivers/net/iavf/iavf.h
index 93c165c62..f4ef92225 100644
--- a/drivers/net/iavf/iavf.h
+++ b/drivers/net/iavf/iavf.h
@@ -19,7 +19,11 @@
 #define IAVF_FRAME_SIZE_MAX       9728
 #define IAVF_QUEUE_BASE_ADDR_UNIT 128
 
-#define IAVF_MAX_NUM_QUEUES       16
+#define IAVF_MAX_NUM_QUEUES_DFLT	 16
+#define IAVF_MAX_NUM_QUEUES_LV		 256
+#define IAVF_RXTX_QUEUE_CHUNKS_NUM	 2
+#define IAVF_CFG_Q_NUM_PER_BUF		 32
+#define IAVF_IRQ_MAP_NUM_PER_BUF	 128
 
 #define IAVF_NUM_MACADDR_MAX      64
 
@@ -104,8 +108,10 @@ struct iavf_fdir_info {
 	struct iavf_fdir_conf conf;
 };
 
-/* TODO: is that correct to assume the max number to be 16 ?*/
-#define IAVF_MAX_MSIX_VECTORS   16
+struct iavf_qv_map {
+	uint16_t queue_id;
+	uint16_t vector_id;
+};
 
 /* Message type read in admin queue from PF */
 enum iavf_aq_result {
@@ -149,14 +155,16 @@ struct iavf_info {
 	uint8_t *rss_key;
 	uint16_t nb_msix;   /* number of MSI-X interrupts on Rx */
 	uint16_t msix_base; /* msix vector base from */
-	/* queue bitmask for each vector */
-	uint16_t rxq_map[IAVF_MAX_MSIX_VECTORS];
+	uint16_t max_rss_qregion; /* max RSS queue region supported by PF */
+	struct iavf_qv_map *qv_map; /* queue vector mapping */
 	struct iavf_flow_list flow_list;
 	rte_spinlock_t flow_ops_lock;
 	struct iavf_parser_list rss_parser_list;
 	struct iavf_parser_list dist_parser_list;
 
 	struct iavf_fdir_info fdir; /* flow director info */
+	/* indicate large VF support enabled or not */
+	bool lv_enabled;
 };
 
 #define IAVF_MAX_PKT_TYPE 1024
@@ -283,13 +291,20 @@ int iavf_enable_vlan_strip(struct iavf_adapter *adapter);
 int iavf_disable_vlan_strip(struct iavf_adapter *adapter);
 int iavf_switch_queue(struct iavf_adapter *adapter, uint16_t qid,
 		     bool rx, bool on);
+int iavf_switch_queue_lv(struct iavf_adapter *adapter, uint16_t qid,
+		     bool rx, bool on);
 int iavf_enable_queues(struct iavf_adapter *adapter);
+int iavf_enable_queues_lv(struct iavf_adapter *adapter);
 int iavf_disable_queues(struct iavf_adapter *adapter);
+int iavf_disable_queues_lv(struct iavf_adapter *adapter);
 int iavf_configure_rss_lut(struct iavf_adapter *adapter);
 int iavf_configure_rss_key(struct iavf_adapter *adapter);
-int iavf_configure_queues(struct iavf_adapter *adapter);
+int iavf_configure_queues(struct iavf_adapter *adapter,
+			uint16_t num_queue_pairs, uint16_t index);
 int iavf_get_supported_rxdid(struct iavf_adapter *adapter);
 int iavf_config_irq_map(struct iavf_adapter *adapter);
+int iavf_config_irq_map_lv(struct iavf_adapter *adapter, uint16_t num,
+			uint16_t index);
 void iavf_add_del_all_mac_addr(struct iavf_adapter *adapter, bool add);
 int iavf_dev_link_update(struct rte_eth_dev *dev,
 			__rte_unused int wait_to_complete);
@@ -310,4 +325,5 @@ int iavf_add_del_mc_addr_list(struct iavf_adapter *adapter,
 			struct rte_ether_addr *mc_addrs,
 			uint32_t mc_addrs_num, bool add);
 int iavf_request_queues(struct iavf_adapter *adapter, uint16_t num);
+int iavf_get_max_rss_queue_region(struct iavf_adapter *adapter);
 #endif /* _IAVF_ETHDEV_H_ */
diff --git a/drivers/net/iavf/iavf_ethdev.c b/drivers/net/iavf/iavf_ethdev.c
index a4a28b885..f9c5ef1c7 100644
--- a/drivers/net/iavf/iavf_ethdev.c
+++ b/drivers/net/iavf/iavf_ethdev.c
@@ -238,7 +238,7 @@ iavf_init_rss(struct iavf_adapter *adapter)
 
 	rss_conf = &adapter->eth_dev->data->dev_conf.rx_adv_conf.rss_conf;
 	nb_q = RTE_MIN(adapter->eth_dev->data->nb_rx_queues,
-		       IAVF_MAX_NUM_QUEUES);
+		       vf->max_rss_qregion);
 
 	if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) {
 		PMD_DRV_LOG(DEBUG, "RSS is not supported");
@@ -284,6 +284,31 @@ iavf_init_rss(struct iavf_adapter *adapter)
 	return 0;
 }
 
+static int
+iavf_queues_req_reset(struct rte_eth_dev *dev, uint16_t num)
+{
+	struct iavf_adapter *ad =
+		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
+	struct iavf_info *vf =  IAVF_DEV_PRIVATE_TO_VF(ad);
+	int ret = 0;
+
+	ret = iavf_request_queues(ad, num);
+	if (ret) {
+		PMD_DRV_LOG(ERR, "request queues from PF failed");
+		return ret;
+	}
+	PMD_DRV_LOG(INFO, "change queue pairs from %u to %u",
+			vf->vsi_res->num_queue_pairs, num);
+
+	ret = iavf_dev_reset(dev);
+	if (ret) {
+		PMD_DRV_LOG(ERR, "vf reset failed");
+		return ret;
+	}
+
+	return 0;
+}
+
 static int
 iavf_dev_configure(struct rte_eth_dev *dev)
 {
@@ -292,6 +317,8 @@ iavf_dev_configure(struct rte_eth_dev *dev)
 	struct iavf_info *vf =  IAVF_DEV_PRIVATE_TO_VF(ad);
 	struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 	uint32_t frame_size = dev->data->mtu + IAVF_ETH_OVERHEAD;
+	uint16_t num_queue_pairs = RTE_MAX(dev->data->nb_rx_queues,
+		dev->data->nb_tx_queues);
 	int ret;
 
 	ad->rx_bulk_alloc_allowed = true;
@@ -314,6 +341,46 @@ iavf_dev_configure(struct rte_eth_dev *dev)
 			return ret;
 	}
 
+	/* Large VF setting */
+	if (num_queue_pairs > IAVF_MAX_NUM_QUEUES_DFLT) {
+		if (!(vf->vf_res->vf_cap_flags &
+				VIRTCHNL_VF_LARGE_NUM_QPAIRS)) {
+			PMD_DRV_LOG(ERR, "large VF is not supported");
+			return -1;
+		}
+
+		if (num_queue_pairs > IAVF_MAX_NUM_QUEUES_LV) {
+			PMD_DRV_LOG(ERR, "queue pairs number cannot be larger "
+				"than %u", IAVF_MAX_NUM_QUEUES_LV);
+			return -1;
+		}
+
+		ret = iavf_queues_req_reset(dev, num_queue_pairs);
+		if (ret)
+			return ret;
+
+		ret = iavf_get_max_rss_queue_region(ad);
+		if (ret) {
+			PMD_INIT_LOG(ERR, "get max rss queue region failed");
+			return ret;
+		}
+
+		vf->lv_enabled = true;
+	} else {
+		/* Check if large VF is already enabled. If so, disable and
+		 * release redundant queue resource.
+		 */
+		if (vf->lv_enabled) {
+			ret = iavf_queues_req_reset(dev, num_queue_pairs);
+			if (ret)
+				return ret;
+
+			vf->lv_enabled = false;
+		}
+		/* if large VF is not required, use default rss queue region */
+		vf->max_rss_qregion = IAVF_MAX_NUM_QUEUES_DFLT;
+	}
+
 	/* Vlan stripping setting */
 	if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN) {
 		if (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
@@ -328,6 +395,7 @@ iavf_dev_configure(struct rte_eth_dev *dev)
 			return -1;
 		}
 	}
+
 	return 0;
 }
 
@@ -410,6 +478,7 @@ static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev,
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter);
+	struct iavf_qv_map *qv_map;
 	uint16_t interval, i;
 	int vec;
 
@@ -430,6 +499,14 @@ static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev,
 		}
 	}
 
+	qv_map = rte_zmalloc("qv_map",
+		dev->data->nb_rx_queues * sizeof(struct iavf_qv_map), 0);
+	if (!qv_map) {
+		PMD_DRV_LOG(ERR, "Failed to allocate %d queue-vector map",
+				dev->data->nb_rx_queues);
+		return -1;
+	}
+
 	if (!dev->data->dev_conf.intr_conf.rxq ||
 	    !rte_intr_dp_is_en(intr_handle)) {
 		/* Rx interrupt disabled, Map interrupt only for writeback */
@@ -460,16 +537,21 @@ static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev,
 		}
 		IAVF_WRITE_FLUSH(hw);
 		/* map all queues to the same interrupt */
-		for (i = 0; i < dev->data->nb_rx_queues; i++)
-			vf->rxq_map[vf->msix_base] |= 1 << i;
+		for (i = 0; i < dev->data->nb_rx_queues; i++) {
+			qv_map[i].queue_id = i;
+			qv_map[i].vector_id = vf->msix_base;
+		}
+		vf->qv_map = qv_map;
 	} else {
 		if (!rte_intr_allow_others(intr_handle)) {
 			vf->nb_msix = 1;
 			vf->msix_base = IAVF_MISC_VEC_ID;
 			for (i = 0; i < dev->data->nb_rx_queues; i++) {
-				vf->rxq_map[vf->msix_base] |= 1 << i;
+				qv_map[i].queue_id = i;
+				qv_map[i].vector_id = vf->msix_base;
 				intr_handle->intr_vec[i] = IAVF_MISC_VEC_ID;
 			}
+			vf->qv_map = qv_map;
 			PMD_DRV_LOG(DEBUG,
 				    "vector %u are mapping to all Rx queues",
 				    vf->msix_base);
@@ -482,21 +564,46 @@ static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev,
 			vf->msix_base = IAVF_RX_VEC_START;
 			vec = IAVF_RX_VEC_START;
 			for (i = 0; i < dev->data->nb_rx_queues; i++) {
-				vf->rxq_map[vec] |= 1 << i;
+				qv_map[i].queue_id = i;
+				qv_map[i].vector_id = vec;
 				intr_handle->intr_vec[i] = vec++;
 				if (vec >= vf->nb_msix)
 					vec = IAVF_RX_VEC_START;
 			}
+			vf->qv_map = qv_map;
 			PMD_DRV_LOG(DEBUG,
 				    "%u vectors are mapping to %u Rx queues",
 				    vf->nb_msix, dev->data->nb_rx_queues);
 		}
 	}
 
-	if (iavf_config_irq_map(adapter)) {
-		PMD_DRV_LOG(ERR, "config interrupt mapping failed");
-		return -1;
+	if (!vf->lv_enabled) {
+		if (iavf_config_irq_map(adapter)) {
+			PMD_DRV_LOG(ERR, "config interrupt mapping failed");
+			return -1;
+		}
+	} else {
+		uint16_t num_qv_maps = dev->data->nb_rx_queues;
+		uint16_t index = 0;
+
+		while (num_qv_maps > IAVF_IRQ_MAP_NUM_PER_BUF) {
+			if (iavf_config_irq_map_lv(adapter,
+					IAVF_IRQ_MAP_NUM_PER_BUF, index)) {
+				PMD_DRV_LOG(ERR, "config interrupt mapping "
+					"for large VF failed");
+				return -1;
+			}
+			num_qv_maps -= IAVF_IRQ_MAP_NUM_PER_BUF;
+			index += IAVF_IRQ_MAP_NUM_PER_BUF;
+		}
+
+		if (iavf_config_irq_map_lv(adapter, num_qv_maps, index)) {
+			PMD_DRV_LOG(ERR, "config interrupt mapping "
+				"for large VF failed");
+			return -1;
+		}
 	}
+
 	return 0;
 }
 
@@ -537,6 +644,8 @@ iavf_dev_start(struct rte_eth_dev *dev)
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	struct rte_intr_handle *intr_handle = dev->intr_handle;
+	uint16_t num_queue_pairs;
+	uint16_t index = 0;
 
 	PMD_INIT_FUNC_TRACE();
 
@@ -545,13 +654,27 @@ iavf_dev_start(struct rte_eth_dev *dev)
 	vf->max_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
 	vf->num_queue_pairs = RTE_MAX(dev->data->nb_rx_queues,
 				      dev->data->nb_tx_queues);
+	num_queue_pairs = vf->num_queue_pairs;
 
 	if (iavf_init_queues(dev) != 0) {
 		PMD_DRV_LOG(ERR, "failed to do Queue init");
 		return -1;
 	}
 
-	if (iavf_configure_queues(adapter) != 0) {
+	/* If needed, send configure queues msg multiple times to make the
+	 * adminq buffer length smaller than the 4K limitation.
+	 */
+	while (num_queue_pairs > IAVF_CFG_Q_NUM_PER_BUF) {
+		if (iavf_configure_queues(adapter,
+				IAVF_CFG_Q_NUM_PER_BUF, index) != 0) {
+			PMD_DRV_LOG(ERR, "configure queues failed");
+			goto err_queue;
+		}
+		num_queue_pairs -= IAVF_CFG_Q_NUM_PER_BUF;
+		index += IAVF_CFG_Q_NUM_PER_BUF;
+	}
+
+	if (iavf_configure_queues(adapter, num_queue_pairs, index) != 0) {
 		PMD_DRV_LOG(ERR, "configure queues failed");
 		goto err_queue;
 	}
@@ -560,6 +683,7 @@ iavf_dev_start(struct rte_eth_dev *dev)
 		PMD_DRV_LOG(ERR, "configure irq failed");
 		goto err_queue;
 	}
+
 	/* re-enable intr again, because efd assign may change */
 	if (dev->data->dev_conf.intr_conf.rxq != 0) {
 		rte_intr_disable(intr_handle);
@@ -624,8 +748,8 @@ iavf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 {
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 
-	dev_info->max_rx_queues = vf->vsi_res->num_queue_pairs;
-	dev_info->max_tx_queues = vf->vsi_res->num_queue_pairs;
+	dev_info->max_rx_queues = IAVF_MAX_NUM_QUEUES_LV;
+	dev_info->max_tx_queues = IAVF_MAX_NUM_QUEUES_LV;
 	dev_info->min_rx_bufsize = IAVF_BUF_SIZE_MIN;
 	dev_info->max_rx_pktlen = IAVF_FRAME_SIZE_MAX;
 	dev_info->max_mtu = dev_info->max_rx_pktlen - IAVF_ETH_OVERHEAD;
@@ -1703,6 +1827,7 @@ iavf_init_vf(struct rte_eth_dev *dev)
 		PMD_INIT_LOG(ERR, "iavf_get_vf_config failed");
 		goto err_alloc;
 	}
+
 	/* Allocate memort for RSS info */
 	if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) {
 		vf->rss_key = rte_zmalloc("rss_key",
diff --git a/drivers/net/iavf/iavf_rxtx.c b/drivers/net/iavf/iavf_rxtx.c
index ca0f5d072..987eb3c4e 100644
--- a/drivers/net/iavf/iavf_rxtx.c
+++ b/drivers/net/iavf/iavf_rxtx.c
@@ -720,6 +720,7 @@ iavf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 	struct iavf_adapter *adapter =
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	struct iavf_rx_queue *rxq;
 	int err = 0;
 
@@ -743,7 +744,11 @@ iavf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 	IAVF_WRITE_FLUSH(hw);
 
 	/* Ready to switch the queue on */
-	err = iavf_switch_queue(adapter, rx_queue_id, true, true);
+	if (!vf->lv_enabled)
+		err = iavf_switch_queue(adapter, rx_queue_id, true, true);
+	else
+		err = iavf_switch_queue_lv(adapter, rx_queue_id, true, true);
+
 	if (err)
 		PMD_DRV_LOG(ERR, "Failed to switch RX queue %u on",
 			    rx_queue_id);
@@ -760,6 +765,7 @@ iavf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 	struct iavf_adapter *adapter =
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	struct iavf_tx_queue *txq;
 	int err = 0;
 
@@ -775,7 +781,10 @@ iavf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 	IAVF_WRITE_FLUSH(hw);
 
 	/* Ready to switch the queue on */
-	err = iavf_switch_queue(adapter, tx_queue_id, false, true);
+	if (!vf->lv_enabled)
+		err = iavf_switch_queue(adapter, tx_queue_id, false, true);
+	else
+		err = iavf_switch_queue_lv(adapter, tx_queue_id, false, true);
 
 	if (err)
 		PMD_DRV_LOG(ERR, "Failed to switch TX queue %u on",
@@ -876,14 +885,22 @@ iavf_stop_queues(struct rte_eth_dev *dev)
 {
 	struct iavf_adapter *adapter =
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	struct iavf_rx_queue *rxq;
 	struct iavf_tx_queue *txq;
 	int ret, i;
 
 	/* Stop All queues */
-	ret = iavf_disable_queues(adapter);
-	if (ret)
-		PMD_DRV_LOG(WARNING, "Fail to stop queues");
+	if (!vf->lv_enabled) {
+		ret = iavf_disable_queues(adapter);
+		if (ret)
+			PMD_DRV_LOG(WARNING, "Fail to stop queues");
+	} else {
+		ret = iavf_disable_queues_lv(adapter);
+		if (ret)
+			PMD_DRV_LOG(WARNING, "Fail to stop queues for large VF");
+	}
+
 
 	for (i = 0; i < dev->data->nb_tx_queues; i++) {
 		txq = dev->data->tx_queues[i];
diff --git a/drivers/net/iavf/iavf_vchnl.c b/drivers/net/iavf/iavf_vchnl.c
index 11a1ff608..b1b0aa000 100644
--- a/drivers/net/iavf/iavf_vchnl.c
+++ b/drivers/net/iavf/iavf_vchnl.c
@@ -457,7 +457,8 @@ iavf_get_vf_resource(struct iavf_adapter *adapter)
 		VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC |
 		VIRTCHNL_VF_OFFLOAD_FDIR_PF |
 		VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF |
-		VIRTCHNL_VF_OFFLOAD_REQ_QUEUES;
+		VIRTCHNL_VF_OFFLOAD_REQ_QUEUES |
+		VIRTCHNL_VF_LARGE_NUM_QPAIRS;
 
 	args.in_args = (uint8_t *)&caps;
 	args.in_args_size = sizeof(caps);
@@ -607,6 +608,138 @@ iavf_switch_queue(struct iavf_adapter *adapter, uint16_t qid,
 	return err;
 }
 
+int
+iavf_enable_queues_lv(struct iavf_adapter *adapter)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct virtchnl_del_ena_dis_queues *queue_select;
+	struct virtchnl_queue_chunk *queue_chunk;
+	struct iavf_cmd_info args;
+	int err, len;
+
+	len = sizeof(struct virtchnl_del_ena_dis_queues) +
+		  sizeof(struct virtchnl_queue_chunk) *
+		  (IAVF_RXTX_QUEUE_CHUNKS_NUM - 1);
+	queue_select = rte_zmalloc("queue_select", len, 0);
+	if (!queue_select)
+		return -ENOMEM;
+
+	queue_chunk = queue_select->chunks.chunks;
+	queue_select->chunks.num_chunks = IAVF_RXTX_QUEUE_CHUNKS_NUM;
+	queue_select->vport_id = vf->vsi_res->vsi_id;
+
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].type = VIRTCHNL_QUEUE_TYPE_TX;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].start_queue_id = 0;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].num_queues =
+		adapter->eth_dev->data->nb_tx_queues;
+
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].type = VIRTCHNL_QUEUE_TYPE_RX;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].start_queue_id = 0;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].num_queues =
+		adapter->eth_dev->data->nb_rx_queues;
+
+	args.ops = VIRTCHNL_OP_ENABLE_QUEUES_V2;
+	args.in_args = (u8 *)queue_select;
+	args.in_args_size = len;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err) {
+		PMD_DRV_LOG(ERR,
+			    "Failed to execute command of OP_ENABLE_QUEUES_V2");
+		return err;
+	}
+	return 0;
+}
+
+int
+iavf_disable_queues_lv(struct iavf_adapter *adapter)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct virtchnl_del_ena_dis_queues *queue_select;
+	struct virtchnl_queue_chunk *queue_chunk;
+	struct iavf_cmd_info args;
+	int err, len;
+
+	len = sizeof(struct virtchnl_del_ena_dis_queues) +
+		  sizeof(struct virtchnl_queue_chunk) *
+		  (IAVF_RXTX_QUEUE_CHUNKS_NUM - 1);
+	queue_select = rte_zmalloc("queue_select", len, 0);
+	if (!queue_select)
+		return -ENOMEM;
+
+	queue_chunk = queue_select->chunks.chunks;
+	queue_select->chunks.num_chunks = IAVF_RXTX_QUEUE_CHUNKS_NUM;
+	queue_select->vport_id = vf->vsi_res->vsi_id;
+
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].type = VIRTCHNL_QUEUE_TYPE_TX;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].start_queue_id = 0;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].num_queues =
+		adapter->eth_dev->data->nb_tx_queues;
+
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].type = VIRTCHNL_QUEUE_TYPE_RX;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].start_queue_id = 0;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].num_queues =
+		adapter->eth_dev->data->nb_rx_queues;
+
+	args.ops = VIRTCHNL_OP_DISABLE_QUEUES_V2;
+	args.in_args = (u8 *)queue_select;
+	args.in_args_size = len;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err) {
+		PMD_DRV_LOG(ERR,
+			    "Failed to execute command of OP_DISABLE_QUEUES_V2");
+		return err;
+	}
+	return 0;
+}
+
+int
+iavf_switch_queue_lv(struct iavf_adapter *adapter, uint16_t qid,
+		 bool rx, bool on)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct virtchnl_del_ena_dis_queues *queue_select;
+	struct virtchnl_queue_chunk *queue_chunk;
+	struct iavf_cmd_info args;
+	int err, len;
+
+	len = sizeof(struct virtchnl_del_ena_dis_queues);
+	queue_select = rte_zmalloc("queue_select", len, 0);
+	if (!queue_select)
+		return -ENOMEM;
+
+	queue_chunk = queue_select->chunks.chunks;
+	queue_select->chunks.num_chunks = 1;
+	queue_select->vport_id = vf->vsi_res->vsi_id;
+
+	if (rx) {
+		queue_chunk->type = VIRTCHNL_QUEUE_TYPE_RX;
+		queue_chunk->start_queue_id = qid;
+		queue_chunk->num_queues = 1;
+	} else {
+		queue_chunk->type = VIRTCHNL_QUEUE_TYPE_TX;
+		queue_chunk->start_queue_id = qid;
+		queue_chunk->num_queues = 1;
+	}
+
+	if (on)
+		args.ops = VIRTCHNL_OP_ENABLE_QUEUES_V2;
+	else
+		args.ops = VIRTCHNL_OP_DISABLE_QUEUES_V2;
+	args.in_args = (u8 *)queue_select;
+	args.in_args_size = len;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err)
+		PMD_DRV_LOG(ERR, "Failed to execute command of %s",
+			    on ? "OP_ENABLE_QUEUES_V2" : "OP_DISABLE_QUEUES_V2");
+	return err;
+}
+
 int
 iavf_configure_rss_lut(struct iavf_adapter *adapter)
 {
@@ -672,7 +805,8 @@ iavf_configure_rss_key(struct iavf_adapter *adapter)
 }
 
 int
-iavf_configure_queues(struct iavf_adapter *adapter)
+iavf_configure_queues(struct iavf_adapter *adapter,
+		uint16_t num_queue_pairs, uint16_t index)
 {
 	struct iavf_rx_queue **rxq =
 		(struct iavf_rx_queue **)adapter->eth_dev->data->rx_queues;
@@ -686,16 +820,16 @@ iavf_configure_queues(struct iavf_adapter *adapter)
 	int err;
 
 	size = sizeof(*vc_config) +
-	       sizeof(vc_config->qpair[0]) * vf->num_queue_pairs;
+	       sizeof(vc_config->qpair[0]) * num_queue_pairs;
 	vc_config = rte_zmalloc("cfg_queue", size, 0);
 	if (!vc_config)
 		return -ENOMEM;
 
 	vc_config->vsi_id = vf->vsi_res->vsi_id;
-	vc_config->num_queue_pairs = vf->num_queue_pairs;
+	vc_config->num_queue_pairs = num_queue_pairs;
 
-	for (i = 0, vc_qp = vc_config->qpair;
-	     i < vf->num_queue_pairs;
+	for (i = index, vc_qp = vc_config->qpair;
+	     i < index + num_queue_pairs;
 	     i++, vc_qp++) {
 		vc_qp->txq.vsi_id = vf->vsi_res->vsi_id;
 		vc_qp->txq.queue_id = i;
@@ -775,13 +909,14 @@ iavf_config_irq_map(struct iavf_adapter *adapter)
 		return -ENOMEM;
 
 	map_info->num_vectors = vf->nb_msix;
-	for (i = 0; i < vf->nb_msix; i++) {
-		vecmap = &map_info->vecmap[i];
+	for (i = 0; i < adapter->eth_dev->data->nb_rx_queues; i++) {
+		vecmap =
+		    &map_info->vecmap[vf->qv_map[i].vector_id - vf->msix_base];
 		vecmap->vsi_id = vf->vsi_res->vsi_id;
 		vecmap->rxitr_idx = IAVF_ITR_INDEX_DEFAULT;
-		vecmap->vector_id = vf->msix_base + i;
+		vecmap->vector_id = vf->qv_map[i].vector_id;
 		vecmap->txq_map = 0;
-		vecmap->rxq_map = vf->rxq_map[vf->msix_base + i];
+		vecmap->rxq_map |= 1 << vf->qv_map[i].queue_id;
 	}
 
 	args.ops = VIRTCHNL_OP_CONFIG_IRQ_MAP;
@@ -797,6 +932,47 @@ iavf_config_irq_map(struct iavf_adapter *adapter)
 	return err;
 }
 
+int
+iavf_config_irq_map_lv(struct iavf_adapter *adapter, uint16_t num,
+		uint16_t index)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct virtchnl_queue_vector_maps *map_info;
+	struct virtchnl_queue_vector *qv_maps;
+	struct iavf_cmd_info args;
+	int len, i, err;
+	int count = 0;
+
+	len = sizeof(struct virtchnl_queue_vector_maps) +
+	      sizeof(struct virtchnl_queue_vector) * (num - 1);
+
+	map_info = rte_zmalloc("map_info", len, 0);
+	if (!map_info)
+		return -ENOMEM;
+
+	map_info->vport_id = vf->vsi_res->vsi_id;
+	map_info->num_qv_maps = num;
+	for (i = index; i < index + map_info->num_qv_maps; i++) {
+		qv_maps = &map_info->qv_maps[count++];
+		qv_maps->itr_idx = VIRTCHNL_ITR_IDX_0;
+		qv_maps->queue_type = VIRTCHNL_QUEUE_TYPE_RX;
+		qv_maps->queue_id = vf->qv_map[i].queue_id;
+		qv_maps->vector_id = vf->qv_map[i].vector_id;
+	}
+
+	args.ops = VIRTCHNL_OP_MAP_QUEUE_VECTOR;
+	args.in_args = (u8 *)map_info;
+	args.in_args_size = len;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err)
+		PMD_DRV_LOG(ERR, "fail to execute command OP_MAP_QUEUE_VECTOR");
+
+	rte_free(map_info);
+	return err;
+}
+
 void
 iavf_add_del_all_mac_addr(struct iavf_adapter *adapter, bool add)
 {
@@ -1272,3 +1448,33 @@ iavf_request_queues(struct iavf_adapter *adapter, uint16_t num)
 
 	return -1;
 }
+
+int
+iavf_get_max_rss_queue_region(struct iavf_adapter *adapter)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct iavf_cmd_info args;
+	uint16_t qregion_width;
+	int err;
+
+	args.ops = VIRTCHNL_OP_GET_MAX_RSS_QREGION;
+	args.in_args = NULL;
+	args.in_args_size = 0;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err) {
+		PMD_DRV_LOG(ERR,
+			    "Failed to execute command of "
+			    "VIRTCHNL_OP_GET_MAX_RSS_QREGION");
+		return err;
+	}
+
+	qregion_width =
+	((struct virtchnl_max_rss_qregion *)args.out_buffer)->qregion_width;
+
+	vf->max_rss_qregion = (uint16_t)(1 << qregion_width);
+
+	return 0;
+}
-- 
2.17.1


^ permalink raw reply	[flat|nested] 45+ messages in thread

* Re: [dpdk-dev] [PATCH v6 1/2] net/iavf: add IAVF request queues function
  2020-10-16  1:43   ` [dpdk-dev] [PATCH v6 1/2] net/iavf: add IAVF request queues function Ting Xu
@ 2020-10-16  8:41     ` Xing, Beilei
  2020-10-18 10:29       ` Xu, Ting
  0 siblings, 1 reply; 45+ messages in thread
From: Xing, Beilei @ 2020-10-16  8:41 UTC (permalink / raw)
  To: Xu, Ting, dev; +Cc: Zhang, Qi Z, Wu, Jingjing



> -----Original Message-----
> From: Xu, Ting <ting.xu@intel.com>
> Sent: Friday, October 16, 2020 9:43 AM
> To: dev@dpdk.org
> Cc: Zhang, Qi Z <qi.z.zhang@intel.com>; Xing, Beilei <beilei.xing@intel.com>;
> Wu, Jingjing <jingjing.wu@intel.com>; Xu, Ting <ting.xu@intel.com>
> Subject: [PATCH v6 1/2] net/iavf: add IAVF request queues function
> 
> Add new virtchnl function to request additional queues from PF. Current
> default queue pairs number when creating a VF is 16. In order to support up to
> 256 queue pairs, enable this request queues function.
> Since request queues command may return event message, modify function
> iavf_read_msg_from_pf to identify event opcode and mark VF reset status.
> 
> Signed-off-by: Ting Xu <ting.xu@intel.com>
> ---
>  drivers/net/iavf/iavf.h        |   9 ++
>  drivers/net/iavf/iavf_ethdev.c |  11 +-  drivers/net/iavf/iavf_vchnl.c  | 226
> +++++++++++++++++++++++++--------
>  3 files changed, 192 insertions(+), 54 deletions(-)
> 
> diff --git a/drivers/net/iavf/iavf.h b/drivers/net/iavf/iavf.h index
> d56611608..93c165c62 100644
> --- a/drivers/net/iavf/iavf.h
> +++ b/drivers/net/iavf/iavf.h
> @@ -107,6 +107,14 @@ struct iavf_fdir_info {
>  /* TODO: is that correct to assume the max number to be 16 ?*/
>  #define IAVF_MAX_MSIX_VECTORS   16
> 
> +/* Message type read in admin queue from PF */ enum iavf_aq_result {
> +	IAVF_MSG_ERR = -1, /* Meet error when accessing admin queue */
> +	IAVF_MSG_NON,      /* Read nothing from admin queue */
> +	IAVF_MSG_SYS,      /* Read system msg from admin queue */
> +	IAVF_MSG_CMD,      /* Read async command result */
> +};

Is there no such message type in shared code?

> +
>  /* Structure to store private data specific for VF instance. */  struct iavf_info {
>  	uint16_t num_queue_pairs;
> @@ -301,4 +309,5 @@ int iavf_add_del_rss_cfg(struct iavf_adapter *adapter,
> int iavf_add_del_mc_addr_list(struct iavf_adapter *adapter,
>  			struct rte_ether_addr *mc_addrs,
>  			uint32_t mc_addrs_num, bool add);
> +int iavf_request_queues(struct iavf_adapter *adapter, uint16_t num);
>  #endif /* _IAVF_ETHDEV_H_ */
> diff --git a/drivers/net/iavf/iavf_ethdev.c b/drivers/net/iavf/iavf_ethdev.c
> index 8b1cf8f1c..a4a28b885 100644
> --- a/drivers/net/iavf/iavf_ethdev.c
> +++ b/drivers/net/iavf/iavf_ethdev.c
> @@ -1282,7 +1282,7 @@ iavf_dev_rx_queue_intr_disable(struct rte_eth_dev
> *dev, uint16_t queue_id)  }
> 
>  static int
> -iavf_check_vf_reset_done(struct iavf_hw *hw)
> +iavf_check_vf_reset_done(struct iavf_hw *hw, struct iavf_info *vf)
>  {
>  	int i, reset;
> 
> @@ -1299,6 +1299,9 @@ iavf_check_vf_reset_done(struct iavf_hw *hw)
>  	if (i >= IAVF_RESET_WAIT_CNT)
>  		return -1;
> 
> +	/* VF is not in reset or reset is completed */
> +	vf->vf_reset = false;

Seems it's not related to the feature.
Is the fix for commit 1eab95fe2e36e191ad85a9aacf82a44e7c8011fc?
If yes, it's better to separate bug fix from the feature.

> +
>  	return 0;
>  }
> 
> @@ -1666,7 +1669,7 @@ iavf_init_vf(struct rte_eth_dev *dev)
>  		goto err;
>  	}
> 
> -	err = iavf_check_vf_reset_done(hw);
> +	err = iavf_check_vf_reset_done(hw, vf);
>  	if (err) {
>  		PMD_INIT_LOG(ERR, "VF is still resetting");
>  		goto err;
> @@ -1911,7 +1914,9 @@ iavf_dev_close(struct rte_eth_dev *dev)
> 
>  	iavf_dev_stop(dev);
>  	iavf_flow_flush(dev, NULL);
> -	iavf_flow_uninit(adapter);
> +	/* if VF is in reset, adminq is disabled, skip the process via adminq */
> +	if (!vf->vf_reset)
> +		iavf_flow_uninit(adapter);

Same as above.

> 
>  	/*
>  	 * disable promiscuous mode before reset vf diff --git
> a/drivers/net/iavf/iavf_vchnl.c b/drivers/net/iavf/iavf_vchnl.c index
> 5e7142893..11a1ff608 100644
> --- a/drivers/net/iavf/iavf_vchnl.c
> +++ b/drivers/net/iavf/iavf_vchnl.c
> @@ -17,6 +17,7 @@
>  #include <rte_eal.h>
>  #include <rte_ether.h>
>  #include <rte_ethdev_driver.h>
> +#include <rte_ethdev_pci.h>
>  #include <rte_dev.h>
> 
>  #include "iavf.h"
> @@ -25,14 +26,54 @@
>  #define MAX_TRY_TIMES 200
>  #define ASQ_DELAY_MS  10
> 
> +static uint32_t
> +iavf_convert_link_speed(enum virtchnl_link_speed virt_link_speed) {
> +	uint32_t speed;
> +
> +	switch (virt_link_speed) {
> +	case VIRTCHNL_LINK_SPEED_100MB:
> +		speed = 100;
> +		break;
> +	case VIRTCHNL_LINK_SPEED_1GB:
> +		speed = 1000;
> +		break;
> +	case VIRTCHNL_LINK_SPEED_10GB:
> +		speed = 10000;
> +		break;
> +	case VIRTCHNL_LINK_SPEED_40GB:
> +		speed = 40000;
> +		break;
> +	case VIRTCHNL_LINK_SPEED_20GB:
> +		speed = 20000;
> +		break;
> +	case VIRTCHNL_LINK_SPEED_25GB:
> +		speed = 25000;
> +		break;
> +	case VIRTCHNL_LINK_SPEED_2_5GB:
> +		speed = 2500;
> +		break;
> +	case VIRTCHNL_LINK_SPEED_5GB:
> +		speed = 5000;
> +		break;
> +	default:
> +		speed = 0;
> +		break;
> +	}
> +
> +	return speed;
> +}
> +
>  /* Read data in admin queue to get msg from pf driver */ -static enum
> iavf_status
> +static enum iavf_aq_result
>  iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,
>  		     uint8_t *buf)
>  {
>  	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter);
>  	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
> +	struct rte_eth_dev *dev = adapter->eth_dev;
>  	struct iavf_arq_event_info event;
> +	enum iavf_aq_result result = IAVF_MSG_NON;
>  	enum virtchnl_ops opcode;
>  	int ret;
> 
> @@ -42,7 +83,9 @@ iavf_read_msg_from_pf(struct iavf_adapter *adapter,
> uint16_t buf_len,
>  	/* Can't read any msg from adminQ */
>  	if (ret) {
>  		PMD_DRV_LOG(DEBUG, "Can't read msg from AQ");
> -		return ret;
> +		if (ret != IAVF_ERR_ADMIN_QUEUE_NO_WORK)
> +			result = IAVF_MSG_ERR;
> +		return result;
>  	}
> 
>  	opcode = (enum
> virtchnl_ops)rte_le_to_cpu_32(event.desc.cookie_high);
> @@ -52,16 +95,51 @@ iavf_read_msg_from_pf(struct iavf_adapter *adapter,
> uint16_t buf_len,
>  	PMD_DRV_LOG(DEBUG, "AQ from pf carries opcode %u, retval %d",
>  		    opcode, vf->cmd_retval);
> 
> -	if (opcode != vf->pend_cmd) {
> -		if (opcode != VIRTCHNL_OP_EVENT) {
> -			PMD_DRV_LOG(WARNING,
> -				    "command mismatch, expect %u, get %u",
> -				    vf->pend_cmd, opcode);
> +	if (opcode == VIRTCHNL_OP_EVENT) {
> +		struct virtchnl_pf_event *vpe =
> +			(struct virtchnl_pf_event *)event.msg_buf;
> +
> +		result = IAVF_MSG_SYS;
> +		switch (vpe->event) {
> +		case VIRTCHNL_EVENT_LINK_CHANGE:
> +			vf->link_up =
> +				vpe->event_data.link_event.link_status;
> +			if (vf->vf_res->vf_cap_flags &
> +				VIRTCHNL_VF_CAP_ADV_LINK_SPEED) {
> +				vf->link_speed =
> +				    vpe-
> >event_data.link_event_adv.link_speed;
> +			} else {
> +				enum virtchnl_link_speed speed;
> +				speed = vpe-
> >event_data.link_event.link_speed;
> +				vf->link_speed =
> iavf_convert_link_speed(speed);
> +			}
> +			iavf_dev_link_update(dev, 0);
> +			PMD_DRV_LOG(INFO, "Link status update:%s",
> +					vf->link_up ? "up" : "down");
> +			break;
> +		case VIRTCHNL_EVENT_RESET_IMPENDING:
> +			vf->vf_reset = true;
> +			PMD_DRV_LOG(INFO, "VF is resetting");
> +			break;
> +		case VIRTCHNL_EVENT_PF_DRIVER_CLOSE:
> +			vf->dev_closed = true;
> +			PMD_DRV_LOG(INFO, "PF driver closed");
> +			break;
> +		default:
> +			PMD_DRV_LOG(ERR, "%s: Unknown event %d from pf",
> +					__func__, vpe->event);
> +		}
> +	}  else {
> +		/* async reply msg on command issued by vf previously */
> +		result = IAVF_MSG_CMD;
> +		if (opcode != vf->pend_cmd) {
> +			PMD_DRV_LOG(WARNING, "command mismatch,
> expect %u, get %u",
> +					vf->pend_cmd, opcode);
> +			result = IAVF_MSG_ERR;
>  		}
> -		return IAVF_ERR_OPCODE_MISMATCH;
>  	}
> 
> -	return IAVF_SUCCESS;
> +	return result;
>  }

How about separate this part which is handling the msg from PF?

> 
>  static int
> @@ -69,6 +147,7 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter,
> struct iavf_cmd_info *args)  {
>  	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter);
>  	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
> +	enum iavf_aq_result result;
>  	enum iavf_status ret;
>  	int err = 0;
>  	int i = 0;
> @@ -97,9 +176,9 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter,
> struct iavf_cmd_info *args)
>  	case VIRTCHNL_OP_GET_SUPPORTED_RXDIDS:
>  		/* for init virtchnl ops, need to poll the response */
>  		do {
> -			ret = iavf_read_msg_from_pf(adapter, args->out_size,
> +			result = iavf_read_msg_from_pf(adapter, args-
> >out_size,
>  						   args->out_buffer);
> -			if (ret == IAVF_SUCCESS)
> +			if (result == IAVF_MSG_CMD)
>  				break;
>  			rte_delay_ms(ASQ_DELAY_MS);
>  		} while (i++ < MAX_TRY_TIMES);
> @@ -111,7 +190,33 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter,
> struct iavf_cmd_info *args)
>  		}
>  		_clear_cmd(vf);
>  		break;
> -
> +	case VIRTCHNL_OP_REQUEST_QUEUES:
> +		/*
> +		 * ignore async reply, only wait for system message,
> +		 * vf_reset = true if get VIRTCHNL_EVENT_RESET_IMPENDING,
> +		 * if not, means request queues failed.
> +		 */
> +		do {
> +			result = iavf_read_msg_from_pf(adapter, args-
> >out_size,
> +						   args->out_buffer);
> +			if (result == IAVF_MSG_SYS && vf->vf_reset) {
> +				break;
> +			} else if (result == IAVF_MSG_CMD ||
> +				result == IAVF_MSG_ERR) {
> +				err = -1;
> +				break;
> +			}
> +			rte_delay_ms(ASQ_DELAY_MS);
> +			/* If don't read msg or read sys event, continue */
> +		} while (i++ < MAX_TRY_TIMES);
> +		if (i >= MAX_TRY_TIMES ||
> +		    vf->cmd_retval != VIRTCHNL_STATUS_SUCCESS) {
> +			err = -1;
> +			PMD_DRV_LOG(ERR, "No response or return failure
> (%d)"
> +				    " for cmd %d", vf->cmd_retval, args->ops);
> +		}
> +		_clear_cmd(vf);
> +		break;
>  	default:
>  		/* For other virtchnl ops in running time,
>  		 * wait for the cmd done flag.
> @@ -136,44 +241,6 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter,
> struct iavf_cmd_info *args)
>  	return err;
>  }
> 
> -static uint32_t
> -iavf_convert_link_speed(enum virtchnl_link_speed virt_link_speed) -{
> -	uint32_t speed;
> -
> -	switch (virt_link_speed) {
> -	case VIRTCHNL_LINK_SPEED_100MB:
> -		speed = 100;
> -		break;
> -	case VIRTCHNL_LINK_SPEED_1GB:
> -		speed = 1000;
> -		break;
> -	case VIRTCHNL_LINK_SPEED_10GB:
> -		speed = 10000;
> -		break;
> -	case VIRTCHNL_LINK_SPEED_40GB:
> -		speed = 40000;
> -		break;
> -	case VIRTCHNL_LINK_SPEED_20GB:
> -		speed = 20000;
> -		break;
> -	case VIRTCHNL_LINK_SPEED_25GB:
> -		speed = 25000;
> -		break;
> -	case VIRTCHNL_LINK_SPEED_2_5GB:
> -		speed = 2500;
> -		break;
> -	case VIRTCHNL_LINK_SPEED_5GB:
> -		speed = 5000;
> -		break;
> -	default:
> -		speed = 0;
> -		break;
> -	}
> -
> -	return speed;
> -}
> -
>  static void
>  iavf_handle_pf_event_msg(struct rte_eth_dev *dev, uint8_t *msg,
>  			uint16_t msglen)
> @@ -389,7 +456,8 @@ iavf_get_vf_resource(struct iavf_adapter *adapter)
>  	caps = IAVF_BASIC_OFFLOAD_CAPS |
> VIRTCHNL_VF_CAP_ADV_LINK_SPEED |
>  		VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC |
>  		VIRTCHNL_VF_OFFLOAD_FDIR_PF |
> -		VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF;
> +		VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF |
> +		VIRTCHNL_VF_OFFLOAD_REQ_QUEUES;
> 
>  	args.in_args = (uint8_t *)&caps;
>  	args.in_args_size = sizeof(caps);
> @@ -1148,3 +1216,59 @@ iavf_add_del_mc_addr_list(struct iavf_adapter
> *adapter,
> 
>  	return 0;
>  }
> +
> +int
> +iavf_request_queues(struct iavf_adapter *adapter, uint16_t num) {
> +	struct rte_eth_dev *dev = adapter->eth_dev;
> +	struct iavf_info *vf =  IAVF_DEV_PRIVATE_TO_VF(adapter);
> +	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
> +	struct virtchnl_vf_res_request vfres;
> +	struct iavf_cmd_info args;
> +	uint16_t num_queue_pairs;
> +	int err;
> +
> +	if (!(vf->vf_res->vf_cap_flags &
> +		VIRTCHNL_VF_OFFLOAD_REQ_QUEUES)) {
> +		PMD_DRV_LOG(ERR, "request queues not supported");
> +		return -1;
> +	}
> +
> +	if (num == 0) {
> +		PMD_DRV_LOG(ERR, "queue number cannot be zero");
> +		return -1;
> +	}
> +	vfres.num_queue_pairs = num;
> +
> +	args.ops = VIRTCHNL_OP_REQUEST_QUEUES;
> +	args.in_args = (u8 *)&vfres;
> +	args.in_args_size = sizeof(vfres);
> +	args.out_buffer = vf->aq_resp;
> +	args.out_size = IAVF_AQ_BUF_SZ;
> +
> +	/*
> +	 * disable interrupt to avoid the admin queue message to be read
> +	 * before iavf_read_msg_from_pf.
> +	 */
> +	rte_intr_disable(&pci_dev->intr_handle);
> +	err = iavf_execute_vf_cmd(adapter, &args);
> +	rte_intr_enable(&pci_dev->intr_handle);
> +	if (err) {
> +		PMD_DRV_LOG(ERR, "fail to execute command
> OP_REQUEST_QUEUES");
> +		return err;
> +	}
> +
> +	/* request queues succeeded, vf is resetting */
> +	if (vf->vf_reset) {
> +		PMD_DRV_LOG(INFO, "vf is resetting");
> +		return 0;
> +	}
> +
> +	/* request additional queues failed, return available number */
> +	num_queue_pairs =
> +	  ((struct virtchnl_vf_res_request *)args.out_buffer)-
> >num_queue_pairs;
> +	PMD_DRV_LOG(ERR, "request queues failed, only %u queues "
> +		"available", num_queue_pairs);
> +
> +	return -1;
> +}
> --
> 2.17.1


^ permalink raw reply	[flat|nested] 45+ messages in thread

* Re: [dpdk-dev] [PATCH v6 1/2] net/iavf: add IAVF request queues function
  2020-10-16  8:41     ` Xing, Beilei
@ 2020-10-18 10:29       ` Xu, Ting
  0 siblings, 0 replies; 45+ messages in thread
From: Xu, Ting @ 2020-10-18 10:29 UTC (permalink / raw)
  To: Xing, Beilei, dev; +Cc: Zhang, Qi Z, Wu, Jingjing

> -----Original Message-----
> From: Xing, Beilei <beilei.xing@intel.com>
> Sent: Friday, October 16, 2020 4:42 PM
> To: Xu, Ting <ting.xu@intel.com>; dev@dpdk.org
> Cc: Zhang, Qi Z <qi.z.zhang@intel.com>; Wu, Jingjing <jingjing.wu@intel.com>
> Subject: RE: [PATCH v6 1/2] net/iavf: add IAVF request queues function
> 
> 
> 
> > -----Original Message-----
> > From: Xu, Ting <ting.xu@intel.com>
> > Sent: Friday, October 16, 2020 9:43 AM
> > To: dev@dpdk.org
> > Cc: Zhang, Qi Z <qi.z.zhang@intel.com>; Xing, Beilei
> > <beilei.xing@intel.com>; Wu, Jingjing <jingjing.wu@intel.com>; Xu,
> > Ting <ting.xu@intel.com>
> > Subject: [PATCH v6 1/2] net/iavf: add IAVF request queues function
> >
> > Add new virtchnl function to request additional queues from PF.
> > Current default queue pairs number when creating a VF is 16. In order
> > to support up to
> > 256 queue pairs, enable this request queues function.
> > Since request queues command may return event message, modify function
> > iavf_read_msg_from_pf to identify event opcode and mark VF reset status.
> >
> > Signed-off-by: Ting Xu <ting.xu@intel.com>
> > ---
> >  drivers/net/iavf/iavf.h        |   9 ++
> >  drivers/net/iavf/iavf_ethdev.c |  11 +-
> > drivers/net/iavf/iavf_vchnl.c  | 226
> > +++++++++++++++++++++++++--------
> >  3 files changed, 192 insertions(+), 54 deletions(-)
> >
> > diff --git a/drivers/net/iavf/iavf.h b/drivers/net/iavf/iavf.h index
> > d56611608..93c165c62 100644
> > --- a/drivers/net/iavf/iavf.h
> > +++ b/drivers/net/iavf/iavf.h
> > @@ -107,6 +107,14 @@ struct iavf_fdir_info {
> >  /* TODO: is that correct to assume the max number to be 16 ?*/
> >  #define IAVF_MAX_MSIX_VECTORS   16
> >
> > +/* Message type read in admin queue from PF */ enum iavf_aq_result {
> > +	IAVF_MSG_ERR = -1, /* Meet error when accessing admin queue */
> > +	IAVF_MSG_NON,      /* Read nothing from admin queue */
> > +	IAVF_MSG_SYS,      /* Read system msg from admin queue */
> > +	IAVF_MSG_CMD,      /* Read async command result */
> > +};
> 
> Is there no such message type in shared code?
> 

Yes, I did not find similar codes suitable for this case.

> > +
> >  /* Structure to store private data specific for VF instance. */  struct
> iavf_info {
> >  	uint16_t num_queue_pairs;
> > @@ -301,4 +309,5 @@ int iavf_add_del_rss_cfg(struct iavf_adapter
> > *adapter, int iavf_add_del_mc_addr_list(struct iavf_adapter *adapter,
> >  			struct rte_ether_addr *mc_addrs,
> >  			uint32_t mc_addrs_num, bool add);
> > +int iavf_request_queues(struct iavf_adapter *adapter, uint16_t num);
> >  #endif /* _IAVF_ETHDEV_H_ */
> > diff --git a/drivers/net/iavf/iavf_ethdev.c
> > b/drivers/net/iavf/iavf_ethdev.c index 8b1cf8f1c..a4a28b885 100644
> > --- a/drivers/net/iavf/iavf_ethdev.c
> > +++ b/drivers/net/iavf/iavf_ethdev.c
> > @@ -1282,7 +1282,7 @@ iavf_dev_rx_queue_intr_disable(struct
> > rte_eth_dev *dev, uint16_t queue_id)  }
> >
> >  static int
> > -iavf_check_vf_reset_done(struct iavf_hw *hw)
> > +iavf_check_vf_reset_done(struct iavf_hw *hw, struct iavf_info *vf)
> >  {
> >  	int i, reset;
> >
> > @@ -1299,6 +1299,9 @@ iavf_check_vf_reset_done(struct iavf_hw *hw)
> >  	if (i >= IAVF_RESET_WAIT_CNT)
> >  		return -1;
> >
> > +	/* VF is not in reset or reset is completed */
> > +	vf->vf_reset = false;
> 
> Seems it's not related to the feature.
> Is the fix for commit 1eab95fe2e36e191ad85a9aacf82a44e7c8011fc?
> If yes, it's better to separate bug fix from the feature.
> 

I missed the vf->vf_reset setting at the bottom of iavf_dev_init(). It is not needed here.

> > +
> >  	return 0;
> >  }
> >
> > @@ -1666,7 +1669,7 @@ iavf_init_vf(struct rte_eth_dev *dev)
> >  		goto err;
> >  	}
> >
> > -	err = iavf_check_vf_reset_done(hw);
> > +	err = iavf_check_vf_reset_done(hw, vf);
> >  	if (err) {
> >  		PMD_INIT_LOG(ERR, "VF is still resetting");
> >  		goto err;
> > @@ -1911,7 +1914,9 @@ iavf_dev_close(struct rte_eth_dev *dev)
> >
> >  	iavf_dev_stop(dev);
> >  	iavf_flow_flush(dev, NULL);
> > -	iavf_flow_uninit(adapter);
> > +	/* if VF is in reset, adminq is disabled, skip the process via adminq */
> > +	if (!vf->vf_reset)
> > +		iavf_flow_uninit(adapter);
> 
> Same as above.
> 
> >
> >  	/*
> >  	 * disable promiscuous mode before reset vf diff --git
> > a/drivers/net/iavf/iavf_vchnl.c b/drivers/net/iavf/iavf_vchnl.c index
> > 5e7142893..11a1ff608 100644
> > --- a/drivers/net/iavf/iavf_vchnl.c
> > +++ b/drivers/net/iavf/iavf_vchnl.c
> > @@ -17,6 +17,7 @@
> >  #include <rte_eal.h>
> >  #include <rte_ether.h>
> >  #include <rte_ethdev_driver.h>
> > +#include <rte_ethdev_pci.h>
> >  #include <rte_dev.h>
> >
> >  #include "iavf.h"
> > @@ -25,14 +26,54 @@
> >  #define MAX_TRY_TIMES 200
> >  #define ASQ_DELAY_MS  10
> >
> > +static uint32_t
> > +iavf_convert_link_speed(enum virtchnl_link_speed virt_link_speed) {
> > +	uint32_t speed;
> > +
> > +	switch (virt_link_speed) {
> > +	case VIRTCHNL_LINK_SPEED_100MB:
> > +		speed = 100;
> > +		break;
> > +	case VIRTCHNL_LINK_SPEED_1GB:
> > +		speed = 1000;
> > +		break;
> > +	case VIRTCHNL_LINK_SPEED_10GB:
> > +		speed = 10000;
> > +		break;
> > +	case VIRTCHNL_LINK_SPEED_40GB:
> > +		speed = 40000;
> > +		break;
> > +	case VIRTCHNL_LINK_SPEED_20GB:
> > +		speed = 20000;
> > +		break;
> > +	case VIRTCHNL_LINK_SPEED_25GB:
> > +		speed = 25000;
> > +		break;
> > +	case VIRTCHNL_LINK_SPEED_2_5GB:
> > +		speed = 2500;
> > +		break;
> > +	case VIRTCHNL_LINK_SPEED_5GB:
> > +		speed = 5000;
> > +		break;
> > +	default:
> > +		speed = 0;
> > +		break;
> > +	}
> > +
> > +	return speed;
> > +}
> > +
> >  /* Read data in admin queue to get msg from pf driver */ -static enum
> > iavf_status
> > +static enum iavf_aq_result
> >  iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,
> >  		     uint8_t *buf)
> >  {
> >  	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter);
> >  	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
> > +	struct rte_eth_dev *dev = adapter->eth_dev;
> >  	struct iavf_arq_event_info event;
> > +	enum iavf_aq_result result = IAVF_MSG_NON;
> >  	enum virtchnl_ops opcode;
> >  	int ret;
> >
> > @@ -42,7 +83,9 @@ iavf_read_msg_from_pf(struct iavf_adapter *adapter,
> > uint16_t buf_len,
> >  	/* Can't read any msg from adminQ */
> >  	if (ret) {
> >  		PMD_DRV_LOG(DEBUG, "Can't read msg from AQ");
> > -		return ret;
> > +		if (ret != IAVF_ERR_ADMIN_QUEUE_NO_WORK)
> > +			result = IAVF_MSG_ERR;
> > +		return result;
> >  	}
> >
> >  	opcode = (enum
> > virtchnl_ops)rte_le_to_cpu_32(event.desc.cookie_high);
> > @@ -52,16 +95,51 @@ iavf_read_msg_from_pf(struct iavf_adapter
> > *adapter, uint16_t buf_len,
> >  	PMD_DRV_LOG(DEBUG, "AQ from pf carries opcode %u, retval %d",
> >  		    opcode, vf->cmd_retval);
> >
> > -	if (opcode != vf->pend_cmd) {
> > -		if (opcode != VIRTCHNL_OP_EVENT) {
> > -			PMD_DRV_LOG(WARNING,
> > -				    "command mismatch, expect %u, get %u",
> > -				    vf->pend_cmd, opcode);
> > +	if (opcode == VIRTCHNL_OP_EVENT) {
> > +		struct virtchnl_pf_event *vpe =
> > +			(struct virtchnl_pf_event *)event.msg_buf;
> > +
> > +		result = IAVF_MSG_SYS;
> > +		switch (vpe->event) {
> > +		case VIRTCHNL_EVENT_LINK_CHANGE:
> > +			vf->link_up =
> > +				vpe->event_data.link_event.link_status;
> > +			if (vf->vf_res->vf_cap_flags &
> > +				VIRTCHNL_VF_CAP_ADV_LINK_SPEED) {
> > +				vf->link_speed =
> > +				    vpe-
> > >event_data.link_event_adv.link_speed;
> > +			} else {
> > +				enum virtchnl_link_speed speed;
> > +				speed = vpe-
> > >event_data.link_event.link_speed;
> > +				vf->link_speed =
> > iavf_convert_link_speed(speed);
> > +			}
> > +			iavf_dev_link_update(dev, 0);
> > +			PMD_DRV_LOG(INFO, "Link status update:%s",
> > +					vf->link_up ? "up" : "down");
> > +			break;
> > +		case VIRTCHNL_EVENT_RESET_IMPENDING:
> > +			vf->vf_reset = true;
> > +			PMD_DRV_LOG(INFO, "VF is resetting");
> > +			break;
> > +		case VIRTCHNL_EVENT_PF_DRIVER_CLOSE:
> > +			vf->dev_closed = true;
> > +			PMD_DRV_LOG(INFO, "PF driver closed");
> > +			break;
> > +		default:
> > +			PMD_DRV_LOG(ERR, "%s: Unknown event %d from pf",
> > +					__func__, vpe->event);
> > +		}
> > +	}  else {
> > +		/* async reply msg on command issued by vf previously */
> > +		result = IAVF_MSG_CMD;
> > +		if (opcode != vf->pend_cmd) {
> > +			PMD_DRV_LOG(WARNING, "command mismatch,
> > expect %u, get %u",
> > +					vf->pend_cmd, opcode);
> > +			result = IAVF_MSG_ERR;
> >  		}
> > -		return IAVF_ERR_OPCODE_MISMATCH;
> >  	}
> >
> > -	return IAVF_SUCCESS;
> > +	return result;
> >  }
> 
> How about separate this part which is handling the msg from PF?
> 

Done

> >
> >  static int
> > @@ -69,6 +147,7 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter,
> > struct iavf_cmd_info *args)  {
> >  	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter);
> >  	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
> > +	enum iavf_aq_result result;
> >  	enum iavf_status ret;
> >  	int err = 0;
> >  	int i = 0;
> > @@ -97,9 +176,9 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter,
> > struct iavf_cmd_info *args)
> >  	case VIRTCHNL_OP_GET_SUPPORTED_RXDIDS:
> >  		/* for init virtchnl ops, need to poll the response */
> >  		do {
> > -			ret = iavf_read_msg_from_pf(adapter, args->out_size,
> > +			result = iavf_read_msg_from_pf(adapter, args-
> > >out_size,
> >  						   args->out_buffer);
> > -			if (ret == IAVF_SUCCESS)
> > +			if (result == IAVF_MSG_CMD)
> >  				break;
> >  			rte_delay_ms(ASQ_DELAY_MS);
> >  		} while (i++ < MAX_TRY_TIMES);
> > @@ -111,7 +190,33 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter,
> > struct iavf_cmd_info *args)
> >  		}
> >  		_clear_cmd(vf);
> >  		break;
> > -
> > +	case VIRTCHNL_OP_REQUEST_QUEUES:
> > +		/*
> > +		 * ignore async reply, only wait for system message,
> > +		 * vf_reset = true if get VIRTCHNL_EVENT_RESET_IMPENDING,
> > +		 * if not, means request queues failed.
> > +		 */
> > +		do {
> > +			result = iavf_read_msg_from_pf(adapter, args-
> > >out_size,
> > +						   args->out_buffer);
> > +			if (result == IAVF_MSG_SYS && vf->vf_reset) {
> > +				break;
> > +			} else if (result == IAVF_MSG_CMD ||
> > +				result == IAVF_MSG_ERR) {
> > +				err = -1;
> > +				break;
> > +			}
> > +			rte_delay_ms(ASQ_DELAY_MS);
> > +			/* If don't read msg or read sys event, continue */
> > +		} while (i++ < MAX_TRY_TIMES);
> > +		if (i >= MAX_TRY_TIMES ||
> > +		    vf->cmd_retval != VIRTCHNL_STATUS_SUCCESS) {
> > +			err = -1;
> > +			PMD_DRV_LOG(ERR, "No response or return failure
> > (%d)"
> > +				    " for cmd %d", vf->cmd_retval, args->ops);
> > +		}
> > +		_clear_cmd(vf);
> > +		break;
> >  	default:
> >  		/* For other virtchnl ops in running time,
> >  		 * wait for the cmd done flag.
> > @@ -136,44 +241,6 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter,
> > struct iavf_cmd_info *args)
> >  	return err;
> >  }
> >
> > -static uint32_t
> > -iavf_convert_link_speed(enum virtchnl_link_speed virt_link_speed) -{
> > -	uint32_t speed;
> > -
> > -	switch (virt_link_speed) {
> > -	case VIRTCHNL_LINK_SPEED_100MB:
> > -		speed = 100;
> > -		break;
> > -	case VIRTCHNL_LINK_SPEED_1GB:
> > -		speed = 1000;
> > -		break;
> > -	case VIRTCHNL_LINK_SPEED_10GB:
> > -		speed = 10000;
> > -		break;
> > -	case VIRTCHNL_LINK_SPEED_40GB:
> > -		speed = 40000;
> > -		break;
> > -	case VIRTCHNL_LINK_SPEED_20GB:
> > -		speed = 20000;
> > -		break;
> > -	case VIRTCHNL_LINK_SPEED_25GB:
> > -		speed = 25000;
> > -		break;
> > -	case VIRTCHNL_LINK_SPEED_2_5GB:
> > -		speed = 2500;
> > -		break;
> > -	case VIRTCHNL_LINK_SPEED_5GB:
> > -		speed = 5000;
> > -		break;
> > -	default:
> > -		speed = 0;
> > -		break;
> > -	}
> > -
> > -	return speed;
> > -}
> > -
> >  static void
> >  iavf_handle_pf_event_msg(struct rte_eth_dev *dev, uint8_t *msg,
> >  			uint16_t msglen)
> > @@ -389,7 +456,8 @@ iavf_get_vf_resource(struct iavf_adapter *adapter)
> >  	caps = IAVF_BASIC_OFFLOAD_CAPS |
> > VIRTCHNL_VF_CAP_ADV_LINK_SPEED |
> >  		VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC |
> >  		VIRTCHNL_VF_OFFLOAD_FDIR_PF |
> > -		VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF;
> > +		VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF |
> > +		VIRTCHNL_VF_OFFLOAD_REQ_QUEUES;
> >
> >  	args.in_args = (uint8_t *)&caps;
> >  	args.in_args_size = sizeof(caps);
> > @@ -1148,3 +1216,59 @@ iavf_add_del_mc_addr_list(struct iavf_adapter
> > *adapter,
> >
> >  	return 0;
> >  }
> > +
> > +int
> > +iavf_request_queues(struct iavf_adapter *adapter, uint16_t num) {
> > +	struct rte_eth_dev *dev = adapter->eth_dev;
> > +	struct iavf_info *vf =  IAVF_DEV_PRIVATE_TO_VF(adapter);
> > +	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
> > +	struct virtchnl_vf_res_request vfres;
> > +	struct iavf_cmd_info args;
> > +	uint16_t num_queue_pairs;
> > +	int err;
> > +
> > +	if (!(vf->vf_res->vf_cap_flags &
> > +		VIRTCHNL_VF_OFFLOAD_REQ_QUEUES)) {
> > +		PMD_DRV_LOG(ERR, "request queues not supported");
> > +		return -1;
> > +	}
> > +
> > +	if (num == 0) {
> > +		PMD_DRV_LOG(ERR, "queue number cannot be zero");
> > +		return -1;
> > +	}
> > +	vfres.num_queue_pairs = num;
> > +
> > +	args.ops = VIRTCHNL_OP_REQUEST_QUEUES;
> > +	args.in_args = (u8 *)&vfres;
> > +	args.in_args_size = sizeof(vfres);
> > +	args.out_buffer = vf->aq_resp;
> > +	args.out_size = IAVF_AQ_BUF_SZ;
> > +
> > +	/*
> > +	 * disable interrupt to avoid the admin queue message to be read
> > +	 * before iavf_read_msg_from_pf.
> > +	 */
> > +	rte_intr_disable(&pci_dev->intr_handle);
> > +	err = iavf_execute_vf_cmd(adapter, &args);
> > +	rte_intr_enable(&pci_dev->intr_handle);
> > +	if (err) {
> > +		PMD_DRV_LOG(ERR, "fail to execute command
> > OP_REQUEST_QUEUES");
> > +		return err;
> > +	}
> > +
> > +	/* request queues succeeded, vf is resetting */
> > +	if (vf->vf_reset) {
> > +		PMD_DRV_LOG(INFO, "vf is resetting");
> > +		return 0;
> > +	}
> > +
> > +	/* request additional queues failed, return available number */
> > +	num_queue_pairs =
> > +	  ((struct virtchnl_vf_res_request *)args.out_buffer)-
> > >num_queue_pairs;
> > +	PMD_DRV_LOG(ERR, "request queues failed, only %u queues "
> > +		"available", num_queue_pairs);
> > +
> > +	return -1;
> > +}
> > --
> > 2.17.1


^ permalink raw reply	[flat|nested] 45+ messages in thread

* [dpdk-dev] [PATCH v7 0/6] enable large VF configuration
  2020-09-09  7:20 [dpdk-dev] [PATCH v1 0/2] enable large VF configuration Ting Xu
                   ` (7 preceding siblings ...)
  2020-10-16  1:43 ` [dpdk-dev] [PATCH v6 0/2] " Ting Xu
@ 2020-10-18 10:34 ` Ting Xu
  2020-10-18 10:34   ` [dpdk-dev] [PATCH v7 1/6] net/iavf: handle virtchnl event message without interrupt Ting Xu
                     ` (5 more replies)
  2020-10-22  6:48 ` [dpdk-dev] [PATCH v8 0/6] enable large VF configuration Ting Xu
  9 siblings, 6 replies; 45+ messages in thread
From: Ting Xu @ 2020-10-18 10:34 UTC (permalink / raw)
  To: dev; +Cc: qi.z.zhang, beilei.xing, jingjing.wu, Ting Xu

This patchset supports to configure up to 256 queue pairs per VF. If
large VF is supported after capability negotiation, VF will request
queues from PF as needed. New virtual channel opcodes and structures
are used to indicate 256 queues, so VF is designed to handle the new
function of configure VSI queues, IRQ mapping and enable/disable queues.
Also, enable VF to query the max RSS queue region for RSS and filter
configuration.

Ting Xu (6):
  net/iavf: handle virtchnl event message without interrupt
  net/iavf: add IAVF request queues function
  net/iavf: negotiate large VF and request more queues
  net/iavf: enable multiple queues configurations for large VF
  net/iavf: enable IRQ mapping configuration for large VF
  net/iavf: add enable/disable queues for large VF

---
v6->v7:
Separate patches.

v4->v6:
Solve queue mapping buffer limitation issue.
Optimize VSI queue configuration

v3->v4:
Optimize handling messages from PF.

v2->v3:
Fix coding style issue.
Move get max RSS queue region after VF reset.
Add request queues capability negotiation.

v1->v2:
Change the communication with kernel PF.

---
 drivers/net/iavf/iavf.h        |  37 ++-
 drivers/net/iavf/iavf_ethdev.c | 144 ++++++++++-
 drivers/net/iavf/iavf_rxtx.c   |  25 +-
 drivers/net/iavf/iavf_vchnl.c  | 450 ++++++++++++++++++++++++++++-----
 4 files changed, 576 insertions(+), 80 deletions(-)

-- 
2.17.1


^ permalink raw reply	[flat|nested] 45+ messages in thread

* [dpdk-dev] [PATCH v7 1/6] net/iavf: handle virtchnl event message without interrupt
  2020-10-18 10:34 ` [dpdk-dev] [PATCH v7 0/6] " Ting Xu
@ 2020-10-18 10:34   ` Ting Xu
  2020-10-21  8:15     ` Xing, Beilei
  2020-10-18 10:34   ` [dpdk-dev] [PATCH v7 2/6] net/iavf: add IAVF request queues function Ting Xu
                     ` (4 subsequent siblings)
  5 siblings, 1 reply; 45+ messages in thread
From: Ting Xu @ 2020-10-18 10:34 UTC (permalink / raw)
  To: dev; +Cc: qi.z.zhang, beilei.xing, jingjing.wu, Ting Xu

Currently, VF only handle virtchnl event message when interrupt is
called. In some cases, event message is supposed to be handled with
interrupt disabled, for example request queues. To solve this issue,
virtchnl event message handling are added in reading message from PF
in this patch.

Signed-off-by: Ting Xu <ting.xu@intel.com>
---
 drivers/net/iavf/iavf.h       |   8 ++
 drivers/net/iavf/iavf_vchnl.c | 138 ++++++++++++++++++++++------------
 2 files changed, 97 insertions(+), 49 deletions(-)

diff --git a/drivers/net/iavf/iavf.h b/drivers/net/iavf/iavf.h
index 3198d85b3..9c16324c1 100644
--- a/drivers/net/iavf/iavf.h
+++ b/drivers/net/iavf/iavf.h
@@ -107,6 +107,14 @@ struct iavf_fdir_info {
 /* TODO: is that correct to assume the max number to be 16 ?*/
 #define IAVF_MAX_MSIX_VECTORS   16
 
+/* Message type read in admin queue from PF */
+enum iavf_aq_result {
+	IAVF_MSG_ERR = -1, /* Meet error when accessing admin queue */
+	IAVF_MSG_NON,      /* Read nothing from admin queue */
+	IAVF_MSG_SYS,      /* Read system msg from admin queue */
+	IAVF_MSG_CMD,      /* Read async command result */
+};
+
 /* Structure to store private data specific for VF instance. */
 struct iavf_info {
 	uint16_t num_queue_pairs;
diff --git a/drivers/net/iavf/iavf_vchnl.c b/drivers/net/iavf/iavf_vchnl.c
index db0b76876..d53aeba5a 100644
--- a/drivers/net/iavf/iavf_vchnl.c
+++ b/drivers/net/iavf/iavf_vchnl.c
@@ -25,14 +25,54 @@
 #define MAX_TRY_TIMES 200
 #define ASQ_DELAY_MS  10
 
+static uint32_t
+iavf_convert_link_speed(enum virtchnl_link_speed virt_link_speed)
+{
+	uint32_t speed;
+
+	switch (virt_link_speed) {
+	case VIRTCHNL_LINK_SPEED_100MB:
+		speed = 100;
+		break;
+	case VIRTCHNL_LINK_SPEED_1GB:
+		speed = 1000;
+		break;
+	case VIRTCHNL_LINK_SPEED_10GB:
+		speed = 10000;
+		break;
+	case VIRTCHNL_LINK_SPEED_40GB:
+		speed = 40000;
+		break;
+	case VIRTCHNL_LINK_SPEED_20GB:
+		speed = 20000;
+		break;
+	case VIRTCHNL_LINK_SPEED_25GB:
+		speed = 25000;
+		break;
+	case VIRTCHNL_LINK_SPEED_2_5GB:
+		speed = 2500;
+		break;
+	case VIRTCHNL_LINK_SPEED_5GB:
+		speed = 5000;
+		break;
+	default:
+		speed = 0;
+		break;
+	}
+
+	return speed;
+}
+
 /* Read data in admin queue to get msg from pf driver */
-static enum iavf_status
+static enum iavf_aq_result
 iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,
 		     uint8_t *buf)
 {
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter);
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct rte_eth_dev *dev = adapter->eth_dev;
 	struct iavf_arq_event_info event;
+	enum iavf_aq_result result = IAVF_MSG_NON;
 	enum virtchnl_ops opcode;
 	int ret;
 
@@ -42,7 +82,9 @@ iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,
 	/* Can't read any msg from adminQ */
 	if (ret) {
 		PMD_DRV_LOG(DEBUG, "Can't read msg from AQ");
-		return ret;
+		if (ret != IAVF_ERR_ADMIN_QUEUE_NO_WORK)
+			result = IAVF_MSG_ERR;
+		return result;
 	}
 
 	opcode = (enum virtchnl_ops)rte_le_to_cpu_32(event.desc.cookie_high);
@@ -52,16 +94,51 @@ iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,
 	PMD_DRV_LOG(DEBUG, "AQ from pf carries opcode %u, retval %d",
 		    opcode, vf->cmd_retval);
 
-	if (opcode != vf->pend_cmd) {
-		if (opcode != VIRTCHNL_OP_EVENT) {
-			PMD_DRV_LOG(WARNING,
-				    "command mismatch, expect %u, get %u",
-				    vf->pend_cmd, opcode);
+	if (opcode == VIRTCHNL_OP_EVENT) {
+		struct virtchnl_pf_event *vpe =
+			(struct virtchnl_pf_event *)event.msg_buf;
+
+		result = IAVF_MSG_SYS;
+		switch (vpe->event) {
+		case VIRTCHNL_EVENT_LINK_CHANGE:
+			vf->link_up =
+				vpe->event_data.link_event.link_status;
+			if (vf->vf_res->vf_cap_flags &
+				VIRTCHNL_VF_CAP_ADV_LINK_SPEED) {
+				vf->link_speed =
+				    vpe->event_data.link_event_adv.link_speed;
+			} else {
+				enum virtchnl_link_speed speed;
+				speed = vpe->event_data.link_event.link_speed;
+				vf->link_speed = iavf_convert_link_speed(speed);
+			}
+			iavf_dev_link_update(dev, 0);
+			PMD_DRV_LOG(INFO, "Link status update:%s",
+					vf->link_up ? "up" : "down");
+			break;
+		case VIRTCHNL_EVENT_RESET_IMPENDING:
+			vf->vf_reset = true;
+			PMD_DRV_LOG(INFO, "VF is resetting");
+			break;
+		case VIRTCHNL_EVENT_PF_DRIVER_CLOSE:
+			vf->dev_closed = true;
+			PMD_DRV_LOG(INFO, "PF driver closed");
+			break;
+		default:
+			PMD_DRV_LOG(ERR, "%s: Unknown event %d from pf",
+					__func__, vpe->event);
+		}
+	}  else {
+		/* async reply msg on command issued by vf previously */
+		result = IAVF_MSG_CMD;
+		if (opcode != vf->pend_cmd) {
+			PMD_DRV_LOG(WARNING, "command mismatch, expect %u, get %u",
+					vf->pend_cmd, opcode);
+			result = IAVF_MSG_ERR;
 		}
-		return IAVF_ERR_OPCODE_MISMATCH;
 	}
 
-	return IAVF_SUCCESS;
+	return result;
 }
 
 static int
@@ -69,6 +146,7 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter, struct iavf_cmd_info *args)
 {
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter);
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	enum iavf_aq_result result;
 	enum iavf_status ret;
 	int err = 0;
 	int i = 0;
@@ -97,9 +175,9 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter, struct iavf_cmd_info *args)
 	case VIRTCHNL_OP_GET_SUPPORTED_RXDIDS:
 		/* for init virtchnl ops, need to poll the response */
 		do {
-			ret = iavf_read_msg_from_pf(adapter, args->out_size,
+			result = iavf_read_msg_from_pf(adapter, args->out_size,
 						   args->out_buffer);
-			if (ret == IAVF_SUCCESS)
+			if (result == IAVF_MSG_CMD)
 				break;
 			rte_delay_ms(ASQ_DELAY_MS);
 		} while (i++ < MAX_TRY_TIMES);
@@ -136,44 +214,6 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter, struct iavf_cmd_info *args)
 	return err;
 }
 
-static uint32_t
-iavf_convert_link_speed(enum virtchnl_link_speed virt_link_speed)
-{
-	uint32_t speed;
-
-	switch (virt_link_speed) {
-	case VIRTCHNL_LINK_SPEED_100MB:
-		speed = 100;
-		break;
-	case VIRTCHNL_LINK_SPEED_1GB:
-		speed = 1000;
-		break;
-	case VIRTCHNL_LINK_SPEED_10GB:
-		speed = 10000;
-		break;
-	case VIRTCHNL_LINK_SPEED_40GB:
-		speed = 40000;
-		break;
-	case VIRTCHNL_LINK_SPEED_20GB:
-		speed = 20000;
-		break;
-	case VIRTCHNL_LINK_SPEED_25GB:
-		speed = 25000;
-		break;
-	case VIRTCHNL_LINK_SPEED_2_5GB:
-		speed = 2500;
-		break;
-	case VIRTCHNL_LINK_SPEED_5GB:
-		speed = 5000;
-		break;
-	default:
-		speed = 0;
-		break;
-	}
-
-	return speed;
-}
-
 static void
 iavf_handle_pf_event_msg(struct rte_eth_dev *dev, uint8_t *msg,
 			uint16_t msglen)
-- 
2.17.1


^ permalink raw reply	[flat|nested] 45+ messages in thread

* [dpdk-dev] [PATCH v7 2/6] net/iavf: add IAVF request queues function
  2020-10-18 10:34 ` [dpdk-dev] [PATCH v7 0/6] " Ting Xu
  2020-10-18 10:34   ` [dpdk-dev] [PATCH v7 1/6] net/iavf: handle virtchnl event message without interrupt Ting Xu
@ 2020-10-18 10:34   ` Ting Xu
  2020-10-18 10:34   ` [dpdk-dev] [PATCH v7 3/6] net/iavf: negotiate large VF and request more queues Ting Xu
                     ` (3 subsequent siblings)
  5 siblings, 0 replies; 45+ messages in thread
From: Ting Xu @ 2020-10-18 10:34 UTC (permalink / raw)
  To: dev; +Cc: qi.z.zhang, beilei.xing, jingjing.wu, Ting Xu

Add new virtchnl function to request additional queues from PF. Current
default queue pairs number when creating a VF is 16. In order to support
up to 256 queue pairs, enable this request queues function.
Since request queues command may return event message to report info,
disable interrupt before request queues to receive the adminq message.

Signed-off-by: Ting Xu <ting.xu@intel.com>
---
 drivers/net/iavf/iavf.h       |  1 +
 drivers/net/iavf/iavf_vchnl.c | 88 ++++++++++++++++++++++++++++++++++-
 2 files changed, 87 insertions(+), 2 deletions(-)

diff --git a/drivers/net/iavf/iavf.h b/drivers/net/iavf/iavf.h
index 9c16324c1..778b6c23c 100644
--- a/drivers/net/iavf/iavf.h
+++ b/drivers/net/iavf/iavf.h
@@ -287,4 +287,5 @@ int iavf_add_del_rss_cfg(struct iavf_adapter *adapter,
 int iavf_add_del_mc_addr_list(struct iavf_adapter *adapter,
 			struct rte_ether_addr *mc_addrs,
 			uint32_t mc_addrs_num, bool add);
+int iavf_request_queues(struct iavf_adapter *adapter, uint16_t num);
 #endif /* _IAVF_ETHDEV_H_ */
diff --git a/drivers/net/iavf/iavf_vchnl.c b/drivers/net/iavf/iavf_vchnl.c
index d53aeba5a..9caf8ee70 100644
--- a/drivers/net/iavf/iavf_vchnl.c
+++ b/drivers/net/iavf/iavf_vchnl.c
@@ -17,6 +17,7 @@
 #include <rte_eal.h>
 #include <rte_ether.h>
 #include <rte_ethdev_driver.h>
+#include <rte_ethdev_pci.h>
 #include <rte_dev.h>
 
 #include "iavf.h"
@@ -189,7 +190,33 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter, struct iavf_cmd_info *args)
 		}
 		_clear_cmd(vf);
 		break;
-
+	case VIRTCHNL_OP_REQUEST_QUEUES:
+		/*
+		 * ignore async reply, only wait for system message,
+		 * vf_reset = true if get VIRTCHNL_EVENT_RESET_IMPENDING,
+		 * if not, means request queues failed.
+		 */
+		do {
+			result = iavf_read_msg_from_pf(adapter, args->out_size,
+						   args->out_buffer);
+			if (result == IAVF_MSG_SYS && vf->vf_reset) {
+				break;
+			} else if (result == IAVF_MSG_CMD ||
+				result == IAVF_MSG_ERR) {
+				err = -1;
+				break;
+			}
+			rte_delay_ms(ASQ_DELAY_MS);
+			/* If don't read msg or read sys event, continue */
+		} while (i++ < MAX_TRY_TIMES);
+		if (i >= MAX_TRY_TIMES ||
+			vf->cmd_retval != VIRTCHNL_STATUS_SUCCESS) {
+			err = -1;
+			PMD_DRV_LOG(ERR, "No response or return failure (%d)"
+				    " for cmd %d", vf->cmd_retval, args->ops);
+		}
+		_clear_cmd(vf);
+		break;
 	default:
 		/* For other virtchnl ops in running time,
 		 * wait for the cmd done flag.
@@ -429,7 +456,8 @@ iavf_get_vf_resource(struct iavf_adapter *adapter)
 	caps = IAVF_BASIC_OFFLOAD_CAPS | VIRTCHNL_VF_CAP_ADV_LINK_SPEED |
 		VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC |
 		VIRTCHNL_VF_OFFLOAD_FDIR_PF |
-		VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF;
+		VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF |
+		VIRTCHNL_VF_OFFLOAD_REQ_QUEUES;
 
 	args.in_args = (uint8_t *)&caps;
 	args.in_args_size = sizeof(caps);
@@ -1186,3 +1214,59 @@ iavf_add_del_mc_addr_list(struct iavf_adapter *adapter,
 
 	return 0;
 }
+
+int
+iavf_request_queues(struct iavf_adapter *adapter, uint16_t num)
+{
+	struct rte_eth_dev *dev = adapter->eth_dev;
+	struct iavf_info *vf =  IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+	struct virtchnl_vf_res_request vfres;
+	struct iavf_cmd_info args;
+	uint16_t num_queue_pairs;
+	int err;
+
+	if (!(vf->vf_res->vf_cap_flags &
+		VIRTCHNL_VF_OFFLOAD_REQ_QUEUES)) {
+		PMD_DRV_LOG(ERR, "request queues not supported");
+		return -1;
+	}
+
+	if (num == 0) {
+		PMD_DRV_LOG(ERR, "queue number cannot be zero");
+		return -1;
+	}
+	vfres.num_queue_pairs = num;
+
+	args.ops = VIRTCHNL_OP_REQUEST_QUEUES;
+	args.in_args = (u8 *)&vfres;
+	args.in_args_size = sizeof(vfres);
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+
+	/*
+	 * disable interrupt to avoid the admin queue message to be read
+	 * before iavf_read_msg_from_pf.
+	 */
+	rte_intr_disable(&pci_dev->intr_handle);
+	err = iavf_execute_vf_cmd(adapter, &args);
+	rte_intr_enable(&pci_dev->intr_handle);
+	if (err) {
+		PMD_DRV_LOG(ERR, "fail to execute command OP_REQUEST_QUEUES");
+		return err;
+	}
+
+	/* request queues succeeded, vf is resetting */
+	if (vf->vf_reset) {
+		PMD_DRV_LOG(INFO, "vf is resetting");
+		return 0;
+	}
+
+	/* request additional queues failed, return available number */
+	num_queue_pairs =
+	  ((struct virtchnl_vf_res_request *)args.out_buffer)->num_queue_pairs;
+	PMD_DRV_LOG(ERR, "request queues failed, only %u queues "
+		"available", num_queue_pairs);
+
+	return -1;
+}
-- 
2.17.1


^ permalink raw reply	[flat|nested] 45+ messages in thread

* [dpdk-dev] [PATCH v7 3/6] net/iavf: negotiate large VF and request more queues
  2020-10-18 10:34 ` [dpdk-dev] [PATCH v7 0/6] " Ting Xu
  2020-10-18 10:34   ` [dpdk-dev] [PATCH v7 1/6] net/iavf: handle virtchnl event message without interrupt Ting Xu
  2020-10-18 10:34   ` [dpdk-dev] [PATCH v7 2/6] net/iavf: add IAVF request queues function Ting Xu
@ 2020-10-18 10:34   ` Ting Xu
  2020-10-18 10:34   ` [dpdk-dev] [PATCH v7 4/6] net/iavf: enable multiple queues configurations for large VF Ting Xu
                     ` (2 subsequent siblings)
  5 siblings, 0 replies; 45+ messages in thread
From: Ting Xu @ 2020-10-18 10:34 UTC (permalink / raw)
  To: dev; +Cc: qi.z.zhang, beilei.xing, jingjing.wu, Ting Xu

Negotiate large VF capability with PF during VF initialization. If large
VF is supported and the number of queues larger than 16 is required, VF
requests additional queues from PF. Mark the state that large VF is
supported.

If the allocated queues number is larger than 16, the max RSS queue
region cannot be 16 anymore. Add the function to query max RSS queue
region from PF, use it in the RSS initialization and future filters
configuration.

Signed-off-by: Ting Xu <ting.xu@intel.com>
---
 drivers/net/iavf/iavf.h        |  7 +++-
 drivers/net/iavf/iavf_ethdev.c | 74 ++++++++++++++++++++++++++++++++--
 drivers/net/iavf/iavf_vchnl.c  | 33 ++++++++++++++-
 3 files changed, 109 insertions(+), 5 deletions(-)

diff --git a/drivers/net/iavf/iavf.h b/drivers/net/iavf/iavf.h
index 778b6c23c..49ccfeece 100644
--- a/drivers/net/iavf/iavf.h
+++ b/drivers/net/iavf/iavf.h
@@ -19,7 +19,8 @@
 #define IAVF_FRAME_SIZE_MAX       9728
 #define IAVF_QUEUE_BASE_ADDR_UNIT 128
 
-#define IAVF_MAX_NUM_QUEUES       16
+#define IAVF_MAX_NUM_QUEUES_DFLT	 16
+#define IAVF_MAX_NUM_QUEUES_LV		 256
 
 #define IAVF_NUM_MACADDR_MAX      64
 
@@ -149,6 +150,7 @@ struct iavf_info {
 	uint8_t *rss_key;
 	uint16_t nb_msix;   /* number of MSI-X interrupts on Rx */
 	uint16_t msix_base; /* msix vector base from */
+	uint16_t max_rss_qregion; /* max RSS queue region supported by PF */
 	/* queue bitmask for each vector */
 	uint16_t rxq_map[IAVF_MAX_MSIX_VECTORS];
 	struct iavf_flow_list flow_list;
@@ -157,6 +159,8 @@ struct iavf_info {
 	struct iavf_parser_list dist_parser_list;
 
 	struct iavf_fdir_info fdir; /* flow director info */
+	/* indicate large VF support enabled or not */
+	bool lv_enabled;
 };
 
 #define IAVF_MAX_PKT_TYPE 1024
@@ -288,4 +292,5 @@ int iavf_add_del_mc_addr_list(struct iavf_adapter *adapter,
 			struct rte_ether_addr *mc_addrs,
 			uint32_t mc_addrs_num, bool add);
 int iavf_request_queues(struct iavf_adapter *adapter, uint16_t num);
+int iavf_get_max_rss_queue_region(struct iavf_adapter *adapter);
 #endif /* _IAVF_ETHDEV_H_ */
diff --git a/drivers/net/iavf/iavf_ethdev.c b/drivers/net/iavf/iavf_ethdev.c
index f5e6e852a..f41be2b7d 100644
--- a/drivers/net/iavf/iavf_ethdev.c
+++ b/drivers/net/iavf/iavf_ethdev.c
@@ -195,7 +195,7 @@ iavf_init_rss(struct iavf_adapter *adapter)
 
 	rss_conf = &adapter->eth_dev->data->dev_conf.rx_adv_conf.rss_conf;
 	nb_q = RTE_MIN(adapter->eth_dev->data->nb_rx_queues,
-		       IAVF_MAX_NUM_QUEUES);
+		       vf->max_rss_qregion);
 
 	if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) {
 		PMD_DRV_LOG(DEBUG, "RSS is not supported");
@@ -241,6 +241,31 @@ iavf_init_rss(struct iavf_adapter *adapter)
 	return 0;
 }
 
+static int
+iavf_queues_req_reset(struct rte_eth_dev *dev, uint16_t num)
+{
+	struct iavf_adapter *ad =
+		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
+	struct iavf_info *vf =  IAVF_DEV_PRIVATE_TO_VF(ad);
+	int ret;
+
+	ret = iavf_request_queues(ad, num);
+	if (ret) {
+		PMD_DRV_LOG(ERR, "request queues from PF failed");
+		return ret;
+	}
+	PMD_DRV_LOG(INFO, "change queue pairs from %u to %u",
+			vf->vsi_res->num_queue_pairs, num);
+
+	ret = iavf_dev_reset(dev);
+	if (ret) {
+		PMD_DRV_LOG(ERR, "vf reset failed");
+		return ret;
+	}
+
+	return 0;
+}
+
 static int
 iavf_dev_configure(struct rte_eth_dev *dev)
 {
@@ -248,6 +273,9 @@ iavf_dev_configure(struct rte_eth_dev *dev)
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct iavf_info *vf =  IAVF_DEV_PRIVATE_TO_VF(ad);
 	struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
+	uint16_t num_queue_pairs = RTE_MAX(dev->data->nb_rx_queues,
+		dev->data->nb_tx_queues);
+	int ret;
 
 	ad->rx_bulk_alloc_allowed = true;
 	/* Initialize to TRUE. If any of Rx queues doesn't meet the
@@ -259,6 +287,46 @@ iavf_dev_configure(struct rte_eth_dev *dev)
 	if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)
 		dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_RSS_HASH;
 
+	/* Large VF setting */
+	if (num_queue_pairs > IAVF_MAX_NUM_QUEUES_DFLT) {
+		if (!(vf->vf_res->vf_cap_flags &
+				VIRTCHNL_VF_LARGE_NUM_QPAIRS)) {
+			PMD_DRV_LOG(ERR, "large VF is not supported");
+			return -1;
+		}
+
+		if (num_queue_pairs > IAVF_MAX_NUM_QUEUES_LV) {
+			PMD_DRV_LOG(ERR, "queue pairs number cannot be larger "
+				"than %u", IAVF_MAX_NUM_QUEUES_LV);
+			return -1;
+		}
+
+		ret = iavf_queues_req_reset(dev, num_queue_pairs);
+		if (ret)
+			return ret;
+
+		ret = iavf_get_max_rss_queue_region(ad);
+		if (ret) {
+			PMD_INIT_LOG(ERR, "get max rss queue region failed");
+			return ret;
+		}
+
+		vf->lv_enabled = true;
+	} else {
+		/* Check if large VF is already enabled. If so, disable and
+		 * release redundant queue resource.
+		 */
+		if (vf->lv_enabled) {
+			ret = iavf_queues_req_reset(dev, num_queue_pairs);
+			if (ret)
+				return ret;
+
+			vf->lv_enabled = false;
+		}
+		/* if large VF is not required, use default rss queue region */
+		vf->max_rss_qregion = IAVF_MAX_NUM_QUEUES_DFLT;
+	}
+
 	/* Vlan stripping setting */
 	if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN) {
 		if (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
@@ -569,8 +637,8 @@ iavf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 {
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 
-	dev_info->max_rx_queues = vf->vsi_res->num_queue_pairs;
-	dev_info->max_tx_queues = vf->vsi_res->num_queue_pairs;
+	dev_info->max_rx_queues = IAVF_MAX_NUM_QUEUES_LV;
+	dev_info->max_tx_queues = IAVF_MAX_NUM_QUEUES_LV;
 	dev_info->min_rx_bufsize = IAVF_BUF_SIZE_MIN;
 	dev_info->max_rx_pktlen = IAVF_FRAME_SIZE_MAX;
 	dev_info->max_mtu = dev_info->max_rx_pktlen - IAVF_ETH_OVERHEAD;
diff --git a/drivers/net/iavf/iavf_vchnl.c b/drivers/net/iavf/iavf_vchnl.c
index 9caf8ee70..e939b275a 100644
--- a/drivers/net/iavf/iavf_vchnl.c
+++ b/drivers/net/iavf/iavf_vchnl.c
@@ -457,7 +457,8 @@ iavf_get_vf_resource(struct iavf_adapter *adapter)
 		VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC |
 		VIRTCHNL_VF_OFFLOAD_FDIR_PF |
 		VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF |
-		VIRTCHNL_VF_OFFLOAD_REQ_QUEUES;
+		VIRTCHNL_VF_OFFLOAD_REQ_QUEUES |
+		VIRTCHNL_VF_LARGE_NUM_QPAIRS;
 
 	args.in_args = (uint8_t *)&caps;
 	args.in_args_size = sizeof(caps);
@@ -1270,3 +1271,33 @@ iavf_request_queues(struct iavf_adapter *adapter, uint16_t num)
 
 	return -1;
 }
+
+int
+iavf_get_max_rss_queue_region(struct iavf_adapter *adapter)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct iavf_cmd_info args;
+	uint16_t qregion_width;
+	int err;
+
+	args.ops = VIRTCHNL_OP_GET_MAX_RSS_QREGION;
+	args.in_args = NULL;
+	args.in_args_size = 0;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err) {
+		PMD_DRV_LOG(ERR,
+			    "Failed to execute command of "
+			    "VIRTCHNL_OP_GET_MAX_RSS_QREGION");
+		return err;
+	}
+
+	qregion_width =
+	((struct virtchnl_max_rss_qregion *)args.out_buffer)->qregion_width;
+
+	vf->max_rss_qregion = (uint16_t)(1 << qregion_width);
+
+	return 0;
+}
-- 
2.17.1


^ permalink raw reply	[flat|nested] 45+ messages in thread

* [dpdk-dev] [PATCH v7 4/6] net/iavf: enable multiple queues configurations for large VF
  2020-10-18 10:34 ` [dpdk-dev] [PATCH v7 0/6] " Ting Xu
                     ` (2 preceding siblings ...)
  2020-10-18 10:34   ` [dpdk-dev] [PATCH v7 3/6] net/iavf: negotiate large VF and request more queues Ting Xu
@ 2020-10-18 10:34   ` Ting Xu
  2020-10-18 10:34   ` [dpdk-dev] [PATCH v7 5/6] net/iavf: enable IRQ mapping configuration " Ting Xu
  2020-10-18 10:34   ` [dpdk-dev] [PATCH v7 6/6] net/iavf: add enable/disable queues " Ting Xu
  5 siblings, 0 replies; 45+ messages in thread
From: Ting Xu @ 2020-10-18 10:34 UTC (permalink / raw)
  To: dev; +Cc: qi.z.zhang, beilei.xing, jingjing.wu, Ting Xu

Since the adminq buffer size has a 4K limitation, the current virtchnl
command VIRTCHNL_OP_CONFIG_VSI_QUEUES cannot send the message only once to
configure up to 256 queues. In this patch, we send the messages multiple
times to make sure that the buffer size is less than 4K each time.

Signed-off-by: Ting Xu <ting.xu@intel.com>
---
 drivers/net/iavf/iavf.h        |  4 +++-
 drivers/net/iavf/iavf_ethdev.c | 18 +++++++++++++++++-
 drivers/net/iavf/iavf_vchnl.c  | 11 ++++++-----
 3 files changed, 26 insertions(+), 7 deletions(-)

diff --git a/drivers/net/iavf/iavf.h b/drivers/net/iavf/iavf.h
index 49ccfeece..1cdac1b10 100644
--- a/drivers/net/iavf/iavf.h
+++ b/drivers/net/iavf/iavf.h
@@ -21,6 +21,7 @@
 
 #define IAVF_MAX_NUM_QUEUES_DFLT	 16
 #define IAVF_MAX_NUM_QUEUES_LV		 256
+#define IAVF_CFG_Q_NUM_PER_BUF		 32
 
 #define IAVF_NUM_MACADDR_MAX      64
 
@@ -269,7 +270,8 @@ int iavf_enable_queues(struct iavf_adapter *adapter);
 int iavf_disable_queues(struct iavf_adapter *adapter);
 int iavf_configure_rss_lut(struct iavf_adapter *adapter);
 int iavf_configure_rss_key(struct iavf_adapter *adapter);
-int iavf_configure_queues(struct iavf_adapter *adapter);
+int iavf_configure_queues(struct iavf_adapter *adapter,
+			uint16_t num_queue_pairs, uint16_t index);
 int iavf_get_supported_rxdid(struct iavf_adapter *adapter);
 int iavf_config_irq_map(struct iavf_adapter *adapter);
 void iavf_add_del_all_mac_addr(struct iavf_adapter *adapter, bool add);
diff --git a/drivers/net/iavf/iavf_ethdev.c b/drivers/net/iavf/iavf_ethdev.c
index f41be2b7d..db334b390 100644
--- a/drivers/net/iavf/iavf_ethdev.c
+++ b/drivers/net/iavf/iavf_ethdev.c
@@ -550,6 +550,8 @@ iavf_dev_start(struct rte_eth_dev *dev)
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	struct rte_intr_handle *intr_handle = dev->intr_handle;
+	uint16_t num_queue_pairs;
+	uint16_t index = 0;
 
 	PMD_INIT_FUNC_TRACE();
 
@@ -558,13 +560,27 @@ iavf_dev_start(struct rte_eth_dev *dev)
 	vf->max_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
 	vf->num_queue_pairs = RTE_MAX(dev->data->nb_rx_queues,
 				      dev->data->nb_tx_queues);
+	num_queue_pairs = vf->num_queue_pairs;
 
 	if (iavf_init_queues(dev) != 0) {
 		PMD_DRV_LOG(ERR, "failed to do Queue init");
 		return -1;
 	}
 
-	if (iavf_configure_queues(adapter) != 0) {
+	/* If needed, send configure queues msg multiple times to make the
+	 * adminq buffer length smaller than the 4K limitation.
+	 */
+	while (num_queue_pairs > IAVF_CFG_Q_NUM_PER_BUF) {
+		if (iavf_configure_queues(adapter,
+				IAVF_CFG_Q_NUM_PER_BUF, index) != 0) {
+			PMD_DRV_LOG(ERR, "configure queues failed");
+			goto err_queue;
+		}
+		num_queue_pairs -= IAVF_CFG_Q_NUM_PER_BUF;
+		index += IAVF_CFG_Q_NUM_PER_BUF;
+	}
+
+	if (iavf_configure_queues(adapter, num_queue_pairs, index) != 0) {
 		PMD_DRV_LOG(ERR, "configure queues failed");
 		goto err_queue;
 	}
diff --git a/drivers/net/iavf/iavf_vchnl.c b/drivers/net/iavf/iavf_vchnl.c
index e939b275a..829963434 100644
--- a/drivers/net/iavf/iavf_vchnl.c
+++ b/drivers/net/iavf/iavf_vchnl.c
@@ -673,7 +673,8 @@ iavf_configure_rss_key(struct iavf_adapter *adapter)
 }
 
 int
-iavf_configure_queues(struct iavf_adapter *adapter)
+iavf_configure_queues(struct iavf_adapter *adapter,
+		uint16_t num_queue_pairs, uint16_t index)
 {
 	struct iavf_rx_queue **rxq =
 		(struct iavf_rx_queue **)adapter->eth_dev->data->rx_queues;
@@ -687,16 +688,16 @@ iavf_configure_queues(struct iavf_adapter *adapter)
 	int err;
 
 	size = sizeof(*vc_config) +
-	       sizeof(vc_config->qpair[0]) * vf->num_queue_pairs;
+	       sizeof(vc_config->qpair[0]) * num_queue_pairs;
 	vc_config = rte_zmalloc("cfg_queue", size, 0);
 	if (!vc_config)
 		return -ENOMEM;
 
 	vc_config->vsi_id = vf->vsi_res->vsi_id;
-	vc_config->num_queue_pairs = vf->num_queue_pairs;
+	vc_config->num_queue_pairs = num_queue_pairs;
 
-	for (i = 0, vc_qp = vc_config->qpair;
-	     i < vf->num_queue_pairs;
+	for (i = index, vc_qp = vc_config->qpair;
+		 i < index + num_queue_pairs;
 	     i++, vc_qp++) {
 		vc_qp->txq.vsi_id = vf->vsi_res->vsi_id;
 		vc_qp->txq.queue_id = i;
-- 
2.17.1


^ permalink raw reply	[flat|nested] 45+ messages in thread

* [dpdk-dev] [PATCH v7 5/6] net/iavf: enable IRQ mapping configuration for large VF
  2020-10-18 10:34 ` [dpdk-dev] [PATCH v7 0/6] " Ting Xu
                     ` (3 preceding siblings ...)
  2020-10-18 10:34   ` [dpdk-dev] [PATCH v7 4/6] net/iavf: enable multiple queues configurations for large VF Ting Xu
@ 2020-10-18 10:34   ` Ting Xu
  2020-10-18 10:34   ` [dpdk-dev] [PATCH v7 6/6] net/iavf: add enable/disable queues " Ting Xu
  5 siblings, 0 replies; 45+ messages in thread
From: Ting Xu @ 2020-10-18 10:34 UTC (permalink / raw)
  To: dev; +Cc: qi.z.zhang, beilei.xing, jingjing.wu, Ting Xu

The current IRQ mapping configuration only supports max 16 queues and
16 MSIX vectors. Change the queue vector mapping structure to indicate
up to 256 queues. A new opcode is used to handle the case with large
number of queues. To avoid adminq buffer size limitation, we support
to send the virtchnl message multiple times if needed.

Signed-off-by: Ting Xu <ting.xu@intel.com>
---
 drivers/net/iavf/iavf.h        | 12 +++++---
 drivers/net/iavf/iavf_ethdev.c | 52 +++++++++++++++++++++++++++++-----
 drivers/net/iavf/iavf_vchnl.c  | 50 +++++++++++++++++++++++++++++---
 3 files changed, 99 insertions(+), 15 deletions(-)

diff --git a/drivers/net/iavf/iavf.h b/drivers/net/iavf/iavf.h
index 1cdac1b10..5e330b215 100644
--- a/drivers/net/iavf/iavf.h
+++ b/drivers/net/iavf/iavf.h
@@ -22,6 +22,7 @@
 #define IAVF_MAX_NUM_QUEUES_DFLT	 16
 #define IAVF_MAX_NUM_QUEUES_LV		 256
 #define IAVF_CFG_Q_NUM_PER_BUF		 32
+#define IAVF_IRQ_MAP_NUM_PER_BUF	 128
 
 #define IAVF_NUM_MACADDR_MAX      64
 
@@ -106,8 +107,10 @@ struct iavf_fdir_info {
 	struct iavf_fdir_conf conf;
 };
 
-/* TODO: is that correct to assume the max number to be 16 ?*/
-#define IAVF_MAX_MSIX_VECTORS   16
+struct iavf_qv_map {
+	uint16_t queue_id;
+	uint16_t vector_id;
+};
 
 /* Message type read in admin queue from PF */
 enum iavf_aq_result {
@@ -152,8 +155,7 @@ struct iavf_info {
 	uint16_t nb_msix;   /* number of MSI-X interrupts on Rx */
 	uint16_t msix_base; /* msix vector base from */
 	uint16_t max_rss_qregion; /* max RSS queue region supported by PF */
-	/* queue bitmask for each vector */
-	uint16_t rxq_map[IAVF_MAX_MSIX_VECTORS];
+	struct iavf_qv_map *qv_map; /* queue vector mapping */
 	struct iavf_flow_list flow_list;
 	rte_spinlock_t flow_ops_lock;
 	struct iavf_parser_list rss_parser_list;
@@ -274,6 +276,8 @@ int iavf_configure_queues(struct iavf_adapter *adapter,
 			uint16_t num_queue_pairs, uint16_t index);
 int iavf_get_supported_rxdid(struct iavf_adapter *adapter);
 int iavf_config_irq_map(struct iavf_adapter *adapter);
+int iavf_config_irq_map_lv(struct iavf_adapter *adapter, uint16_t num,
+			uint16_t index);
 void iavf_add_del_all_mac_addr(struct iavf_adapter *adapter, bool add);
 int iavf_dev_link_update(struct rte_eth_dev *dev,
 			__rte_unused int wait_to_complete);
diff --git a/drivers/net/iavf/iavf_ethdev.c b/drivers/net/iavf/iavf_ethdev.c
index db334b390..77c49ed03 100644
--- a/drivers/net/iavf/iavf_ethdev.c
+++ b/drivers/net/iavf/iavf_ethdev.c
@@ -423,6 +423,7 @@ static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev,
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter);
+	struct iavf_qv_map *qv_map;
 	uint16_t interval, i;
 	int vec;
 
@@ -443,6 +444,14 @@ static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev,
 		}
 	}
 
+	qv_map = rte_zmalloc("qv_map",
+		dev->data->nb_rx_queues * sizeof(struct iavf_qv_map), 0);
+	if (!qv_map) {
+		PMD_DRV_LOG(ERR, "Failed to allocate %d queue-vector map",
+				dev->data->nb_rx_queues);
+		return -1;
+	}
+
 	if (!dev->data->dev_conf.intr_conf.rxq ||
 	    !rte_intr_dp_is_en(intr_handle)) {
 		/* Rx interrupt disabled, Map interrupt only for writeback */
@@ -473,16 +482,21 @@ static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev,
 		}
 		IAVF_WRITE_FLUSH(hw);
 		/* map all queues to the same interrupt */
-		for (i = 0; i < dev->data->nb_rx_queues; i++)
-			vf->rxq_map[vf->msix_base] |= 1 << i;
+		for (i = 0; i < dev->data->nb_rx_queues; i++) {
+			qv_map[i].queue_id = i;
+			qv_map[i].vector_id = vf->msix_base;
+		}
+		vf->qv_map = qv_map;
 	} else {
 		if (!rte_intr_allow_others(intr_handle)) {
 			vf->nb_msix = 1;
 			vf->msix_base = IAVF_MISC_VEC_ID;
 			for (i = 0; i < dev->data->nb_rx_queues; i++) {
-				vf->rxq_map[vf->msix_base] |= 1 << i;
+				qv_map[i].queue_id = i;
+				qv_map[i].vector_id = vf->msix_base;
 				intr_handle->intr_vec[i] = IAVF_MISC_VEC_ID;
 			}
+			vf->qv_map = qv_map;
 			PMD_DRV_LOG(DEBUG,
 				    "vector %u are mapping to all Rx queues",
 				    vf->msix_base);
@@ -495,20 +509,44 @@ static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev,
 			vf->msix_base = IAVF_RX_VEC_START;
 			vec = IAVF_RX_VEC_START;
 			for (i = 0; i < dev->data->nb_rx_queues; i++) {
-				vf->rxq_map[vec] |= 1 << i;
+				qv_map[i].queue_id = i;
+				qv_map[i].vector_id = vec;
 				intr_handle->intr_vec[i] = vec++;
 				if (vec >= vf->nb_msix)
 					vec = IAVF_RX_VEC_START;
 			}
+			vf->qv_map = qv_map;
 			PMD_DRV_LOG(DEBUG,
 				    "%u vectors are mapping to %u Rx queues",
 				    vf->nb_msix, dev->data->nb_rx_queues);
 		}
 	}
 
-	if (iavf_config_irq_map(adapter)) {
-		PMD_DRV_LOG(ERR, "config interrupt mapping failed");
-		return -1;
+	if (!vf->lv_enabled) {
+		if (iavf_config_irq_map(adapter)) {
+			PMD_DRV_LOG(ERR, "config interrupt mapping failed");
+			return -1;
+		}
+	} else {
+		uint16_t num_qv_maps = dev->data->nb_rx_queues;
+		uint16_t index = 0;
+
+		while (num_qv_maps > IAVF_IRQ_MAP_NUM_PER_BUF) {
+			if (iavf_config_irq_map_lv(adapter,
+					IAVF_IRQ_MAP_NUM_PER_BUF, index)) {
+				PMD_DRV_LOG(ERR, "config interrupt mapping "
+					"for large VF failed");
+				return -1;
+			}
+			num_qv_maps -= IAVF_IRQ_MAP_NUM_PER_BUF;
+			index += IAVF_IRQ_MAP_NUM_PER_BUF;
+		}
+
+		if (iavf_config_irq_map_lv(adapter, num_qv_maps, index)) {
+			PMD_DRV_LOG(ERR, "config interrupt mapping "
+				"for large VF failed");
+			return -1;
+		}
 	}
 	return 0;
 }
diff --git a/drivers/net/iavf/iavf_vchnl.c b/drivers/net/iavf/iavf_vchnl.c
index 829963434..bc7e4f83f 100644
--- a/drivers/net/iavf/iavf_vchnl.c
+++ b/drivers/net/iavf/iavf_vchnl.c
@@ -775,13 +775,14 @@ iavf_config_irq_map(struct iavf_adapter *adapter)
 		return -ENOMEM;
 
 	map_info->num_vectors = vf->nb_msix;
-	for (i = 0; i < vf->nb_msix; i++) {
-		vecmap = &map_info->vecmap[i];
+	for (i = 0; i < adapter->eth_dev->data->nb_rx_queues; i++) {
+		vecmap =
+		    &map_info->vecmap[vf->qv_map[i].vector_id - vf->msix_base];
 		vecmap->vsi_id = vf->vsi_res->vsi_id;
 		vecmap->rxitr_idx = IAVF_ITR_INDEX_DEFAULT;
-		vecmap->vector_id = vf->msix_base + i;
+		vecmap->vector_id = vf->qv_map[i].vector_id;
 		vecmap->txq_map = 0;
-		vecmap->rxq_map = vf->rxq_map[vf->msix_base + i];
+		vecmap->rxq_map |= 1 << vf->qv_map[i].queue_id;
 	}
 
 	args.ops = VIRTCHNL_OP_CONFIG_IRQ_MAP;
@@ -797,6 +798,47 @@ iavf_config_irq_map(struct iavf_adapter *adapter)
 	return err;
 }
 
+int
+iavf_config_irq_map_lv(struct iavf_adapter *adapter, uint16_t num,
+		uint16_t index)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct virtchnl_queue_vector_maps *map_info;
+	struct virtchnl_queue_vector *qv_maps;
+	struct iavf_cmd_info args;
+	int len, i, err;
+	int count = 0;
+
+	len = sizeof(struct virtchnl_queue_vector_maps) +
+	      sizeof(struct virtchnl_queue_vector) * (num - 1);
+
+	map_info = rte_zmalloc("map_info", len, 0);
+	if (!map_info)
+		return -ENOMEM;
+
+	map_info->vport_id = vf->vsi_res->vsi_id;
+	map_info->num_qv_maps = num;
+	for (i = index; i < index + map_info->num_qv_maps; i++) {
+		qv_maps = &map_info->qv_maps[count++];
+		qv_maps->itr_idx = VIRTCHNL_ITR_IDX_0;
+		qv_maps->queue_type = VIRTCHNL_QUEUE_TYPE_RX;
+		qv_maps->queue_id = vf->qv_map[i].queue_id;
+		qv_maps->vector_id = vf->qv_map[i].vector_id;
+	}
+
+	args.ops = VIRTCHNL_OP_MAP_QUEUE_VECTOR;
+	args.in_args = (u8 *)map_info;
+	args.in_args_size = len;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err)
+		PMD_DRV_LOG(ERR, "fail to execute command OP_MAP_QUEUE_VECTOR");
+
+	rte_free(map_info);
+	return err;
+}
+
 void
 iavf_add_del_all_mac_addr(struct iavf_adapter *adapter, bool add)
 {
-- 
2.17.1


^ permalink raw reply	[flat|nested] 45+ messages in thread

* [dpdk-dev] [PATCH v7 6/6] net/iavf: add enable/disable queues for large VF
  2020-10-18 10:34 ` [dpdk-dev] [PATCH v7 0/6] " Ting Xu
                     ` (4 preceding siblings ...)
  2020-10-18 10:34   ` [dpdk-dev] [PATCH v7 5/6] net/iavf: enable IRQ mapping configuration " Ting Xu
@ 2020-10-18 10:34   ` Ting Xu
  5 siblings, 0 replies; 45+ messages in thread
From: Ting Xu @ 2020-10-18 10:34 UTC (permalink / raw)
  To: dev; +Cc: qi.z.zhang, beilei.xing, jingjing.wu, Ting Xu

The current virtchnl structure for enable/disable queues only supports
max 32 queue pairs. Use a new opcode and structure to indicate up to 256
queue pairs, in order to enable/disable queues in large VF case.

Signed-off-by: Ting Xu <ting.xu@intel.com>
---
 drivers/net/iavf/iavf.h       |   5 ++
 drivers/net/iavf/iavf_rxtx.c  |  25 ++++++-
 drivers/net/iavf/iavf_vchnl.c | 132 ++++++++++++++++++++++++++++++++++
 3 files changed, 159 insertions(+), 3 deletions(-)

diff --git a/drivers/net/iavf/iavf.h b/drivers/net/iavf/iavf.h
index 5e330b215..3d3b0da5d 100644
--- a/drivers/net/iavf/iavf.h
+++ b/drivers/net/iavf/iavf.h
@@ -23,6 +23,7 @@
 #define IAVF_MAX_NUM_QUEUES_LV		 256
 #define IAVF_CFG_Q_NUM_PER_BUF		 32
 #define IAVF_IRQ_MAP_NUM_PER_BUF	 128
+#define IAVF_RXTX_QUEUE_CHUNKS_NUM	 2
 
 #define IAVF_NUM_MACADDR_MAX      64
 
@@ -268,8 +269,12 @@ int iavf_enable_vlan_strip(struct iavf_adapter *adapter);
 int iavf_disable_vlan_strip(struct iavf_adapter *adapter);
 int iavf_switch_queue(struct iavf_adapter *adapter, uint16_t qid,
 		     bool rx, bool on);
+int iavf_switch_queue_lv(struct iavf_adapter *adapter, uint16_t qid,
+		     bool rx, bool on);
 int iavf_enable_queues(struct iavf_adapter *adapter);
+int iavf_enable_queues_lv(struct iavf_adapter *adapter);
 int iavf_disable_queues(struct iavf_adapter *adapter);
+int iavf_disable_queues_lv(struct iavf_adapter *adapter);
 int iavf_configure_rss_lut(struct iavf_adapter *adapter);
 int iavf_configure_rss_key(struct iavf_adapter *adapter);
 int iavf_configure_queues(struct iavf_adapter *adapter,
diff --git a/drivers/net/iavf/iavf_rxtx.c b/drivers/net/iavf/iavf_rxtx.c
index 1b0efe043..ed052dda1 100644
--- a/drivers/net/iavf/iavf_rxtx.c
+++ b/drivers/net/iavf/iavf_rxtx.c
@@ -531,6 +531,7 @@ iavf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 {
 	struct iavf_adapter *adapter =
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 	struct iavf_rx_queue *rxq;
 	int err = 0;
@@ -555,7 +556,11 @@ iavf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 	IAVF_WRITE_FLUSH(hw);
 
 	/* Ready to switch the queue on */
-	err = iavf_switch_queue(adapter, rx_queue_id, true, true);
+	if (!vf->lv_enabled)
+		err = iavf_switch_queue(adapter, rx_queue_id, true, true);
+	else
+		err = iavf_switch_queue_lv(adapter, rx_queue_id, true, true);
+
 	if (err)
 		PMD_DRV_LOG(ERR, "Failed to switch RX queue %u on",
 			    rx_queue_id);
@@ -571,6 +576,7 @@ iavf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 {
 	struct iavf_adapter *adapter =
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 	struct iavf_tx_queue *txq;
 	int err = 0;
@@ -587,7 +593,10 @@ iavf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 	IAVF_WRITE_FLUSH(hw);
 
 	/* Ready to switch the queue on */
-	err = iavf_switch_queue(adapter, tx_queue_id, false, true);
+	if (!vf->lv_enabled)
+		err = iavf_switch_queue(adapter, tx_queue_id, false, true);
+	else
+		err = iavf_switch_queue_lv(adapter, tx_queue_id, false, true);
 
 	if (err)
 		PMD_DRV_LOG(ERR, "Failed to switch TX queue %u on",
@@ -688,12 +697,22 @@ iavf_stop_queues(struct rte_eth_dev *dev)
 {
 	struct iavf_adapter *adapter =
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	struct iavf_rx_queue *rxq;
 	struct iavf_tx_queue *txq;
 	int ret, i;
 
 	/* Stop All queues */
-	ret = iavf_disable_queues(adapter);
+	if (!vf->lv_enabled) {
+		ret = iavf_disable_queues(adapter);
+		if (ret)
+			PMD_DRV_LOG(WARNING, "Fail to stop queues");
+	} else {
+		ret = iavf_disable_queues_lv(adapter);
+		if (ret)
+			PMD_DRV_LOG(WARNING, "Fail to stop queues for large VF");
+	}
+
 	if (ret)
 		PMD_DRV_LOG(WARNING, "Fail to stop queues");
 
diff --git a/drivers/net/iavf/iavf_vchnl.c b/drivers/net/iavf/iavf_vchnl.c
index bc7e4f83f..42d8c47f9 100644
--- a/drivers/net/iavf/iavf_vchnl.c
+++ b/drivers/net/iavf/iavf_vchnl.c
@@ -608,6 +608,138 @@ iavf_switch_queue(struct iavf_adapter *adapter, uint16_t qid,
 	return err;
 }
 
+int
+iavf_enable_queues_lv(struct iavf_adapter *adapter)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct virtchnl_del_ena_dis_queues *queue_select;
+	struct virtchnl_queue_chunk *queue_chunk;
+	struct iavf_cmd_info args;
+	int err, len;
+
+	len = sizeof(struct virtchnl_del_ena_dis_queues) +
+		  sizeof(struct virtchnl_queue_chunk) *
+		  (IAVF_RXTX_QUEUE_CHUNKS_NUM - 1);
+	queue_select = rte_zmalloc("queue_select", len, 0);
+	if (!queue_select)
+		return -ENOMEM;
+
+	queue_chunk = queue_select->chunks.chunks;
+	queue_select->chunks.num_chunks = IAVF_RXTX_QUEUE_CHUNKS_NUM;
+	queue_select->vport_id = vf->vsi_res->vsi_id;
+
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].type = VIRTCHNL_QUEUE_TYPE_TX;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].start_queue_id = 0;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].num_queues =
+		adapter->eth_dev->data->nb_tx_queues;
+
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].type = VIRTCHNL_QUEUE_TYPE_RX;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].start_queue_id = 0;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].num_queues =
+		adapter->eth_dev->data->nb_rx_queues;
+
+	args.ops = VIRTCHNL_OP_ENABLE_QUEUES_V2;
+	args.in_args = (u8 *)queue_select;
+	args.in_args_size = len;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err) {
+		PMD_DRV_LOG(ERR,
+			    "Failed to execute command of OP_ENABLE_QUEUES_V2");
+		return err;
+	}
+	return 0;
+}
+
+int
+iavf_disable_queues_lv(struct iavf_adapter *adapter)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct virtchnl_del_ena_dis_queues *queue_select;
+	struct virtchnl_queue_chunk *queue_chunk;
+	struct iavf_cmd_info args;
+	int err, len;
+
+	len = sizeof(struct virtchnl_del_ena_dis_queues) +
+		  sizeof(struct virtchnl_queue_chunk) *
+		  (IAVF_RXTX_QUEUE_CHUNKS_NUM - 1);
+	queue_select = rte_zmalloc("queue_select", len, 0);
+	if (!queue_select)
+		return -ENOMEM;
+
+	queue_chunk = queue_select->chunks.chunks;
+	queue_select->chunks.num_chunks = IAVF_RXTX_QUEUE_CHUNKS_NUM;
+	queue_select->vport_id = vf->vsi_res->vsi_id;
+
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].type = VIRTCHNL_QUEUE_TYPE_TX;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].start_queue_id = 0;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].num_queues =
+		adapter->eth_dev->data->nb_tx_queues;
+
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].type = VIRTCHNL_QUEUE_TYPE_RX;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].start_queue_id = 0;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].num_queues =
+		adapter->eth_dev->data->nb_rx_queues;
+
+	args.ops = VIRTCHNL_OP_DISABLE_QUEUES_V2;
+	args.in_args = (u8 *)queue_select;
+	args.in_args_size = len;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err) {
+		PMD_DRV_LOG(ERR,
+			    "Failed to execute command of OP_DISABLE_QUEUES_V2");
+		return err;
+	}
+	return 0;
+}
+
+int
+iavf_switch_queue_lv(struct iavf_adapter *adapter, uint16_t qid,
+		 bool rx, bool on)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct virtchnl_del_ena_dis_queues *queue_select;
+	struct virtchnl_queue_chunk *queue_chunk;
+	struct iavf_cmd_info args;
+	int err, len;
+
+	len = sizeof(struct virtchnl_del_ena_dis_queues);
+	queue_select = rte_zmalloc("queue_select", len, 0);
+	if (!queue_select)
+		return -ENOMEM;
+
+	queue_chunk = queue_select->chunks.chunks;
+	queue_select->chunks.num_chunks = 1;
+	queue_select->vport_id = vf->vsi_res->vsi_id;
+
+	if (rx) {
+		queue_chunk->type = VIRTCHNL_QUEUE_TYPE_RX;
+		queue_chunk->start_queue_id = qid;
+		queue_chunk->num_queues = 1;
+	} else {
+		queue_chunk->type = VIRTCHNL_QUEUE_TYPE_TX;
+		queue_chunk->start_queue_id = qid;
+		queue_chunk->num_queues = 1;
+	}
+
+	if (on)
+		args.ops = VIRTCHNL_OP_ENABLE_QUEUES_V2;
+	else
+		args.ops = VIRTCHNL_OP_DISABLE_QUEUES_V2;
+	args.in_args = (u8 *)queue_select;
+	args.in_args_size = len;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err)
+		PMD_DRV_LOG(ERR, "Failed to execute command of %s",
+			    on ? "OP_ENABLE_QUEUES_V2" : "OP_DISABLE_QUEUES_V2");
+	return err;
+}
+
 int
 iavf_configure_rss_lut(struct iavf_adapter *adapter)
 {
-- 
2.17.1


^ permalink raw reply	[flat|nested] 45+ messages in thread

* Re: [dpdk-dev] [PATCH v7 1/6] net/iavf: handle virtchnl event message without interrupt
  2020-10-18 10:34   ` [dpdk-dev] [PATCH v7 1/6] net/iavf: handle virtchnl event message without interrupt Ting Xu
@ 2020-10-21  8:15     ` Xing, Beilei
  0 siblings, 0 replies; 45+ messages in thread
From: Xing, Beilei @ 2020-10-21  8:15 UTC (permalink / raw)
  To: Xu, Ting, dev; +Cc: Zhang, Qi Z, Wu, Jingjing



> -----Original Message-----
> From: Xu, Ting <ting.xu@intel.com>
> Sent: Sunday, October 18, 2020 6:34 PM
> To: dev@dpdk.org
> Cc: Zhang, Qi Z <qi.z.zhang@intel.com>; Xing, Beilei <beilei.xing@intel.com>;
> Wu, Jingjing <jingjing.wu@intel.com>; Xu, Ting <ting.xu@intel.com>
> Subject: [PATCH v7 1/6] net/iavf: handle virtchnl event message without
> interrupt
> 
The patch looks fine to me except the some typo in commit log:
> Currently, VF only handle virtchnl event message when interrupt is called. In
> some cases, event message is supposed to be handled with interrupt disabled,
> for example request queues. To solve this issue, virtchnl event message
> handling are added in reading message from PF in this patch.

It's better to explain why we need the asynchronous way, but not use the example of request queues

> 
> Signed-off-by: Ting Xu <ting.xu@intel.com>
> ---
>  drivers/net/iavf/iavf.h       |   8 ++
>  drivers/net/iavf/iavf_vchnl.c | 138 ++++++++++++++++++++++------------
>  2 files changed, 97 insertions(+), 49 deletions(-)
> 

^ permalink raw reply	[flat|nested] 45+ messages in thread

* [dpdk-dev] [PATCH v8 0/6] enable large VF configuration
  2020-09-09  7:20 [dpdk-dev] [PATCH v1 0/2] enable large VF configuration Ting Xu
                   ` (8 preceding siblings ...)
  2020-10-18 10:34 ` [dpdk-dev] [PATCH v7 0/6] " Ting Xu
@ 2020-10-22  6:48 ` Ting Xu
  2020-10-22  6:48   ` [dpdk-dev] [PATCH v8 1/6] net/iavf: handle virtchnl event message without interrupt Ting Xu
                     ` (6 more replies)
  9 siblings, 7 replies; 45+ messages in thread
From: Ting Xu @ 2020-10-22  6:48 UTC (permalink / raw)
  To: dev; +Cc: qi.z.zhang, beilei.xing, jingjing.wu, Ting Xu

This patchset supports to configure up to 256 queue pairs per VF. If
large VF is supported after capability negotiation, VF will request
queues from PF as needed. New virtual channel opcodes and structures
are used to indicate 256 queues, so VF is designed to handle the new
function of configure VSI queues, IRQ mapping and enable/disable queues.
Also, enable VF to query the max RSS queue region for RSS and filter
configuration.

Ting Xu (6):
  net/iavf: handle virtchnl event message without interrupt
  net/iavf: add IAVF request queues function
  net/iavf: negotiate large VF and request more queues
  net/iavf: enable multiple queues configurations for large VF
  net/iavf: enable IRQ mapping configuration for large VF
  net/iavf: add enable/disable queues for large VF

---
v7->v8:
Modify commit logs.
Make log sentences in one line.

v6->v7:
Separate patches.

v4->v6:
Solve queue mapping buffer limitation issue.
Optimize VSI queue configuration

v3->v4:
Optimize handling messages from PF.

v2->v3:
Fix coding style issue.
Move get max RSS queue region after VF reset.
Add request queues capability negotiation.

v1->v2:
Change the communication with kernel PF.

---
 drivers/net/iavf/iavf.h        |  37 ++-
 drivers/net/iavf/iavf_ethdev.c | 142 ++++++++++-
 drivers/net/iavf/iavf_rxtx.c   |  25 +-
 drivers/net/iavf/iavf_vchnl.c  | 448 ++++++++++++++++++++++++++++-----
 4 files changed, 572 insertions(+), 80 deletions(-)

-- 
2.17.1


^ permalink raw reply	[flat|nested] 45+ messages in thread

* [dpdk-dev] [PATCH v8 1/6] net/iavf: handle virtchnl event message without interrupt
  2020-10-22  6:48 ` [dpdk-dev] [PATCH v8 0/6] enable large VF configuration Ting Xu
@ 2020-10-22  6:48   ` Ting Xu
  2020-10-22  6:48   ` [dpdk-dev] [PATCH v8 2/6] net/iavf: add IAVF request queues function Ting Xu
                     ` (5 subsequent siblings)
  6 siblings, 0 replies; 45+ messages in thread
From: Ting Xu @ 2020-10-22  6:48 UTC (permalink / raw)
  To: dev; +Cc: qi.z.zhang, beilei.xing, jingjing.wu, Ting Xu

Currently, VF can only handle virtchnl event message by calling interrupt.
It is not available in two cases:
1. If the event message comes during VF initialization before interrupt
is enabled, this message will not be handled correctly.
2. Some virtchnl commands need to receive the event message and handle
it with interrupt disabled.
To solve this issue, we add the virtchnl event message handling in the
process of reading vitchnl messages in adminq from PF.

Signed-off-by: Ting Xu <ting.xu@intel.com>
---
 drivers/net/iavf/iavf.h       |   8 ++
 drivers/net/iavf/iavf_vchnl.c | 138 ++++++++++++++++++++++------------
 2 files changed, 97 insertions(+), 49 deletions(-)

diff --git a/drivers/net/iavf/iavf.h b/drivers/net/iavf/iavf.h
index 3198d85b3..9c16324c1 100644
--- a/drivers/net/iavf/iavf.h
+++ b/drivers/net/iavf/iavf.h
@@ -107,6 +107,14 @@ struct iavf_fdir_info {
 /* TODO: is that correct to assume the max number to be 16 ?*/
 #define IAVF_MAX_MSIX_VECTORS   16
 
+/* Message type read in admin queue from PF */
+enum iavf_aq_result {
+	IAVF_MSG_ERR = -1, /* Meet error when accessing admin queue */
+	IAVF_MSG_NON,      /* Read nothing from admin queue */
+	IAVF_MSG_SYS,      /* Read system msg from admin queue */
+	IAVF_MSG_CMD,      /* Read async command result */
+};
+
 /* Structure to store private data specific for VF instance. */
 struct iavf_info {
 	uint16_t num_queue_pairs;
diff --git a/drivers/net/iavf/iavf_vchnl.c b/drivers/net/iavf/iavf_vchnl.c
index a2295f879..b62c8683c 100644
--- a/drivers/net/iavf/iavf_vchnl.c
+++ b/drivers/net/iavf/iavf_vchnl.c
@@ -25,14 +25,54 @@
 #define MAX_TRY_TIMES 200
 #define ASQ_DELAY_MS  10
 
+static uint32_t
+iavf_convert_link_speed(enum virtchnl_link_speed virt_link_speed)
+{
+	uint32_t speed;
+
+	switch (virt_link_speed) {
+	case VIRTCHNL_LINK_SPEED_100MB:
+		speed = 100;
+		break;
+	case VIRTCHNL_LINK_SPEED_1GB:
+		speed = 1000;
+		break;
+	case VIRTCHNL_LINK_SPEED_10GB:
+		speed = 10000;
+		break;
+	case VIRTCHNL_LINK_SPEED_40GB:
+		speed = 40000;
+		break;
+	case VIRTCHNL_LINK_SPEED_20GB:
+		speed = 20000;
+		break;
+	case VIRTCHNL_LINK_SPEED_25GB:
+		speed = 25000;
+		break;
+	case VIRTCHNL_LINK_SPEED_2_5GB:
+		speed = 2500;
+		break;
+	case VIRTCHNL_LINK_SPEED_5GB:
+		speed = 5000;
+		break;
+	default:
+		speed = 0;
+		break;
+	}
+
+	return speed;
+}
+
 /* Read data in admin queue to get msg from pf driver */
-static enum iavf_status
+static enum iavf_aq_result
 iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,
 		     uint8_t *buf)
 {
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter);
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct rte_eth_dev *dev = adapter->eth_dev;
 	struct iavf_arq_event_info event;
+	enum iavf_aq_result result = IAVF_MSG_NON;
 	enum virtchnl_ops opcode;
 	int ret;
 
@@ -42,7 +82,9 @@ iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,
 	/* Can't read any msg from adminQ */
 	if (ret) {
 		PMD_DRV_LOG(DEBUG, "Can't read msg from AQ");
-		return ret;
+		if (ret != IAVF_ERR_ADMIN_QUEUE_NO_WORK)
+			result = IAVF_MSG_ERR;
+		return result;
 	}
 
 	opcode = (enum virtchnl_ops)rte_le_to_cpu_32(event.desc.cookie_high);
@@ -52,16 +94,51 @@ iavf_read_msg_from_pf(struct iavf_adapter *adapter, uint16_t buf_len,
 	PMD_DRV_LOG(DEBUG, "AQ from pf carries opcode %u, retval %d",
 		    opcode, vf->cmd_retval);
 
-	if (opcode != vf->pend_cmd) {
-		if (opcode != VIRTCHNL_OP_EVENT) {
-			PMD_DRV_LOG(WARNING,
-				    "command mismatch, expect %u, get %u",
-				    vf->pend_cmd, opcode);
+	if (opcode == VIRTCHNL_OP_EVENT) {
+		struct virtchnl_pf_event *vpe =
+			(struct virtchnl_pf_event *)event.msg_buf;
+
+		result = IAVF_MSG_SYS;
+		switch (vpe->event) {
+		case VIRTCHNL_EVENT_LINK_CHANGE:
+			vf->link_up =
+				vpe->event_data.link_event.link_status;
+			if (vf->vf_res->vf_cap_flags &
+				VIRTCHNL_VF_CAP_ADV_LINK_SPEED) {
+				vf->link_speed =
+				    vpe->event_data.link_event_adv.link_speed;
+			} else {
+				enum virtchnl_link_speed speed;
+				speed = vpe->event_data.link_event.link_speed;
+				vf->link_speed = iavf_convert_link_speed(speed);
+			}
+			iavf_dev_link_update(dev, 0);
+			PMD_DRV_LOG(INFO, "Link status update:%s",
+					vf->link_up ? "up" : "down");
+			break;
+		case VIRTCHNL_EVENT_RESET_IMPENDING:
+			vf->vf_reset = true;
+			PMD_DRV_LOG(INFO, "VF is resetting");
+			break;
+		case VIRTCHNL_EVENT_PF_DRIVER_CLOSE:
+			vf->dev_closed = true;
+			PMD_DRV_LOG(INFO, "PF driver closed");
+			break;
+		default:
+			PMD_DRV_LOG(ERR, "%s: Unknown event %d from pf",
+					__func__, vpe->event);
+		}
+	}  else {
+		/* async reply msg on command issued by vf previously */
+		result = IAVF_MSG_CMD;
+		if (opcode != vf->pend_cmd) {
+			PMD_DRV_LOG(WARNING, "command mismatch, expect %u, get %u",
+					vf->pend_cmd, opcode);
+			result = IAVF_MSG_ERR;
 		}
-		return IAVF_ERR_OPCODE_MISMATCH;
 	}
 
-	return IAVF_SUCCESS;
+	return result;
 }
 
 static int
@@ -69,6 +146,7 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter, struct iavf_cmd_info *args)
 {
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter);
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	enum iavf_aq_result result;
 	enum iavf_status ret;
 	int err = 0;
 	int i = 0;
@@ -97,9 +175,9 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter, struct iavf_cmd_info *args)
 	case VIRTCHNL_OP_GET_SUPPORTED_RXDIDS:
 		/* for init virtchnl ops, need to poll the response */
 		do {
-			ret = iavf_read_msg_from_pf(adapter, args->out_size,
+			result = iavf_read_msg_from_pf(adapter, args->out_size,
 						   args->out_buffer);
-			if (ret == IAVF_SUCCESS)
+			if (result == IAVF_MSG_CMD)
 				break;
 			rte_delay_ms(ASQ_DELAY_MS);
 		} while (i++ < MAX_TRY_TIMES);
@@ -136,44 +214,6 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter, struct iavf_cmd_info *args)
 	return err;
 }
 
-static uint32_t
-iavf_convert_link_speed(enum virtchnl_link_speed virt_link_speed)
-{
-	uint32_t speed;
-
-	switch (virt_link_speed) {
-	case VIRTCHNL_LINK_SPEED_100MB:
-		speed = 100;
-		break;
-	case VIRTCHNL_LINK_SPEED_1GB:
-		speed = 1000;
-		break;
-	case VIRTCHNL_LINK_SPEED_10GB:
-		speed = 10000;
-		break;
-	case VIRTCHNL_LINK_SPEED_40GB:
-		speed = 40000;
-		break;
-	case VIRTCHNL_LINK_SPEED_20GB:
-		speed = 20000;
-		break;
-	case VIRTCHNL_LINK_SPEED_25GB:
-		speed = 25000;
-		break;
-	case VIRTCHNL_LINK_SPEED_2_5GB:
-		speed = 2500;
-		break;
-	case VIRTCHNL_LINK_SPEED_5GB:
-		speed = 5000;
-		break;
-	default:
-		speed = 0;
-		break;
-	}
-
-	return speed;
-}
-
 static void
 iavf_handle_pf_event_msg(struct rte_eth_dev *dev, uint8_t *msg,
 			uint16_t msglen)
-- 
2.17.1


^ permalink raw reply	[flat|nested] 45+ messages in thread

* [dpdk-dev] [PATCH v8 2/6] net/iavf: add IAVF request queues function
  2020-10-22  6:48 ` [dpdk-dev] [PATCH v8 0/6] enable large VF configuration Ting Xu
  2020-10-22  6:48   ` [dpdk-dev] [PATCH v8 1/6] net/iavf: handle virtchnl event message without interrupt Ting Xu
@ 2020-10-22  6:48   ` Ting Xu
  2020-10-23 10:07     ` Ferruh Yigit
  2020-10-23 10:11     ` Ferruh Yigit
  2020-10-22  6:48   ` [dpdk-dev] [PATCH v8 3/6] net/iavf: negotiate large VF and request more queues Ting Xu
                     ` (4 subsequent siblings)
  6 siblings, 2 replies; 45+ messages in thread
From: Ting Xu @ 2020-10-22  6:48 UTC (permalink / raw)
  To: dev; +Cc: qi.z.zhang, beilei.xing, jingjing.wu, Ting Xu

Add a new virtchnl function to request additional queues from PF. Current
default queue pairs number when creating a VF is 16. In order to support
up to 256 queue pairs per VF, enable this request queues function.

When requesting queues succeeds, PF will return an event message. If it
is handled by interrupt first, the request queues command cannot receive
the correct PF response and will wait until timeout. Therefore, disable
interrupt before requesting queues in order to handle the event message
asynchronously.

Signed-off-by: Ting Xu <ting.xu@intel.com>
---
 drivers/net/iavf/iavf.h       |  1 +
 drivers/net/iavf/iavf_vchnl.c | 88 ++++++++++++++++++++++++++++++++++-
 2 files changed, 87 insertions(+), 2 deletions(-)

diff --git a/drivers/net/iavf/iavf.h b/drivers/net/iavf/iavf.h
index 9c16324c1..778b6c23c 100644
--- a/drivers/net/iavf/iavf.h
+++ b/drivers/net/iavf/iavf.h
@@ -287,4 +287,5 @@ int iavf_add_del_rss_cfg(struct iavf_adapter *adapter,
 int iavf_add_del_mc_addr_list(struct iavf_adapter *adapter,
 			struct rte_ether_addr *mc_addrs,
 			uint32_t mc_addrs_num, bool add);
+int iavf_request_queues(struct iavf_adapter *adapter, uint16_t num);
 #endif /* _IAVF_ETHDEV_H_ */
diff --git a/drivers/net/iavf/iavf_vchnl.c b/drivers/net/iavf/iavf_vchnl.c
index b62c8683c..323e2a843 100644
--- a/drivers/net/iavf/iavf_vchnl.c
+++ b/drivers/net/iavf/iavf_vchnl.c
@@ -17,6 +17,7 @@
 #include <rte_eal.h>
 #include <rte_ether.h>
 #include <rte_ethdev_driver.h>
+#include <rte_ethdev_pci.h>
 #include <rte_dev.h>
 
 #include "iavf.h"
@@ -189,7 +190,33 @@ iavf_execute_vf_cmd(struct iavf_adapter *adapter, struct iavf_cmd_info *args)
 		}
 		_clear_cmd(vf);
 		break;
-
+	case VIRTCHNL_OP_REQUEST_QUEUES:
+		/*
+		 * ignore async reply, only wait for system message,
+		 * vf_reset = true if get VIRTCHNL_EVENT_RESET_IMPENDING,
+		 * if not, means request queues failed.
+		 */
+		do {
+			result = iavf_read_msg_from_pf(adapter, args->out_size,
+						   args->out_buffer);
+			if (result == IAVF_MSG_SYS && vf->vf_reset) {
+				break;
+			} else if (result == IAVF_MSG_CMD ||
+				result == IAVF_MSG_ERR) {
+				err = -1;
+				break;
+			}
+			rte_delay_ms(ASQ_DELAY_MS);
+			/* If don't read msg or read sys event, continue */
+		} while (i++ < MAX_TRY_TIMES);
+		if (i >= MAX_TRY_TIMES ||
+			vf->cmd_retval != VIRTCHNL_STATUS_SUCCESS) {
+			err = -1;
+			PMD_DRV_LOG(ERR, "No response or return failure (%d)"
+				    " for cmd %d", vf->cmd_retval, args->ops);
+		}
+		_clear_cmd(vf);
+		break;
 	default:
 		/* For other virtchnl ops in running time,
 		 * wait for the cmd done flag.
@@ -429,7 +456,8 @@ iavf_get_vf_resource(struct iavf_adapter *adapter)
 	caps = IAVF_BASIC_OFFLOAD_CAPS | VIRTCHNL_VF_CAP_ADV_LINK_SPEED |
 		VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC |
 		VIRTCHNL_VF_OFFLOAD_FDIR_PF |
-		VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF;
+		VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF |
+		VIRTCHNL_VF_OFFLOAD_REQ_QUEUES;
 
 	args.in_args = (uint8_t *)&caps;
 	args.in_args_size = sizeof(caps);
@@ -1183,3 +1211,59 @@ iavf_add_del_mc_addr_list(struct iavf_adapter *adapter,
 
 	return 0;
 }
+
+int
+iavf_request_queues(struct iavf_adapter *adapter, uint16_t num)
+{
+	struct rte_eth_dev *dev = adapter->eth_dev;
+	struct iavf_info *vf =  IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+	struct virtchnl_vf_res_request vfres;
+	struct iavf_cmd_info args;
+	uint16_t num_queue_pairs;
+	int err;
+
+	if (!(vf->vf_res->vf_cap_flags &
+		VIRTCHNL_VF_OFFLOAD_REQ_QUEUES)) {
+		PMD_DRV_LOG(ERR, "request queues not supported");
+		return -1;
+	}
+
+	if (num == 0) {
+		PMD_DRV_LOG(ERR, "queue number cannot be zero");
+		return -1;
+	}
+	vfres.num_queue_pairs = num;
+
+	args.ops = VIRTCHNL_OP_REQUEST_QUEUES;
+	args.in_args = (u8 *)&vfres;
+	args.in_args_size = sizeof(vfres);
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+
+	/*
+	 * disable interrupt to avoid the admin queue message to be read
+	 * before iavf_read_msg_from_pf.
+	 */
+	rte_intr_disable(&pci_dev->intr_handle);
+	err = iavf_execute_vf_cmd(adapter, &args);
+	rte_intr_enable(&pci_dev->intr_handle);
+	if (err) {
+		PMD_DRV_LOG(ERR, "fail to execute command OP_REQUEST_QUEUES");
+		return err;
+	}
+
+	/* request queues succeeded, vf is resetting */
+	if (vf->vf_reset) {
+		PMD_DRV_LOG(INFO, "vf is resetting");
+		return 0;
+	}
+
+	/* request additional queues failed, return available number */
+	num_queue_pairs =
+	  ((struct virtchnl_vf_res_request *)args.out_buffer)->num_queue_pairs;
+	PMD_DRV_LOG(ERR, "request queues failed, only %u queues "
+		"available", num_queue_pairs);
+
+	return -1;
+}
-- 
2.17.1


^ permalink raw reply	[flat|nested] 45+ messages in thread

* [dpdk-dev] [PATCH v8 3/6] net/iavf: negotiate large VF and request more queues
  2020-10-22  6:48 ` [dpdk-dev] [PATCH v8 0/6] enable large VF configuration Ting Xu
  2020-10-22  6:48   ` [dpdk-dev] [PATCH v8 1/6] net/iavf: handle virtchnl event message without interrupt Ting Xu
  2020-10-22  6:48   ` [dpdk-dev] [PATCH v8 2/6] net/iavf: add IAVF request queues function Ting Xu
@ 2020-10-22  6:48   ` Ting Xu
  2020-10-22  6:49   ` [dpdk-dev] [PATCH v8 4/6] net/iavf: enable multiple queues configurations for large VF Ting Xu
                     ` (3 subsequent siblings)
  6 siblings, 0 replies; 45+ messages in thread
From: Ting Xu @ 2020-10-22  6:48 UTC (permalink / raw)
  To: dev; +Cc: qi.z.zhang, beilei.xing, jingjing.wu, Ting Xu

Negotiate large VF capability with PF during VF initialization. If large
VF is supported and the number of queues larger than 16 is required, VF
requests additional queues from PF. Mark the state that large VF is
supported.

If the allocated queues number is larger than 16, the max RSS queue
region cannot be 16 anymore. Add the function to query max RSS queue
region from PF, use it in the RSS initialization and future filters
configuration.

Signed-off-by: Ting Xu <ting.xu@intel.com>
---
 drivers/net/iavf/iavf.h        |  7 +++-
 drivers/net/iavf/iavf_ethdev.c | 74 ++++++++++++++++++++++++++++++++--
 drivers/net/iavf/iavf_vchnl.c  | 31 +++++++++++++-
 3 files changed, 107 insertions(+), 5 deletions(-)

diff --git a/drivers/net/iavf/iavf.h b/drivers/net/iavf/iavf.h
index 778b6c23c..49ccfeece 100644
--- a/drivers/net/iavf/iavf.h
+++ b/drivers/net/iavf/iavf.h
@@ -19,7 +19,8 @@
 #define IAVF_FRAME_SIZE_MAX       9728
 #define IAVF_QUEUE_BASE_ADDR_UNIT 128
 
-#define IAVF_MAX_NUM_QUEUES       16
+#define IAVF_MAX_NUM_QUEUES_DFLT	 16
+#define IAVF_MAX_NUM_QUEUES_LV		 256
 
 #define IAVF_NUM_MACADDR_MAX      64
 
@@ -149,6 +150,7 @@ struct iavf_info {
 	uint8_t *rss_key;
 	uint16_t nb_msix;   /* number of MSI-X interrupts on Rx */
 	uint16_t msix_base; /* msix vector base from */
+	uint16_t max_rss_qregion; /* max RSS queue region supported by PF */
 	/* queue bitmask for each vector */
 	uint16_t rxq_map[IAVF_MAX_MSIX_VECTORS];
 	struct iavf_flow_list flow_list;
@@ -157,6 +159,8 @@ struct iavf_info {
 	struct iavf_parser_list dist_parser_list;
 
 	struct iavf_fdir_info fdir; /* flow director info */
+	/* indicate large VF support enabled or not */
+	bool lv_enabled;
 };
 
 #define IAVF_MAX_PKT_TYPE 1024
@@ -288,4 +292,5 @@ int iavf_add_del_mc_addr_list(struct iavf_adapter *adapter,
 			struct rte_ether_addr *mc_addrs,
 			uint32_t mc_addrs_num, bool add);
 int iavf_request_queues(struct iavf_adapter *adapter, uint16_t num);
+int iavf_get_max_rss_queue_region(struct iavf_adapter *adapter);
 #endif /* _IAVF_ETHDEV_H_ */
diff --git a/drivers/net/iavf/iavf_ethdev.c b/drivers/net/iavf/iavf_ethdev.c
index a5b06e6bd..87082d1cc 100644
--- a/drivers/net/iavf/iavf_ethdev.c
+++ b/drivers/net/iavf/iavf_ethdev.c
@@ -209,7 +209,7 @@ iavf_init_rss(struct iavf_adapter *adapter)
 
 	rss_conf = &adapter->eth_dev->data->dev_conf.rx_adv_conf.rss_conf;
 	nb_q = RTE_MIN(adapter->eth_dev->data->nb_rx_queues,
-		       IAVF_MAX_NUM_QUEUES);
+		       vf->max_rss_qregion);
 
 	if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) {
 		PMD_DRV_LOG(DEBUG, "RSS is not supported");
@@ -255,6 +255,31 @@ iavf_init_rss(struct iavf_adapter *adapter)
 	return 0;
 }
 
+static int
+iavf_queues_req_reset(struct rte_eth_dev *dev, uint16_t num)
+{
+	struct iavf_adapter *ad =
+		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
+	struct iavf_info *vf =  IAVF_DEV_PRIVATE_TO_VF(ad);
+	int ret;
+
+	ret = iavf_request_queues(ad, num);
+	if (ret) {
+		PMD_DRV_LOG(ERR, "request queues from PF failed");
+		return ret;
+	}
+	PMD_DRV_LOG(INFO, "change queue pairs from %u to %u",
+			vf->vsi_res->num_queue_pairs, num);
+
+	ret = iavf_dev_reset(dev);
+	if (ret) {
+		PMD_DRV_LOG(ERR, "vf reset failed");
+		return ret;
+	}
+
+	return 0;
+}
+
 static int
 iavf_dev_configure(struct rte_eth_dev *dev)
 {
@@ -262,6 +287,9 @@ iavf_dev_configure(struct rte_eth_dev *dev)
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct iavf_info *vf =  IAVF_DEV_PRIVATE_TO_VF(ad);
 	struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
+	uint16_t num_queue_pairs = RTE_MAX(dev->data->nb_rx_queues,
+		dev->data->nb_tx_queues);
+	int ret;
 
 	ad->rx_bulk_alloc_allowed = true;
 	/* Initialize to TRUE. If any of Rx queues doesn't meet the
@@ -273,6 +301,46 @@ iavf_dev_configure(struct rte_eth_dev *dev)
 	if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)
 		dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_RSS_HASH;
 
+	/* Large VF setting */
+	if (num_queue_pairs > IAVF_MAX_NUM_QUEUES_DFLT) {
+		if (!(vf->vf_res->vf_cap_flags &
+				VIRTCHNL_VF_LARGE_NUM_QPAIRS)) {
+			PMD_DRV_LOG(ERR, "large VF is not supported");
+			return -1;
+		}
+
+		if (num_queue_pairs > IAVF_MAX_NUM_QUEUES_LV) {
+			PMD_DRV_LOG(ERR, "queue pairs number cannot be larger than %u",
+				IAVF_MAX_NUM_QUEUES_LV);
+			return -1;
+		}
+
+		ret = iavf_queues_req_reset(dev, num_queue_pairs);
+		if (ret)
+			return ret;
+
+		ret = iavf_get_max_rss_queue_region(ad);
+		if (ret) {
+			PMD_INIT_LOG(ERR, "get max rss queue region failed");
+			return ret;
+		}
+
+		vf->lv_enabled = true;
+	} else {
+		/* Check if large VF is already enabled. If so, disable and
+		 * release redundant queue resource.
+		 */
+		if (vf->lv_enabled) {
+			ret = iavf_queues_req_reset(dev, num_queue_pairs);
+			if (ret)
+				return ret;
+
+			vf->lv_enabled = false;
+		}
+		/* if large VF is not required, use default rss queue region */
+		vf->max_rss_qregion = IAVF_MAX_NUM_QUEUES_DFLT;
+	}
+
 	/* Vlan stripping setting */
 	if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN) {
 		if (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
@@ -586,8 +654,8 @@ iavf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 {
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 
-	dev_info->max_rx_queues = vf->vsi_res->num_queue_pairs;
-	dev_info->max_tx_queues = vf->vsi_res->num_queue_pairs;
+	dev_info->max_rx_queues = IAVF_MAX_NUM_QUEUES_LV;
+	dev_info->max_tx_queues = IAVF_MAX_NUM_QUEUES_LV;
 	dev_info->min_rx_bufsize = IAVF_BUF_SIZE_MIN;
 	dev_info->max_rx_pktlen = IAVF_FRAME_SIZE_MAX;
 	dev_info->max_mtu = dev_info->max_rx_pktlen - IAVF_ETH_OVERHEAD;
diff --git a/drivers/net/iavf/iavf_vchnl.c b/drivers/net/iavf/iavf_vchnl.c
index 323e2a843..2c49a0e76 100644
--- a/drivers/net/iavf/iavf_vchnl.c
+++ b/drivers/net/iavf/iavf_vchnl.c
@@ -457,7 +457,8 @@ iavf_get_vf_resource(struct iavf_adapter *adapter)
 		VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC |
 		VIRTCHNL_VF_OFFLOAD_FDIR_PF |
 		VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF |
-		VIRTCHNL_VF_OFFLOAD_REQ_QUEUES;
+		VIRTCHNL_VF_OFFLOAD_REQ_QUEUES |
+		VIRTCHNL_VF_LARGE_NUM_QPAIRS;
 
 	args.in_args = (uint8_t *)&caps;
 	args.in_args_size = sizeof(caps);
@@ -1267,3 +1268,31 @@ iavf_request_queues(struct iavf_adapter *adapter, uint16_t num)
 
 	return -1;
 }
+
+int
+iavf_get_max_rss_queue_region(struct iavf_adapter *adapter)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct iavf_cmd_info args;
+	uint16_t qregion_width;
+	int err;
+
+	args.ops = VIRTCHNL_OP_GET_MAX_RSS_QREGION;
+	args.in_args = NULL;
+	args.in_args_size = 0;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err) {
+		PMD_DRV_LOG(ERR, "Failed to execute command of VIRTCHNL_OP_GET_MAX_RSS_QREGION");
+		return err;
+	}
+
+	qregion_width =
+	((struct virtchnl_max_rss_qregion *)args.out_buffer)->qregion_width;
+
+	vf->max_rss_qregion = (uint16_t)(1 << qregion_width);
+
+	return 0;
+}
-- 
2.17.1


^ permalink raw reply	[flat|nested] 45+ messages in thread

* [dpdk-dev] [PATCH v8 4/6] net/iavf: enable multiple queues configurations for large VF
  2020-10-22  6:48 ` [dpdk-dev] [PATCH v8 0/6] enable large VF configuration Ting Xu
                     ` (2 preceding siblings ...)
  2020-10-22  6:48   ` [dpdk-dev] [PATCH v8 3/6] net/iavf: negotiate large VF and request more queues Ting Xu
@ 2020-10-22  6:49   ` Ting Xu
  2020-10-22  6:49   ` [dpdk-dev] [PATCH v8 5/6] net/iavf: enable IRQ mapping configuration " Ting Xu
                     ` (2 subsequent siblings)
  6 siblings, 0 replies; 45+ messages in thread
From: Ting Xu @ 2020-10-22  6:49 UTC (permalink / raw)
  To: dev; +Cc: qi.z.zhang, beilei.xing, jingjing.wu, Ting Xu

Since the adminq buffer size has a 4K limitation, the current virtchnl
command VIRTCHNL_OP_CONFIG_VSI_QUEUES cannot send the message only once to
configure up to 256 queues. In this patch, we send the messages multiple
times to make sure that the buffer size is less than 4K each time.

Signed-off-by: Ting Xu <ting.xu@intel.com>
---
 drivers/net/iavf/iavf.h        |  4 +++-
 drivers/net/iavf/iavf_ethdev.c | 18 +++++++++++++++++-
 drivers/net/iavf/iavf_vchnl.c  | 11 ++++++-----
 3 files changed, 26 insertions(+), 7 deletions(-)

diff --git a/drivers/net/iavf/iavf.h b/drivers/net/iavf/iavf.h
index 49ccfeece..1cdac1b10 100644
--- a/drivers/net/iavf/iavf.h
+++ b/drivers/net/iavf/iavf.h
@@ -21,6 +21,7 @@
 
 #define IAVF_MAX_NUM_QUEUES_DFLT	 16
 #define IAVF_MAX_NUM_QUEUES_LV		 256
+#define IAVF_CFG_Q_NUM_PER_BUF		 32
 
 #define IAVF_NUM_MACADDR_MAX      64
 
@@ -269,7 +270,8 @@ int iavf_enable_queues(struct iavf_adapter *adapter);
 int iavf_disable_queues(struct iavf_adapter *adapter);
 int iavf_configure_rss_lut(struct iavf_adapter *adapter);
 int iavf_configure_rss_key(struct iavf_adapter *adapter);
-int iavf_configure_queues(struct iavf_adapter *adapter);
+int iavf_configure_queues(struct iavf_adapter *adapter,
+			uint16_t num_queue_pairs, uint16_t index);
 int iavf_get_supported_rxdid(struct iavf_adapter *adapter);
 int iavf_config_irq_map(struct iavf_adapter *adapter);
 void iavf_add_del_all_mac_addr(struct iavf_adapter *adapter, bool add);
diff --git a/drivers/net/iavf/iavf_ethdev.c b/drivers/net/iavf/iavf_ethdev.c
index 87082d1cc..db9449590 100644
--- a/drivers/net/iavf/iavf_ethdev.c
+++ b/drivers/net/iavf/iavf_ethdev.c
@@ -564,6 +564,8 @@ iavf_dev_start(struct rte_eth_dev *dev)
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	struct rte_intr_handle *intr_handle = dev->intr_handle;
+	uint16_t num_queue_pairs;
+	uint16_t index = 0;
 
 	PMD_INIT_FUNC_TRACE();
 
@@ -572,13 +574,27 @@ iavf_dev_start(struct rte_eth_dev *dev)
 	vf->max_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
 	vf->num_queue_pairs = RTE_MAX(dev->data->nb_rx_queues,
 				      dev->data->nb_tx_queues);
+	num_queue_pairs = vf->num_queue_pairs;
 
 	if (iavf_init_queues(dev) != 0) {
 		PMD_DRV_LOG(ERR, "failed to do Queue init");
 		return -1;
 	}
 
-	if (iavf_configure_queues(adapter) != 0) {
+	/* If needed, send configure queues msg multiple times to make the
+	 * adminq buffer length smaller than the 4K limitation.
+	 */
+	while (num_queue_pairs > IAVF_CFG_Q_NUM_PER_BUF) {
+		if (iavf_configure_queues(adapter,
+				IAVF_CFG_Q_NUM_PER_BUF, index) != 0) {
+			PMD_DRV_LOG(ERR, "configure queues failed");
+			goto err_queue;
+		}
+		num_queue_pairs -= IAVF_CFG_Q_NUM_PER_BUF;
+		index += IAVF_CFG_Q_NUM_PER_BUF;
+	}
+
+	if (iavf_configure_queues(adapter, num_queue_pairs, index) != 0) {
 		PMD_DRV_LOG(ERR, "configure queues failed");
 		goto err_queue;
 	}
diff --git a/drivers/net/iavf/iavf_vchnl.c b/drivers/net/iavf/iavf_vchnl.c
index 2c49a0e76..7e6abeca9 100644
--- a/drivers/net/iavf/iavf_vchnl.c
+++ b/drivers/net/iavf/iavf_vchnl.c
@@ -673,7 +673,8 @@ iavf_configure_rss_key(struct iavf_adapter *adapter)
 }
 
 int
-iavf_configure_queues(struct iavf_adapter *adapter)
+iavf_configure_queues(struct iavf_adapter *adapter,
+		uint16_t num_queue_pairs, uint16_t index)
 {
 	struct iavf_rx_queue **rxq =
 		(struct iavf_rx_queue **)adapter->eth_dev->data->rx_queues;
@@ -687,16 +688,16 @@ iavf_configure_queues(struct iavf_adapter *adapter)
 	int err;
 
 	size = sizeof(*vc_config) +
-	       sizeof(vc_config->qpair[0]) * vf->num_queue_pairs;
+	       sizeof(vc_config->qpair[0]) * num_queue_pairs;
 	vc_config = rte_zmalloc("cfg_queue", size, 0);
 	if (!vc_config)
 		return -ENOMEM;
 
 	vc_config->vsi_id = vf->vsi_res->vsi_id;
-	vc_config->num_queue_pairs = vf->num_queue_pairs;
+	vc_config->num_queue_pairs = num_queue_pairs;
 
-	for (i = 0, vc_qp = vc_config->qpair;
-	     i < vf->num_queue_pairs;
+	for (i = index, vc_qp = vc_config->qpair;
+		 i < index + num_queue_pairs;
 	     i++, vc_qp++) {
 		vc_qp->txq.vsi_id = vf->vsi_res->vsi_id;
 		vc_qp->txq.queue_id = i;
-- 
2.17.1


^ permalink raw reply	[flat|nested] 45+ messages in thread

* [dpdk-dev] [PATCH v8 5/6] net/iavf: enable IRQ mapping configuration for large VF
  2020-10-22  6:48 ` [dpdk-dev] [PATCH v8 0/6] enable large VF configuration Ting Xu
                     ` (3 preceding siblings ...)
  2020-10-22  6:49   ` [dpdk-dev] [PATCH v8 4/6] net/iavf: enable multiple queues configurations for large VF Ting Xu
@ 2020-10-22  6:49   ` Ting Xu
  2020-10-22  6:49   ` [dpdk-dev] [PATCH v8 6/6] net/iavf: add enable/disable queues " Ting Xu
  2020-10-22  6:54   ` [dpdk-dev] [PATCH v8 0/6] enable large VF configuration Xing, Beilei
  6 siblings, 0 replies; 45+ messages in thread
From: Ting Xu @ 2020-10-22  6:49 UTC (permalink / raw)
  To: dev; +Cc: qi.z.zhang, beilei.xing, jingjing.wu, Ting Xu

The current IRQ mapping configuration only supports max 16 queues and
16 MSIX vectors. Change the queue vector mapping structure to indicate
up to 256 queues. A new opcode is used to handle the case with large
number of queues. To avoid adminq buffer size limitation, we support
to send the virtchnl message multiple times if needed.

Signed-off-by: Ting Xu <ting.xu@intel.com>
---
 drivers/net/iavf/iavf.h        | 12 +++++---
 drivers/net/iavf/iavf_ethdev.c | 50 +++++++++++++++++++++++++++++-----
 drivers/net/iavf/iavf_vchnl.c  | 50 +++++++++++++++++++++++++++++++---
 3 files changed, 97 insertions(+), 15 deletions(-)

diff --git a/drivers/net/iavf/iavf.h b/drivers/net/iavf/iavf.h
index 1cdac1b10..5e330b215 100644
--- a/drivers/net/iavf/iavf.h
+++ b/drivers/net/iavf/iavf.h
@@ -22,6 +22,7 @@
 #define IAVF_MAX_NUM_QUEUES_DFLT	 16
 #define IAVF_MAX_NUM_QUEUES_LV		 256
 #define IAVF_CFG_Q_NUM_PER_BUF		 32
+#define IAVF_IRQ_MAP_NUM_PER_BUF	 128
 
 #define IAVF_NUM_MACADDR_MAX      64
 
@@ -106,8 +107,10 @@ struct iavf_fdir_info {
 	struct iavf_fdir_conf conf;
 };
 
-/* TODO: is that correct to assume the max number to be 16 ?*/
-#define IAVF_MAX_MSIX_VECTORS   16
+struct iavf_qv_map {
+	uint16_t queue_id;
+	uint16_t vector_id;
+};
 
 /* Message type read in admin queue from PF */
 enum iavf_aq_result {
@@ -152,8 +155,7 @@ struct iavf_info {
 	uint16_t nb_msix;   /* number of MSI-X interrupts on Rx */
 	uint16_t msix_base; /* msix vector base from */
 	uint16_t max_rss_qregion; /* max RSS queue region supported by PF */
-	/* queue bitmask for each vector */
-	uint16_t rxq_map[IAVF_MAX_MSIX_VECTORS];
+	struct iavf_qv_map *qv_map; /* queue vector mapping */
 	struct iavf_flow_list flow_list;
 	rte_spinlock_t flow_ops_lock;
 	struct iavf_parser_list rss_parser_list;
@@ -274,6 +276,8 @@ int iavf_configure_queues(struct iavf_adapter *adapter,
 			uint16_t num_queue_pairs, uint16_t index);
 int iavf_get_supported_rxdid(struct iavf_adapter *adapter);
 int iavf_config_irq_map(struct iavf_adapter *adapter);
+int iavf_config_irq_map_lv(struct iavf_adapter *adapter, uint16_t num,
+			uint16_t index);
 void iavf_add_del_all_mac_addr(struct iavf_adapter *adapter, bool add);
 int iavf_dev_link_update(struct rte_eth_dev *dev,
 			__rte_unused int wait_to_complete);
diff --git a/drivers/net/iavf/iavf_ethdev.c b/drivers/net/iavf/iavf_ethdev.c
index db9449590..92dd14171 100644
--- a/drivers/net/iavf/iavf_ethdev.c
+++ b/drivers/net/iavf/iavf_ethdev.c
@@ -437,6 +437,7 @@ static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev,
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
 	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter);
+	struct iavf_qv_map *qv_map;
 	uint16_t interval, i;
 	int vec;
 
@@ -457,6 +458,14 @@ static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev,
 		}
 	}
 
+	qv_map = rte_zmalloc("qv_map",
+		dev->data->nb_rx_queues * sizeof(struct iavf_qv_map), 0);
+	if (!qv_map) {
+		PMD_DRV_LOG(ERR, "Failed to allocate %d queue-vector map",
+				dev->data->nb_rx_queues);
+		return -1;
+	}
+
 	if (!dev->data->dev_conf.intr_conf.rxq ||
 	    !rte_intr_dp_is_en(intr_handle)) {
 		/* Rx interrupt disabled, Map interrupt only for writeback */
@@ -487,16 +496,21 @@ static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev,
 		}
 		IAVF_WRITE_FLUSH(hw);
 		/* map all queues to the same interrupt */
-		for (i = 0; i < dev->data->nb_rx_queues; i++)
-			vf->rxq_map[vf->msix_base] |= 1 << i;
+		for (i = 0; i < dev->data->nb_rx_queues; i++) {
+			qv_map[i].queue_id = i;
+			qv_map[i].vector_id = vf->msix_base;
+		}
+		vf->qv_map = qv_map;
 	} else {
 		if (!rte_intr_allow_others(intr_handle)) {
 			vf->nb_msix = 1;
 			vf->msix_base = IAVF_MISC_VEC_ID;
 			for (i = 0; i < dev->data->nb_rx_queues; i++) {
-				vf->rxq_map[vf->msix_base] |= 1 << i;
+				qv_map[i].queue_id = i;
+				qv_map[i].vector_id = vf->msix_base;
 				intr_handle->intr_vec[i] = IAVF_MISC_VEC_ID;
 			}
+			vf->qv_map = qv_map;
 			PMD_DRV_LOG(DEBUG,
 				    "vector %u are mapping to all Rx queues",
 				    vf->msix_base);
@@ -509,20 +523,42 @@ static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev,
 			vf->msix_base = IAVF_RX_VEC_START;
 			vec = IAVF_RX_VEC_START;
 			for (i = 0; i < dev->data->nb_rx_queues; i++) {
-				vf->rxq_map[vec] |= 1 << i;
+				qv_map[i].queue_id = i;
+				qv_map[i].vector_id = vec;
 				intr_handle->intr_vec[i] = vec++;
 				if (vec >= vf->nb_msix)
 					vec = IAVF_RX_VEC_START;
 			}
+			vf->qv_map = qv_map;
 			PMD_DRV_LOG(DEBUG,
 				    "%u vectors are mapping to %u Rx queues",
 				    vf->nb_msix, dev->data->nb_rx_queues);
 		}
 	}
 
-	if (iavf_config_irq_map(adapter)) {
-		PMD_DRV_LOG(ERR, "config interrupt mapping failed");
-		return -1;
+	if (!vf->lv_enabled) {
+		if (iavf_config_irq_map(adapter)) {
+			PMD_DRV_LOG(ERR, "config interrupt mapping failed");
+			return -1;
+		}
+	} else {
+		uint16_t num_qv_maps = dev->data->nb_rx_queues;
+		uint16_t index = 0;
+
+		while (num_qv_maps > IAVF_IRQ_MAP_NUM_PER_BUF) {
+			if (iavf_config_irq_map_lv(adapter,
+					IAVF_IRQ_MAP_NUM_PER_BUF, index)) {
+				PMD_DRV_LOG(ERR, "config interrupt mapping for large VF failed");
+				return -1;
+			}
+			num_qv_maps -= IAVF_IRQ_MAP_NUM_PER_BUF;
+			index += IAVF_IRQ_MAP_NUM_PER_BUF;
+		}
+
+		if (iavf_config_irq_map_lv(adapter, num_qv_maps, index)) {
+			PMD_DRV_LOG(ERR, "config interrupt mapping for large VF failed");
+			return -1;
+		}
 	}
 	return 0;
 }
diff --git a/drivers/net/iavf/iavf_vchnl.c b/drivers/net/iavf/iavf_vchnl.c
index 7e6abeca9..145a4ff9e 100644
--- a/drivers/net/iavf/iavf_vchnl.c
+++ b/drivers/net/iavf/iavf_vchnl.c
@@ -775,13 +775,14 @@ iavf_config_irq_map(struct iavf_adapter *adapter)
 		return -ENOMEM;
 
 	map_info->num_vectors = vf->nb_msix;
-	for (i = 0; i < vf->nb_msix; i++) {
-		vecmap = &map_info->vecmap[i];
+	for (i = 0; i < adapter->eth_dev->data->nb_rx_queues; i++) {
+		vecmap =
+		    &map_info->vecmap[vf->qv_map[i].vector_id - vf->msix_base];
 		vecmap->vsi_id = vf->vsi_res->vsi_id;
 		vecmap->rxitr_idx = IAVF_ITR_INDEX_DEFAULT;
-		vecmap->vector_id = vf->msix_base + i;
+		vecmap->vector_id = vf->qv_map[i].vector_id;
 		vecmap->txq_map = 0;
-		vecmap->rxq_map = vf->rxq_map[vf->msix_base + i];
+		vecmap->rxq_map |= 1 << vf->qv_map[i].queue_id;
 	}
 
 	args.ops = VIRTCHNL_OP_CONFIG_IRQ_MAP;
@@ -797,6 +798,47 @@ iavf_config_irq_map(struct iavf_adapter *adapter)
 	return err;
 }
 
+int
+iavf_config_irq_map_lv(struct iavf_adapter *adapter, uint16_t num,
+		uint16_t index)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct virtchnl_queue_vector_maps *map_info;
+	struct virtchnl_queue_vector *qv_maps;
+	struct iavf_cmd_info args;
+	int len, i, err;
+	int count = 0;
+
+	len = sizeof(struct virtchnl_queue_vector_maps) +
+	      sizeof(struct virtchnl_queue_vector) * (num - 1);
+
+	map_info = rte_zmalloc("map_info", len, 0);
+	if (!map_info)
+		return -ENOMEM;
+
+	map_info->vport_id = vf->vsi_res->vsi_id;
+	map_info->num_qv_maps = num;
+	for (i = index; i < index + map_info->num_qv_maps; i++) {
+		qv_maps = &map_info->qv_maps[count++];
+		qv_maps->itr_idx = VIRTCHNL_ITR_IDX_0;
+		qv_maps->queue_type = VIRTCHNL_QUEUE_TYPE_RX;
+		qv_maps->queue_id = vf->qv_map[i].queue_id;
+		qv_maps->vector_id = vf->qv_map[i].vector_id;
+	}
+
+	args.ops = VIRTCHNL_OP_MAP_QUEUE_VECTOR;
+	args.in_args = (u8 *)map_info;
+	args.in_args_size = len;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err)
+		PMD_DRV_LOG(ERR, "fail to execute command OP_MAP_QUEUE_VECTOR");
+
+	rte_free(map_info);
+	return err;
+}
+
 void
 iavf_add_del_all_mac_addr(struct iavf_adapter *adapter, bool add)
 {
-- 
2.17.1


^ permalink raw reply	[flat|nested] 45+ messages in thread

* [dpdk-dev] [PATCH v8 6/6] net/iavf: add enable/disable queues for large VF
  2020-10-22  6:48 ` [dpdk-dev] [PATCH v8 0/6] enable large VF configuration Ting Xu
                     ` (4 preceding siblings ...)
  2020-10-22  6:49   ` [dpdk-dev] [PATCH v8 5/6] net/iavf: enable IRQ mapping configuration " Ting Xu
@ 2020-10-22  6:49   ` Ting Xu
  2020-10-22  6:54   ` [dpdk-dev] [PATCH v8 0/6] enable large VF configuration Xing, Beilei
  6 siblings, 0 replies; 45+ messages in thread
From: Ting Xu @ 2020-10-22  6:49 UTC (permalink / raw)
  To: dev; +Cc: qi.z.zhang, beilei.xing, jingjing.wu, Ting Xu

The current virtchnl structure for enable/disable queues only supports
max 32 queue pairs. Use a new opcode and structure to indicate up to 256
queue pairs, in order to enable/disable queues in large VF case.

Signed-off-by: Ting Xu <ting.xu@intel.com>
---
 drivers/net/iavf/iavf.h       |   5 ++
 drivers/net/iavf/iavf_rxtx.c  |  25 ++++++-
 drivers/net/iavf/iavf_vchnl.c | 132 ++++++++++++++++++++++++++++++++++
 3 files changed, 159 insertions(+), 3 deletions(-)

diff --git a/drivers/net/iavf/iavf.h b/drivers/net/iavf/iavf.h
index 5e330b215..3d3b0da5d 100644
--- a/drivers/net/iavf/iavf.h
+++ b/drivers/net/iavf/iavf.h
@@ -23,6 +23,7 @@
 #define IAVF_MAX_NUM_QUEUES_LV		 256
 #define IAVF_CFG_Q_NUM_PER_BUF		 32
 #define IAVF_IRQ_MAP_NUM_PER_BUF	 128
+#define IAVF_RXTX_QUEUE_CHUNKS_NUM	 2
 
 #define IAVF_NUM_MACADDR_MAX      64
 
@@ -268,8 +269,12 @@ int iavf_enable_vlan_strip(struct iavf_adapter *adapter);
 int iavf_disable_vlan_strip(struct iavf_adapter *adapter);
 int iavf_switch_queue(struct iavf_adapter *adapter, uint16_t qid,
 		     bool rx, bool on);
+int iavf_switch_queue_lv(struct iavf_adapter *adapter, uint16_t qid,
+		     bool rx, bool on);
 int iavf_enable_queues(struct iavf_adapter *adapter);
+int iavf_enable_queues_lv(struct iavf_adapter *adapter);
 int iavf_disable_queues(struct iavf_adapter *adapter);
+int iavf_disable_queues_lv(struct iavf_adapter *adapter);
 int iavf_configure_rss_lut(struct iavf_adapter *adapter);
 int iavf_configure_rss_key(struct iavf_adapter *adapter);
 int iavf_configure_queues(struct iavf_adapter *adapter,
diff --git a/drivers/net/iavf/iavf_rxtx.c b/drivers/net/iavf/iavf_rxtx.c
index 7d4f4ed48..6635f7fd9 100644
--- a/drivers/net/iavf/iavf_rxtx.c
+++ b/drivers/net/iavf/iavf_rxtx.c
@@ -532,6 +532,7 @@ iavf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 {
 	struct iavf_adapter *adapter =
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 	struct iavf_rx_queue *rxq;
 	int err = 0;
@@ -556,7 +557,11 @@ iavf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 	IAVF_WRITE_FLUSH(hw);
 
 	/* Ready to switch the queue on */
-	err = iavf_switch_queue(adapter, rx_queue_id, true, true);
+	if (!vf->lv_enabled)
+		err = iavf_switch_queue(adapter, rx_queue_id, true, true);
+	else
+		err = iavf_switch_queue_lv(adapter, rx_queue_id, true, true);
+
 	if (err)
 		PMD_DRV_LOG(ERR, "Failed to switch RX queue %u on",
 			    rx_queue_id);
@@ -572,6 +577,7 @@ iavf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 {
 	struct iavf_adapter *adapter =
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 	struct iavf_tx_queue *txq;
 	int err = 0;
@@ -588,7 +594,10 @@ iavf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 	IAVF_WRITE_FLUSH(hw);
 
 	/* Ready to switch the queue on */
-	err = iavf_switch_queue(adapter, tx_queue_id, false, true);
+	if (!vf->lv_enabled)
+		err = iavf_switch_queue(adapter, tx_queue_id, false, true);
+	else
+		err = iavf_switch_queue_lv(adapter, tx_queue_id, false, true);
 
 	if (err)
 		PMD_DRV_LOG(ERR, "Failed to switch TX queue %u on",
@@ -689,12 +698,22 @@ iavf_stop_queues(struct rte_eth_dev *dev)
 {
 	struct iavf_adapter *adapter =
 		IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
 	struct iavf_rx_queue *rxq;
 	struct iavf_tx_queue *txq;
 	int ret, i;
 
 	/* Stop All queues */
-	ret = iavf_disable_queues(adapter);
+	if (!vf->lv_enabled) {
+		ret = iavf_disable_queues(adapter);
+		if (ret)
+			PMD_DRV_LOG(WARNING, "Fail to stop queues");
+	} else {
+		ret = iavf_disable_queues_lv(adapter);
+		if (ret)
+			PMD_DRV_LOG(WARNING, "Fail to stop queues for large VF");
+	}
+
 	if (ret)
 		PMD_DRV_LOG(WARNING, "Fail to stop queues");
 
diff --git a/drivers/net/iavf/iavf_vchnl.c b/drivers/net/iavf/iavf_vchnl.c
index 145a4ff9e..54d9917c0 100644
--- a/drivers/net/iavf/iavf_vchnl.c
+++ b/drivers/net/iavf/iavf_vchnl.c
@@ -608,6 +608,138 @@ iavf_switch_queue(struct iavf_adapter *adapter, uint16_t qid,
 	return err;
 }
 
+int
+iavf_enable_queues_lv(struct iavf_adapter *adapter)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct virtchnl_del_ena_dis_queues *queue_select;
+	struct virtchnl_queue_chunk *queue_chunk;
+	struct iavf_cmd_info args;
+	int err, len;
+
+	len = sizeof(struct virtchnl_del_ena_dis_queues) +
+		  sizeof(struct virtchnl_queue_chunk) *
+		  (IAVF_RXTX_QUEUE_CHUNKS_NUM - 1);
+	queue_select = rte_zmalloc("queue_select", len, 0);
+	if (!queue_select)
+		return -ENOMEM;
+
+	queue_chunk = queue_select->chunks.chunks;
+	queue_select->chunks.num_chunks = IAVF_RXTX_QUEUE_CHUNKS_NUM;
+	queue_select->vport_id = vf->vsi_res->vsi_id;
+
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].type = VIRTCHNL_QUEUE_TYPE_TX;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].start_queue_id = 0;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].num_queues =
+		adapter->eth_dev->data->nb_tx_queues;
+
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].type = VIRTCHNL_QUEUE_TYPE_RX;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].start_queue_id = 0;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].num_queues =
+		adapter->eth_dev->data->nb_rx_queues;
+
+	args.ops = VIRTCHNL_OP_ENABLE_QUEUES_V2;
+	args.in_args = (u8 *)queue_select;
+	args.in_args_size = len;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err) {
+		PMD_DRV_LOG(ERR,
+			    "Failed to execute command of OP_ENABLE_QUEUES_V2");
+		return err;
+	}
+	return 0;
+}
+
+int
+iavf_disable_queues_lv(struct iavf_adapter *adapter)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct virtchnl_del_ena_dis_queues *queue_select;
+	struct virtchnl_queue_chunk *queue_chunk;
+	struct iavf_cmd_info args;
+	int err, len;
+
+	len = sizeof(struct virtchnl_del_ena_dis_queues) +
+		  sizeof(struct virtchnl_queue_chunk) *
+		  (IAVF_RXTX_QUEUE_CHUNKS_NUM - 1);
+	queue_select = rte_zmalloc("queue_select", len, 0);
+	if (!queue_select)
+		return -ENOMEM;
+
+	queue_chunk = queue_select->chunks.chunks;
+	queue_select->chunks.num_chunks = IAVF_RXTX_QUEUE_CHUNKS_NUM;
+	queue_select->vport_id = vf->vsi_res->vsi_id;
+
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].type = VIRTCHNL_QUEUE_TYPE_TX;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].start_queue_id = 0;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_TX].num_queues =
+		adapter->eth_dev->data->nb_tx_queues;
+
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].type = VIRTCHNL_QUEUE_TYPE_RX;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].start_queue_id = 0;
+	queue_chunk[VIRTCHNL_QUEUE_TYPE_RX].num_queues =
+		adapter->eth_dev->data->nb_rx_queues;
+
+	args.ops = VIRTCHNL_OP_DISABLE_QUEUES_V2;
+	args.in_args = (u8 *)queue_select;
+	args.in_args_size = len;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err) {
+		PMD_DRV_LOG(ERR,
+			    "Failed to execute command of OP_DISABLE_QUEUES_V2");
+		return err;
+	}
+	return 0;
+}
+
+int
+iavf_switch_queue_lv(struct iavf_adapter *adapter, uint16_t qid,
+		 bool rx, bool on)
+{
+	struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter);
+	struct virtchnl_del_ena_dis_queues *queue_select;
+	struct virtchnl_queue_chunk *queue_chunk;
+	struct iavf_cmd_info args;
+	int err, len;
+
+	len = sizeof(struct virtchnl_del_ena_dis_queues);
+	queue_select = rte_zmalloc("queue_select", len, 0);
+	if (!queue_select)
+		return -ENOMEM;
+
+	queue_chunk = queue_select->chunks.chunks;
+	queue_select->chunks.num_chunks = 1;
+	queue_select->vport_id = vf->vsi_res->vsi_id;
+
+	if (rx) {
+		queue_chunk->type = VIRTCHNL_QUEUE_TYPE_RX;
+		queue_chunk->start_queue_id = qid;
+		queue_chunk->num_queues = 1;
+	} else {
+		queue_chunk->type = VIRTCHNL_QUEUE_TYPE_TX;
+		queue_chunk->start_queue_id = qid;
+		queue_chunk->num_queues = 1;
+	}
+
+	if (on)
+		args.ops = VIRTCHNL_OP_ENABLE_QUEUES_V2;
+	else
+		args.ops = VIRTCHNL_OP_DISABLE_QUEUES_V2;
+	args.in_args = (u8 *)queue_select;
+	args.in_args_size = len;
+	args.out_buffer = vf->aq_resp;
+	args.out_size = IAVF_AQ_BUF_SZ;
+	err = iavf_execute_vf_cmd(adapter, &args);
+	if (err)
+		PMD_DRV_LOG(ERR, "Failed to execute command of %s",
+			    on ? "OP_ENABLE_QUEUES_V2" : "OP_DISABLE_QUEUES_V2");
+	return err;
+}
+
 int
 iavf_configure_rss_lut(struct iavf_adapter *adapter)
 {
-- 
2.17.1


^ permalink raw reply	[flat|nested] 45+ messages in thread

* Re: [dpdk-dev] [PATCH v8 0/6] enable large VF configuration
  2020-10-22  6:48 ` [dpdk-dev] [PATCH v8 0/6] enable large VF configuration Ting Xu
                     ` (5 preceding siblings ...)
  2020-10-22  6:49   ` [dpdk-dev] [PATCH v8 6/6] net/iavf: add enable/disable queues " Ting Xu
@ 2020-10-22  6:54   ` Xing, Beilei
  2020-10-22  8:50     ` Zhang, Qi Z
  6 siblings, 1 reply; 45+ messages in thread
From: Xing, Beilei @ 2020-10-22  6:54 UTC (permalink / raw)
  To: Xu, Ting, dev; +Cc: Zhang, Qi Z, Wu, Jingjing



> -----Original Message-----
> From: Xu, Ting <ting.xu@intel.com>
> Sent: Thursday, October 22, 2020 2:49 PM
> To: dev@dpdk.org
> Cc: Zhang, Qi Z <qi.z.zhang@intel.com>; Xing, Beilei <beilei.xing@intel.com>;
> Wu, Jingjing <jingjing.wu@intel.com>; Xu, Ting <ting.xu@intel.com>
> Subject: [PATCH v8 0/6] enable large VF configuration
> 
> This patchset supports to configure up to 256 queue pairs per VF. If large VF is
> supported after capability negotiation, VF will request queues from PF as
> needed. New virtual channel opcodes and structures are used to indicate 256
> queues, so VF is designed to handle the new function of configure VSI queues,
> IRQ mapping and enable/disable queues.
> Also, enable VF to query the max RSS queue region for RSS and filter
> configuration.
> 
> Ting Xu (6):
>   net/iavf: handle virtchnl event message without interrupt
>   net/iavf: add IAVF request queues function
>   net/iavf: negotiate large VF and request more queues
>   net/iavf: enable multiple queues configurations for large VF
>   net/iavf: enable IRQ mapping configuration for large VF
>   net/iavf: add enable/disable queues for large VF
> 
> ---
> v7->v8:
> Modify commit logs.
> Make log sentences in one line.
> 
> v6->v7:
> Separate patches.
> 
> v4->v6:
> Solve queue mapping buffer limitation issue.
> Optimize VSI queue configuration
> 
> v3->v4:
> Optimize handling messages from PF.
> 
> v2->v3:
> Fix coding style issue.
> Move get max RSS queue region after VF reset.
> Add request queues capability negotiation.
> 
> v1->v2:
> Change the communication with kernel PF.
> 
> ---
>  drivers/net/iavf/iavf.h        |  37 ++-
>  drivers/net/iavf/iavf_ethdev.c | 142 ++++++++++-
>  drivers/net/iavf/iavf_rxtx.c   |  25 +-
>  drivers/net/iavf/iavf_vchnl.c  | 448 ++++++++++++++++++++++++++++-----
>  4 files changed, 572 insertions(+), 80 deletions(-)
> 
> --
> 2.17.1

Acked-by: Beilei Xing <beilei.xing@intel.com>


^ permalink raw reply	[flat|nested] 45+ messages in thread

* Re: [dpdk-dev] [PATCH v8 0/6] enable large VF configuration
  2020-10-22  6:54   ` [dpdk-dev] [PATCH v8 0/6] enable large VF configuration Xing, Beilei
@ 2020-10-22  8:50     ` Zhang, Qi Z
  0 siblings, 0 replies; 45+ messages in thread
From: Zhang, Qi Z @ 2020-10-22  8:50 UTC (permalink / raw)
  To: Xing, Beilei, Xu, Ting, dev; +Cc: Wu, Jingjing



> -----Original Message-----
> From: Xing, Beilei <beilei.xing@intel.com>
> Sent: Thursday, October 22, 2020 2:55 PM
> To: Xu, Ting <ting.xu@intel.com>; dev@dpdk.org
> Cc: Zhang, Qi Z <qi.z.zhang@intel.com>; Wu, Jingjing <jingjing.wu@intel.com>
> Subject: RE: [PATCH v8 0/6] enable large VF configuration
> 
> 
> 
> > -----Original Message-----
> > From: Xu, Ting <ting.xu@intel.com>
> > Sent: Thursday, October 22, 2020 2:49 PM
> > To: dev@dpdk.org
> > Cc: Zhang, Qi Z <qi.z.zhang@intel.com>; Xing, Beilei
> > <beilei.xing@intel.com>; Wu, Jingjing <jingjing.wu@intel.com>; Xu,
> > Ting <ting.xu@intel.com>
> > Subject: [PATCH v8 0/6] enable large VF configuration
> >
> > This patchset supports to configure up to 256 queue pairs per VF. If
> > large VF is supported after capability negotiation, VF will request
> > queues from PF as needed. New virtual channel opcodes and structures
> > are used to indicate 256 queues, so VF is designed to handle the new
> > function of configure VSI queues, IRQ mapping and enable/disable queues.
> > Also, enable VF to query the max RSS queue region for RSS and filter
> > configuration.
> >
> > Ting Xu (6):
> >   net/iavf: handle virtchnl event message without interrupt
> >   net/iavf: add IAVF request queues function
> >   net/iavf: negotiate large VF and request more queues
> >   net/iavf: enable multiple queues configurations for large VF
> >   net/iavf: enable IRQ mapping configuration for large VF
> >   net/iavf: add enable/disable queues for large VF
> >
> > ---
> > v7->v8:
> > Modify commit logs.
> > Make log sentences in one line.
> >
> > v6->v7:
> > Separate patches.
> >
> > v4->v6:
> > Solve queue mapping buffer limitation issue.
> > Optimize VSI queue configuration
> >
> > v3->v4:
> > Optimize handling messages from PF.
> >
> > v2->v3:
> > Fix coding style issue.
> > Move get max RSS queue region after VF reset.
> > Add request queues capability negotiation.
> >
> > v1->v2:
> > Change the communication with kernel PF.
> >
> > ---
> >  drivers/net/iavf/iavf.h        |  37 ++-
> >  drivers/net/iavf/iavf_ethdev.c | 142 ++++++++++-
> >  drivers/net/iavf/iavf_rxtx.c   |  25 +-
> >  drivers/net/iavf/iavf_vchnl.c  | 448
> > ++++++++++++++++++++++++++++-----
> >  4 files changed, 572 insertions(+), 80 deletions(-)
> >
> > --
> > 2.17.1
> 
> Acked-by: Beilei Xing <beilei.xing@intel.com>

Applied to dpdk-next-net-intel.

Thanks
Qi
> 


^ permalink raw reply	[flat|nested] 45+ messages in thread

* Re: [dpdk-dev] [PATCH v8 2/6] net/iavf: add IAVF request queues function
  2020-10-22  6:48   ` [dpdk-dev] [PATCH v8 2/6] net/iavf: add IAVF request queues function Ting Xu
@ 2020-10-23 10:07     ` Ferruh Yigit
  2020-10-25  2:28       ` Xu, Ting
  2020-10-23 10:11     ` Ferruh Yigit
  1 sibling, 1 reply; 45+ messages in thread
From: Ferruh Yigit @ 2020-10-23 10:07 UTC (permalink / raw)
  To: Ting Xu, dev; +Cc: qi.z.zhang, beilei.xing, jingjing.wu

On 10/22/2020 7:48 AM, Ting Xu wrote:
> Add a new virtchnl function to request additional queues from PF. Current
> default queue pairs number when creating a VF is 16. In order to support
> up to 256 queue pairs per VF, enable this request queues function.
> 
> When requesting queues succeeds, PF will return an event message. If it
> is handled by interrupt first, the request queues command cannot receive
> the correct PF response and will wait until timeout.

Why interrupt is not working? Is the PF not generating an interrupt when this 
request/command executed?

> Therefore, disable
> interrupt before requesting queues in order to handle the event message
> asynchronously.
> 
> Signed-off-by: Ting Xu <ting.xu@intel.com>

<...>


^ permalink raw reply	[flat|nested] 45+ messages in thread

* Re: [dpdk-dev] [PATCH v8 2/6] net/iavf: add IAVF request queues function
  2020-10-22  6:48   ` [dpdk-dev] [PATCH v8 2/6] net/iavf: add IAVF request queues function Ting Xu
  2020-10-23 10:07     ` Ferruh Yigit
@ 2020-10-23 10:11     ` Ferruh Yigit
  2020-10-23 10:17       ` Ferruh Yigit
  1 sibling, 1 reply; 45+ messages in thread
From: Ferruh Yigit @ 2020-10-23 10:11 UTC (permalink / raw)
  To: Ting Xu, dev; +Cc: qi.z.zhang, beilei.xing, jingjing.wu

On 10/22/2020 7:48 AM, Ting Xu wrote:
> Add a new virtchnl function to request additional queues from PF. Current
> default queue pairs number when creating a VF is 16. In order to support
> up to 256 queue pairs per VF, enable this request queues function.
> 

If there is a "256" queue pair limit, should it be added into the function, 
right now it gets "uint16_t num" parameter?

> When requesting queues succeeds, PF will return an event message. If it
> is handled by interrupt first, the request queues command cannot receive
> the correct PF response and will wait until timeout. Therefore, disable
> interrupt before requesting queues in order to handle the event message
> asynchronously.
> 
> Signed-off-by: Ting Xu <ting.xu@intel.com>

<...>


^ permalink raw reply	[flat|nested] 45+ messages in thread

* Re: [dpdk-dev] [PATCH v8 2/6] net/iavf: add IAVF request queues function
  2020-10-23 10:11     ` Ferruh Yigit
@ 2020-10-23 10:17       ` Ferruh Yigit
  0 siblings, 0 replies; 45+ messages in thread
From: Ferruh Yigit @ 2020-10-23 10:17 UTC (permalink / raw)
  To: Ting Xu, dev; +Cc: qi.z.zhang, beilei.xing, jingjing.wu

On 10/23/2020 11:11 AM, Ferruh Yigit wrote:
> On 10/22/2020 7:48 AM, Ting Xu wrote:
>> Add a new virtchnl function to request additional queues from PF. Current
>> default queue pairs number when creating a VF is 16. In order to support
>> up to 256 queue pairs per VF, enable this request queues function.
>>
> 
> If there is a "256" queue pair limit, should it be added into the function, 
> right now it gets "uint16_t num" parameter?
> 

Ahh, I can see it is enforced by caller in next patch, that is OK.

>> When requesting queues succeeds, PF will return an event message. If it
>> is handled by interrupt first, the request queues command cannot receive
>> the correct PF response and will wait until timeout. Therefore, disable
>> interrupt before requesting queues in order to handle the event message
>> asynchronously.
>>
>> Signed-off-by: Ting Xu <ting.xu@intel.com>
> 
> <...>
> 


^ permalink raw reply	[flat|nested] 45+ messages in thread

* Re: [dpdk-dev] [PATCH v8 2/6] net/iavf: add IAVF request queues function
  2020-10-23 10:07     ` Ferruh Yigit
@ 2020-10-25  2:28       ` Xu, Ting
  0 siblings, 0 replies; 45+ messages in thread
From: Xu, Ting @ 2020-10-25  2:28 UTC (permalink / raw)
  To: Yigit, Ferruh, dev; +Cc: Zhang, Qi Z, Xing, Beilei, Wu, Jingjing

> -----Original Message-----
> From: Ferruh Yigit <ferruh.yigit@intel.com>
> Sent: Friday, October 23, 2020 6:07 PM
> To: Xu, Ting <ting.xu@intel.com>; dev@dpdk.org
> Cc: Zhang, Qi Z <qi.z.zhang@intel.com>; Xing, Beilei <beilei.xing@intel.com>;
> Wu, Jingjing <jingjing.wu@intel.com>
> Subject: Re: [dpdk-dev] [PATCH v8 2/6] net/iavf: add IAVF request queues
> function
> 
> On 10/22/2020 7:48 AM, Ting Xu wrote:
> > Add a new virtchnl function to request additional queues from PF.
> > Current default queue pairs number when creating a VF is 16. In order
> > to support up to 256 queue pairs per VF, enable this request queues
> function.
> >
> > When requesting queues succeeds, PF will return an event message. If
> > it is handled by interrupt first, the request queues command cannot
> > receive the correct PF response and will wait until timeout.
> 
> Why interrupt is not working? Is the PF not generating an interrupt when this
> request/command executed?
> 

If request queues command succeeds, PF will reset VF and return event message, which will trigger interrupt in the legacy way; if request queues fails, PF will return VIRTCHNL_OP_REQUEST_QUEUES message and report available queues in iavf_read_msg_from_pf() . Therefore, after we send request queues command, we cannot leave to wait for interrupt, but should loop in iavf_execute_vf_cmd() to read adminq message, including event and VIRTCHNL_OP_REQUEST_QUEUES message. So interrupt should be disabled.

> > Therefore, disable
> > interrupt before requesting queues in order to handle the event
> > message asynchronously.
> >
> > Signed-off-by: Ting Xu <ting.xu@intel.com>
> 
> <...>


^ permalink raw reply	[flat|nested] 45+ messages in thread

* [dpdk-dev] [PATCH v3 0/2] enable large VF configuration
@ 2020-09-28  9:56 Ting Xu
  0 siblings, 0 replies; 45+ messages in thread
From: Ting Xu @ 2020-09-28  9:56 UTC (permalink / raw)
  To: dev; +Cc: qi.z.zhang, beilei.xing, jingjing.wu, Ting Xu

This patchset supports to configure up to 256 queue pairs per VF. If
large VF is supported after capability negotiation, VF will request
queues from PF as needed. New virtual channel opcodes and structures
are used to indicate 256 queues, so VF is designed to handle the new
function of configure VSI queues, IRQ mapping and enable/disable queues.
Also, enable VF to query the max RSS queue region.

Ting Xu (2):
  net/iavf: add IAVF request queues function
  net/iavf: enable large VF configuration

---
v2->v3:
Fix coding style issue.
Move get max RSS queue region after VF reset.
Add request queues capability negotiation.

v1->v2:
Change the communication with kernel PF.

---
 drivers/net/iavf/iavf.h        |  40 ++-
 drivers/net/iavf/iavf_ethdev.c | 102 +++++++-
 drivers/net/iavf/iavf_rxtx.c   |  27 +-
 drivers/net/iavf/iavf_vchnl.c  | 444 ++++++++++++++++++++++++++++++---
 4 files changed, 558 insertions(+), 55 deletions(-)

-- 
2.17.1


^ permalink raw reply	[flat|nested] 45+ messages in thread

end of thread, other threads:[~2020-10-25  2:28 UTC | newest]

Thread overview: 45+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2020-09-09  7:20 [dpdk-dev] [PATCH v1 0/2] enable large VF configuration Ting Xu
2020-09-09  7:20 ` [dpdk-dev] [PATCH v1 1/2] net/iavf: add IAVF request queues Ting Xu
2020-09-09  7:20 ` [dpdk-dev] [PATCH v1 2/2] net/iavf: enable large VF configuration Ting Xu
2020-09-25  5:59 ` [dpdk-dev] [PATCH v2 0/2] " Ting Xu
2020-09-25  5:59   ` [dpdk-dev] [PATCH v2 1/2] net/iavf: add IAVF request queues function Ting Xu
2020-09-25  5:59   ` [dpdk-dev] [PATCH v2 2/2] net/iavf: enable large VF configuration Ting Xu
2020-09-27 12:42 ` [dpdk-dev] [PATCH v3 0/2] " Ting Xu
2020-09-27 12:42   ` [dpdk-dev] [PATCH v3 1/2] net/iavf: add IAVF request queues function Ting Xu
2020-09-27 12:42   ` [dpdk-dev] [PATCH v3 2/2] net/iavf: enable large VF configuration Ting Xu
2020-10-15  5:21 ` [dpdk-dev] [PATCH v4 0/2] " Ting Xu
2020-10-15  5:21   ` [dpdk-dev] [PATCH v4 1/2] net/iavf: add IAVF request queues function Ting Xu
2020-10-15  5:21   ` [dpdk-dev] [PATCH v4 2/2] net/iavf: enable large VF configuration Ting Xu
2020-10-16  1:21 ` [dpdk-dev] [PATCH v5 0/2] " Ting Xu
2020-10-16  1:21   ` [dpdk-dev] [PATCH v5 1/2] net/iavf: add IAVF request queues function Ting Xu
2020-10-16  1:21   ` [dpdk-dev] [PATCH v5 2/2] net/iavf: enable large VF configuration Ting Xu
2020-10-16  1:34 ` [dpdk-dev] [PATCH v5 0/2] " Ting Xu
2020-10-16  1:34   ` [dpdk-dev] [PATCH v5 1/2] net/iavf: add IAVF request queues function Ting Xu
2020-10-16  1:34   ` [dpdk-dev] [PATCH v5 2/2] net/iavf: enable large VF configuration Ting Xu
2020-10-16  1:43 ` [dpdk-dev] [PATCH v6 0/2] " Ting Xu
2020-10-16  1:43   ` [dpdk-dev] [PATCH v6 1/2] net/iavf: add IAVF request queues function Ting Xu
2020-10-16  8:41     ` Xing, Beilei
2020-10-18 10:29       ` Xu, Ting
2020-10-16  1:43   ` [dpdk-dev] [PATCH v6 2/2] net/iavf: enable large VF configuration Ting Xu
2020-10-18 10:34 ` [dpdk-dev] [PATCH v7 0/6] " Ting Xu
2020-10-18 10:34   ` [dpdk-dev] [PATCH v7 1/6] net/iavf: handle virtchnl event message without interrupt Ting Xu
2020-10-21  8:15     ` Xing, Beilei
2020-10-18 10:34   ` [dpdk-dev] [PATCH v7 2/6] net/iavf: add IAVF request queues function Ting Xu
2020-10-18 10:34   ` [dpdk-dev] [PATCH v7 3/6] net/iavf: negotiate large VF and request more queues Ting Xu
2020-10-18 10:34   ` [dpdk-dev] [PATCH v7 4/6] net/iavf: enable multiple queues configurations for large VF Ting Xu
2020-10-18 10:34   ` [dpdk-dev] [PATCH v7 5/6] net/iavf: enable IRQ mapping configuration " Ting Xu
2020-10-18 10:34   ` [dpdk-dev] [PATCH v7 6/6] net/iavf: add enable/disable queues " Ting Xu
2020-10-22  6:48 ` [dpdk-dev] [PATCH v8 0/6] enable large VF configuration Ting Xu
2020-10-22  6:48   ` [dpdk-dev] [PATCH v8 1/6] net/iavf: handle virtchnl event message without interrupt Ting Xu
2020-10-22  6:48   ` [dpdk-dev] [PATCH v8 2/6] net/iavf: add IAVF request queues function Ting Xu
2020-10-23 10:07     ` Ferruh Yigit
2020-10-25  2:28       ` Xu, Ting
2020-10-23 10:11     ` Ferruh Yigit
2020-10-23 10:17       ` Ferruh Yigit
2020-10-22  6:48   ` [dpdk-dev] [PATCH v8 3/6] net/iavf: negotiate large VF and request more queues Ting Xu
2020-10-22  6:49   ` [dpdk-dev] [PATCH v8 4/6] net/iavf: enable multiple queues configurations for large VF Ting Xu
2020-10-22  6:49   ` [dpdk-dev] [PATCH v8 5/6] net/iavf: enable IRQ mapping configuration " Ting Xu
2020-10-22  6:49   ` [dpdk-dev] [PATCH v8 6/6] net/iavf: add enable/disable queues " Ting Xu
2020-10-22  6:54   ` [dpdk-dev] [PATCH v8 0/6] enable large VF configuration Xing, Beilei
2020-10-22  8:50     ` Zhang, Qi Z
2020-09-28  9:56 [dpdk-dev] [PATCH v3 0/2] " Ting Xu

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).