[dpdk-dev] [PATCH v4 11/12] vhost: alloc core to virtq

[Ouyang Changchun <changchun.ouyang@intel.com>]

This patch allocates the core on the granularity of virtq instead of virtio device.
This allows vhost having the capability of polling different virtq with different core,
which shows better performance on vhost/virtio ports with more cores.

Add 2 API: rte_vhost_core_id_get and rte_vhost_core_id_set.

Signed-off-by: Changchun Ouyang <changchun.ouyang@intel.com>
---
It is added since v4.

 examples/vhost/Makefile           |   4 +-
 examples/vhost/main.c             | 243 ++++++++++++++++++++------------------
 examples/vhost/main.h             |   3 +-
 lib/librte_vhost/rte_virtio_net.h |  25 ++++
 lib/librte_vhost/virtio-net.c     |  22 ++++
 5 files changed, 178 insertions(+), 119 deletions(-)

diff --git a/examples/vhost/Makefile b/examples/vhost/Makefile
index c269466..32a3dec 100644
--- a/examples/vhost/Makefile
+++ b/examples/vhost/Makefile
@@ -50,8 +50,8 @@ APP = vhost-switch
 # all source are stored in SRCS-y
 SRCS-y := main.c
 
-CFLAGS += -O2 -D_FILE_OFFSET_BITS=64
-CFLAGS += $(WERROR_FLAGS)
+CFLAGS += -O0 -g -D_FILE_OFFSET_BITS=64
+CFLAGS += $(WERROR_FLAGS) -Wno-maybe-uninitialized
 
 include $(RTE_SDK)/mk/rte.extapp.mk
 
diff --git a/examples/vhost/main.c b/examples/vhost/main.c
index 54f9648..0a36c61 100644
--- a/examples/vhost/main.c
+++ b/examples/vhost/main.c
@@ -1386,60 +1386,58 @@ switch_worker(__attribute__((unused)) void *arg)
 			}
 			if (likely(vdev->ready == DEVICE_RX)) {
 				/*Handle guest RX*/
-				for (i = 0; i < rxq; i++) {
-					rx_count = rte_eth_rx_burst(ports[0],
-						vdev->vmdq_rx_q + i, pkts_burst, MAX_PKT_BURST);
+				uint16_t q_idx = dev_ll->work_q_idx;
+				rx_count = rte_eth_rx_burst(ports[0],
+					vdev->vmdq_rx_q + q_idx, pkts_burst, MAX_PKT_BURST);
 
-					if (rx_count) {
-						/*
-						* Retry is enabled and the queue is full then we wait and retry to avoid packet loss
-						* Here MAX_PKT_BURST must be less than virtio queue size
-						*/
-						if (enable_retry && unlikely(rx_count > rte_vring_available_entries(dev,
-											VIRTIO_RXQ + i * VIRTIO_QNUM))) {
-							for (retry = 0; retry < burst_rx_retry_num; retry++) {
-								rte_delay_us(burst_rx_delay_time);
-								if (rx_count <= rte_vring_available_entries(dev,
-											VIRTIO_RXQ + i * VIRTIO_QNUM))
-									break;
-							}
-						}
-						ret_count = rte_vhost_enqueue_burst(dev, VIRTIO_RXQ + i * VIRTIO_QNUM,
-											pkts_burst, rx_count);
-						if (enable_stats) {
-							rte_atomic64_add(
-							&dev_statistics[dev_ll->vdev->dev->device_fh].qp_stats[i].rx_total_atomic,
-							rx_count);
-							rte_atomic64_add(
-							&dev_statistics[dev_ll->vdev->dev->device_fh].qp_stats[i].rx_atomic, ret_count);
-						}
-						while (likely(rx_count)) {
-							rx_count--;
-							rte_pktmbuf_free(pkts_burst[rx_count]);
+				if (rx_count) {
+					/*
+					* Retry is enabled and the queue is full then we wait and retry to avoid packet loss
+					* Here MAX_PKT_BURST must be less than virtio queue size
+					*/
+					if (enable_retry && unlikely(rx_count > rte_vring_available_entries(dev,
+										VIRTIO_RXQ + q_idx * VIRTIO_QNUM))) {
+						for (retry = 0; retry < burst_rx_retry_num; retry++) {
+							rte_delay_us(burst_rx_delay_time);
+							if (rx_count <= rte_vring_available_entries(dev,
+										VIRTIO_RXQ + q_idx * VIRTIO_QNUM))
+								break;
 						}
 					}
+					ret_count = rte_vhost_enqueue_burst(dev, VIRTIO_RXQ + q_idx * VIRTIO_QNUM,
+										pkts_burst, rx_count);
+					if (enable_stats) {
+						rte_atomic64_add(
+						&dev_statistics[dev_ll->vdev->dev->device_fh].qp_stats[q_idx].rx_total_atomic,
+						rx_count);
+						rte_atomic64_add(
+						&dev_statistics[dev_ll->vdev->dev->device_fh].qp_stats[q_idx].rx_atomic, ret_count);
+					}
+					while (likely(rx_count)) {
+						rx_count--;
+						rte_pktmbuf_free(pkts_burst[rx_count]);
+					}
 				}
 			}
 
 			if (likely(!vdev->remove)) {
 				/* Handle guest TX*/
-				for (i = 0; i < rxq; i++) {
-					tx_count = rte_vhost_dequeue_burst(dev, VIRTIO_TXQ + i * VIRTIO_QNUM,
-							mbuf_pool, pkts_burst, MAX_PKT_BURST);
-					/*
-					 * If this is the first received packet we need to learn
-					 * the MAC and setup VMDQ
-					 */
-					if (unlikely(vdev->ready == DEVICE_MAC_LEARNING) && tx_count) {
-						if (vdev->remove || (link_vmdq(vdev, pkts_burst[0]) == -1)) {
-							while (tx_count)
-								rte_pktmbuf_free(pkts_burst[--tx_count]);
-						}
+				uint16_t q_idx = dev_ll->work_q_idx;
+				tx_count = rte_vhost_dequeue_burst(dev, VIRTIO_TXQ + q_idx * VIRTIO_QNUM,
+						mbuf_pool, pkts_burst, MAX_PKT_BURST);
+				/*
+				 * If this is the first received packet we need to learn
+				 * the MAC and setup VMDQ
+				 */
+				if (unlikely(vdev->ready == DEVICE_MAC_LEARNING) && tx_count) {
+					if (vdev->remove || (link_vmdq(vdev, pkts_burst[0]) == -1)) {
+						while (tx_count)
+							rte_pktmbuf_free(pkts_burst[--tx_count]);
 					}
-					while (tx_count)
-						virtio_tx_route(vdev, pkts_burst[--tx_count],
-								(uint16_t)dev->device_fh, i);
 				}
+				while (tx_count)
+					virtio_tx_route(vdev, pkts_burst[--tx_count],
+						(uint16_t)dev->device_fh, q_idx);
 			}
 
 			/*move to the next device in the list*/
@@ -2427,6 +2425,7 @@ destroy_device (volatile struct virtio_net *dev)
 	struct virtio_net_data_ll *ll_main_dev_last = NULL;
 	struct vhost_dev *vdev;
 	int lcore;
+	uint32_t i;
 
 	dev->flags &= ~VIRTIO_DEV_RUNNING;
 
@@ -2438,61 +2437,73 @@ destroy_device (volatile struct virtio_net *dev)
 	}
 
 	/* Search for entry to be removed from lcore ll */
-	ll_lcore_dev_cur = lcore_info[vdev->coreid].lcore_ll->ll_root_used;
-	while (ll_lcore_dev_cur != NULL) {
-		if (ll_lcore_dev_cur->vdev == vdev) {
-			break;
-		} else {
-			ll_lcore_dev_last = ll_lcore_dev_cur;
-			ll_lcore_dev_cur = ll_lcore_dev_cur->next;
+	for (i = 0; i < rxq; i++) {
+		uint16_t core_id = rte_vhost_core_id_get(dev, i);
+
+		ll_lcore_dev_cur = lcore_info[core_id].lcore_ll->ll_root_used;
+
+		while (ll_lcore_dev_cur != NULL) {
+			if (ll_lcore_dev_cur->vdev == vdev) {
+				break;
+			} else {
+				ll_lcore_dev_last = ll_lcore_dev_cur;
+				ll_lcore_dev_cur = ll_lcore_dev_cur->next;
+			}
 		}
-	}
 
-	if (ll_lcore_dev_cur == NULL) {
-		RTE_LOG(ERR, VHOST_CONFIG,
-			"(%"PRIu64") Failed to find the dev to be destroy.\n",
-			dev->device_fh);
-		return;
-	}
+		if (ll_lcore_dev_cur == NULL) {
+			RTE_LOG(ERR, VHOST_CONFIG,
+				"(%"PRIu64") Failed to find the dev to be destroy.\n",
+				dev->device_fh);
+			if (i == 0)
+				return;
+			else
+				break;
+		}
 
-	/* Search for entry to be removed from main ll */
-	ll_main_dev_cur = ll_root_used;
-	ll_main_dev_last = NULL;
-	while (ll_main_dev_cur != NULL) {
-		if (ll_main_dev_cur->vdev == vdev) {
-			break;
-		} else {
-			ll_main_dev_last = ll_main_dev_cur;
-			ll_main_dev_cur = ll_main_dev_cur->next;
+		/* Search for entry to be removed from main ll */
+		if (i == 0) {
+			ll_main_dev_cur = ll_root_used;
+			ll_main_dev_last = NULL;
+			while (ll_main_dev_cur != NULL) {
+				if (ll_main_dev_cur->vdev == vdev) {
+					break;
+				} else {
+					ll_main_dev_last = ll_main_dev_cur;
+					ll_main_dev_cur = ll_main_dev_cur->next;
+				}
+			}
 		}
-	}
 
-	/* Remove entries from the lcore and main ll. */
-	rm_data_ll_entry(&lcore_info[vdev->coreid].lcore_ll->ll_root_used, ll_lcore_dev_cur, ll_lcore_dev_last);
-	rm_data_ll_entry(&ll_root_used, ll_main_dev_cur, ll_main_dev_last);
+		/* Remove entries from the lcore and main ll. */
+		rm_data_ll_entry(&lcore_info[core_id].lcore_ll->ll_root_used, ll_lcore_dev_cur, ll_lcore_dev_last);
+		if (i == 0)
+			rm_data_ll_entry(&ll_root_used, ll_main_dev_cur, ll_main_dev_last);
 
-	/* Set the dev_removal_flag on each lcore. */
-	RTE_LCORE_FOREACH_SLAVE(lcore) {
-		lcore_info[lcore].lcore_ll->dev_removal_flag = REQUEST_DEV_REMOVAL;
-	}
+		/* Set the dev_removal_flag on each lcore. */
+		RTE_LCORE_FOREACH_SLAVE(lcore) {
+			lcore_info[lcore].lcore_ll->dev_removal_flag = REQUEST_DEV_REMOVAL;
+		}
 
-	/*
-	 * Once each core has set the dev_removal_flag to ACK_DEV_REMOVAL we can be sure that
-	 * they can no longer access the device removed from the linked lists and that the devices
-	 * are no longer in use.
-	 */
-	RTE_LCORE_FOREACH_SLAVE(lcore) {
-		while (lcore_info[lcore].lcore_ll->dev_removal_flag != ACK_DEV_REMOVAL) {
-			rte_pause();
+		/*
+		 * Once each core has set the dev_removal_flag to ACK_DEV_REMOVAL we can be sure that
+		 * they can no longer access the device removed from the linked lists and that the devices
+		 * are no longer in use.
+		 */
+		RTE_LCORE_FOREACH_SLAVE(lcore) {
+			while (lcore_info[lcore].lcore_ll->dev_removal_flag != ACK_DEV_REMOVAL)
+				rte_pause();
 		}
-	}
 
-	/* Add the entries back to the lcore and main free ll.*/
-	put_data_ll_free_entry(&lcore_info[vdev->coreid].lcore_ll->ll_root_free, ll_lcore_dev_cur);
-	put_data_ll_free_entry(&ll_root_free, ll_main_dev_cur);
+		/* Add the entries back to the lcore and main free ll.*/
+		put_data_ll_free_entry(&lcore_info[core_id].lcore_ll->ll_root_free, ll_lcore_dev_cur);
 
-	/* Decrement number of device on the lcore. */
-	lcore_info[vdev->coreid].lcore_ll->device_num--;
+		if (i == 0)
+			put_data_ll_free_entry(&ll_root_free, ll_main_dev_cur);
+
+		/* Decrement number of device on the lcore. */
+		lcore_info[core_id].lcore_ll->device_num--;
+	}
 
 	RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") Device has been removed from data core\n", dev->device_fh);
 
@@ -2846,42 +2857,44 @@ new_device (struct virtio_net *dev)
 	vdev->remove = 0;
 
 	/* Find a suitable lcore to add the device. */
-	RTE_LCORE_FOREACH_SLAVE(lcore) {
-		if (lcore_info[lcore].lcore_ll->device_num < device_num_min) {
-			device_num_min = lcore_info[lcore].lcore_ll->device_num;
-			core_add = lcore;
+	for (i = 0; i < rxq; i++) {
+		device_num_min = num_devices;
+		RTE_LCORE_FOREACH_SLAVE(lcore) {
+			if (lcore_info[lcore].lcore_ll->device_num < device_num_min) {
+				device_num_min = lcore_info[lcore].lcore_ll->device_num;
+				core_add = lcore;
+			}
 		}
-	}
-	/* Add device to lcore ll */
-	ll_dev = get_data_ll_free_entry(&lcore_info[core_add].lcore_ll->ll_root_free);
-	if (ll_dev == NULL) {
-		RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") Failed to add device to data core\n", dev->device_fh);
-		vdev->ready = DEVICE_SAFE_REMOVE;
-		destroy_device(dev);
-		rte_free(vdev->regions_hpa);
-		rte_free(vdev);
-		return -1;
-	}
-	ll_dev->vdev = vdev;
-	vdev->coreid = core_add;
+		/* Add device to lcore ll */
+		ll_dev = get_data_ll_free_entry(&lcore_info[core_add].lcore_ll->ll_root_free);
+		if (ll_dev == NULL) {
+			RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") Failed to add device to data core\n", dev->device_fh);
+			vdev->ready = DEVICE_SAFE_REMOVE;
+			destroy_device(dev);
+			rte_free(vdev->regions_hpa);
+			rte_free(vdev);
+			return -1;
+		}
+		ll_dev->vdev = vdev;
+		ll_dev->work_q_idx = i;
+		rte_vhost_core_id_set(dev, i, core_add);
+		add_data_ll_entry(&lcore_info[core_add].lcore_ll->ll_root_used, ll_dev);
 
-	add_data_ll_entry(&lcore_info[vdev->coreid].lcore_ll->ll_root_used, ll_dev);
+		/* Disable notifications. */
+		rte_vhost_enable_guest_notification(dev, i * VIRTIO_QNUM + VIRTIO_RXQ, 0);
+		rte_vhost_enable_guest_notification(dev, i * VIRTIO_QNUM + VIRTIO_TXQ, 0);
+		lcore_info[core_add].lcore_ll->device_num++;
+		RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") Device has been added to data core %d for vq: %d\n",
+			dev->device_fh, core_add, i);
+	}
 
 	/* Initialize device stats */
 	if (enable_stats)
 		memset(dev_statistics[dev->device_fh].qp_stats, 0,
 			VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX * sizeof(struct qp_statistics));
 
-	/* Disable notifications. */
-	for (i = 0; i < rxq; i++) {
-		rte_vhost_enable_guest_notification(dev, i * VIRTIO_QNUM + VIRTIO_RXQ, 0);
-		rte_vhost_enable_guest_notification(dev, i * VIRTIO_QNUM + VIRTIO_TXQ, 0);
-	}
-
-	lcore_info[vdev->coreid].lcore_ll->device_num++;
 	dev->flags |= VIRTIO_DEV_RUNNING;
 
-	RTE_LOG(INFO, VHOST_DATA, "(%"PRIu64") Device has been added to data core %d\n", dev->device_fh, vdev->coreid);
 
 	return 0;
 }
diff --git a/examples/vhost/main.h b/examples/vhost/main.h
index d04e2be..42336bc 100644
--- a/examples/vhost/main.h
+++ b/examples/vhost/main.h
@@ -82,8 +82,6 @@ struct vhost_dev {
 	uint16_t vmdq_rx_q;
 	/**< Vlan tag assigned to the pool */
 	uint32_t vlan_tag;
-	/**< Data core that the device is added to. */
-	uint16_t coreid;
 	/**< A device is set as ready if the MAC address has been set. */
 	volatile uint8_t ready;
 	/**< Device is marked for removal from the data core. */
@@ -94,6 +92,7 @@ struct virtio_net_data_ll
 {
 	struct vhost_dev		*vdev;	/* Pointer to device created by configuration core. */
 	struct virtio_net_data_ll	*next;  /* Pointer to next device in linked list. */
+	uint32_t work_q_idx;
 };
 
 /*
diff --git a/lib/librte_vhost/rte_virtio_net.h b/lib/librte_vhost/rte_virtio_net.h
index e16ad3a..93d3e27 100644
--- a/lib/librte_vhost/rte_virtio_net.h
+++ b/lib/librte_vhost/rte_virtio_net.h
@@ -89,6 +89,7 @@ struct vhost_virtqueue {
 	eventfd_t		callfd;			/**< Used to notify the guest (trigger interrupt). */
 	eventfd_t		kickfd;			/**< Currently unused as polling mode is enabled. */
 	uint32_t		enabled;		/**< Indicate the queue is enabled or not. */
+	uint16_t		core_id;		/**< Data core that the vq is added to. */
 	struct buf_vector	buf_vec[BUF_VECTOR_MAX];	/**< for scatter RX. */
 } __rte_cache_aligned;
 
@@ -241,8 +242,32 @@ uint16_t rte_vhost_dequeue_burst(struct virtio_net *dev, uint16_t queue_id,
 
 /**
  * This function get the queue pair number of one vhost device.
+ * @param dev
+ *  virtio-net device
  * @return
  *  num of queue pair of specified virtio device.
  */
 uint16_t rte_vhost_qp_num_get(struct virtio_net *dev);
+
+/**
+ * This function get the data core id for queue pair in one vhost device.
+ * @param dev
+ *  virtio-net device
+ * @param queue_id
+ *  virtio queue index in mq case
+ * @return
+ *  core id of queue pair of specified virtio device.
+ */
+uint16_t rte_vhost_core_id_get(volatile struct virtio_net *dev, uint16_t queue_id);
+
+/**
+ * This function set the data core id for queue pair in one vhost device.
+ * @param dev
+ *  virtio-net device
+ * @param queue_id
+ *  virtio queue index in mq case
+ * @param core_id
+ *  data core id for virtio queue pair in mq case
+ */
+void rte_vhost_core_id_set(struct virtio_net *dev, uint16_t queue_id, uint16_t core_id);
 #endif /* _VIRTIO_NET_H_ */
diff --git a/lib/librte_vhost/virtio-net.c b/lib/librte_vhost/virtio-net.c
index 24d0c53..d4c55c6 100644
--- a/lib/librte_vhost/virtio-net.c
+++ b/lib/librte_vhost/virtio-net.c
@@ -965,6 +965,28 @@ uint16_t rte_vhost_qp_num_get(struct virtio_net *dev)
 	return dev->virt_qp_nb;
 }
 
+uint16_t rte_vhost_core_id_get(volatile struct virtio_net *dev, uint16_t queue_id)
+{
+	if (dev == NULL)
+		return 0;
+
+	if (dev->virtqueue == NULL || dev->virtqueue[queue_id] == NULL)
+		return 0;
+
+	return dev->virtqueue[queue_id]->core_id;
+}
+
+void rte_vhost_core_id_set(struct virtio_net *dev, uint16_t queue_id, uint16_t core_id)
+{
+	if (dev == NULL)
+		return;
+
+	if (dev->virtqueue == NULL || dev->virtqueue[queue_id] == NULL)
+		return;
+
+	dev->virtqueue[queue_id]->core_id = core_id;
+}
+
 /*
  * Register ops so that we can add/remove device to data core.
  */
-- 
1.8.4.2