DPDK patches and discussions
 help / color / mirror / Atom feed
From: Junjie Chen <junjie.j.chen@intel.com>
To: jianfeng.tan@intel.com, maxime.coquelin@redhat.com, mtetsuyah@gmail.com
Cc: dev@dpdk.org, Junjie Chen <junjie.j.chen@intel.com>
Subject: [dpdk-dev] [PATCH v4] vhost: add support for interrupt mode
Date: Wed,  4 Apr 2018 05:43:10 -0400	[thread overview]
Message-ID: <1522834990-146451-1-git-send-email-junjie.j.chen@intel.com> (raw)
In-Reply-To: <1522773213-132764-1-git-send-email-junjie.j.chen@intel.com>

In some cases we want vhost dequeue work in interrupt mode to
release cpus to others when no data to transmit. So we install
interrupt handler of vhost device and interrupt vectors for each
rx queue when creating new backend according to vhost intrerupt
configuration. Thus, applications could register a epoll event fd
to associate rx queues with interrupt vectors.

Signed-off-by: Junjie Chen <junjie.j.chen@intel.com>
---
Changes in v4:
- revert back license change
Changes in v3:
- handle failure in the middle of intr setup.
- use vhost API to enable interrupt.
- rebase to check rxq existence.
- update vhost API to support guest notification.
Changes in v2:
- update rx queue index.
- fill efd_counter_size for intr handler.
- update log.
 drivers/net/vhost/rte_eth_vhost.c | 167 ++++++++++++++++++++++++++++++++++++--
 lib/librte_vhost/vhost.c          |  13 ++-
 2 files changed, 164 insertions(+), 16 deletions(-)

diff --git a/drivers/net/vhost/rte_eth_vhost.c b/drivers/net/vhost/rte_eth_vhost.c
index 11b6076..9a23c12 100644
--- a/drivers/net/vhost/rte_eth_vhost.c
+++ b/drivers/net/vhost/rte_eth_vhost.c
@@ -554,28 +554,174 @@ update_queuing_status(struct rte_eth_dev *dev)
 	}
 }
 
+static int
+eth_rxq_intr_enable(struct rte_eth_dev *dev, uint16_t qid)
+{
+	struct rte_vhost_vring vring;
+	struct vhost_queue *vq;
+	int ret = 0;
+
+	vq = dev->data->rx_queues[qid];
+	if (!vq) {
+		RTE_LOG(ERR, PMD, "rxq%d is not setup yet\n", qid);
+		return -1;
+	}
+
+	ret = rte_vhost_get_vhost_vring(vq->vid, (qid << 1) + 1, &vring);
+	if (ret < 0) {
+		RTE_LOG(ERR, PMD, "Failed to get rxq%d's vring\n", qid);
+		return ret;
+	}
+	RTE_LOG(INFO, PMD, "Enable interrupt for rxq%d\n", qid);
+	rte_vhost_enable_guest_notification(vq->vid, (qid << 1) + 1, 1);
+	rte_wmb();
+
+	return ret;
+}
+
+static int
+eth_rxq_intr_disable(struct rte_eth_dev *dev, uint16_t qid)
+{
+	struct rte_vhost_vring vring;
+	struct vhost_queue *vq;
+	int ret = 0;
+
+	vq = dev->data->rx_queues[qid];
+	if (!vq) {
+		RTE_LOG(ERR, PMD, "rxq%d is not setup yet\n", qid);
+		return -1;
+	}
+
+	ret = rte_vhost_get_vhost_vring(vq->vid, (qid << 1) + 1, &vring);
+	if (ret < 0) {
+		RTE_LOG(ERR, PMD, "Failed to get rxq%d's vring", qid);
+		return ret;
+	}
+	RTE_LOG(INFO, PMD, "Disable interrupt for rxq%d\n", qid);
+	rte_vhost_enable_guest_notification(vq->vid, (qid << 1) + 1, 0);
+	rte_wmb();
+
+	return 0;
+}
+
 static void
+eth_vhost_uninstall_intr(struct rte_eth_dev *dev)
+{
+	struct rte_intr_handle *intr_handle = dev->intr_handle;
+
+	if (intr_handle) {
+		if (intr_handle->intr_vec)
+			free(intr_handle->intr_vec);
+		free(intr_handle);
+	}
+
+	dev->intr_handle = NULL;
+}
+
+static int
+eth_vhost_install_intr(struct rte_eth_dev *dev)
+{
+	struct rte_vhost_vring vring;
+	struct vhost_queue *vq;
+	int count = 0;
+	int nb_rxq = dev->data->nb_rx_queues;
+	int i;
+	int ret;
+
+	/* uninstall firstly if we are reconnecting */
+	if (dev->intr_handle)
+		eth_vhost_uninstall_intr(dev);
+
+	dev->intr_handle = malloc(sizeof(*dev->intr_handle));
+	if (!dev->intr_handle) {
+		RTE_LOG(ERR, PMD, "Fail to allocate intr_handle\n");
+		return -ENOMEM;
+	}
+	memset(dev->intr_handle, 0, sizeof(*dev->intr_handle));
+
+	dev->intr_handle->efd_counter_size = sizeof(uint64_t);
+
+	dev->intr_handle->intr_vec =
+		malloc(nb_rxq * sizeof(dev->intr_handle->intr_vec[0]));
+
+	if (!dev->intr_handle->intr_vec) {
+		RTE_LOG(ERR, PMD,
+			"Failed to allocate memory for interrupt vector\n");
+		free(dev->intr_handle);
+		return -ENOMEM;
+	}
+
+	RTE_LOG(INFO, PMD, "Prepare intr vec\n");
+	for (i = 0; i < nb_rxq; i++) {
+		vq = dev->data->rx_queues[i];
+		if (!vq) {
+			RTE_LOG(INFO, PMD, "rxq-%d not setup yet, skip!\n", i);
+			continue;
+		}
+
+		ret = rte_vhost_get_vhost_vring(vq->vid, (i << 1) + 1, &vring);
+		if (ret < 0) {
+			RTE_LOG(INFO, PMD,
+				"Failed to get rxq-%d's vring, skip!\n", i);
+			continue;
+		}
+
+		if (vring.kickfd < 0) {
+			RTE_LOG(INFO, PMD,
+				"rxq-%d's kickfd is invalid, skip!\n", i);
+			continue;
+		}
+		dev->intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + i;
+		dev->intr_handle->efds[i] = vring.kickfd;
+		count++;
+		RTE_LOG(INFO, PMD, "Installed intr vec for rxq-%d\n", i);
+	}
+
+	dev->intr_handle->nb_efd = count;
+	dev->intr_handle->max_intr = count + 1;
+	dev->intr_handle->type = RTE_INTR_HANDLE_VDEV;
+
+	return 0;
+}
+
+static int
 queue_setup(struct rte_eth_dev *eth_dev, struct pmd_internal *internal)
 {
 	struct vhost_queue *vq;
+	struct rte_eth_conf *dev_conf = &eth_dev->data->dev_conf;
+	int vid = internal->vid;
 	int i;
 
 	for (i = 0; i < eth_dev->data->nb_rx_queues; i++) {
 		vq = eth_dev->data->rx_queues[i];
 		if (!vq)
 			continue;
-		vq->vid = internal->vid;
+		vq->vid = vid;
 		vq->internal = internal;
 		vq->port = eth_dev->data->port_id;
 	}
+
 	for (i = 0; i < eth_dev->data->nb_tx_queues; i++) {
 		vq = eth_dev->data->tx_queues[i];
 		if (!vq)
 			continue;
-		vq->vid = internal->vid;
+		vq->vid = vid;
 		vq->internal = internal;
 		vq->port = eth_dev->data->port_id;
 	}
+
+	for (i = 0; i < rte_vhost_get_vring_num(vid); i++)
+		rte_vhost_enable_guest_notification(vid, i, 0);
+
+	if (dev_conf->intr_conf.rxq) {
+		if (eth_vhost_install_intr(eth_dev) < 0) {
+			RTE_LOG(INFO, PMD,
+				"Failed to install interrupt handler.");
+				return -1;
+		}
+	}
+
+	return 0;
 }
 
 static int
@@ -584,7 +730,6 @@ new_device(int vid)
 	struct rte_eth_dev *eth_dev;
 	struct internal_list *list;
 	struct pmd_internal *internal;
-	unsigned i;
 	char ifname[PATH_MAX];
 #ifdef RTE_LIBRTE_VHOST_NUMA
 	int newnode;
@@ -608,16 +753,13 @@ new_device(int vid)
 
 	internal->vid = vid;
 	if (eth_dev->data->rx_queues && eth_dev->data->tx_queues) {
-		queue_setup(eth_dev, internal);
-		rte_atomic32_set(&internal->dev_attached, 1);
+		if (!queue_setup(eth_dev, internal))
+			rte_atomic32_set(&internal->dev_attached, 1);
 	} else {
 		RTE_LOG(INFO, PMD, "RX/TX queues have not setup yet\n");
 		rte_atomic32_set(&internal->dev_attached, 0);
 	}
 
-	for (i = 0; i < rte_vhost_get_vring_num(vid); i++)
-		rte_vhost_enable_guest_notification(vid, i, 0);
-
 	rte_vhost_get_mtu(vid, &eth_dev->data->mtu);
 
 	eth_dev->data->dev_link.link_status = ETH_LINK_UP;
@@ -681,6 +823,8 @@ destroy_device(int vid)
 
 	RTE_LOG(INFO, PMD, "Vhost device %d destroyed\n", vid);
 
+	eth_vhost_uninstall_intr(eth_dev);
+
 	_rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC, NULL);
 }
 
@@ -793,7 +937,10 @@ eth_dev_start(struct rte_eth_dev *eth_dev)
 	struct pmd_internal *internal = eth_dev->data->dev_private;
 
 	if (unlikely(rte_atomic32_read(&internal->dev_attached) == 0)) {
-		queue_setup(eth_dev, internal);
+		if (!queue_setup(eth_dev, internal)) {
+			RTE_LOG(ERR, PMD, "Failed to set up rx queue\n");
+			return -1;
+		}
 		rte_atomic32_set(&internal->dev_attached, 1);
 	}
 
@@ -1030,6 +1177,8 @@ static const struct eth_dev_ops ops = {
 	.xstats_reset = vhost_dev_xstats_reset,
 	.xstats_get = vhost_dev_xstats_get,
 	.xstats_get_names = vhost_dev_xstats_get_names,
+	.rx_queue_intr_enable = eth_rxq_intr_enable,
+	.rx_queue_intr_disable = eth_rxq_intr_disable,
 };
 
 static struct rte_vdev_driver pmd_vhost_drv;
diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c
index f6f12a0..68e9340 100644
--- a/lib/librte_vhost/vhost.c
+++ b/lib/librte_vhost/vhost.c
@@ -545,16 +545,15 @@ rte_vhost_enable_guest_notification(int vid, uint16_t queue_id, int enable)
 {
 	struct virtio_net *dev = get_device(vid);
 
-	if (dev == NULL)
+	if (!dev)
 		return -1;
 
-	if (enable) {
-		RTE_LOG(ERR, VHOST_CONFIG,
-			"guest notification isn't supported.\n");
-		return -1;
-	}
+	if (enable)
+		dev->virtqueue[queue_id]->used->flags &=
+		    ~VRING_USED_F_NO_NOTIFY;
+	else
+		dev->virtqueue[queue_id]->used->flags = VRING_USED_F_NO_NOTIFY;
 
-	dev->virtqueue[queue_id]->used->flags = VRING_USED_F_NO_NOTIFY;
 	return 0;
 }
 
-- 
2.0.1

  reply	other threads:[~2018-04-04  2:04 UTC|newest]

Thread overview: 16+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2018-02-22  9:59 [dpdk-dev] [PATCH] " Junjie Chen
2018-03-29 14:37 ` [dpdk-dev] [PATCH v2] " Junjie Chen
2018-03-31  4:00   ` Tan, Jianfeng
2018-04-02  8:51     ` Chen, Junjie J
2018-04-03 16:33   ` [dpdk-dev] [PATCH v3] " Junjie Chen
2018-04-04  9:43     ` Junjie Chen [this message]
2018-04-08 16:50       ` [dpdk-dev] [PATCH v5] " Junjie Chen
2018-04-12 16:28         ` [dpdk-dev] [PATCH v6 1/2] " Junjie Chen
2018-04-12 12:18           ` Tan, Jianfeng
2018-04-13  9:15           ` Maxime Coquelin
2018-04-13 10:07             ` Chen, Junjie J
2018-04-12 16:29         ` [dpdk-dev] [PATCH v6 2/2] net/vhost: update license to SPDX format Junjie Chen
2018-04-12 16:43         ` Junjie Chen
2018-04-12 12:21           ` Maxime Coquelin
2018-04-19  1:07           ` Takanari Hayama
2018-04-19 14:55             ` Maxime Coquelin

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=1522834990-146451-1-git-send-email-junjie.j.chen@intel.com \
    --to=junjie.j.chen@intel.com \
    --cc=dev@dpdk.org \
    --cc=jianfeng.tan@intel.com \
    --cc=maxime.coquelin@redhat.com \
    --cc=mtetsuyah@gmail.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).