From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mga03.intel.com (mga03.intel.com [134.134.136.65]) by dpdk.org (Postfix) with ESMTP id 9885A5320 for ; Mon, 13 Jun 2016 08:39:18 +0200 (CEST) Received: from fmsmga001.fm.intel.com ([10.253.24.23]) by orsmga103.jf.intel.com with ESMTP; 12 Jun 2016 23:39:18 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.26,465,1459839600"; d="scan'208";a="986193325" Received: from dpdk06.sh.intel.com ([10.239.128.225]) by fmsmga001.fm.intel.com with ESMTP; 12 Jun 2016 23:39:16 -0700 From: Jianfeng Tan To: dev@dpdk.org Cc: Jianfeng Tan , Huawei Xie , rich.lane@bigswitch.com, yuanhan.liu@linux.intel.com, mst@redhat.com, nakajima.yoshihiro@lab.ntt.co.jp, p.fedin@samsung.com, ann.zhuangyanying@huawei.com, mukawa@igel.co.jp, nhorman@tuxdriver.com Date: Mon, 13 Jun 2016 06:38:59 +0000 Message-Id: <1465799943-138577-3-git-send-email-jianfeng.tan@intel.com> X-Mailer: git-send-email 2.1.4 In-Reply-To: <1465799943-138577-1-git-send-email-jianfeng.tan@intel.com> References: <1446748276-132087-1-git-send-email-jianfeng.tan@intel.com> <1465799943-138577-1-git-send-email-jianfeng.tan@intel.com> Subject: [dpdk-dev] [PATCH v8 2/6] virtio: enable use virtual address to fill desc X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: patches and discussions about DPDK List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 13 Jun 2016 06:39:19 -0000 This patch is related to how to calculate relative address for vhost backend. The principle is that: based on one or multiple shared memory regions, vhost maintains a reference system with the frontend start address, backend start address, and length for each segment, so that each frontend address (GPA, Guest Physical Address) can be translated into vhost-recognizable backend address. To make the address translation efficient, we need to maintain as few regions as possible. In the case of VM, GPA is always locally continuous. But for some other case, like virtio-user, we use virtual address here. It basically means: a. when set_base_addr, VA address is used; b. when preparing RX's descriptors, VA address is used; c. when transmitting packets, VA is filled in TX's descriptors; d. in TX and CQ's header, VA is used. Signed-off-by: Huawei Xie Signed-off-by: Jianfeng Tan Acked-by: Neil Horman --- drivers/net/virtio/virtio_ethdev.c | 43 +++++++++++++++++++++++---------- drivers/net/virtio/virtio_rxtx.c | 5 ++-- drivers/net/virtio/virtio_rxtx_simple.c | 13 +++++----- drivers/net/virtio/virtqueue.h | 10 ++++++++ 4 files changed, 49 insertions(+), 22 deletions(-) diff --git a/drivers/net/virtio/virtio_ethdev.c b/drivers/net/virtio/virtio_ethdev.c index 841949b..9ccce79 100644 --- a/drivers/net/virtio/virtio_ethdev.c +++ b/drivers/net/virtio/virtio_ethdev.c @@ -431,9 +431,6 @@ int virtio_dev_queue_setup(struct rte_eth_dev *dev, rxvq->mz = mz; *pvq = rxvq; } else if (queue_type == VTNET_TQ) { - struct virtio_tx_region *txr; - unsigned int i; - txvq = (struct virtnet_tx *)RTE_PTR_ADD(vq, sz_vq); txvq->vq = vq; txvq->port_id = dev->data->port_id; @@ -442,6 +439,36 @@ int virtio_dev_queue_setup(struct rte_eth_dev *dev, txvq->virtio_net_hdr_mz = hdr_mz; txvq->virtio_net_hdr_mem = hdr_mz->phys_addr; + *pvq = txvq; + } else if (queue_type == VTNET_CQ) { + cvq = (struct virtnet_ctl *)RTE_PTR_ADD(vq, sz_vq); + cvq->vq = vq; + cvq->mz = mz; + cvq->virtio_net_hdr_mz = hdr_mz; + cvq->virtio_net_hdr_mem = hdr_mz->phys_addr; + memset(cvq->virtio_net_hdr_mz->addr, 0, PAGE_SIZE); + *pvq = cvq; + } + + /* For virtio-user case (that is when dev->pci_dev is NULL), we use + * virtual address. And we need properly set _offset_, please see + * MBUF_DATA_DMA_ADDR in virtqueue.h for more information. + */ + if (dev->pci_dev) + vq->offset = offsetof(struct rte_mbuf, buf_physaddr); + else { + vq->vq_ring_mem = (uintptr_t)mz->addr; + vq->offset = offsetof(struct rte_mbuf, buf_addr); + if (queue_type == VTNET_TQ) + txvq->virtio_net_hdr_mem = (uintptr_t)hdr_mz->addr; + else if (queue_type == VTNET_CQ) + cvq->virtio_net_hdr_mem = (uintptr_t)hdr_mz->addr; + } + + if (queue_type == VTNET_TQ) { + struct virtio_tx_region *txr; + unsigned int i; + txr = hdr_mz->addr; memset(txr, 0, vq_size * sizeof(*txr)); for (i = 0; i < vq_size; i++) { @@ -457,16 +484,6 @@ int virtio_dev_queue_setup(struct rte_eth_dev *dev, start_dp->len = hw->vtnet_hdr_size; start_dp->flags = VRING_DESC_F_NEXT; } - - *pvq = txvq; - } else if (queue_type == VTNET_CQ) { - cvq = (struct virtnet_ctl *)RTE_PTR_ADD(vq, sz_vq); - cvq->vq = vq; - cvq->mz = mz; - cvq->virtio_net_hdr_mz = hdr_mz; - cvq->virtio_net_hdr_mem = hdr_mz->phys_addr; - memset(cvq->virtio_net_hdr_mz->addr, 0, PAGE_SIZE); - *pvq = cvq; } if (hw->vtpci_ops->setup_queue(hw, vq) < 0) { diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c index 28266d2..b96d0cb 100644 --- a/drivers/net/virtio/virtio_rxtx.c +++ b/drivers/net/virtio/virtio_rxtx.c @@ -193,8 +193,7 @@ virtqueue_enqueue_recv_refill(struct virtqueue *vq, struct rte_mbuf *cookie) start_dp = vq->vq_ring.desc; start_dp[idx].addr = - (uint64_t)(cookie->buf_physaddr + RTE_PKTMBUF_HEADROOM - - hw->vtnet_hdr_size); + MBUF_DATA_DMA_ADDR(cookie, vq->offset) - hw->vtnet_hdr_size; start_dp[idx].len = cookie->buf_len - RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size; start_dp[idx].flags = VRING_DESC_F_WRITE; @@ -266,7 +265,7 @@ virtqueue_enqueue_xmit(struct virtnet_tx *txvq, struct rte_mbuf *cookie, } do { - start_dp[idx].addr = rte_mbuf_data_dma_addr(cookie); + start_dp[idx].addr = MBUF_DATA_DMA_ADDR(cookie, vq->offset); start_dp[idx].len = cookie->data_len; start_dp[idx].flags = cookie->next ? VRING_DESC_F_NEXT : 0; idx = start_dp[idx].next; diff --git a/drivers/net/virtio/virtio_rxtx_simple.c b/drivers/net/virtio/virtio_rxtx_simple.c index fdd655d..7b50119 100644 --- a/drivers/net/virtio/virtio_rxtx_simple.c +++ b/drivers/net/virtio/virtio_rxtx_simple.c @@ -80,8 +80,8 @@ virtqueue_enqueue_recv_refill_simple(struct virtqueue *vq, vq->sw_ring[desc_idx] = cookie; start_dp = vq->vq_ring.desc; - start_dp[desc_idx].addr = (uint64_t)((uintptr_t)cookie->buf_physaddr + - RTE_PKTMBUF_HEADROOM - vq->hw->vtnet_hdr_size); + start_dp[desc_idx].addr = MBUF_DATA_DMA_ADDR(cookie, vq->offset) - + vq->hw->vtnet_hdr_size; start_dp[desc_idx].len = cookie->buf_len - RTE_PKTMBUF_HEADROOM + vq->hw->vtnet_hdr_size; @@ -120,8 +120,8 @@ virtio_rxq_rearm_vec(struct virtnet_rx *rxvq) *(uint64_t *)p = rxvq->mbuf_initializer; start_dp[i].addr = - (uint64_t)((uintptr_t)sw_ring[i]->buf_physaddr + - RTE_PKTMBUF_HEADROOM - vq->hw->vtnet_hdr_size); + MBUF_DATA_DMA_ADDR(sw_ring[i], vq->offset) - + vq->hw->vtnet_hdr_size; start_dp[i].len = sw_ring[i]->buf_len - RTE_PKTMBUF_HEADROOM + vq->hw->vtnet_hdr_size; } @@ -369,7 +369,7 @@ virtio_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts, vq->vq_descx[desc_idx + i].cookie = tx_pkts[i]; for (i = 0; i < nb_tail; i++) { start_dp[desc_idx].addr = - rte_mbuf_data_dma_addr(*tx_pkts); + MBUF_DATA_DMA_ADDR(*tx_pkts, vq->offset); start_dp[desc_idx].len = (*tx_pkts)->pkt_len; tx_pkts++; desc_idx++; @@ -380,7 +380,8 @@ virtio_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts, for (i = 0; i < nb_commit; i++) vq->vq_descx[desc_idx + i].cookie = tx_pkts[i]; for (i = 0; i < nb_commit; i++) { - start_dp[desc_idx].addr = rte_mbuf_data_dma_addr(*tx_pkts); + start_dp[desc_idx].addr = + MBUF_DATA_DMA_ADDR(*tx_pkts, vq->offset); start_dp[desc_idx].len = (*tx_pkts)->pkt_len; tx_pkts++; desc_idx++; diff --git a/drivers/net/virtio/virtqueue.h b/drivers/net/virtio/virtqueue.h index 17c1ea1..455aaaf 100644 --- a/drivers/net/virtio/virtqueue.h +++ b/drivers/net/virtio/virtqueue.h @@ -66,6 +66,14 @@ struct rte_mbuf; #define VIRTQUEUE_MAX_NAME_SZ 32 +#ifdef RTE_VIRTIO_USER +#define MBUF_DATA_DMA_ADDR(mb, offset) \ + ((uint64_t)((uintptr_t)(*(void **)((uintptr_t)mb + offset)) \ + + (mb)->data_off)) +#else /* RTE_VIRTIO_USER */ +#define MBUF_DATA_DMA_ADDR(mb, offset) rte_mbuf_data_dma_addr(mb) +#endif /* RTE_VIRTIO_USER */ + #define VTNET_SQ_RQ_QUEUE_IDX 0 #define VTNET_SQ_TQ_QUEUE_IDX 1 #define VTNET_SQ_CQ_QUEUE_IDX 2 @@ -175,6 +183,7 @@ struct virtqueue { unsigned int vq_ring_size; phys_addr_t vq_ring_mem; /**< physical address of vring */ + /**< use virtual address for virtio-user. */ /** * Head of the free chain in the descriptor table. If @@ -184,6 +193,7 @@ struct virtqueue { uint16_t vq_desc_head_idx; uint16_t vq_desc_tail_idx; uint16_t vq_queue_index; /**< PCI queue index */ + uint16_t offset; /**< relative offset to obtain addr in mbuf */ uint16_t *notify_addr; int configured; struct rte_mbuf **sw_ring; /**< RX software ring. */ -- 2.1.4