From mboxrd@z Thu Jan  1 00:00:00 1970
Return-Path: <changchun.ouyang@intel.com>
Received: from mga14.intel.com (mga14.intel.com [192.55.52.115])
 by dpdk.org (Postfix) with ESMTP id A46EAE72
 for <dev@dpdk.org>; Sun,  6 Sep 2015 10:37:13 +0200 (CEST)
Received: from fmsmga002.fm.intel.com ([10.253.24.26])
 by fmsmga103.fm.intel.com with ESMTP; 06 Sep 2015 01:37:12 -0700
X-ExtLoop1: 1
X-IronPort-AV: E=Sophos;i="5.17,478,1437462000"; d="scan'208";a="799377321"
Received: from pgsmsx102.gar.corp.intel.com ([10.221.44.80])
 by fmsmga002.fm.intel.com with ESMTP; 06 Sep 2015 01:37:12 -0700
Received: from shsmsx151.ccr.corp.intel.com (10.239.6.50) by
 PGSMSX102.gar.corp.intel.com (10.221.44.80) with Microsoft SMTP Server (TLS)
 id 14.3.224.2; Sun, 6 Sep 2015 16:36:13 +0800
Received: from shsmsx102.ccr.corp.intel.com ([169.254.2.206]) by
 SHSMSX151.ccr.corp.intel.com ([169.254.3.143]) with mapi id 14.03.0224.002;
 Sun, 6 Sep 2015 16:36:12 +0800
From: "Ouyang, Changchun" <changchun.ouyang@intel.com>
To: Stephen Hemminger <stephen@networkplumber.org>, "Xie, Huawei"
 <huawei.xie@intel.com>
Thread-Topic: [PATCH 3/4] virtio: use indirect ring elements
Thread-Index: AQHQ51TG5SiLKqn2L0KGjDKr+OzoSZ4vLHow
Date: Sun, 6 Sep 2015 08:36:10 +0000
Message-ID: <F52918179C57134FAEC9EA62FA2F962511CFDA18@shsmsx102.ccr.corp.intel.com>
References: <1441400308-5725-1-git-send-email-stephen@networkplumber.org>
 <1441400308-5725-4-git-send-email-stephen@networkplumber.org>
In-Reply-To: <1441400308-5725-4-git-send-email-stephen@networkplumber.org>
Accept-Language: zh-CN, en-US
Content-Language: en-US
X-MS-Has-Attach: 
X-MS-TNEF-Correlator: 
x-originating-ip: [10.239.127.40]
Content-Type: text/plain; charset="us-ascii"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0
Cc: "dev@dpdk.org" <dev@dpdk.org>
Subject: Re: [dpdk-dev] [PATCH 3/4] virtio: use indirect ring elements
X-BeenThere: dev@dpdk.org
X-Mailman-Version: 2.1.15
Precedence: list
List-Id: patches and discussions about DPDK <dev.dpdk.org>
List-Unsubscribe: <http://dpdk.org/ml/options/dev>,
 <mailto:dev-request@dpdk.org?subject=unsubscribe>
List-Archive: <http://dpdk.org/ml/archives/dev/>
List-Post: <mailto:dev@dpdk.org>
List-Help: <mailto:dev-request@dpdk.org?subject=help>
List-Subscribe: <http://dpdk.org/ml/listinfo/dev>,
 <mailto:dev-request@dpdk.org?subject=subscribe>
X-List-Received-Date: Sun, 06 Sep 2015 08:37:14 -0000



> -----Original Message-----
> From: Stephen Hemminger [mailto:stephen@networkplumber.org]
> Sent: Saturday, September 5, 2015 4:58 AM
> To: Xie, Huawei; Ouyang, Changchun
> Cc: dev@dpdk.org; Stephen Hemminger
> Subject: [PATCH 3/4] virtio: use indirect ring elements
>=20
> The virtio ring in QEMU/KVM is usually limited to 256 entries and the nor=
mal
> way that virtio driver was queuing mbufs required nsegs + 1 ring elements=
.
> By using the indirect ring element feature if available, each packet will=
 take
> only one ring slot even for multi-segment packets.
>=20
> Signed-off-by: Stephen Hemminger <stephen@networkplumber.org>
> ---
>  drivers/net/virtio/virtio_ethdev.c | 11 +++++---
> drivers/net/virtio/virtio_ethdev.h |  3 ++-
>  drivers/net/virtio/virtio_rxtx.c   | 51 ++++++++++++++++++++++++++++++-
> -------
>  drivers/net/virtio/virtqueue.h     |  8 ++++++
>  4 files changed, 57 insertions(+), 16 deletions(-)
>=20
> diff --git a/drivers/net/virtio/virtio_ethdev.c
> b/drivers/net/virtio/virtio_ethdev.c
> index 465d3cd..bcfb87b 100644
> --- a/drivers/net/virtio/virtio_ethdev.c
> +++ b/drivers/net/virtio/virtio_ethdev.c
> @@ -359,12 +359,15 @@ int virtio_dev_queue_setup(struct rte_eth_dev
> *dev,
>  	if (queue_type =3D=3D VTNET_TQ) {

Do we also need implement indirect ring elements for RX path?

>  		/*
>  		 * For each xmit packet, allocate a virtio_net_hdr
> +		 * and indirect ring elements
>  		 */
>  		snprintf(vq_name, sizeof(vq_name),
> "port%d_tvq%d_hdrzone",
> -			dev->data->port_id, queue_idx);
> -		vq->virtio_net_hdr_mz =3D
> rte_memzone_reserve_aligned(vq_name,
> -			vq_size * hw->vtnet_hdr_size,
> -			socket_id, 0, RTE_CACHE_LINE_SIZE);
> +			 dev->data->port_id, queue_idx);
> +
> +		vq->virtio_net_hdr_mz =3D
> +			rte_memzone_reserve_aligned(vq_name,
> +						    vq_size * sizeof(struct
> virtio_tx_region),
> +						    socket_id, 0,
> RTE_CACHE_LINE_SIZE);
>  		if (vq->virtio_net_hdr_mz =3D=3D NULL) {
>  			if (rte_errno =3D=3D EEXIST)
>  				vq->virtio_net_hdr_mz =3D
> diff --git a/drivers/net/virtio/virtio_ethdev.h
> b/drivers/net/virtio/virtio_ethdev.h
> index 9026d42..07a9265 100644
> --- a/drivers/net/virtio/virtio_ethdev.h
> +++ b/drivers/net/virtio/virtio_ethdev.h
> @@ -64,7 +64,8 @@
>  	 1u << VIRTIO_NET_F_CTRL_VQ	  |	\
>  	 1u << VIRTIO_NET_F_CTRL_RX	  |	\
>  	 1u << VIRTIO_NET_F_CTRL_VLAN	  |	\
> -	 1u << VIRTIO_NET_F_MRG_RXBUF)
> +	 1u << VIRTIO_NET_F_MRG_RXBUF     |	\
> +	 1u << VIRTIO_RING_F_INDIRECT_DESC)
>=20
>  /*
>   * CQ function prototype
> diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio=
_rxtx.c
> index dbe6665..8979695 100644
> --- a/drivers/net/virtio/virtio_rxtx.c
> +++ b/drivers/net/virtio/virtio_rxtx.c
> @@ -199,14 +199,15 @@ virtqueue_enqueue_recv_refill(struct virtqueue
> *vq, struct rte_mbuf *cookie)  }
>=20
>  static int
> -virtqueue_enqueue_xmit(struct virtqueue *txvq, struct rte_mbuf *cookie)
> +virtqueue_enqueue_xmit(struct virtqueue *txvq, struct rte_mbuf *cookie,
> +		       int use_indirect)
>  {
>  	struct vq_desc_extra *dxp;
>  	struct vring_desc *start_dp;
>  	uint16_t seg_num =3D cookie->nb_segs;
> -	uint16_t needed =3D 1 + seg_num;
> +	uint16_t needed =3D use_indirect ? 1 : 1 + seg_num;

Do we need check if seg_num > VIRTIO_MAX_TX_INDIRECT?
That mean one slot is not enough for the whole big packet even it is indire=
ct ring.

>  	uint16_t head_idx, idx;
> -	uint16_t head_size =3D txvq->hw->vtnet_hdr_size;
> +	unsigned long offs;
>=20
>  	if (unlikely(txvq->vq_free_cnt =3D=3D 0))
>  		return -ENOSPC;
> @@ -220,11 +221,26 @@ virtqueue_enqueue_xmit(struct virtqueue *txvq,
> struct rte_mbuf *cookie)
>  	dxp =3D &txvq->vq_descx[idx];
>  	dxp->cookie =3D (void *)cookie;
>  	dxp->ndescs =3D needed;
> -
>  	start_dp =3D txvq->vq_ring.desc;
> -	start_dp[idx].addr =3D
> -		txvq->virtio_net_hdr_mem + idx * head_size;
> -	start_dp[idx].len =3D (uint32_t)head_size;
> +
> +	if (use_indirect) {
> +		offs =3D offsetof(struct virtio_tx_region, tx_indir)
> +			+ idx * sizeof(struct virtio_tx_region);
> +
> +		start_dp[idx].addr =3D txvq->virtio_net_hdr_mem + offs;
> +		start_dp[idx].len =3D sizeof(struct vring_desc);
> +		start_dp[idx].flags =3D VRING_DESC_F_INDIRECT;
> +
> +		start_dp =3D (struct vring_desc *)
> +			((char *)txvq->virtio_net_hdr_mz->addr + offs);
> +		idx =3D 0;
> +	}
> +
> +	offs =3D offsetof(struct virtio_tx_region, tx_hdr)
> +		+ idx * sizeof(struct virtio_tx_region);
> +
> +	start_dp[idx].addr =3D txvq->virtio_net_hdr_mem + offs;
> +	start_dp[idx].len =3D txvq->hw->vtnet_hdr_size;
>  	start_dp[idx].flags =3D VRING_DESC_F_NEXT;
>=20
>  	for (; ((seg_num > 0) && (cookie !=3D NULL)); seg_num--) { @@ -236,7
> +252,12 @@ virtqueue_enqueue_xmit(struct virtqueue *txvq, struct
> rte_mbuf *cookie)
>  	}
>=20
>  	start_dp[idx].flags &=3D ~VRING_DESC_F_NEXT;
> -	idx =3D start_dp[idx].next;
> +
> +	if (use_indirect)
> +		idx =3D txvq->vq_ring.desc[head_idx].next;
> +	else
> +		idx =3D start_dp[idx].next;
> +
>  	txvq->vq_desc_head_idx =3D idx;
>  	if (txvq->vq_desc_head_idx =3D=3D VQ_RING_DESC_CHAIN_END)
>  		txvq->vq_desc_tail_idx =3D idx;
> @@ -760,7 +781,15 @@ virtio_xmit_pkts(void *tx_queue, struct rte_mbuf
> **tx_pkts, uint16_t nb_pkts)
>=20
>  	for (nb_tx =3D 0; nb_tx < nb_pkts; nb_tx++) {
>  		struct rte_mbuf *txm =3D tx_pkts[nb_tx];
> -		int need =3D txm->nb_segs - txvq->vq_free_cnt + 1;
> +		int use_indirect, slots, need;
> +
> +		use_indirect =3D vtpci_with_feature(txvq->hw,
> +
> VIRTIO_RING_F_INDIRECT_DESC)
> +			&& (txm->nb_segs < VIRTIO_MAX_TX_INDIRECT);
> +
> +		/* How many ring entries are needed to this Tx? */
> +		slots =3D use_indirect ? 1 : 1 + txm->nb_segs;
> +		need =3D slots - txvq->vq_free_cnt;
>=20
>  		/* Positive value indicates it need free vring descriptors */
>  		if (need > 0) {
> @@ -769,7 +798,7 @@ virtio_xmit_pkts(void *tx_queue, struct rte_mbuf
> **tx_pkts, uint16_t nb_pkts)
>  			need =3D RTE_MIN(need, (int)nb_used);
>=20
>  			virtio_xmit_cleanup(txvq, need);
> -			need =3D txm->nb_segs - txvq->vq_free_cnt + 1;
> +			need =3D slots - txvq->vq_free_cnt;
>  			if (unlikely(need > 0)) {
>  				PMD_TX_LOG(ERR,
>  					   "No free tx descriptors to transmit");
> @@ -787,7 +816,7 @@ virtio_xmit_pkts(void *tx_queue, struct rte_mbuf
> **tx_pkts, uint16_t nb_pkts)
>  		}
>=20
>  		/* Enqueue Packet buffers */
> -		error =3D virtqueue_enqueue_xmit(txvq, txm);
> +		error =3D virtqueue_enqueue_xmit(txvq, txm, use_indirect);
>  		if (unlikely(error)) {
>  			if (error =3D=3D ENOSPC)
>  				PMD_TX_LOG(ERR, "virtqueue_enqueue
> Free count =3D 0"); diff --git a/drivers/net/virtio/virtqueue.h
> b/drivers/net/virtio/virtqueue.h index 7789411..a9410b4 100644
> --- a/drivers/net/virtio/virtqueue.h
> +++ b/drivers/net/virtio/virtqueue.h
> @@ -237,6 +237,14 @@ struct virtio_net_hdr_mrg_rxbuf {
>  	uint16_t num_buffers; /**< Number of merged rx buffers */  };
>=20
> +/* Region reserved to allow for transmit header and indirect ring */
> +#define VIRTIO_MAX_TX_INDIRECT 8 struct virtio_tx_region {
> +	struct virtio_net_hdr_mrg_rxbuf tx_hdr;
> +	struct vring_desc tx_indir[VIRTIO_MAX_TX_INDIRECT]
> +			   __attribute__((__aligned__(16))); };
> +
>  /**
>   * Tell the backend not to interrupt us.
>   */
> --
> 2.1.4