Re: [dpdk-dev] [RFC PATCH 6/6] ixgbe: PMD for bifurc ixgbe net device

["Liang, Cunming" <cunming.liang@intel.com>]

> -----Original Message-----
> From: Richardson, Bruce
> Sent: Tuesday, November 25, 2014 10:34 PM
> To: Liang, Cunming
> Cc: dev@dpdk.org
> Subject: Re: [dpdk-dev] [RFC PATCH 6/6] ixgbe: PMD for bifurc ixgbe net device
> 
> On Tue, Nov 25, 2014 at 10:11:22PM +0800, Cunming Liang wrote:
> > Signed-off-by: Cunming Liang <cunming.liang@intel.com>
> > ---
> >  lib/librte_pmd_ixgbe/Makefile          |  13 +-
> >  lib/librte_pmd_ixgbe/ixgbe_bifurcate.c | 303
> +++++++++++++++++++++++++++++++++
> >  lib/librte_pmd_ixgbe/ixgbe_bifurcate.h |  57 +++++++
> >  lib/librte_pmd_ixgbe/ixgbe_rxtx.c      |  40 ++++-
> >  lib/librte_pmd_ixgbe/ixgbe_rxtx.h      |  10 ++
> >  5 files changed, 415 insertions(+), 8 deletions(-)
> >  create mode 100644 lib/librte_pmd_ixgbe/ixgbe_bifurcate.c
> >  create mode 100644 lib/librte_pmd_ixgbe/ixgbe_bifurcate.h
> >
> 
> These changes are the ones that I'm not too sure about. I'd prefer if all
> material for the bifurcated driver be kept within the librte_pmd_bifurc directory.
[Liang, Cunming] I haven't a librte_pmd_bifurc library. 
So far the purpose of librte_bifurc is for device scan, not used as a pmd.
During driver probe, depend on device id, it asks for correct pmd from 'librte_pmd_ixgbe, librte_pmd_i40e'.

> Is it possible to leave ixgbe largely unmodified and simply have the new
> bifurcated driver pull in the needed ixgbe (and later i40e) functions at
> compile time i.e. refer from one Makefile to the sources in the other
> driver's directory?
[Liang, Cunming] Nice point. If we have single directory gathering all direct ring access.
e.g. We have aka "librte_pmd_bifurc", inside it, we'll have bifurc_ixgbe, bifurc_i40e, ...
Each of them still depend on other libraries like librte_pmd_ixgbe/librte_pmd_i40e.
We may remove the internal dependence inside one pmd driver, but between libraries we add more.

> My thinking is that the bifurcated driver is so significantly different in
> the way it works, and the limits on it's functionality e.g. no direct filter
> support or queue management, that it's best kept completely separate and only
> "borrow" the needed descriptor read/write functions from the other drivers as is
> needed.
> 
> Just my 2c. I'm curious as to what others think.
> 
> /Bruce
> 
> > diff --git a/lib/librte_pmd_ixgbe/Makefile b/lib/librte_pmd_ixgbe/Makefile
> > index 3588047..6867f17 100644
> > --- a/lib/librte_pmd_ixgbe/Makefile
> > +++ b/lib/librte_pmd_ixgbe/Makefile
> > @@ -37,7 +37,7 @@ include $(RTE_SDK)/mk/rte.vars.mk
> >  LIB = librte_pmd_ixgbe.a
> >
> >  CFLAGS += -O3
> > -CFLAGS += $(WERROR_FLAGS)
> > +CFLAGS += $(WERROR_FLAGS) -Wno-cast-qual
> >
> >  ifeq ($(CC), icc)
> >  #
> > @@ -108,10 +108,21 @@ SRCS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) +=
> ixgbe_bypass.c
> >  SRCS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += ixgbe_82599_bypass.c
> >  endif
> >
> > +ifeq ($(CONFIG_RTE_LIBRTE_BIFURC),y)
> > +ifeq ($(CONFIG_RTE_EXEC_ENV_LINUXAPP),y)
> > +SRCS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += ixgbe_bifurcate.c
> > +endif
> > +endif
> >
> >  # this lib depends upon:
> >  DEPDIRS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += lib/librte_eal
> lib/librte_ether
> >  DEPDIRS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += lib/librte_mempool
> lib/librte_mbuf
> >  DEPDIRS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += lib/librte_net
> lib/librte_malloc
> > +ifeq ($(CONFIG_RTE_LIBRTE_BIFURC),y)
> > +ifeq ($(CONFIG_RTE_EXEC_ENV_LINUXAPP),y)
> > +DEPDIRS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += lib/librte_bifurc
> > +endif
> > +endif
> > +
> >
> >  include $(RTE_SDK)/mk/rte.lib.mk
> > diff --git a/lib/librte_pmd_ixgbe/ixgbe_bifurcate.c
> b/lib/librte_pmd_ixgbe/ixgbe_bifurcate.c
> > new file mode 100644
> > index 0000000..84c445a
> > --- /dev/null
> > +++ b/lib/librte_pmd_ixgbe/ixgbe_bifurcate.c
> > @@ -0,0 +1,303 @@
> > +/*-
> > + *   BSD LICENSE
> > + *
> > + *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
> > + *   All rights reserved.
> > + *
> > + *   Redistribution and use in source and binary forms, with or without
> > + *   modification, are permitted provided that the following conditions
> > + *   are met:
> > + *
> > + *     * Redistributions of source code must retain the above copyright
> > + *       notice, this list of conditions and the following disclaimer.
> > + *     * Redistributions in binary form must reproduce the above copyright
> > + *       notice, this list of conditions and the following disclaimer in
> > + *       the documentation and/or other materials provided with the
> > + *       distribution.
> > + *     * Neither the name of Intel Corporation nor the names of its
> > + *       contributors may be used to endorse or promote products derived
> > + *       from this software without specific prior written permission.
> > + *
> > + *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
> CONTRIBUTORS
> > + *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT
> NOT
> > + *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
> FITNESS FOR
> > + *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
> COPYRIGHT
> > + *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
> INCIDENTAL,
> > + *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
> NOT
> > + *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
> OF USE,
> > + *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
> ON ANY
> > + *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> > + *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
> THE USE
> > + *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
> DAMAGE.
> > + */
> > +
> > +#include <rte_eal.h>
> > +#include <rte_malloc.h>
> > +#include <rte_memzone.h>
> > +#include <rte_dev.h>
> > +#include <rte_pci.h>
> > +#include <rte_ethdev.h>
> > +#include <rte_kvargs.h>
> > +#include <rte_bifurc.h>
> > +
> > +#include <linux/if_ether.h>
> > +#include <linux/if_packet.h>
> > +#include <arpa/inet.h>
> > +#include <net/if.h>
> > +#include <sys/types.h>
> > +#include <sys/socket.h>
> > +#include <sys/ioctl.h>
> > +#include <sys/mman.h>
> > +#include <unistd.h>
> > +#include <poll.h>
> > +#include <errno.h>
> > +
> > +#include "ixgbe_logs.h"
> > +#include "ixgbe_ethdev.h"
> > +#include "ixgbe/ixgbe_api.h"
> > +
> > +#include <rte_mbuf.h>
> > +#include <rte_bifurc.h>
> > +#include "ixgbe_rxtx.h"
> > +
> > +static int
> > +ixgbe_dev_bfc_configure(struct rte_eth_dev *dev __rte_unused)
> > +{
> > +	return 0;
> > +}
> > +
> > +static void
> > +ixgbe_dev_bfc_info(struct rte_eth_dev *dev,
> > +		   struct rte_eth_dev_info *dev_info)
> > +{
> > +	rte_bifurc_ethdev_get_info(dev, dev_info);
> > +}
> > +
> > +static void
> > +ixgbe_dev_bfc_stats_get(__rte_unused struct rte_eth_dev *dev,
> > +			__rte_unused struct rte_eth_stats *igb_stats)
> > +{
> > +	return;
> > +}
> > +
> > +static int
> > +ixgbe_dev_bfc_start(struct rte_eth_dev *dev)
> > +{
> > +	int err;
> > +
> > +	/* initialize transmission unit */
> > +	ixgbe_dev_tx_init(dev);
> > +
> > +	/* This can fail when allocating mbufs for descriptor rings */
> > +	err = ixgbe_dev_rx_init(dev);
> > +	if (err) {
> > +		PMD_INIT_LOG(ERR, "Unable to initialize RX hardware\n");
> > +		goto error;
> > +	}
> > +
> > +	ixgbe_dev_rxtx_start(dev);
> > +
> > +	return 0;
> > +
> > +error:
> > +	PMD_INIT_LOG(ERR, "failure in ixgbe_dev_start(): %d", err);
> > +	ixgbe_dev_clear_queues(dev);
> > +	return -EIO;
> > +}
> > +
> > +static void
> > +ixgbe_dev_bfc_stop(struct rte_eth_dev *dev)
> > +{
> > +	unsigned i;
> > +
> > +	PMD_INIT_FUNC_TRACE();
> > +
> > +	for (i = 0; i < dev->data->nb_tx_queues; i++)
> > +		ixgbe_dev_tx_queue_stop(dev, i);
> > +
> > +	for (i = 0; i < dev->data->nb_rx_queues; i++)
> > +		ixgbe_dev_rx_queue_stop(dev, i);
> > +}
> > +
> > +static void
> > +ixgbe_dev_bfc_close(struct rte_eth_dev *dev)
> > +{
> > +	ixgbe_dev_bfc_stop(dev);
> > +
> > +	rte_bifurc_ethdev_free(dev);
> > +}
> > +
> > +static inline int
> > +rte_ixgbe_dev_atomic_write_link_status(struct rte_eth_dev *dev,
> > +				struct rte_eth_link *link)
> > +{
> > +	struct rte_eth_link *dst = &(dev->data->dev_link);
> > +	struct rte_eth_link *src = link;
> > +
> > +	if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
> > +					*(uint64_t *)src) == 0)
> > +		return -1;
> > +
> > +	return 0;
> > +}
> > +
> > +static int
> > +ixgbe_dev_bfc_link_update(__rte_unused struct rte_eth_dev *dev,
> > +			  __rte_unused int wait_to_complete)
> > +{
> > +	struct rte_eth_link link;
> > +
> > +	link.link_status = 1;
> > +	link.link_duplex = ETH_LINK_FULL_DUPLEX;
> > +	link.link_speed = ETH_LINK_SPEED_10000;
> > +
> > +	rte_ixgbe_dev_atomic_write_link_status(dev, &link);
> > +
> > +	return 0;
> > +}
> > +
> > +static int
> > +ixgbe_dev_bfc_rx_queue_setup(struct rte_eth_dev *dev,
> > +			 uint16_t queue_idx,
> > +			 uint16_t nb_desc,
> > +			 unsigned int socket_id,
> > +			 const struct rte_eth_rxconf *rx_conf,
> > +			 struct rte_mempool *mp)
> > +{
> > +	uint16_t offset = rte_bifurc_qp_base(dev);
> > +	return ixgbe_dev_rxq_setup(dev, queue_idx, offset, nb_desc,
> > +				   socket_id, rx_conf, mp);
> > +}
> > +
> > +static int
> > +ixgbe_dev_bfc_tx_queue_setup(struct rte_eth_dev *dev,
> > +			     uint16_t queue_idx,
> > +			     uint16_t nb_desc,
> > +			     unsigned int socket_id,
> > +			     const struct rte_eth_txconf *tx_conf)
> > +{
> > +	uint16_t offset = rte_bifurc_qp_base(dev);
> > +	return ixgbe_dev_txq_setup(dev, queue_idx, offset,
> > +				   nb_desc, socket_id, tx_conf);
> > +}
> > +
> > +static struct eth_dev_ops ixgbe_bifurc_ops = {
> > +	.dev_start = ixgbe_dev_bfc_start,
> > +	.dev_stop = ixgbe_dev_bfc_stop,
> > +	.dev_close = ixgbe_dev_bfc_close,
> > +	.dev_configure = ixgbe_dev_bfc_configure,
> > +	.dev_infos_get = ixgbe_dev_bfc_info,
> > +	.rx_queue_setup = ixgbe_dev_bfc_rx_queue_setup,
> > +	.tx_queue_setup = ixgbe_dev_bfc_tx_queue_setup,
> > +	.rx_queue_release = ixgbe_dev_rx_queue_release,
> > +	.tx_queue_release = ixgbe_dev_tx_queue_release,
> > +	.link_update = ixgbe_dev_bfc_link_update,
> > +	.stats_get = ixgbe_dev_bfc_stats_get,
> > +	.stats_reset = NULL,
> > +};
> > +
> > +static int
> > +eth_ixgbe_bifurc_dev_init(struct eth_driver *eth_drv __rte_unused,
> > +			  struct rte_eth_dev *eth_dev)
> > +{
> > +	struct rte_pci_device *pci_dev;
> > +	struct ixgbe_hw *hw =
> > +		IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
> > +	int diag;
> > +
> > +	PMD_INIT_FUNC_TRACE();
> > +
> > +	eth_dev->dev_ops = &ixgbe_bifurc_ops;
> > +	eth_dev->rx_pkt_burst = &ixgbe_recv_pkts;
> > +	eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts;
> > +
> > +	/* for secondary processes, we don't initialise any further as primary
> > +	 * has already done this work. Only check we don't need a different
> > +	 * RX function */
> > +	if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
> > +		if (eth_dev->data->scattered_rx)
> > +			eth_dev->rx_pkt_burst = ixgbe_recv_scattered_pkts;
> > +		return 0;
> > +	}
> > +	pci_dev = eth_dev->pci_dev;
> > +
> > +	/* Vendor and Device ID need to be set before init of shared code */
> > +	hw->device_id = pci_dev->id.device_id;
> > +	hw->vendor_id = pci_dev->id.vendor_id;
> > +	hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
> > +
> > +#ifdef RTE_LIBRTE_IXGBE_ALLOW_UNSUPPORTED_SFP
> > +	hw->allow_unsupported_sfp = 1;
> > +#endif
> > +
> > +	/* Initialize the shared code (base driver) */
> > +#ifdef RTE_NIC_BYPASS
> > +	diag = ixgbe_bypass_init_shared_code(hw);
> > +#else
> > +	diag = ixgbe_init_shared_code(hw);
> > +#endif /* RTE_NIC_BYPASS */
> > +
> > +	if (diag != IXGBE_SUCCESS) {
> > +		PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
> > +		return -EIO;
> > +	}
> > +
> > +	/* Allocate memory for storing MAC addresses */
> > +	eth_dev->data->mac_addrs = rte_zmalloc("ixgbe", ETHER_ADDR_LEN *
> > +			hw->mac.num_rar_entries, 0);
> > +	if (eth_dev->data->mac_addrs == NULL) {
> > +		PMD_INIT_LOG(ERR,
> > +			"Failed to allocate %u bytes needed to store "
> > +			"MAC addresses",
> > +			ETHER_ADDR_LEN * hw->mac.num_rar_entries);
> > +		return -ENOMEM;
> > +	}
> > +	rte_bifurc_mac_addr(eth_dev, &eth_dev->data->mac_addrs[0]);
> > +
> > +	return diag;
> > +}
> > +
> > +/*
> > + * The set of PCI devices this driver supports
> > + */
> > +static struct rte_pci_id pci_id_ixgbe_map[] = {
> > +
> > +#define RTE_PCI_DEV_ID_DECL_IXGBE(vend, dev) {RTE_PCI_DEVICE(vend,
> dev)},
> > +#define RTE_PCI_DEV_ID_DECL_IXGBEVF(vend, dev) {RTE_PCI_DEVICE(vend,
> dev)},
> > +#include "rte_pci_dev_ids.h"
> > +
> > +{ .vendor_id = 0, /* sentinel */ },
> > +};
> > +
> > +static struct eth_driver rte_ixgbe_bifurc_pmd = {
> > +	{
> > +		.name = "rte_ixgbe_bifurc_pmd",
> > +		.id_table = pci_id_ixgbe_map,
> > +		.drv_flags = RTE_PCI_DRV_NEED_MAPPING |
> > +			RTE_PCI_DRV_BIFURC,
> > +	},
> > +	.eth_dev_init = eth_ixgbe_bifurc_dev_init,
> > +	.dev_private_size = sizeof(struct ixgbe_adapter),
> > +};
> > +
> > +/*
> > + * Driver initialization routine.
> > + * Invoked once at EAL init time.
> > + * Register itself as the [Poll Mode] Driver of PCI IXGBE devices.
> > + */
> > +static int
> > +rte_ixgbe_bifurc_pmd_init(const char *name __rte_unused,
> > +			  const char *params __rte_unused)
> > +{
> > +	PMD_INIT_FUNC_TRACE();
> > +
> > +	rte_eth_driver_register(&rte_ixgbe_bifurc_pmd);
> > +	return 0;
> > +}
> > +
> > +static struct rte_driver rte_ixgbe_bifurc_driver = {
> > +	.type = PMD_PDEV,
> > +	.init = rte_ixgbe_bifurc_pmd_init,
> > +};
> > +
> > +PMD_REGISTER_DRIVER(rte_ixgbe_bifurc_driver);
> > diff --git a/lib/librte_pmd_ixgbe/ixgbe_bifurcate.h
> b/lib/librte_pmd_ixgbe/ixgbe_bifurcate.h
> > new file mode 100644
> > index 0000000..d40b21d
> > --- /dev/null
> > +++ b/lib/librte_pmd_ixgbe/ixgbe_bifurcate.h
> > @@ -0,0 +1,57 @@
> > +/*-
> > + *   BSD LICENSE
> > + *
> > + *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
> > + *   All rights reserved.
> > + *
> > + *   Redistribution and use in source and binary forms, with or without
> > + *   modification, are permitted provided that the following conditions
> > + *   are met:
> > + *
> > + *     * Redistributions of source code must retain the above copyright
> > + *       notice, this list of conditions and the following disclaimer.
> > + *     * Redistributions in binary form must reproduce the above copyright
> > + *       notice, this list of conditions and the following disclaimer in
> > + *       the documentation and/or other materials provided with the
> > + *       distribution.
> > + *     * Neither the name of Intel Corporation nor the names of its
> > + *       contributors may be used to endorse or promote products derived
> > + *       from this software without specific prior written permission.
> > + *
> > + *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
> CONTRIBUTORS
> > + *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT
> NOT
> > + *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
> FITNESS FOR
> > + *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
> COPYRIGHT
> > + *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
> INCIDENTAL,
> > + *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
> NOT
> > + *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
> OF USE,
> > + *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
> ON ANY
> > + *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> > + *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
> THE USE
> > + *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
> DAMAGE.
> > + */
> > +
> > +#ifndef _IXGBE_BIFFURCATE_H_
> > +#define _IXGBE_BIFFURCATE_H_
> > +
> > +#ifdef __cplusplus
> > +extern "C" {
> > +#endif
> > +
> > +#define IXGBE_82599_MAX_RX_QUEUES 128
> > +
> > +#define RTE_PMD_PACKET_RING_SPLITOFF_LOWER_LIMIT 32
> > +#define RTE_PMD_PACKET_MAX_RING_PAIRS
> IXGBE_82599_MAX_RX_QUEUES
> > +
> > +
> > +/**
> > + * For use by the EAL only. Called as part of EAL init to set up any dummy NICs
> > + * configured on command line.
> > + */
> > +int rte_ixgbe_bfc_pmd_init(const char *name, const char *params);
> > +
> > +#ifdef __cplusplus
> > +}
> > +#endif
> > +
> > +#endif
> > diff --git a/lib/librte_pmd_ixgbe/ixgbe_rxtx.c
> b/lib/librte_pmd_ixgbe/ixgbe_rxtx.c
> > index e240376..2d32907 100644
> > --- a/lib/librte_pmd_ixgbe/ixgbe_rxtx.c
> > +++ b/lib/librte_pmd_ixgbe/ixgbe_rxtx.c
> > @@ -100,6 +100,12 @@ rte_rxmbuf_alloc(struct rte_mempool *mp)
> >  	return (m);
> >  }
> >
> > +static inline uint16_t
> > +ixgbe_dev_queue_offset(struct rte_eth_dev *dev)
> > +{
> > +	return (RTE_ETH_DEV_SRIOV(dev).active == 0) ?
> > +		0 : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx;
> > +}
> >
> >  #if 1
> >  #define RTE_PMD_USE_PREFETCH
> > @@ -1726,6 +1732,17 @@ ixgbe_dev_tx_queue_setup(struct rte_eth_dev
> *dev,
> >  			 unsigned int socket_id,
> >  			 const struct rte_eth_txconf *tx_conf)
> >  {
> > +	uint16_t offset = ixgbe_dev_queue_offset(dev);
> > +	return ixgbe_dev_txq_setup(dev, queue_idx, offset,
> > +				   nb_desc, socket_id, tx_conf);
> > +}
> > +
> > +int
> > +ixgbe_dev_txq_setup(struct rte_eth_dev *dev,
> > +		    uint16_t queue_idx, uint16_t offset,
> > +		    uint16_t nb_desc, unsigned int socket_id,
> > +		    const struct rte_eth_txconf *tx_conf)
> > +{
> >  	const struct rte_memzone *tz;
> >  	struct igb_tx_queue *txq;
> >  	struct ixgbe_hw     *hw;
> > @@ -1849,8 +1866,7 @@ ixgbe_dev_tx_queue_setup(struct rte_eth_dev
> *dev,
> >  	txq->hthresh = tx_conf->tx_thresh.hthresh;
> >  	txq->wthresh = tx_conf->tx_thresh.wthresh;
> >  	txq->queue_id = queue_idx;
> > -	txq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ?
> > -		queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx);
> > +	txq->reg_idx  = queue_idx + offset;
> >  	txq->port_id = dev->data->port_id;
> >  	txq->txq_flags = tx_conf->txq_flags;
> >  	txq->ops = &def_txq_ops;
> > @@ -2083,6 +2099,18 @@ ixgbe_dev_rx_queue_setup(struct rte_eth_dev
> *dev,
> >  			 const struct rte_eth_rxconf *rx_conf,
> >  			 struct rte_mempool *mp)
> >  {
> > +	uint16_t offset = ixgbe_dev_queue_offset(dev);
> > +	return ixgbe_dev_rxq_setup(dev, queue_idx, offset, nb_desc,
> > +				   socket_id, rx_conf, mp);
> > +}
> > +
> > +int
> > +ixgbe_dev_rxq_setup(struct rte_eth_dev *dev,
> > +		    uint16_t queue_idx, uint16_t offset,
> > +		    uint16_t nb_desc, unsigned int socket_id,
> > +		    const struct rte_eth_rxconf *rx_conf,
> > +		    struct rte_mempool *mp)
> > +{
> >  	const struct rte_memzone *rz;
> >  	struct igb_rx_queue *rxq;
> >  	struct ixgbe_hw     *hw;
> > @@ -2118,8 +2146,7 @@ ixgbe_dev_rx_queue_setup(struct rte_eth_dev
> *dev,
> >  	rxq->nb_rx_desc = nb_desc;
> >  	rxq->rx_free_thresh = rx_conf->rx_free_thresh;
> >  	rxq->queue_id = queue_idx;
> > -	rxq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ?
> > -		queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx);
> > +	rxq->reg_idx  =	queue_idx + offset;
> >  	rxq->port_id = dev->data->port_id;
> >  	rxq->crc_len = (uint8_t) ((dev->data->dev_conf.rxmode.hw_strip_crc) ?
> >  							0 : ETHER_CRC_LEN);
> > @@ -3402,9 +3429,9 @@ ixgbe_dev_rx_init(struct rte_eth_dev *dev)
> >  	uint32_t fctrl;
> >  	uint32_t hlreg0;
> >  	uint32_t maxfrs;
> > -	uint32_t srrctl;
> >  	uint32_t rdrxctl;
> >  	uint32_t rxcsum;
> > +	uint32_t srrctl;
> >  	uint16_t buf_size;
> >  	uint16_t i;
> >
> > @@ -3684,9 +3711,9 @@ ixgbe_dev_rxtx_start(struct rte_eth_dev *dev)
> >  	struct ixgbe_hw     *hw;
> >  	struct igb_tx_queue *txq;
> >  	struct igb_rx_queue *rxq;
> > -	uint32_t txdctl;
> >  	uint32_t dmatxctl;
> >  	uint32_t rxctrl;
> > +	uint32_t txdctl;
> >  	uint16_t i;
> >
> >  	PMD_INIT_FUNC_TRACE();
> > @@ -3731,7 +3758,6 @@ ixgbe_dev_rxtx_start(struct rte_eth_dev *dev)
> >  	if (hw->mac.type == ixgbe_mac_82599EB &&
> >  			dev->data->dev_conf.lpbk_mode == IXGBE_LPBK_82599_TX_RX)
> >  		ixgbe_setup_loopback_link_82599(hw);
> > -
> >  }
> >
> >  /*
> > diff --git a/lib/librte_pmd_ixgbe/ixgbe_rxtx.h
> b/lib/librte_pmd_ixgbe/ixgbe_rxtx.h
> > index eb89715..aeffb5f 100644
> > --- a/lib/librte_pmd_ixgbe/ixgbe_rxtx.h
> > +++ b/lib/librte_pmd_ixgbe/ixgbe_rxtx.h
> > @@ -243,6 +243,16 @@ struct ixgbe_txq_ops {
> >  			 IXGBE_ADVTXD_DCMD_DEXT |\
> >  			 IXGBE_ADVTXD_DCMD_EOP)
> >
> > +int ixgbe_dev_txq_setup(struct rte_eth_dev *dev,
> > +			uint16_t queue_idx, uint16_t offset,
> > +			uint16_t nb_desc, unsigned int socket_id,
> > +			const struct rte_eth_txconf *tx_conf);
> > +int ixgbe_dev_rxq_setup(struct rte_eth_dev *dev,
> > +			uint16_t queue_idx, uint16_t offset,
> > +			uint16_t nb_desc, unsigned int socket_id,
> > +			const struct rte_eth_rxconf *rx_conf,
> > +			struct rte_mempool *mp);
> > +
> >  #ifdef RTE_IXGBE_INC_VECTOR
> >  uint16_t ixgbe_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
> >  		uint16_t nb_pkts);
> > --
> > 1.8.1.4
> >