Re: [dpdk-dev] [RFC PATCH 6/6] ixgbe: PMD for bifurc ixgbe net device

[Bruce Richardson <bruce.richardson@intel.com>]

On Tue, Nov 25, 2014 at 10:11:22PM +0800, Cunming Liang wrote:
> Signed-off-by: Cunming Liang <cunming.liang@intel.com>
> ---
>  lib/librte_pmd_ixgbe/Makefile          |  13 +-
>  lib/librte_pmd_ixgbe/ixgbe_bifurcate.c | 303 +++++++++++++++++++++++++++++++++
>  lib/librte_pmd_ixgbe/ixgbe_bifurcate.h |  57 +++++++
>  lib/librte_pmd_ixgbe/ixgbe_rxtx.c      |  40 ++++-
>  lib/librte_pmd_ixgbe/ixgbe_rxtx.h      |  10 ++
>  5 files changed, 415 insertions(+), 8 deletions(-)
>  create mode 100644 lib/librte_pmd_ixgbe/ixgbe_bifurcate.c
>  create mode 100644 lib/librte_pmd_ixgbe/ixgbe_bifurcate.h
> 

These changes are the ones that I'm not too sure about. I'd prefer if all
material for the bifurcated driver be kept within the librte_pmd_bifurc directory.
Is it possible to leave ixgbe largely unmodified and simply have the new
bifurcated driver pull in the needed ixgbe (and later i40e) functions at
compile time i.e. refer from one Makefile to the sources in the other
driver's directory?
My thinking is that the bifurcated driver is so significantly different in
the way it works, and the limits on it's functionality e.g. no direct filter
support or queue management, that it's best kept completely separate and only
"borrow" the needed descriptor read/write functions from the other drivers as is
needed.

Just my 2c. I'm curious as to what others think.

/Bruce

> diff --git a/lib/librte_pmd_ixgbe/Makefile b/lib/librte_pmd_ixgbe/Makefile
> index 3588047..6867f17 100644
> --- a/lib/librte_pmd_ixgbe/Makefile
> +++ b/lib/librte_pmd_ixgbe/Makefile
> @@ -37,7 +37,7 @@ include $(RTE_SDK)/mk/rte.vars.mk
>  LIB = librte_pmd_ixgbe.a
>  
>  CFLAGS += -O3
> -CFLAGS += $(WERROR_FLAGS)
> +CFLAGS += $(WERROR_FLAGS) -Wno-cast-qual
>  
>  ifeq ($(CC), icc)
>  #
> @@ -108,10 +108,21 @@ SRCS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += ixgbe_bypass.c
>  SRCS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += ixgbe_82599_bypass.c
>  endif
>  
> +ifeq ($(CONFIG_RTE_LIBRTE_BIFURC),y)
> +ifeq ($(CONFIG_RTE_EXEC_ENV_LINUXAPP),y)
> +SRCS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += ixgbe_bifurcate.c
> +endif
> +endif
>  
>  # this lib depends upon:
>  DEPDIRS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += lib/librte_eal lib/librte_ether
>  DEPDIRS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += lib/librte_mempool lib/librte_mbuf
>  DEPDIRS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += lib/librte_net lib/librte_malloc
> +ifeq ($(CONFIG_RTE_LIBRTE_BIFURC),y)
> +ifeq ($(CONFIG_RTE_EXEC_ENV_LINUXAPP),y)
> +DEPDIRS-$(CONFIG_RTE_LIBRTE_IXGBE_PMD) += lib/librte_bifurc
> +endif
> +endif
> +
>  
>  include $(RTE_SDK)/mk/rte.lib.mk
> diff --git a/lib/librte_pmd_ixgbe/ixgbe_bifurcate.c b/lib/librte_pmd_ixgbe/ixgbe_bifurcate.c
> new file mode 100644
> index 0000000..84c445a
> --- /dev/null
> +++ b/lib/librte_pmd_ixgbe/ixgbe_bifurcate.c
> @@ -0,0 +1,303 @@
> +/*-
> + *   BSD LICENSE
> + *
> + *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
> + *   All rights reserved.
> + *
> + *   Redistribution and use in source and binary forms, with or without
> + *   modification, are permitted provided that the following conditions
> + *   are met:
> + *
> + *     * Redistributions of source code must retain the above copyright
> + *       notice, this list of conditions and the following disclaimer.
> + *     * Redistributions in binary form must reproduce the above copyright
> + *       notice, this list of conditions and the following disclaimer in
> + *       the documentation and/or other materials provided with the
> + *       distribution.
> + *     * Neither the name of Intel Corporation nor the names of its
> + *       contributors may be used to endorse or promote products derived
> + *       from this software without specific prior written permission.
> + *
> + *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> + *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> + *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> + *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> + *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> + *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> + *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> + *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> + *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> + *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> + *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> + */
> +
> +#include <rte_eal.h>
> +#include <rte_malloc.h>
> +#include <rte_memzone.h>
> +#include <rte_dev.h>
> +#include <rte_pci.h>
> +#include <rte_ethdev.h>
> +#include <rte_kvargs.h>
> +#include <rte_bifurc.h>
> +
> +#include <linux/if_ether.h>
> +#include <linux/if_packet.h>
> +#include <arpa/inet.h>
> +#include <net/if.h>
> +#include <sys/types.h>
> +#include <sys/socket.h>
> +#include <sys/ioctl.h>
> +#include <sys/mman.h>
> +#include <unistd.h>
> +#include <poll.h>
> +#include <errno.h>
> +
> +#include "ixgbe_logs.h"
> +#include "ixgbe_ethdev.h"
> +#include "ixgbe/ixgbe_api.h"
> +
> +#include <rte_mbuf.h>
> +#include <rte_bifurc.h>
> +#include "ixgbe_rxtx.h"
> +
> +static int
> +ixgbe_dev_bfc_configure(struct rte_eth_dev *dev __rte_unused)
> +{
> +	return 0;
> +}
> +
> +static void
> +ixgbe_dev_bfc_info(struct rte_eth_dev *dev,
> +		   struct rte_eth_dev_info *dev_info)
> +{
> +	rte_bifurc_ethdev_get_info(dev, dev_info);
> +}
> +
> +static void
> +ixgbe_dev_bfc_stats_get(__rte_unused struct rte_eth_dev *dev,
> +			__rte_unused struct rte_eth_stats *igb_stats)
> +{
> +	return;
> +}
> +
> +static int
> +ixgbe_dev_bfc_start(struct rte_eth_dev *dev)
> +{
> +	int err;
> +
> +	/* initialize transmission unit */
> +	ixgbe_dev_tx_init(dev);
> +
> +	/* This can fail when allocating mbufs for descriptor rings */
> +	err = ixgbe_dev_rx_init(dev);
> +	if (err) {
> +		PMD_INIT_LOG(ERR, "Unable to initialize RX hardware\n");
> +		goto error;
> +	}
> +
> +	ixgbe_dev_rxtx_start(dev);
> +
> +	return 0;
> +
> +error:
> +	PMD_INIT_LOG(ERR, "failure in ixgbe_dev_start(): %d", err);
> +	ixgbe_dev_clear_queues(dev);
> +	return -EIO;
> +}
> +
> +static void
> +ixgbe_dev_bfc_stop(struct rte_eth_dev *dev)
> +{
> +	unsigned i;
> +
> +	PMD_INIT_FUNC_TRACE();
> +
> +	for (i = 0; i < dev->data->nb_tx_queues; i++)
> +		ixgbe_dev_tx_queue_stop(dev, i);
> +
> +	for (i = 0; i < dev->data->nb_rx_queues; i++)
> +		ixgbe_dev_rx_queue_stop(dev, i);
> +}
> +
> +static void
> +ixgbe_dev_bfc_close(struct rte_eth_dev *dev)
> +{
> +	ixgbe_dev_bfc_stop(dev);
> +
> +	rte_bifurc_ethdev_free(dev);
> +}
> +
> +static inline int
> +rte_ixgbe_dev_atomic_write_link_status(struct rte_eth_dev *dev,
> +				struct rte_eth_link *link)
> +{
> +	struct rte_eth_link *dst = &(dev->data->dev_link);
> +	struct rte_eth_link *src = link;
> +
> +	if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
> +					*(uint64_t *)src) == 0)
> +		return -1;
> +
> +	return 0;
> +}
> +
> +static int
> +ixgbe_dev_bfc_link_update(__rte_unused struct rte_eth_dev *dev,
> +			  __rte_unused int wait_to_complete)
> +{
> +	struct rte_eth_link link;
> +
> +	link.link_status = 1;
> +	link.link_duplex = ETH_LINK_FULL_DUPLEX;
> +	link.link_speed = ETH_LINK_SPEED_10000;
> +
> +	rte_ixgbe_dev_atomic_write_link_status(dev, &link);
> +
> +	return 0;
> +}
> +
> +static int
> +ixgbe_dev_bfc_rx_queue_setup(struct rte_eth_dev *dev,
> +			 uint16_t queue_idx,
> +			 uint16_t nb_desc,
> +			 unsigned int socket_id,
> +			 const struct rte_eth_rxconf *rx_conf,
> +			 struct rte_mempool *mp)
> +{
> +	uint16_t offset = rte_bifurc_qp_base(dev);
> +	return ixgbe_dev_rxq_setup(dev, queue_idx, offset, nb_desc,
> +				   socket_id, rx_conf, mp);
> +}
> +
> +static int
> +ixgbe_dev_bfc_tx_queue_setup(struct rte_eth_dev *dev,
> +			     uint16_t queue_idx,
> +			     uint16_t nb_desc,
> +			     unsigned int socket_id,
> +			     const struct rte_eth_txconf *tx_conf)
> +{
> +	uint16_t offset = rte_bifurc_qp_base(dev);
> +	return ixgbe_dev_txq_setup(dev, queue_idx, offset,
> +				   nb_desc, socket_id, tx_conf);
> +}
> +
> +static struct eth_dev_ops ixgbe_bifurc_ops = {
> +	.dev_start = ixgbe_dev_bfc_start,
> +	.dev_stop = ixgbe_dev_bfc_stop,
> +	.dev_close = ixgbe_dev_bfc_close,
> +	.dev_configure = ixgbe_dev_bfc_configure,
> +	.dev_infos_get = ixgbe_dev_bfc_info,
> +	.rx_queue_setup = ixgbe_dev_bfc_rx_queue_setup,
> +	.tx_queue_setup = ixgbe_dev_bfc_tx_queue_setup,
> +	.rx_queue_release = ixgbe_dev_rx_queue_release,
> +	.tx_queue_release = ixgbe_dev_tx_queue_release,
> +	.link_update = ixgbe_dev_bfc_link_update,
> +	.stats_get = ixgbe_dev_bfc_stats_get,
> +	.stats_reset = NULL,
> +};
> +
> +static int
> +eth_ixgbe_bifurc_dev_init(struct eth_driver *eth_drv __rte_unused,
> +			  struct rte_eth_dev *eth_dev)
> +{
> +	struct rte_pci_device *pci_dev;
> +	struct ixgbe_hw *hw =
> +		IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
> +	int diag;
> +
> +	PMD_INIT_FUNC_TRACE();
> +
> +	eth_dev->dev_ops = &ixgbe_bifurc_ops;
> +	eth_dev->rx_pkt_burst = &ixgbe_recv_pkts;
> +	eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts;
> +
> +	/* for secondary processes, we don't initialise any further as primary
> +	 * has already done this work. Only check we don't need a different
> +	 * RX function */
> +	if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
> +		if (eth_dev->data->scattered_rx)
> +			eth_dev->rx_pkt_burst = ixgbe_recv_scattered_pkts;
> +		return 0;
> +	}
> +	pci_dev = eth_dev->pci_dev;
> +
> +	/* Vendor and Device ID need to be set before init of shared code */
> +	hw->device_id = pci_dev->id.device_id;
> +	hw->vendor_id = pci_dev->id.vendor_id;
> +	hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
> +
> +#ifdef RTE_LIBRTE_IXGBE_ALLOW_UNSUPPORTED_SFP
> +	hw->allow_unsupported_sfp = 1;
> +#endif
> +
> +	/* Initialize the shared code (base driver) */
> +#ifdef RTE_NIC_BYPASS
> +	diag = ixgbe_bypass_init_shared_code(hw);
> +#else
> +	diag = ixgbe_init_shared_code(hw);
> +#endif /* RTE_NIC_BYPASS */
> +
> +	if (diag != IXGBE_SUCCESS) {
> +		PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
> +		return -EIO;
> +	}
> +
> +	/* Allocate memory for storing MAC addresses */
> +	eth_dev->data->mac_addrs = rte_zmalloc("ixgbe", ETHER_ADDR_LEN *
> +			hw->mac.num_rar_entries, 0);
> +	if (eth_dev->data->mac_addrs == NULL) {
> +		PMD_INIT_LOG(ERR,
> +			"Failed to allocate %u bytes needed to store "
> +			"MAC addresses",
> +			ETHER_ADDR_LEN * hw->mac.num_rar_entries);
> +		return -ENOMEM;
> +	}
> +	rte_bifurc_mac_addr(eth_dev, &eth_dev->data->mac_addrs[0]);
> +
> +	return diag;
> +}
> +
> +/*
> + * The set of PCI devices this driver supports
> + */
> +static struct rte_pci_id pci_id_ixgbe_map[] = {
> +
> +#define RTE_PCI_DEV_ID_DECL_IXGBE(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
> +#define RTE_PCI_DEV_ID_DECL_IXGBEVF(vend, dev) {RTE_PCI_DEVICE(vend, dev)},
> +#include "rte_pci_dev_ids.h"
> +
> +{ .vendor_id = 0, /* sentinel */ },
> +};
> +
> +static struct eth_driver rte_ixgbe_bifurc_pmd = {
> +	{
> +		.name = "rte_ixgbe_bifurc_pmd",
> +		.id_table = pci_id_ixgbe_map,
> +		.drv_flags = RTE_PCI_DRV_NEED_MAPPING |
> +			RTE_PCI_DRV_BIFURC,
> +	},
> +	.eth_dev_init = eth_ixgbe_bifurc_dev_init,
> +	.dev_private_size = sizeof(struct ixgbe_adapter),
> +};
> +
> +/*
> + * Driver initialization routine.
> + * Invoked once at EAL init time.
> + * Register itself as the [Poll Mode] Driver of PCI IXGBE devices.
> + */
> +static int
> +rte_ixgbe_bifurc_pmd_init(const char *name __rte_unused,
> +			  const char *params __rte_unused)
> +{
> +	PMD_INIT_FUNC_TRACE();
> +
> +	rte_eth_driver_register(&rte_ixgbe_bifurc_pmd);
> +	return 0;
> +}
> +
> +static struct rte_driver rte_ixgbe_bifurc_driver = {
> +	.type = PMD_PDEV,
> +	.init = rte_ixgbe_bifurc_pmd_init,
> +};
> +
> +PMD_REGISTER_DRIVER(rte_ixgbe_bifurc_driver);
> diff --git a/lib/librte_pmd_ixgbe/ixgbe_bifurcate.h b/lib/librte_pmd_ixgbe/ixgbe_bifurcate.h
> new file mode 100644
> index 0000000..d40b21d
> --- /dev/null
> +++ b/lib/librte_pmd_ixgbe/ixgbe_bifurcate.h
> @@ -0,0 +1,57 @@
> +/*-
> + *   BSD LICENSE
> + *
> + *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
> + *   All rights reserved.
> + *
> + *   Redistribution and use in source and binary forms, with or without
> + *   modification, are permitted provided that the following conditions
> + *   are met:
> + *
> + *     * Redistributions of source code must retain the above copyright
> + *       notice, this list of conditions and the following disclaimer.
> + *     * Redistributions in binary form must reproduce the above copyright
> + *       notice, this list of conditions and the following disclaimer in
> + *       the documentation and/or other materials provided with the
> + *       distribution.
> + *     * Neither the name of Intel Corporation nor the names of its
> + *       contributors may be used to endorse or promote products derived
> + *       from this software without specific prior written permission.
> + *
> + *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> + *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> + *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> + *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> + *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> + *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> + *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> + *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> + *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> + *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> + *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> + */
> +
> +#ifndef _IXGBE_BIFFURCATE_H_
> +#define _IXGBE_BIFFURCATE_H_
> +
> +#ifdef __cplusplus
> +extern "C" {
> +#endif
> +
> +#define IXGBE_82599_MAX_RX_QUEUES 128
> +
> +#define RTE_PMD_PACKET_RING_SPLITOFF_LOWER_LIMIT 32
> +#define RTE_PMD_PACKET_MAX_RING_PAIRS  IXGBE_82599_MAX_RX_QUEUES
> +
> +
> +/**
> + * For use by the EAL only. Called as part of EAL init to set up any dummy NICs
> + * configured on command line.
> + */
> +int rte_ixgbe_bfc_pmd_init(const char *name, const char *params);
> +
> +#ifdef __cplusplus
> +}
> +#endif
> +
> +#endif
> diff --git a/lib/librte_pmd_ixgbe/ixgbe_rxtx.c b/lib/librte_pmd_ixgbe/ixgbe_rxtx.c
> index e240376..2d32907 100644
> --- a/lib/librte_pmd_ixgbe/ixgbe_rxtx.c
> +++ b/lib/librte_pmd_ixgbe/ixgbe_rxtx.c
> @@ -100,6 +100,12 @@ rte_rxmbuf_alloc(struct rte_mempool *mp)
>  	return (m);
>  }
>  
> +static inline uint16_t
> +ixgbe_dev_queue_offset(struct rte_eth_dev *dev)
> +{
> +	return (RTE_ETH_DEV_SRIOV(dev).active == 0) ?
> +		0 : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx;
> +}
>  
>  #if 1
>  #define RTE_PMD_USE_PREFETCH
> @@ -1726,6 +1732,17 @@ ixgbe_dev_tx_queue_setup(struct rte_eth_dev *dev,
>  			 unsigned int socket_id,
>  			 const struct rte_eth_txconf *tx_conf)
>  {
> +	uint16_t offset = ixgbe_dev_queue_offset(dev);
> +	return ixgbe_dev_txq_setup(dev, queue_idx, offset,
> +				   nb_desc, socket_id, tx_conf);
> +}
> +
> +int
> +ixgbe_dev_txq_setup(struct rte_eth_dev *dev,
> +		    uint16_t queue_idx, uint16_t offset,
> +		    uint16_t nb_desc, unsigned int socket_id,
> +		    const struct rte_eth_txconf *tx_conf)
> +{
>  	const struct rte_memzone *tz;
>  	struct igb_tx_queue *txq;
>  	struct ixgbe_hw     *hw;
> @@ -1849,8 +1866,7 @@ ixgbe_dev_tx_queue_setup(struct rte_eth_dev *dev,
>  	txq->hthresh = tx_conf->tx_thresh.hthresh;
>  	txq->wthresh = tx_conf->tx_thresh.wthresh;
>  	txq->queue_id = queue_idx;
> -	txq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ?
> -		queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx);
> +	txq->reg_idx  = queue_idx + offset;
>  	txq->port_id = dev->data->port_id;
>  	txq->txq_flags = tx_conf->txq_flags;
>  	txq->ops = &def_txq_ops;
> @@ -2083,6 +2099,18 @@ ixgbe_dev_rx_queue_setup(struct rte_eth_dev *dev,
>  			 const struct rte_eth_rxconf *rx_conf,
>  			 struct rte_mempool *mp)
>  {
> +	uint16_t offset = ixgbe_dev_queue_offset(dev);
> +	return ixgbe_dev_rxq_setup(dev, queue_idx, offset, nb_desc,
> +				   socket_id, rx_conf, mp);
> +}
> +
> +int
> +ixgbe_dev_rxq_setup(struct rte_eth_dev *dev,
> +		    uint16_t queue_idx, uint16_t offset,
> +		    uint16_t nb_desc, unsigned int socket_id,
> +		    const struct rte_eth_rxconf *rx_conf,
> +		    struct rte_mempool *mp)
> +{
>  	const struct rte_memzone *rz;
>  	struct igb_rx_queue *rxq;
>  	struct ixgbe_hw     *hw;
> @@ -2118,8 +2146,7 @@ ixgbe_dev_rx_queue_setup(struct rte_eth_dev *dev,
>  	rxq->nb_rx_desc = nb_desc;
>  	rxq->rx_free_thresh = rx_conf->rx_free_thresh;
>  	rxq->queue_id = queue_idx;
> -	rxq->reg_idx = (uint16_t)((RTE_ETH_DEV_SRIOV(dev).active == 0) ?
> -		queue_idx : RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx + queue_idx);
> +	rxq->reg_idx  =	queue_idx + offset;
>  	rxq->port_id = dev->data->port_id;
>  	rxq->crc_len = (uint8_t) ((dev->data->dev_conf.rxmode.hw_strip_crc) ?
>  							0 : ETHER_CRC_LEN);
> @@ -3402,9 +3429,9 @@ ixgbe_dev_rx_init(struct rte_eth_dev *dev)
>  	uint32_t fctrl;
>  	uint32_t hlreg0;
>  	uint32_t maxfrs;
> -	uint32_t srrctl;
>  	uint32_t rdrxctl;
>  	uint32_t rxcsum;
> +	uint32_t srrctl;
>  	uint16_t buf_size;
>  	uint16_t i;
>  
> @@ -3684,9 +3711,9 @@ ixgbe_dev_rxtx_start(struct rte_eth_dev *dev)
>  	struct ixgbe_hw     *hw;
>  	struct igb_tx_queue *txq;
>  	struct igb_rx_queue *rxq;
> -	uint32_t txdctl;
>  	uint32_t dmatxctl;
>  	uint32_t rxctrl;
> +	uint32_t txdctl;
>  	uint16_t i;
>  
>  	PMD_INIT_FUNC_TRACE();
> @@ -3731,7 +3758,6 @@ ixgbe_dev_rxtx_start(struct rte_eth_dev *dev)
>  	if (hw->mac.type == ixgbe_mac_82599EB &&
>  			dev->data->dev_conf.lpbk_mode == IXGBE_LPBK_82599_TX_RX)
>  		ixgbe_setup_loopback_link_82599(hw);
> -
>  }
>  
>  /*
> diff --git a/lib/librte_pmd_ixgbe/ixgbe_rxtx.h b/lib/librte_pmd_ixgbe/ixgbe_rxtx.h
> index eb89715..aeffb5f 100644
> --- a/lib/librte_pmd_ixgbe/ixgbe_rxtx.h
> +++ b/lib/librte_pmd_ixgbe/ixgbe_rxtx.h
> @@ -243,6 +243,16 @@ struct ixgbe_txq_ops {
>  			 IXGBE_ADVTXD_DCMD_DEXT |\
>  			 IXGBE_ADVTXD_DCMD_EOP)
>  
> +int ixgbe_dev_txq_setup(struct rte_eth_dev *dev,
> +			uint16_t queue_idx, uint16_t offset,
> +			uint16_t nb_desc, unsigned int socket_id,
> +			const struct rte_eth_txconf *tx_conf);
> +int ixgbe_dev_rxq_setup(struct rte_eth_dev *dev,
> +			uint16_t queue_idx, uint16_t offset,
> +			uint16_t nb_desc, unsigned int socket_id,
> +			const struct rte_eth_rxconf *rx_conf,
> +			struct rte_mempool *mp);
> +
>  #ifdef RTE_IXGBE_INC_VECTOR
>  uint16_t ixgbe_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
>  		uint16_t nb_pkts);
> -- 
> 1.8.1.4
>