[dpdk-dev] [PATCH v2 20/21] net/atlantic: RX side structures and implementation

[Igor Russkikh <igor.russkikh@aquantia.com>]

Signed-off-by: Igor Russkikh <igor.russkikh@aquantia.com>
---
 drivers/net/atlantic/Makefile     |   6 +
 drivers/net/atlantic/atl_ethdev.c |  44 +++
 drivers/net/atlantic/atl_ethdev.h |  82 ++++
 drivers/net/atlantic/atl_rxtx.c   | 787 ++++++++++++++++++++++++++++++++++++++
 drivers/net/atlantic/meson.build  |   6 +
 5 files changed, 925 insertions(+)
 create mode 100644 drivers/net/atlantic/atl_rxtx.c

diff --git a/drivers/net/atlantic/Makefile b/drivers/net/atlantic/Makefile
index 8af64d082..b84c8a988 100644
--- a/drivers/net/atlantic/Makefile
+++ b/drivers/net/atlantic/Makefile
@@ -31,7 +31,13 @@ VPATH += $(SRCDIR)/hw_atl
 #
 # all source are stored in SRCS-y
 #
+SRCS-$(CONFIG_RTE_LIBRTE_ATLANTIC_PMD) += atl_rxtx.c
 SRCS-$(CONFIG_RTE_LIBRTE_ATLANTIC_PMD) += atl_ethdev.c
+SRCS-$(CONFIG_RTE_LIBRTE_ATLANTIC_PMD) += atl_hw_regs.c
+SRCS-$(CONFIG_RTE_LIBRTE_ATLANTIC_PMD) += hw_atl_utils.c
+SRCS-$(CONFIG_RTE_LIBRTE_ATLANTIC_PMD) += hw_atl_llh.c
+SRCS-$(CONFIG_RTE_LIBRTE_ATLANTIC_PMD) += hw_atl_utils_fw2x.c
+SRCS-$(CONFIG_RTE_LIBRTE_ATLANTIC_PMD) += hw_atl_b0.c
 SRCS-$(CONFIG_RTE_LIBRTE_ATLANTIC_PMD) += rte_pmd_atlantic.c
 
 # install this header file
diff --git a/drivers/net/atlantic/atl_ethdev.c b/drivers/net/atlantic/atl_ethdev.c
index 01700b4ad..8f81761f3 100644
--- a/drivers/net/atlantic/atl_ethdev.c
+++ b/drivers/net/atlantic/atl_ethdev.c
@@ -284,6 +284,8 @@ static const struct eth_dev_ops atl_eth_dev_ops = {
 	.xstats_get_names     = atl_dev_xstats_get_names,
 	.stats_reset	      = atl_dev_stats_reset,
 	.xstats_reset	      = atl_dev_xstats_reset,
+	.queue_stats_mapping_set = atl_dev_queue_stats_mapping_set,
+
 	.fw_version_get       = atl_fw_version_get,
 	.dev_infos_get	      = atl_dev_info_get,
 	.dev_supported_ptypes_get = atl_dev_supported_ptypes_get,
@@ -296,6 +298,20 @@ static const struct eth_dev_ops atl_eth_dev_ops = {
 	.vlan_tpid_set        = atl_vlan_tpid_set,
 	.vlan_strip_queue_set = atl_vlan_strip_queue_set,
 
+	/* Queue Control */
+	.rx_queue_start	      = atl_rx_queue_start,
+	.rx_queue_stop	      = atl_rx_queue_stop,
+	.rx_queue_setup       = atl_rx_queue_setup,
+	.rx_queue_release     = atl_rx_queue_release,
+
+	.rx_queue_intr_enable = atl_dev_rx_queue_intr_enable,
+	.rx_queue_intr_disable = atl_dev_rx_queue_intr_disable,
+
+	.rx_queue_count       = atl_rx_queue_count,
+	.rx_descriptor_done   = atl_dev_rx_descriptor_done,
+	.rx_descriptor_status = atl_dev_rx_descriptor_status,
+	.tx_descriptor_status = atl_dev_tx_descriptor_status,
+
 	/* LEDs */
 	.dev_led_on           = atl_dev_led_on,
 	.dev_led_off          = atl_dev_led_off,
@@ -314,6 +330,7 @@ static const struct eth_dev_ops atl_eth_dev_ops = {
 	.mac_addr_remove      = atl_remove_mac_addr,
 	.mac_addr_set	      = atl_set_default_mac_addr,
 	.set_mc_addr_list     = atl_dev_set_mc_addr_list,
+	.rxq_info_get	      = atl_rxq_info_get,
 	.reta_update          = atl_reta_update,
 	.reta_query           = atl_reta_query,
 	.rss_hash_update      = atl_rss_hash_update,
@@ -631,6 +648,19 @@ atl_dev_start(struct rte_eth_dev *dev)
 		}
 	}
 
+	/* This can fail when allocating mbufs for descriptor rings */
+	err = atl_rx_init(dev);
+	if (err) {
+		PMD_INIT_LOG(ERR, "Unable to initialize RX hardware");
+		goto error;
+	}
+
+	err = atl_start_queues(dev);
+	if (err < 0) {
+		PMD_INIT_LOG(ERR, "Unable to start rxtx queues");
+		goto error;
+	}
+
 	err = hw->aq_fw_ops->update_link_status(hw);
 
 	if (err)
@@ -716,6 +746,9 @@ atl_dev_stop(struct rte_eth_dev *dev)
 	/* reset the NIC */
 	atl_reset_hw(hw);
 	hw->adapter_stopped = 0;
+
+	atl_stop_queues(dev);
+
 	/* Clear stored conf */
 	dev->data->scattered_rx = 0;
 	dev->data->lro = 0;
@@ -775,6 +808,8 @@ atl_dev_close(struct rte_eth_dev *dev)
 
 	atl_dev_stop(dev);
 	hw->adapter_stopped = 1;
+
+	atl_free_queues(dev);
 }
 
 static int
@@ -879,6 +914,15 @@ atl_dev_xstats_reset(struct rte_eth_dev *dev __rte_unused)
 {
 }
 
+static int
+atl_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev __rte_unused,
+					     uint16_t queue_id __rte_unused,
+					     uint8_t stat_idx __rte_unused,
+					     uint8_t is_rx __rte_unused)
+{
+	/* The mapping is hardcoded: queue 0 -> stat 0, etc */
+	return 0;
+}
 
 static int
 atl_fw_version_get(struct rte_eth_dev *dev, char *fw_version, size_t fw_size)
diff --git a/drivers/net/atlantic/atl_ethdev.h b/drivers/net/atlantic/atl_ethdev.h
index 3ebef1f43..614757b7a 100644
--- a/drivers/net/atlantic/atl_ethdev.h
+++ b/drivers/net/atlantic/atl_ethdev.h
@@ -13,6 +13,17 @@
 #include "atl_types.h"
 #include "hw_atl/hw_atl_utils.h"
 
+#define ATL_RSS_OFFLOAD_ALL ( \
+	ETH_RSS_IPV4 | \
+	ETH_RSS_NONFRAG_IPV4_TCP | \
+	ETH_RSS_NONFRAG_IPV4_UDP | \
+	ETH_RSS_IPV6 | \
+	ETH_RSS_NONFRAG_IPV6_TCP | \
+	ETH_RSS_NONFRAG_IPV6_UDP | \
+	ETH_RSS_IPV6_EX | \
+	ETH_RSS_IPV6_TCP_EX | \
+	ETH_RSS_IPV6_UDP_EX)
+
 #define ATL_FLAG_NEED_LINK_UPDATE (uint32_t)(1 << 0)
 #define ATL_FLAG_NEED_LINK_CONFIG (uint32_t)(4 << 0)
 
@@ -46,6 +57,77 @@ struct atl_adapter {
 #define ATL_DEV_PRIVATE_TO_CFG(adapter) \
 	(&((struct atl_adapter *)adapter)->hw_cfg)
 
+extern const struct rte_flow_ops atl_flow_ops;
+
+/*
+ * RX/TX function prototypes
+ */
+void atl_rx_queue_release(void *rxq);
+void atl_tx_queue_release(void *txq);
+
+int atl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t rx_queue_id,
+		uint16_t nb_rx_desc, unsigned int socket_id,
+		const struct rte_eth_rxconf *rx_conf,
+		struct rte_mempool *mb_pool);
+
+int atl_tx_queue_setup(struct rte_eth_dev *dev, uint16_t tx_queue_id,
+		uint16_t nb_tx_desc, unsigned int socket_id,
+		const struct rte_eth_txconf *tx_conf);
+
+uint32_t atl_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id);
+
+int atl_dev_rx_descriptor_done(void *rx_queue, uint16_t offset);
+int atl_dev_rx_descriptor_status(void *rx_queue, uint16_t offset);
+int atl_dev_tx_descriptor_status(void *tx_queue, uint16_t offset);
+
+int atl_dev_rx_queue_intr_enable(struct rte_eth_dev *eth_dev,
+				 uint16_t queue_id);
+int atl_dev_rx_queue_intr_disable(struct rte_eth_dev *eth_dev,
+				  uint16_t queue_id);
+
+int atl_rx_init(struct rte_eth_dev *dev);
+int atl_tx_init(struct rte_eth_dev *dev);
+
+int atl_start_queues(struct rte_eth_dev *dev);
+int atl_stop_queues(struct rte_eth_dev *dev);
+void atl_free_queues(struct rte_eth_dev *dev);
+
+int atl_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id);
+int atl_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id);
+
+int atl_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id);
+int atl_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id);
+
+void atl_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
+	struct rte_eth_rxq_info *qinfo);
+
+void atl_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
+	struct rte_eth_txq_info *qinfo);
+
+uint16_t atl_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
+		uint16_t nb_pkts);
+
+uint16_t atl_recv_pkts_bulk_alloc(void *rx_queue, struct rte_mbuf **rx_pkts,
+				    uint16_t nb_pkts);
+
+uint16_t atl_recv_pkts_lro_single_alloc(void *rx_queue,
+		struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
+uint16_t atl_recv_pkts_lro_bulk_alloc(void *rx_queue,
+		struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
+
+uint16_t atl_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
+		uint16_t nb_pkts);
+
+uint16_t atl_xmit_pkts_simple(void *tx_queue, struct rte_mbuf **tx_pkts,
+		uint16_t nb_pkts);
+
+uint16_t atl_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
+		uint16_t nb_pkts);
+
 int
 atl_dev_led_control(struct rte_eth_dev *dev, int control);
+
+bool
+is_atl_supported(struct rte_eth_dev *dev);
+
 #endif /* _ATLANTIC_ETHDEV_H_ */
diff --git a/drivers/net/atlantic/atl_rxtx.c b/drivers/net/atlantic/atl_rxtx.c
new file mode 100644
index 000000000..77009e095
--- /dev/null
+++ b/drivers/net/atlantic/atl_rxtx.c
@@ -0,0 +1,787 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Aquantia Corporation
+ */
+
+#include <sys/queue.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <stdint.h>
+#include <stdarg.h>
+#include <inttypes.h>
+
+#include <rte_interrupts.h>
+#include <rte_byteorder.h>
+#include <rte_common.h>
+#include <rte_log.h>
+#include <rte_debug.h>
+#include <rte_pci.h>
+#include <rte_memory.h>
+#include <rte_memcpy.h>
+#include <rte_memzone.h>
+#include <rte_launch.h>
+#include <rte_eal.h>
+#include <rte_per_lcore.h>
+#include <rte_lcore.h>
+#include <rte_atomic.h>
+#include <rte_branch_prediction.h>
+#include <rte_mempool.h>
+#include <rte_malloc.h>
+#include <rte_mbuf.h>
+#include <rte_ether.h>
+#include <rte_ethdev_driver.h>
+#include <rte_prefetch.h>
+#include <rte_udp.h>
+#include <rte_tcp.h>
+#include <rte_sctp.h>
+#include <rte_net.h>
+#include <rte_string_fns.h>
+#include "atl_ethdev.h"
+#include "atl_hw_regs.h"
+
+#include "atl_logs.h"
+#include "hw_atl/hw_atl_llh.h"
+#include "hw_atl/hw_atl_b0.h"
+#include "hw_atl/hw_atl_b0_internal.h"
+
+/**
+ * Structure associated with each descriptor of the RX ring of a RX queue.
+ */
+struct atl_rx_entry {
+	struct rte_mbuf *mbuf;
+};
+
+/**
+ * Structure associated with each RX queue.
+ */
+struct atl_rx_queue {
+	struct rte_mempool	*mb_pool;
+	struct hw_atl_rxd_s	*hw_ring;
+	uint64_t		hw_ring_phys_addr;
+	struct atl_rx_entry	*sw_ring;
+	uint16_t		nb_rx_desc;
+	uint16_t		rx_tail;
+	uint16_t		nb_rx_hold;
+	uint16_t		rx_free_thresh;
+	uint16_t		queue_id;
+	uint16_t		port_id;
+	uint16_t		buff_size;
+	bool			l3_csum_enabled;
+	bool			l4_csum_enabled;
+};
+
+static inline void
+atl_reset_rx_queue(struct atl_rx_queue *rxq)
+{
+	struct hw_atl_rxd_s *rxd = NULL;
+	int i;
+
+	PMD_INIT_FUNC_TRACE();
+
+	for (i = 0; i < rxq->nb_rx_desc; i++) {
+		rxd = (struct hw_atl_rxd_s *)&rxq->hw_ring[i];
+		rxd->buf_addr = 0;
+		rxd->hdr_addr = 0;
+	}
+
+	rxq->rx_tail = 0;
+}
+
+int
+atl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t rx_queue_id,
+		   uint16_t nb_rx_desc, unsigned int socket_id,
+		   const struct rte_eth_rxconf *rx_conf,
+		   struct rte_mempool *mb_pool)
+{
+	struct atl_rx_queue *rxq;
+	const struct rte_memzone *mz;
+
+	PMD_INIT_FUNC_TRACE();
+
+	/* make sure a valid number of descriptors have been requested */
+	if (nb_rx_desc < AQ_HW_MIN_RX_RING_SIZE ||
+			nb_rx_desc > AQ_HW_MAX_RX_RING_SIZE) {
+		PMD_INIT_LOG(ERR, "Number of Rx descriptors must be "
+		"less than or equal to %d, "
+		"greater than or equal to %d", AQ_HW_MAX_RX_RING_SIZE,
+		AQ_HW_MIN_RX_RING_SIZE);
+		return -EINVAL;
+	}
+
+	/*
+	 * if this queue existed already, free the associated memory. The
+	 * queue cannot be reused in case we need to allocate memory on
+	 * different socket than was previously used.
+	 */
+	if (dev->data->rx_queues[rx_queue_id] != NULL) {
+		atl_rx_queue_release(dev->data->rx_queues[rx_queue_id]);
+		dev->data->rx_queues[rx_queue_id] = NULL;
+	}
+
+	/* allocate memory for the queue structure */
+	rxq = rte_zmalloc_socket("atlantic Rx queue", sizeof(*rxq),
+				 RTE_CACHE_LINE_SIZE, socket_id);
+	if (rxq == NULL) {
+		PMD_INIT_LOG(ERR, "Cannot allocate queue structure");
+		return -ENOMEM;
+	}
+
+	/* setup queue */
+	rxq->mb_pool = mb_pool;
+	rxq->nb_rx_desc = nb_rx_desc;
+	rxq->port_id = dev->data->port_id;
+	rxq->queue_id = rx_queue_id;
+	rxq->rx_free_thresh = rx_conf->rx_free_thresh;
+
+	rxq->l3_csum_enabled = dev->data->dev_conf.rxmode.offloads &
+		DEV_RX_OFFLOAD_IPV4_CKSUM;
+	rxq->l4_csum_enabled = dev->data->dev_conf.rxmode.offloads &
+		(DEV_RX_OFFLOAD_UDP_CKSUM | DEV_RX_OFFLOAD_TCP_CKSUM);
+
+	/* allocate memory for the software ring */
+	rxq->sw_ring = rte_zmalloc_socket("atlantic sw rx ring",
+				nb_rx_desc * sizeof(struct atl_rx_entry),
+				RTE_CACHE_LINE_SIZE, socket_id);
+	if (rxq->sw_ring == NULL) {
+		PMD_INIT_LOG(ERR, "Cannot allocate software ring");
+		rte_free(rxq);
+		return -ENOMEM;
+	}
+
+	/*
+	 * allocate memory for the hardware descriptor ring. A memzone large
+	 * enough to hold the maximum ring size is requested to allow for
+	 * resizing in later calls to the queue setup function.
+	 */
+	mz = rte_eth_dma_zone_reserve(dev, "rx hw_ring", rx_queue_id,
+				      HW_ATL_B0_MAX_RXD *
+					sizeof(struct hw_atl_rxd_s),
+				      128, socket_id);
+	if (mz == NULL) {
+		PMD_INIT_LOG(ERR, "Cannot allocate hardware ring");
+		rte_free(rxq->sw_ring);
+		rte_free(rxq);
+		return -ENOMEM;
+	}
+	rxq->hw_ring = mz->addr;
+	rxq->hw_ring_phys_addr = mz->iova;
+
+	atl_reset_rx_queue(rxq);
+
+	dev->data->rx_queues[rx_queue_id] = rxq;
+	return 0;
+}
+
+int
+atl_rx_init(struct rte_eth_dev *eth_dev)
+{
+	struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
+	struct aq_rss_parameters *rss_params = &hw->aq_nic_cfg->aq_rss;
+	struct atl_rx_queue *rxq;
+	uint64_t base_addr = 0;
+	int i = 0;
+	int err = 0;
+
+	PMD_INIT_FUNC_TRACE();
+
+	for (i = 0; i < eth_dev->data->nb_rx_queues; i++) {
+		rxq = eth_dev->data->rx_queues[i];
+		base_addr = rxq->hw_ring_phys_addr;
+
+		/* Take requested pool mbuf size and adapt
+		 * descriptor buffer to best fit
+		 */
+		int buff_size = rte_pktmbuf_data_room_size(rxq->mb_pool) -
+				RTE_PKTMBUF_HEADROOM;
+
+		buff_size = RTE_ALIGN_FLOOR(buff_size, 1024);
+		if (buff_size > HW_ATL_B0_RXD_BUF_SIZE_MAX) {
+			PMD_INIT_LOG(WARNING,
+				"queue %d: mem pool buff size is too big\n",
+				rxq->queue_id);
+			buff_size = HW_ATL_B0_RXD_BUF_SIZE_MAX;
+		}
+		if (buff_size < 1024) {
+			PMD_INIT_LOG(ERR,
+				"queue %d: mem pool buff size is too small\n",
+				rxq->queue_id);
+			return -EINVAL;
+		}
+		rxq->buff_size = buff_size;
+
+		err = hw_atl_b0_hw_ring_rx_init(hw, base_addr, rxq->queue_id,
+						rxq->nb_rx_desc, buff_size, 0,
+						rxq->port_id);
+	}
+
+	for (i = rss_params->indirection_table_size; i--;)
+		rss_params->indirection_table[i] = i &
+			(eth_dev->data->nb_rx_queues - 1);
+	hw_atl_b0_hw_rss_set(hw, rss_params);
+	return err;
+}
+
+static int
+atl_alloc_rx_queue_mbufs(struct atl_rx_queue *rxq)
+{
+	struct atl_rx_entry *rx_entry = rxq->sw_ring;
+	struct hw_atl_rxd_s *rxd;
+	uint64_t dma_addr = 0;
+	uint32_t i = 0;
+
+	PMD_INIT_FUNC_TRACE();
+
+	/* fill Rx ring */
+	for (i = 0; i < rxq->nb_rx_desc; i++) {
+		struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mb_pool);
+
+		if (mbuf == NULL) {
+			PMD_INIT_LOG(ERR, "mbuf alloca failed for rx queue %u",
+				     (unsigned int)rxq->queue_id);
+			return -ENOMEM;
+		}
+
+		mbuf->data_off = RTE_PKTMBUF_HEADROOM;
+		mbuf->port = rxq->port_id;
+
+		dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf));
+		rxd = (struct hw_atl_rxd_s *)&rxq->hw_ring[i];
+		rxd->buf_addr = dma_addr;
+		rxd->hdr_addr = 0;
+		rx_entry[i].mbuf = mbuf;
+	}
+
+	return 0;
+}
+
+static void
+atl_rx_queue_release_mbufs(struct atl_rx_queue *rxq)
+{
+	int i;
+
+	PMD_INIT_FUNC_TRACE();
+
+	if (rxq->sw_ring != NULL) {
+		for (i = 0; i < rxq->nb_rx_desc; i++) {
+			if (rxq->sw_ring[i].mbuf != NULL) {
+				rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
+				rxq->sw_ring[i].mbuf = NULL;
+			}
+		}
+	}
+}
+
+int
+atl_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
+{
+	struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct atl_rx_queue *rxq = NULL;
+
+	PMD_INIT_FUNC_TRACE();
+
+	if (rx_queue_id < dev->data->nb_rx_queues) {
+		rxq = dev->data->rx_queues[rx_queue_id];
+
+		if (atl_alloc_rx_queue_mbufs(rxq) != 0) {
+			PMD_INIT_LOG(ERR, "Allocate mbufs for queue %d failed",
+				     rx_queue_id);
+			return -1;
+		}
+
+		hw_atl_b0_hw_ring_rx_start(hw, rx_queue_id);
+
+		rte_wmb();
+		hw_atl_reg_rx_dma_desc_tail_ptr_set(hw, rxq->nb_rx_desc - 1,
+						    rx_queue_id);
+		dev->data->rx_queue_state[rx_queue_id] =
+			RTE_ETH_QUEUE_STATE_STARTED;
+	} else {
+		return -1;
+	}
+
+	return 0;
+}
+
+int
+atl_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
+{
+	struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct atl_rx_queue *rxq = NULL;
+
+	PMD_INIT_FUNC_TRACE();
+
+	if (rx_queue_id < dev->data->nb_rx_queues) {
+		rxq = dev->data->rx_queues[rx_queue_id];
+
+		hw_atl_b0_hw_ring_rx_stop(hw, rx_queue_id);
+
+		atl_rx_queue_release_mbufs(rxq);
+		atl_reset_rx_queue(rxq);
+
+		dev->data->rx_queue_state[rx_queue_id] =
+			RTE_ETH_QUEUE_STATE_STOPPED;
+	} else {
+		return -1;
+	}
+
+	return 0;
+}
+
+void
+atl_rx_queue_release(void *rx_queue)
+{
+	PMD_INIT_FUNC_TRACE();
+
+	if (rx_queue != NULL) {
+		struct atl_rx_queue *rxq = (struct atl_rx_queue *)rx_queue;
+
+		atl_rx_queue_release_mbufs(rxq);
+		rte_free(rxq->sw_ring);
+		rte_free(rxq);
+	}
+}
+
+void
+atl_free_queues(struct rte_eth_dev *dev)
+{
+	unsigned int i;
+
+	PMD_INIT_FUNC_TRACE();
+
+	for (i = 0; i < dev->data->nb_rx_queues; i++) {
+		atl_rx_queue_release(dev->data->rx_queues[i]);
+		dev->data->rx_queues[i] = 0;
+	}
+	dev->data->nb_rx_queues = 0;
+}
+
+int
+atl_start_queues(struct rte_eth_dev *dev)
+{
+	int i;
+
+	PMD_INIT_FUNC_TRACE();
+
+	for (i = 0; i < dev->data->nb_rx_queues; i++) {
+		if (atl_rx_queue_start(dev, i) != 0) {
+			PMD_DRV_LOG(ERR, "Start Rx queue %d failed", i);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+int
+atl_stop_queues(struct rte_eth_dev *dev)
+{
+	int i;
+
+	PMD_INIT_FUNC_TRACE();
+
+	for (i = 0; i < dev->data->nb_rx_queues; i++) {
+		if (atl_rx_queue_stop(dev, i) != 0) {
+			PMD_DRV_LOG(ERR, "Stop Rx queue %d failed", i);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+void
+atl_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
+		 struct rte_eth_rxq_info *qinfo)
+{
+	struct atl_rx_queue *rxq;
+
+	PMD_INIT_FUNC_TRACE();
+
+	rxq = dev->data->rx_queues[queue_id];
+
+	qinfo->mp = rxq->mb_pool;
+	qinfo->scattered_rx = dev->data->scattered_rx;
+	qinfo->nb_desc = rxq->nb_rx_desc;
+}
+
+/* Return Rx queue avail count */
+
+uint32_t
+atl_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id)
+{
+	struct atl_rx_queue *rxq;
+
+	PMD_INIT_FUNC_TRACE();
+
+	if (rx_queue_id >= dev->data->nb_rx_queues) {
+		PMD_DRV_LOG(ERR, "Invalid RX queue id=%d", rx_queue_id);
+		return 0;
+	}
+
+	rxq = dev->data->rx_queues[rx_queue_id];
+
+	if (rxq == NULL)
+		return 0;
+
+	return rxq->nb_rx_desc - rxq->nb_rx_hold;
+}
+
+int
+atl_dev_rx_descriptor_done(void *rx_queue, uint16_t offset)
+{
+	struct atl_rx_queue *rxq = rx_queue;
+	struct hw_atl_rxd_wb_s *rxd;
+	uint32_t idx;
+
+	PMD_INIT_FUNC_TRACE();
+
+	if (unlikely(offset >= rxq->nb_rx_desc))
+		return 0;
+
+	idx = rxq->rx_tail + offset;
+
+	if (idx >= rxq->nb_rx_desc)
+		idx -= rxq->nb_rx_desc;
+
+	rxd = (struct hw_atl_rxd_wb_s *)&rxq->hw_ring[idx];
+
+	return rxd->dd;
+}
+
+int
+atl_dev_rx_descriptor_status(void *rx_queue, uint16_t offset)
+{
+	struct atl_rx_queue *rxq = rx_queue;
+	struct hw_atl_rxd_wb_s *rxd;
+	uint32_t idx;
+
+	PMD_INIT_FUNC_TRACE();
+
+	if (unlikely(offset >= rxq->nb_rx_desc))
+		return -EINVAL;
+
+	if (offset >= rxq->nb_rx_desc - rxq->nb_rx_hold)
+		return RTE_ETH_RX_DESC_UNAVAIL;
+
+	idx = rxq->rx_tail + offset;
+
+	if (idx >= rxq->nb_rx_desc)
+		idx -= rxq->nb_rx_desc;
+
+	rxd = (struct hw_atl_rxd_wb_s *)&rxq->hw_ring[idx];
+
+	if (rxd->dd)
+		return RTE_ETH_RX_DESC_DONE;
+
+	return RTE_ETH_RX_DESC_AVAIL;
+}
+
+static int
+atl_rx_enable_intr(struct rte_eth_dev *dev, uint16_t queue_id, bool enable)
+{
+	struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+	struct atl_rx_queue *rxq;
+
+	PMD_INIT_FUNC_TRACE();
+
+	if (queue_id >= dev->data->nb_rx_queues) {
+		PMD_DRV_LOG(ERR, "Invalid RX queue id=%d", queue_id);
+		return -EINVAL;
+	}
+
+	rxq = dev->data->rx_queues[queue_id];
+
+	if (rxq == NULL)
+		return 0;
+
+	/* Mapping interrupt vector */
+	hw_atl_itr_irq_map_en_rx_set(hw, enable, queue_id);
+
+	return 0;
+}
+
+int
+atl_dev_rx_queue_intr_enable(struct rte_eth_dev *eth_dev, uint16_t queue_id)
+{
+	return atl_rx_enable_intr(eth_dev, queue_id, true);
+}
+
+int
+atl_dev_rx_queue_intr_disable(struct rte_eth_dev *eth_dev, uint16_t queue_id)
+{
+	return atl_rx_enable_intr(eth_dev, queue_id, false);
+}
+
+static uint64_t
+atl_desc_to_offload_flags(struct atl_rx_queue *rxq,
+			  struct hw_atl_rxd_wb_s *rxd_wb)
+{
+	uint64_t mbuf_flags = 0;
+
+	PMD_INIT_FUNC_TRACE();
+
+	/* IPv4 ? */
+	if (rxq->l3_csum_enabled && ((rxd_wb->pkt_type & 0x3) == 0)) {
+		/* IPv4 csum error ? */
+		if (rxd_wb->rx_stat & BIT(1))
+			mbuf_flags |= PKT_RX_IP_CKSUM_BAD;
+		else
+			mbuf_flags |= PKT_RX_IP_CKSUM_GOOD;
+	} else {
+		mbuf_flags |= PKT_RX_IP_CKSUM_UNKNOWN;
+	}
+
+	/* CSUM calculated ? */
+	if (rxq->l4_csum_enabled && (rxd_wb->rx_stat & BIT(3))) {
+		if (rxd_wb->rx_stat & BIT(2))
+			mbuf_flags |= PKT_RX_L4_CKSUM_BAD;
+		else
+			mbuf_flags |= PKT_RX_L4_CKSUM_GOOD;
+	} else {
+		mbuf_flags |= PKT_RX_L4_CKSUM_UNKNOWN;
+	}
+
+	return mbuf_flags;
+}
+
+static uint32_t
+atl_desc_to_pkt_type(struct hw_atl_rxd_wb_s *rxd_wb)
+{
+	uint32_t type = RTE_PTYPE_UNKNOWN;
+	uint16_t l2_l3_type = rxd_wb->pkt_type & 0x3;
+	uint16_t l4_type = (rxd_wb->pkt_type & 0x1C) >> 2;
+
+	switch (l2_l3_type) {
+	case 0:
+		type = RTE_PTYPE_L3_IPV4;
+		break;
+	case 1:
+		type = RTE_PTYPE_L3_IPV6;
+		break;
+	case 2:
+		type = RTE_PTYPE_L2_ETHER;
+		break;
+	case 3:
+		type = RTE_PTYPE_L2_ETHER_ARP;
+		break;
+	}
+
+	switch (l4_type) {
+	case 0:
+		type |= RTE_PTYPE_L4_TCP;
+		break;
+	case 1:
+		type |= RTE_PTYPE_L4_UDP;
+		break;
+	case 2:
+		type |= RTE_PTYPE_L4_SCTP;
+		break;
+	case 3:
+		type |= RTE_PTYPE_L4_ICMP;
+		break;
+	}
+
+	if (rxd_wb->pkt_type & BIT(5))
+		type |= RTE_PTYPE_L2_ETHER_VLAN;
+
+	return type;
+}
+
+uint16_t
+atl_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+{
+	struct atl_rx_queue *rxq = (struct atl_rx_queue *)rx_queue;
+	struct rte_eth_dev *dev = &rte_eth_devices[rxq->port_id];
+	struct atl_adapter *adapter =
+		ATL_DEV_TO_ADAPTER(&rte_eth_devices[rxq->port_id]);
+	struct aq_hw_s *hw = ATL_DEV_PRIVATE_TO_HW(adapter);
+	struct atl_rx_entry *sw_ring = rxq->sw_ring;
+
+	struct rte_mbuf *new_mbuf;
+	struct rte_mbuf *rx_mbuf, *rx_mbuf_prev, *rx_mbuf_first;
+	struct atl_rx_entry *rx_entry;
+	uint16_t nb_rx = 0;
+	uint16_t nb_hold = 0;
+	struct hw_atl_rxd_wb_s rxd_wb;
+	struct hw_atl_rxd_s *rxd = NULL;
+	uint16_t tail = rxq->rx_tail;
+	uint64_t dma_addr;
+	uint16_t pkt_len = 0;
+
+	while (nb_rx < nb_pkts) {
+		uint16_t eop_tail = tail;
+
+		rxd = (struct hw_atl_rxd_s *)&rxq->hw_ring[tail];
+		rxd_wb = *(struct hw_atl_rxd_wb_s *)rxd;
+
+		if (!rxd_wb.dd) { /* RxD is not done */
+			break;
+		}
+
+		PMD_RX_LOG(ERR, "port_id=%u queue_id=%u tail=%u "
+			   "eop=0x%x pkt_len=%u hash=0x%x hash_type=0x%x",
+			   (unsigned int)rxq->port_id,
+			   (unsigned int)rxq->queue_id,
+			   (unsigned int)tail, (unsigned int)rxd_wb.eop,
+			   (unsigned int)rte_le_to_cpu_16(rxd_wb.pkt_len),
+			rxd_wb.rss_hash, rxd_wb.rss_type);
+
+		/* RxD is not done */
+		if (!rxd_wb.eop) {
+			while (true) {
+				struct hw_atl_rxd_wb_s *eop_rxwbd;
+
+				eop_tail = (eop_tail + 1) % rxq->nb_rx_desc;
+				eop_rxwbd = (struct hw_atl_rxd_wb_s *)
+					&rxq->hw_ring[eop_tail];
+				if (!eop_rxwbd->dd) {
+					/* no EOP received yet */
+					eop_tail = tail;
+					break;
+				}
+				if (eop_rxwbd->dd && eop_rxwbd->eop)
+					break;
+			}
+			/* No EOP in ring */
+			if (eop_tail == tail)
+				break;
+		}
+		rx_mbuf_prev = NULL;
+		rx_mbuf_first = NULL;
+
+		/* Run through packet segments */
+		while (true) {
+			new_mbuf = rte_mbuf_raw_alloc(rxq->mb_pool);
+			if (new_mbuf == NULL) {
+				PMD_RX_LOG(ERR,
+				   "RX mbuf alloc failed port_id=%u "
+				   "queue_id=%u", (unsigned int)rxq->port_id,
+				   (unsigned int)rxq->queue_id);
+				dev->data->rx_mbuf_alloc_failed++;
+				adapter->sw_stats.rx_nombuf++;
+				goto err_stop;
+			}
+
+			nb_hold++;
+			rx_entry = &sw_ring[tail];
+
+			rx_mbuf = rx_entry->mbuf;
+			rx_entry->mbuf = new_mbuf;
+			dma_addr = rte_cpu_to_le_64(
+				rte_mbuf_data_iova_default(new_mbuf));
+
+			/* setup RX descriptor */
+			rxd->hdr_addr = 0;
+			rxd->buf_addr = dma_addr;
+
+			/*
+			 * Initialize the returned mbuf.
+			 * 1) setup generic mbuf fields:
+			 *	  - number of segments,
+			 *	  - next segment,
+			 *	  - packet length,
+			 *	  - RX port identifier.
+			 * 2) integrate hardware offload data, if any:
+			 *	<  - RSS flag & hash,
+			 *	  - IP checksum flag,
+			 *	  - VLAN TCI, if any,
+			 *	  - error flags.
+			 */
+			pkt_len = (uint16_t)rte_le_to_cpu_16(rxd_wb.pkt_len);
+			rx_mbuf->data_off = RTE_PKTMBUF_HEADROOM;
+			rte_prefetch1((char *)rx_mbuf->buf_addr +
+				rx_mbuf->data_off);
+			rx_mbuf->nb_segs = 0;
+			rx_mbuf->next = NULL;
+			rx_mbuf->pkt_len = pkt_len;
+			rx_mbuf->data_len = pkt_len;
+			if (rxd_wb.eop) {
+				u16 remainder_len = pkt_len % rxq->buff_size;
+				if (!remainder_len)
+					remainder_len = rxq->buff_size;
+				rx_mbuf->data_len = remainder_len;
+			} else {
+				rx_mbuf->data_len = pkt_len > rxq->buff_size ?
+						rxq->buff_size : pkt_len;
+			}
+			rx_mbuf->port = rxq->port_id;
+
+			rx_mbuf->hash.rss = rxd_wb.rss_hash;
+
+			rx_mbuf->vlan_tci = rxd_wb.vlan;
+
+			rx_mbuf->ol_flags =
+				atl_desc_to_offload_flags(rxq, &rxd_wb);
+			rx_mbuf->packet_type = atl_desc_to_pkt_type(&rxd_wb);
+
+			if (!rx_mbuf_first)
+				rx_mbuf_first = rx_mbuf;
+			rx_mbuf_first->nb_segs++;
+
+			if (rx_mbuf_prev)
+				rx_mbuf_prev->next = rx_mbuf;
+			rx_mbuf_prev = rx_mbuf;
+
+			tail = (tail + 1) % rxq->nb_rx_desc;
+			/* Prefetch next mbufs */
+			rte_prefetch0(sw_ring[tail].mbuf);
+			if ((tail & 0x3) == 0) {
+				rte_prefetch0(&sw_ring[tail]);
+				rte_prefetch0(&sw_ring[tail]);
+			}
+
+			/* filled mbuf_first */
+			if (rxd_wb.eop)
+				break;
+			rxd = (struct hw_atl_rxd_s *)&rxq->hw_ring[tail];
+			rxd_wb = *(struct hw_atl_rxd_wb_s *)rxd;
+		};
+
+		/*
+		 * Store the mbuf address into the next entry of the array
+		 * of returned packets.
+		 */
+		rx_pkts[nb_rx++] = rx_mbuf_first;
+		adapter->sw_stats.q_ipackets[rxq->queue_id]++;
+		adapter->sw_stats.q_ibytes[rxq->queue_id] +=
+			rx_mbuf_first->pkt_len;
+
+		PMD_RX_LOG(ERR, "add mbuf segs=%d pkt_len=%d",
+			rx_mbuf_first->nb_segs,
+			rx_mbuf_first->pkt_len);
+	}
+
+err_stop:
+
+	rxq->rx_tail = tail;
+
+	/*
+	 * If the number of free RX descriptors is greater than the RX free
+	 * threshold of the queue, advance the Receive Descriptor Tail (RDT)
+	 * register.
+	 * Update the RDT with the value of the last processed RX descriptor
+	 * minus 1, to guarantee that the RDT register is never equal to the
+	 * RDH register, which creates a "full" ring situtation from the
+	 * hardware point of view...
+	 */
+	nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold);
+	if (nb_hold > rxq->rx_free_thresh) {
+		PMD_RX_LOG(ERR, "port_id=%u queue_id=%u rx_tail=%u "
+			"nb_hold=%u nb_rx=%u",
+			(unsigned int)rxq->port_id, (unsigned int)rxq->queue_id,
+			(unsigned int)tail, (unsigned int)nb_hold,
+			(unsigned int)nb_rx);
+		tail = (uint16_t)((tail == 0) ?
+			(rxq->nb_rx_desc - 1) : (tail - 1));
+
+		hw_atl_reg_rx_dma_desc_tail_ptr_set(hw, tail, rxq->queue_id);
+
+		nb_hold = 0;
+	}
+
+	rxq->nb_rx_hold = nb_hold;
+
+	return nb_rx;
+}
diff --git a/drivers/net/atlantic/meson.build b/drivers/net/atlantic/meson.build
index 19fa41cd3..42821f35a 100644
--- a/drivers/net/atlantic/meson.build
+++ b/drivers/net/atlantic/meson.build
@@ -4,8 +4,14 @@
 #subdir('hw_atl')
 
 sources = files(
+	'atl_rxtx.c',
 	'atl_ethdev.c',
+	'atl_hw_regs.c',
 	'rte_pmd_atlantic.c',
+	'hw_atl/hw_atl_b0.c',
+	'hw_atl/hw_atl_llh.c',
+	'hw_atl/hw_atl_utils_fw2x.c',
+	'hw_atl/hw_atl_utils.c',
 )
 
 deps += ['hash', 'eal']
-- 
2.13.3.windows.1