dev start/stop implementations, start/stop the rx/tx queues.

Signed-off-by: Junlong Wang <wang.junlong1@zte.com.cn>
---
 doc/guides/nics/features/zxdh.ini |  2 +
 doc/guides/nics/zxdh.rst          |  2 +
 drivers/net/zxdh/zxdh_ethdev.c    | 61 +++++++++++++++++++++
 drivers/net/zxdh/zxdh_pci.c       | 24 ++++++++
 drivers/net/zxdh/zxdh_pci.h       |  1 +
 drivers/net/zxdh/zxdh_queue.c     | 91 +++++++++++++++++++++++++++++++
 drivers/net/zxdh/zxdh_queue.h     | 69 +++++++++++++++++++++++
 drivers/net/zxdh/zxdh_rxtx.h      | 14 ++---
 8 files changed, 256 insertions(+), 8 deletions(-)

diff --git a/doc/guides/nics/features/zxdh.ini b/doc/guides/nics/features/zxdh.ini
index 05c8091ed7..7b72be5f25 100644
--- a/doc/guides/nics/features/zxdh.ini
+++ b/doc/guides/nics/features/zxdh.ini
@@ -7,3 +7,5 @@
 Linux                = Y
 x86-64               = Y
 ARMv8                = Y
+SR-IOV               = Y
+Multiprocess aware   = Y
diff --git a/doc/guides/nics/zxdh.rst b/doc/guides/nics/zxdh.rst
index 2144753d75..eb970a888f 100644
--- a/doc/guides/nics/zxdh.rst
+++ b/doc/guides/nics/zxdh.rst
@@ -18,6 +18,8 @@ Features
 Features of the ZXDH PMD are:
 
 - Multi arch support: x86_64, ARMv8.
+- Multiple queues for TX and RX
+- SR-IOV VF
 
 
 Driver compilation and testing
diff --git a/drivers/net/zxdh/zxdh_ethdev.c b/drivers/net/zxdh/zxdh_ethdev.c
index 521d7ed433..59ee942bdd 100644
--- a/drivers/net/zxdh/zxdh_ethdev.c
+++ b/drivers/net/zxdh/zxdh_ethdev.c
@@ -899,12 +899,35 @@ zxdh_tables_uninit(struct rte_eth_dev *dev)
     return ret;
 }
 
+static int
+zxdh_dev_stop(struct rte_eth_dev *dev)
+{
+    int ret = 0;
+
+    if (dev->data->dev_started == 0)
+        return 0;
+
+    ret = zxdh_intr_disable(dev);
+    if (ret) {
+        PMD_DRV_LOG(ERR, "intr disable failed");
+        return -EINVAL;
+    }
+
+    return 0;
+}
+
 static int
 zxdh_dev_close(struct rte_eth_dev *dev)
 {
     struct zxdh_hw *hw = dev->data->dev_private;
     int ret = 0;
 
+    ret = zxdh_dev_stop(dev);
+    if (ret != 0) {
+        PMD_DRV_LOG(ERR, " stop port %s failed.", dev->device->name);
+        return -1;
+    }
+
     ret = zxdh_tables_uninit(dev);
     if (ret != 0) {
         PMD_DRV_LOG(ERR, "%s :tables uninit %s failed ", __func__, dev->device->name);
@@ -928,9 +951,47 @@ zxdh_dev_close(struct rte_eth_dev *dev)
     return ret;
 }
 
+static int
+zxdh_dev_start(struct rte_eth_dev *dev)
+{
+    struct zxdh_hw *hw = dev->data->dev_private;
+    struct zxdh_virtqueue *vq;
+    int32_t ret;
+    uint16_t logic_qidx;
+    uint16_t i;
+
+    for (i = 0; i < dev->data->nb_rx_queues; i++) {
+        logic_qidx = 2 * i + ZXDH_RQ_QUEUE_IDX;
+        ret = zxdh_dev_rx_queue_setup_finish(dev, logic_qidx);
+        if (ret < 0)
+            return ret;
+    }
+    ret = zxdh_intr_enable(dev);
+    if (ret) {
+        PMD_DRV_LOG(ERR, "interrupt enable failed");
+        return -EINVAL;
+    }
+
+    for (i = 0; i < dev->data->nb_rx_queues; i++) {
+        logic_qidx = 2 * i + ZXDH_RQ_QUEUE_IDX;
+        vq = hw->vqs[logic_qidx];
+        /* Flush the old packets */
+        zxdh_queue_rxvq_flush(vq);
+        zxdh_queue_notify(vq);
+    }
+    for (i = 0; i < dev->data->nb_tx_queues; i++) {
+        logic_qidx = 2 * i + ZXDH_TQ_QUEUE_IDX;
+        vq = hw->vqs[logic_qidx];
+        zxdh_queue_notify(vq);
+    }
+    return 0;
+}
+
 /* dev_ops for zxdh, bare necessities for basic operation */
 static const struct eth_dev_ops zxdh_eth_dev_ops = {
     .dev_configure             = zxdh_dev_configure,
+    .dev_start                 = zxdh_dev_start,
+    .dev_stop                 = zxdh_dev_stop,
     .dev_close                 = zxdh_dev_close,
     .dev_infos_get             = zxdh_dev_infos_get,
     .rx_queue_setup             = zxdh_dev_rx_queue_setup,
diff --git a/drivers/net/zxdh/zxdh_pci.c b/drivers/net/zxdh/zxdh_pci.c
index 250e67d560..83164a5c79 100644
--- a/drivers/net/zxdh/zxdh_pci.c
+++ b/drivers/net/zxdh/zxdh_pci.c
@@ -202,6 +202,29 @@ zxdh_del_queue(struct zxdh_hw *hw, struct zxdh_virtqueue *vq)
     rte_write16(0, &hw->common_cfg->queue_enable);
 }
 
+static void
+zxdh_notify_queue(struct zxdh_hw *hw, struct zxdh_virtqueue *vq)
+{
+    uint32_t notify_data = 0;
+
+    if (!zxdh_pci_with_feature(hw, ZXDH_F_NOTIFICATION_DATA)) {
+        rte_write16(vq->vq_queue_index, vq->notify_addr);
+        return;
+    }
+
+    if (zxdh_pci_with_feature(hw, ZXDH_F_RING_PACKED)) {
+        notify_data = ((uint32_t)(!!(vq->vq_packed.cached_flags &
+                        ZXDH_VRING_PACKED_DESC_F_AVAIL)) << 31) |
+                        ((uint32_t)vq->vq_avail_idx << 16) |
+                        vq->vq_queue_index;
+    } else {
+        notify_data = ((uint32_t)vq->vq_avail_idx << 16) | vq->vq_queue_index;
+    }
+    PMD_DRV_LOG(DEBUG, "queue:%d notify_data 0x%x notify_addr 0x%p",
+                 vq->vq_queue_index, notify_data, vq->notify_addr);
+    rte_write32(notify_data, vq->notify_addr);
+}
+
 const struct zxdh_pci_ops zxdh_dev_pci_ops = {
     .read_dev_cfg   = zxdh_read_dev_config,
     .write_dev_cfg  = zxdh_write_dev_config,
@@ -216,6 +239,7 @@ const struct zxdh_pci_ops zxdh_dev_pci_ops = {
     .set_queue_num  = zxdh_set_queue_num,
     .setup_queue    = zxdh_setup_queue,
     .del_queue      = zxdh_del_queue,
+    .notify_queue   = zxdh_notify_queue,
 };
 
 uint8_t
diff --git a/drivers/net/zxdh/zxdh_pci.h b/drivers/net/zxdh/zxdh_pci.h
index e3f13cb17d..5c5f72b90e 100644
--- a/drivers/net/zxdh/zxdh_pci.h
+++ b/drivers/net/zxdh/zxdh_pci.h
@@ -144,6 +144,7 @@ struct zxdh_pci_ops {
 
     int32_t  (*setup_queue)(struct zxdh_hw *hw, struct zxdh_virtqueue *vq);
     void     (*del_queue)(struct zxdh_hw *hw, struct zxdh_virtqueue *vq);
+    void     (*notify_queue)(struct zxdh_hw *hw, struct zxdh_virtqueue *vq);
 };
 
 struct zxdh_hw_internal {
diff --git a/drivers/net/zxdh/zxdh_queue.c b/drivers/net/zxdh/zxdh_queue.c
index af21f046ad..8c8f2605f6 100644
--- a/drivers/net/zxdh/zxdh_queue.c
+++ b/drivers/net/zxdh/zxdh_queue.c
@@ -274,3 +274,94 @@ zxdh_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
     zxdh_queue_disable_intr(vq);
     return 0;
 }
+
+int32_t zxdh_enqueue_recv_refill_packed(struct zxdh_virtqueue *vq,
+            struct rte_mbuf **cookie, uint16_t num)
+{
+    struct zxdh_vring_packed_desc *start_dp = vq->vq_packed.ring.desc;
+    struct zxdh_hw *hw = vq->hw;
+    struct zxdh_vq_desc_extra *dxp;
+    uint16_t flags = vq->vq_packed.cached_flags;
+    int32_t i;
+    uint16_t idx;
+
+    for (i = 0; i < num; i++) {
+        idx = vq->vq_avail_idx;
+        dxp = &vq->vq_descx[idx];
+        dxp->cookie = (void *)cookie[i];
+        dxp->ndescs = 1;
+        /* rx pkt fill in data_off */
+        start_dp[idx].addr = rte_mbuf_iova_get(cookie[i]) + RTE_PKTMBUF_HEADROOM;
+        start_dp[idx].len = cookie[i]->buf_len - RTE_PKTMBUF_HEADROOM;
+        vq->vq_desc_head_idx = dxp->next;
+        if (vq->vq_desc_head_idx == ZXDH_VQ_RING_DESC_CHAIN_END)
+            vq->vq_desc_tail_idx = vq->vq_desc_head_idx;
+        zxdh_queue_store_flags_packed(&start_dp[idx], flags, hw->weak_barriers);
+        if (++vq->vq_avail_idx >= vq->vq_nentries) {
+            vq->vq_avail_idx -= vq->vq_nentries;
+            vq->vq_packed.cached_flags ^= ZXDH_VRING_PACKED_DESC_F_AVAIL_USED;
+            flags = vq->vq_packed.cached_flags;
+        }
+    }
+    vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num);
+    return 0;
+}
+
+int32_t zxdh_dev_rx_queue_setup_finish(struct rte_eth_dev *dev, uint16_t logic_qidx)
+{
+    struct zxdh_hw *hw = dev->data->dev_private;
+    struct zxdh_virtqueue *vq = hw->vqs[logic_qidx];
+    struct zxdh_virtnet_rx *rxvq = &vq->rxq;
+    uint16_t desc_idx;
+    int32_t error = 0;
+
+    /* Allocate blank mbufs for the each rx descriptor */
+    memset(&rxvq->fake_mbuf, 0, sizeof(rxvq->fake_mbuf));
+    for (desc_idx = 0; desc_idx < ZXDH_MBUF_BURST_SZ; desc_idx++)
+        vq->sw_ring[vq->vq_nentries + desc_idx] = &rxvq->fake_mbuf;
+
+    while (!zxdh_queue_full(vq)) {
+        uint16_t free_cnt = vq->vq_free_cnt;
+
+        free_cnt = RTE_MIN(ZXDH_MBUF_BURST_SZ, free_cnt);
+        struct rte_mbuf *new_pkts[free_cnt];
+
+        if (likely(rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt) == 0)) {
+            error = zxdh_enqueue_recv_refill_packed(vq, new_pkts, free_cnt);
+            if (unlikely(error)) {
+                int32_t i;
+                for (i = 0; i < free_cnt; i++)
+                    rte_pktmbuf_free(new_pkts[i]);
+            }
+        } else {
+            PMD_DRV_LOG(ERR, "port %d rxq %d allocated bufs from %s failed",
+                hw->port_id, logic_qidx, rxvq->mpool->name);
+            break;
+        }
+    }
+    return 0;
+}
+
+void zxdh_queue_rxvq_flush(struct zxdh_virtqueue *vq)
+{
+    struct zxdh_vq_desc_extra *dxp = NULL;
+    uint16_t i = 0;
+    struct zxdh_vring_packed_desc *descs = vq->vq_packed.ring.desc;
+    int32_t cnt = 0;
+
+    i = vq->vq_used_cons_idx;
+    while (zxdh_desc_used(&descs[i], vq) && cnt++ < vq->vq_nentries) {
+        dxp = &vq->vq_descx[descs[i].id];
+        if (dxp->cookie != NULL) {
+            rte_pktmbuf_free(dxp->cookie);
+            dxp->cookie = NULL;
+        }
+        vq->vq_free_cnt++;
+        vq->vq_used_cons_idx++;
+        if (vq->vq_used_cons_idx >= vq->vq_nentries) {
+            vq->vq_used_cons_idx -= vq->vq_nentries;
+            vq->vq_packed.used_wrap_counter ^= 1;
+        }
+        i = vq->vq_used_cons_idx;
+    }
+}
diff --git a/drivers/net/zxdh/zxdh_queue.h b/drivers/net/zxdh/zxdh_queue.h
index 2f602d894f..6513aec3f0 100644
--- a/drivers/net/zxdh/zxdh_queue.h
+++ b/drivers/net/zxdh/zxdh_queue.h
@@ -25,6 +25,11 @@ enum { ZXDH_VTNET_RQ = 0, ZXDH_VTNET_TQ = 1 };
 #define ZXDH_VRING_DESC_F_WRITE           2
 /* This flag means the descriptor was made available by the driver */
 #define ZXDH_VRING_PACKED_DESC_F_AVAIL   (1 << (7))
+#define ZXDH_VRING_PACKED_DESC_F_USED    (1 << (15))
+
+/* Frequently used combinations */
+#define ZXDH_VRING_PACKED_DESC_F_AVAIL_USED \
+        (ZXDH_VRING_PACKED_DESC_F_AVAIL | ZXDH_VRING_PACKED_DESC_F_USED)
 
 #define ZXDH_RING_EVENT_FLAGS_ENABLE      0x0
 #define ZXDH_RING_EVENT_FLAGS_DISABLE     0x1
@@ -33,6 +38,9 @@ enum { ZXDH_VTNET_RQ = 0, ZXDH_VTNET_TQ = 1 };
 #define ZXDH_VQ_RING_DESC_CHAIN_END       32768
 #define ZXDH_QUEUE_DEPTH                  1024
 
+#define ZXDH_RQ_QUEUE_IDX                 0
+#define ZXDH_TQ_QUEUE_IDX                 1
+
 /*
  * ring descriptors: 16 bytes.
  * These can chain together via "next".
@@ -290,6 +298,63 @@ zxdh_mb(uint8_t weak_barriers)
         rte_mb();
 }
 
+static inline int32_t
+zxdh_queue_full(const struct zxdh_virtqueue *vq)
+{
+    return (vq->vq_free_cnt == 0);
+}
+
+static inline void
+zxdh_queue_store_flags_packed(struct zxdh_vring_packed_desc *dp,
+        uint16_t flags, uint8_t weak_barriers)
+    {
+    if (weak_barriers) {
+    #ifdef RTE_ARCH_X86_64
+        rte_io_wmb();
+        dp->flags = flags;
+    #else
+        rte_atomic_store_explicit(&dp->flags, flags, rte_memory_order_release);
+    #endif
+    } else {
+        rte_io_wmb();
+        dp->flags = flags;
+    }
+}
+
+static inline uint16_t
+zxdh_queue_fetch_flags_packed(struct zxdh_vring_packed_desc *dp,
+        uint8_t weak_barriers)
+    {
+    uint16_t flags;
+    if (weak_barriers) {
+    #ifdef RTE_ARCH_X86_64
+        flags = dp->flags;
+        rte_io_rmb();
+    #else
+        flags = rte_atomic_load_explicit(&dp->flags, rte_memory_order_acquire);
+    #endif
+    } else {
+        flags = dp->flags;
+        rte_io_rmb();
+    }
+
+    return flags;
+}
+
+static inline int32_t
+zxdh_desc_used(struct zxdh_vring_packed_desc *desc, struct zxdh_virtqueue *vq)
+{
+    uint16_t flags = zxdh_queue_fetch_flags_packed(desc, vq->hw->weak_barriers);
+    uint16_t used = !!(flags & ZXDH_VRING_PACKED_DESC_F_USED);
+    uint16_t avail = !!(flags & ZXDH_VRING_PACKED_DESC_F_AVAIL);
+    return avail == used && used == vq->vq_packed.used_wrap_counter;
+}
+
+static inline void zxdh_queue_notify(struct zxdh_virtqueue *vq)
+{
+    ZXDH_VTPCI_OPS(vq->hw)->notify_queue(vq->hw, vq);
+}
+
 struct rte_mbuf *zxdh_queue_detach_unused(struct zxdh_virtqueue *vq);
 int32_t zxdh_free_queues(struct rte_eth_dev *dev);
 int32_t zxdh_get_queue_type(uint16_t vtpci_queue_idx);
@@ -306,5 +371,9 @@ int32_t zxdh_dev_rx_queue_setup(struct rte_eth_dev *dev,
             struct rte_mempool *mp);
 int32_t zxdh_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id);
 int32_t zxdh_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id);
+int32_t zxdh_dev_rx_queue_setup_finish(struct rte_eth_dev *dev, uint16_t logic_qidx);
+void zxdh_queue_rxvq_flush(struct zxdh_virtqueue *vq);
+int32_t zxdh_enqueue_recv_refill_packed(struct zxdh_virtqueue *vq,
+            struct rte_mbuf **cookie, uint16_t num);
 
 #endif /* ZXDH_QUEUE_H */
diff --git a/drivers/net/zxdh/zxdh_rxtx.h b/drivers/net/zxdh/zxdh_rxtx.h
index de9353b223..8c7f734805 100644
--- a/drivers/net/zxdh/zxdh_rxtx.h
+++ b/drivers/net/zxdh/zxdh_rxtx.h
@@ -20,21 +20,19 @@ struct zxdh_virtnet_stats {
     uint64_t size_bins[8];
 };
 
-struct zxdh_virtnet_rx {
+struct __rte_cache_aligned zxdh_virtnet_rx {
     struct zxdh_virtqueue         *vq;
-
-    /* dummy mbuf, for wraparound when processing RX ring. */
-    struct rte_mbuf           fake_mbuf;
-
     uint64_t                  mbuf_initializer; /* value to init mbufs. */
     struct rte_mempool       *mpool;            /* mempool for mbuf allocation */
     uint16_t                  queue_id;         /* DPDK queue index. */
     uint16_t                  port_id;          /* Device port identifier. */
     struct zxdh_virtnet_stats      stats;
     const struct rte_memzone *mz;               /* mem zone to populate RX ring. */
-} __rte_packed;
+    /* dummy mbuf, for wraparound when processing RX ring. */
+    struct rte_mbuf           fake_mbuf;
+};
 
-struct zxdh_virtnet_tx {
+struct __rte_cache_aligned zxdh_virtnet_tx {
     struct zxdh_virtqueue         *vq;
     const struct rte_memzone *zxdh_net_hdr_mz;  /* memzone to populate hdr. */
     rte_iova_t                zxdh_net_hdr_mem; /* hdr for each xmit packet */
@@ -42,6 +40,6 @@ struct zxdh_virtnet_tx {
     uint16_t                  port_id;            /* Device port identifier. */
     struct zxdh_virtnet_stats      stats;
     const struct rte_memzone *mz;                 /* mem zone to populate TX ring. */
-} __rte_packed;
+};
 
 #endif  /* ZXDH_RXTX_H */
-- 
2.27.0