* [dpdk-dev] [PATCH] ENIC PMD receive path performance improvements.
2016-02-16 5:36 [dpdk-dev] [PATCH] Enic PMD Rx performance improvements johndale
@ 2016-02-16 5:36 ` johndale
2016-02-18 1:31 ` [dpdk-dev] [PATCH] Enic PMD Rx " John Daley (johndale)
1 sibling, 0 replies; 3+ messages in thread
From: johndale @ 2016-02-16 5:36 UTC (permalink / raw)
To: dev; +Cc: johndale
- Simplify and reduce code path length of receive function.
- Put all receive stuff in enic_rx.c
- Reduce the number of posted_index updates (pay attention to rx_free_thresh)
- Remove the container structure around the rq mbuf ring
- Prefetch next Mbuf and descriptors while processing the current one
- Lookup table for converting CQ flags to mbuf flags.
Signed-off-by: johndale <johndale@cisco.com>
---
drivers/net/enic/Makefile | 1 +
drivers/net/enic/base/vnic_rq.c | 99 ++--------
drivers/net/enic/base/vnic_rq.h | 147 +-------------
drivers/net/enic/enic.h | 16 +-
drivers/net/enic/enic_ethdev.c | 21 +-
drivers/net/enic/enic_main.c | 319 +++++++++----------------------
drivers/net/enic/enic_res.h | 16 +-
drivers/net/enic/enic_rx.c | 413 ++++++++++++++++++++++++++++++++++++++++
8 files changed, 555 insertions(+), 477 deletions(-)
create mode 100644 drivers/net/enic/enic_rx.c
diff --git a/drivers/net/enic/Makefile b/drivers/net/enic/Makefile
index f0ee093..f316274 100644
--- a/drivers/net/enic/Makefile
+++ b/drivers/net/enic/Makefile
@@ -53,6 +53,7 @@ VPATH += $(SRCDIR)/src
#
SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_ethdev.c
SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_main.c
+SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_rx.c
SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_clsf.c
SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_res.c
SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += base/vnic_cq.c
diff --git a/drivers/net/enic/base/vnic_rq.c b/drivers/net/enic/base/vnic_rq.c
index 1441604..cb62c5e 100644
--- a/drivers/net/enic/base/vnic_rq.c
+++ b/drivers/net/enic/base/vnic_rq.c
@@ -35,77 +35,21 @@
#include "vnic_dev.h"
#include "vnic_rq.h"
-static int vnic_rq_alloc_bufs(struct vnic_rq *rq)
-{
- struct vnic_rq_buf *buf;
- unsigned int i, j, count = rq->ring.desc_count;
- unsigned int blks = VNIC_RQ_BUF_BLKS_NEEDED(count);
-
- for (i = 0; i < blks; i++) {
- rq->bufs[i] = kzalloc(VNIC_RQ_BUF_BLK_SZ(count), GFP_ATOMIC);
- if (!rq->bufs[i])
- return -ENOMEM;
- }
-
- for (i = 0; i < blks; i++) {
- buf = rq->bufs[i];
- for (j = 0; j < VNIC_RQ_BUF_BLK_ENTRIES(count); j++) {
- buf->index = i * VNIC_RQ_BUF_BLK_ENTRIES(count) + j;
- buf->desc = (u8 *)rq->ring.descs +
- rq->ring.desc_size * buf->index;
- if (buf->index + 1 == count) {
- buf->next = rq->bufs[0];
- break;
- } else if (j + 1 == VNIC_RQ_BUF_BLK_ENTRIES(count)) {
- buf->next = rq->bufs[i + 1];
- } else {
- buf->next = buf + 1;
- buf++;
- }
- }
- }
-
- rq->to_use = rq->to_clean = rq->bufs[0];
-
- return 0;
-}
-
-int vnic_rq_mem_size(struct vnic_rq *rq, unsigned int desc_count,
- unsigned int desc_size)
-{
- int mem_size = 0;
-
- mem_size += vnic_dev_desc_ring_size(&rq->ring, desc_count, desc_size);
-
- mem_size += VNIC_RQ_BUF_BLKS_NEEDED(rq->ring.desc_count) *
- VNIC_RQ_BUF_BLK_SZ(rq->ring.desc_count);
-
- return mem_size;
-}
-
void vnic_rq_free(struct vnic_rq *rq)
{
struct vnic_dev *vdev;
- unsigned int i;
vdev = rq->vdev;
vnic_dev_free_desc_ring(vdev, &rq->ring);
- for (i = 0; i < VNIC_RQ_BUF_BLKS_MAX; i++) {
- if (rq->bufs[i]) {
- kfree(rq->bufs[i]);
- rq->bufs[i] = NULL;
- }
- }
-
rq->ctrl = NULL;
}
int vnic_rq_alloc(struct vnic_dev *vdev, struct vnic_rq *rq, unsigned int index,
unsigned int desc_count, unsigned int desc_size)
{
- int err;
+ int rc;
char res_name[NAME_MAX];
static int instance;
@@ -121,18 +65,9 @@ int vnic_rq_alloc(struct vnic_dev *vdev, struct vnic_rq *rq, unsigned int index,
vnic_rq_disable(rq);
snprintf(res_name, sizeof(res_name), "%d-rq-%d", instance++, index);
- err = vnic_dev_alloc_desc_ring(vdev, &rq->ring, desc_count, desc_size,
+ rc = vnic_dev_alloc_desc_ring(vdev, &rq->ring, desc_count, desc_size,
rq->socket_id, res_name);
- if (err)
- return err;
-
- err = vnic_rq_alloc_bufs(rq);
- if (err) {
- vnic_rq_free(rq);
- return err;
- }
-
- return 0;
+ return rc;
}
void vnic_rq_init_start(struct vnic_rq *rq, unsigned int cq_index,
@@ -154,9 +89,6 @@ void vnic_rq_init_start(struct vnic_rq *rq, unsigned int cq_index,
iowrite32(fetch_index, &rq->ctrl->fetch_index);
iowrite32(posted_index, &rq->ctrl->posted_index);
- rq->to_use = rq->to_clean =
- &rq->bufs[fetch_index / VNIC_RQ_BUF_BLK_ENTRIES(count)]
- [fetch_index % VNIC_RQ_BUF_BLK_ENTRIES(count)];
}
void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index,
@@ -176,6 +108,8 @@ void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index,
fetch_index, fetch_index,
error_interrupt_enable,
error_interrupt_offset);
+ rq->rxst_idx = 0;
+ rq->tot_pkts = 0;
}
void vnic_rq_error_out(struct vnic_rq *rq, unsigned int error)
@@ -212,21 +146,20 @@ int vnic_rq_disable(struct vnic_rq *rq)
}
void vnic_rq_clean(struct vnic_rq *rq,
- void (*buf_clean)(struct vnic_rq *rq, struct vnic_rq_buf *buf))
+ void (*buf_clean)(struct rte_mbuf **buf))
{
- struct vnic_rq_buf *buf;
- u32 fetch_index;
+ struct rte_mbuf **buf;
+ u32 fetch_index, i;
unsigned int count = rq->ring.desc_count;
- buf = rq->to_clean;
-
- while (vnic_rq_desc_used(rq) > 0) {
+ buf = &rq->mbuf_ring[0];
- (*buf_clean)(rq, buf);
-
- buf = rq->to_clean = buf->next;
- rq->ring.desc_avail++;
+ for (i = 0; i < count; i++) {
+ (*buf_clean)(buf);
+ buf++;
}
+ rq->ring.desc_avail = count - 1;
+ rq->rx_nb_hold = 0;
/* Use current fetch_index as the ring starting point */
fetch_index = ioread32(&rq->ctrl->fetch_index);
@@ -235,9 +168,7 @@ void vnic_rq_clean(struct vnic_rq *rq,
/* Hardware surprise removal: reset fetch_index */
fetch_index = 0;
}
- rq->to_use = rq->to_clean =
- &rq->bufs[fetch_index / VNIC_RQ_BUF_BLK_ENTRIES(count)]
- [fetch_index % VNIC_RQ_BUF_BLK_ENTRIES(count)];
+
iowrite32(fetch_index, &rq->ctrl->posted_index);
vnic_dev_clear_desc_ring(&rq->ring);
diff --git a/drivers/net/enic/base/vnic_rq.h b/drivers/net/enic/base/vnic_rq.h
index 0f5c3c1..e083ccc 100644
--- a/drivers/net/enic/base/vnic_rq.h
+++ b/drivers/net/enic/base/vnic_rq.h
@@ -66,42 +66,22 @@ struct vnic_rq_ctrl {
u32 pad10;
};
-/* Break the vnic_rq_buf allocations into blocks of 32/64 entries */
-#define VNIC_RQ_BUF_MIN_BLK_ENTRIES 32
-#define VNIC_RQ_BUF_DFLT_BLK_ENTRIES 64
-#define VNIC_RQ_BUF_BLK_ENTRIES(entries) \
- ((unsigned int)((entries < VNIC_RQ_BUF_DFLT_BLK_ENTRIES) ? \
- VNIC_RQ_BUF_MIN_BLK_ENTRIES : VNIC_RQ_BUF_DFLT_BLK_ENTRIES))
-#define VNIC_RQ_BUF_BLK_SZ(entries) \
- (VNIC_RQ_BUF_BLK_ENTRIES(entries) * sizeof(struct vnic_rq_buf))
-#define VNIC_RQ_BUF_BLKS_NEEDED(entries) \
- DIV_ROUND_UP(entries, VNIC_RQ_BUF_BLK_ENTRIES(entries))
-#define VNIC_RQ_BUF_BLKS_MAX VNIC_RQ_BUF_BLKS_NEEDED(4096)
-
-struct vnic_rq_buf {
- struct vnic_rq_buf *next;
- dma_addr_t dma_addr;
- void *os_buf;
- unsigned int os_buf_index;
- unsigned int len;
- unsigned int index;
- void *desc;
- uint64_t wr_id;
-};
-
struct vnic_rq {
unsigned int index;
+ unsigned int posted_index;
struct vnic_dev *vdev;
- struct vnic_rq_ctrl __iomem *ctrl; /* memory-mapped */
+ struct vnic_rq_ctrl __iomem *ctrl; /* memory-mapped */
struct vnic_dev_ring ring;
- struct vnic_rq_buf *bufs[VNIC_RQ_BUF_BLKS_MAX];
- struct vnic_rq_buf *to_use;
- struct vnic_rq_buf *to_clean;
+ struct rte_mbuf **mbuf_ring; /* array of allocated mbufs */
+ unsigned int mbuf_next_idx; /* next mb to consume */
void *os_buf_head;
unsigned int pkts_outstanding;
-
+ uint16_t rx_nb_hold;
+ uint16_t rx_free_thresh;
unsigned int socket_id;
struct rte_mempool *mp;
+ uint16_t rxst_idx;
+ uint32_t tot_pkts;
};
static inline unsigned int vnic_rq_desc_avail(struct vnic_rq *rq)
@@ -116,119 +96,13 @@ static inline unsigned int vnic_rq_desc_used(struct vnic_rq *rq)
return rq->ring.desc_count - rq->ring.desc_avail - 1;
}
-static inline void *vnic_rq_next_desc(struct vnic_rq *rq)
-{
- return rq->to_use->desc;
-}
-
-static inline unsigned int vnic_rq_next_index(struct vnic_rq *rq)
-{
- return rq->to_use->index;
-}
-
-static inline void vnic_rq_post(struct vnic_rq *rq,
- void *os_buf, unsigned int os_buf_index,
- dma_addr_t dma_addr, unsigned int len,
- uint64_t wrid)
-{
- struct vnic_rq_buf *buf = rq->to_use;
-
- buf->os_buf = os_buf;
- buf->os_buf_index = os_buf_index;
- buf->dma_addr = dma_addr;
- buf->len = len;
- buf->wr_id = wrid;
-
- buf = buf->next;
- rq->to_use = buf;
- rq->ring.desc_avail--;
-
- /* Move the posted_index every nth descriptor
- */
-
-#ifndef VNIC_RQ_RETURN_RATE
-#define VNIC_RQ_RETURN_RATE 0xf /* keep 2^n - 1 */
-#endif
-
- if ((buf->index & VNIC_RQ_RETURN_RATE) == 0) {
- /* Adding write memory barrier prevents compiler and/or CPU
- * reordering, thus avoiding descriptor posting before
- * descriptor is initialized. Otherwise, hardware can read
- * stale descriptor fields.
- */
- wmb();
- iowrite32(buf->index, &rq->ctrl->posted_index);
- }
-}
-
-static inline void vnic_rq_post_commit(struct vnic_rq *rq,
- void *os_buf, unsigned int os_buf_index,
- dma_addr_t dma_addr, unsigned int len)
-{
- struct vnic_rq_buf *buf = rq->to_use;
-
- buf->os_buf = os_buf;
- buf->os_buf_index = os_buf_index;
- buf->dma_addr = dma_addr;
- buf->len = len;
-
- buf = buf->next;
- rq->to_use = buf;
- rq->ring.desc_avail--;
-
- /* Move the posted_index every descriptor
- */
-
- /* Adding write memory barrier prevents compiler and/or CPU
- * reordering, thus avoiding descriptor posting before
- * descriptor is initialized. Otherwise, hardware can read
- * stale descriptor fields.
- */
- wmb();
- iowrite32(buf->index, &rq->ctrl->posted_index);
-}
-static inline void vnic_rq_return_descs(struct vnic_rq *rq, unsigned int count)
-{
- rq->ring.desc_avail += count;
-}
enum desc_return_options {
VNIC_RQ_RETURN_DESC,
VNIC_RQ_DEFER_RETURN_DESC,
};
-static inline int vnic_rq_service(struct vnic_rq *rq,
- struct cq_desc *cq_desc, u16 completed_index,
- int desc_return, int (*buf_service)(struct vnic_rq *rq,
- struct cq_desc *cq_desc, struct vnic_rq_buf *buf,
- int skipped, void *opaque), void *opaque)
-{
- struct vnic_rq_buf *buf;
- int skipped;
- int eop = 0;
-
- buf = rq->to_clean;
- while (1) {
-
- skipped = (buf->index != completed_index);
-
- if ((*buf_service)(rq, cq_desc, buf, skipped, opaque))
- eop++;
-
- if (desc_return == VNIC_RQ_RETURN_DESC)
- rq->ring.desc_avail++;
-
- rq->to_clean = buf->next;
-
- if (!skipped)
- break;
-
- buf = rq->to_clean;
- }
- return eop;
-}
-
static inline int vnic_rq_fill(struct vnic_rq *rq,
int (*buf_fill)(struct vnic_rq *rq))
{
@@ -274,8 +148,5 @@ unsigned int vnic_rq_error_status(struct vnic_rq *rq);
void vnic_rq_enable(struct vnic_rq *rq);
int vnic_rq_disable(struct vnic_rq *rq);
void vnic_rq_clean(struct vnic_rq *rq,
- void (*buf_clean)(struct vnic_rq *rq, struct vnic_rq_buf *buf));
-int vnic_rq_mem_size(struct vnic_rq *rq, unsigned int desc_count,
- unsigned int desc_size);
-
+ void (*buf_clean)(struct rte_mbuf **buf));
#endif /* _VNIC_RQ_H_ */
diff --git a/drivers/net/enic/enic.h b/drivers/net/enic/enic.h
index 9e78305..8c914f5 100644
--- a/drivers/net/enic/enic.h
+++ b/drivers/net/enic/enic.h
@@ -45,6 +45,7 @@
#include "vnic_nic.h"
#include "vnic_rss.h"
#include "enic_res.h"
+#include "cq_enet_desc.h"
#define DRV_NAME "enic_pmd"
#define DRV_DESCRIPTION "Cisco VIC Ethernet NIC Poll-mode Driver"
@@ -154,6 +155,16 @@ static inline struct enic *pmd_priv(struct rte_eth_dev *eth_dev)
return (struct enic *)eth_dev->data->dev_private;
}
+#define RTE_LIBRTE_ENIC_ASSERT_ENABLE
+#ifdef RTE_LIBRTE_ENIC_ASSERT_ENABLE
+#define ASSERT(x) do { \
+ if (!(x)) \
+ rte_panic("ENIC: x"); \
+} while (0)
+#else
+#define ASSERT(x)
+#endif
+
extern void enic_fdir_stats_get(struct enic *enic,
struct rte_eth_fdir_stats *stats);
extern int enic_fdir_add_fltr(struct enic *enic,
@@ -193,9 +204,10 @@ extern void enic_send_pkt(struct enic *enic, struct vnic_wq *wq,
uint16_t ol_flags, uint16_t vlan_tag);
extern void enic_post_wq_index(struct vnic_wq *wq);
-extern int enic_poll(struct vnic_rq *rq, struct rte_mbuf **rx_pkts,
- unsigned int budget, unsigned int *work_done);
extern int enic_probe(struct enic *enic);
extern int enic_clsf_init(struct enic *enic);
extern void enic_clsf_destroy(struct enic *enic);
+uint16_t enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts);
+
#endif /* _ENIC_H_ */
diff --git a/drivers/net/enic/enic_ethdev.c b/drivers/net/enic/enic_ethdev.c
index 2a88043..e54919b 100644
--- a/drivers/net/enic/enic_ethdev.c
+++ b/drivers/net/enic/enic_ethdev.c
@@ -270,6 +270,10 @@ static int enicpmd_dev_rx_queue_setup(struct rte_eth_dev *eth_dev,
return ret;
}
+ enic->rq[queue_idx].rx_free_thresh = rx_conf->rx_free_thresh;
+ dev_debug(enic, "Set queue_id:%u free thresh:%u\n", queue_idx,
+ enic->rq[queue_idx].rx_free_thresh);
+
return enicpmd_dev_setup_intr(enic);
}
@@ -429,6 +433,9 @@ static void enicpmd_dev_info_get(struct rte_eth_dev *eth_dev,
DEV_TX_OFFLOAD_IPV4_CKSUM |
DEV_TX_OFFLOAD_UDP_CKSUM |
DEV_TX_OFFLOAD_TCP_CKSUM;
+ device_info->default_rxconf = (struct rte_eth_rxconf) {
+ .rx_free_thresh = ENIC_DEFAULT_RX_FREE_THRESH
+ };
}
static void enicpmd_dev_promiscuous_enable(struct rte_eth_dev *eth_dev)
@@ -538,18 +545,6 @@ static uint16_t enicpmd_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
return index;
}
-static uint16_t enicpmd_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
- uint16_t nb_pkts)
-{
- struct vnic_rq *rq = (struct vnic_rq *)rx_queue;
- unsigned int work_done;
-
- if (enic_poll(rq, rx_pkts, (unsigned int)nb_pkts, &work_done))
- dev_err(enic, "error in enicpmd poll\n");
-
- return work_done;
-}
-
static const struct eth_dev_ops enicpmd_eth_dev_ops = {
.dev_configure = enicpmd_dev_configure,
.dev_start = enicpmd_dev_start,
@@ -606,7 +601,7 @@ static int eth_enicpmd_dev_init(struct rte_eth_dev *eth_dev)
enic->port_id = eth_dev->data->port_id;
enic->rte_dev = eth_dev;
eth_dev->dev_ops = &enicpmd_eth_dev_ops;
- eth_dev->rx_pkt_burst = &enicpmd_recv_pkts;
+ eth_dev->rx_pkt_burst = &enic_recv_pkts;
eth_dev->tx_pkt_burst = &enicpmd_xmit_pkts;
pdev = eth_dev->pci_dev;
diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c
index 07a9810..d897fcf 100644
--- a/drivers/net/enic/enic_main.c
+++ b/drivers/net/enic/enic_main.c
@@ -60,6 +60,17 @@
#include "vnic_nic.h"
#include "enic_vnic_wq.h"
+static inline struct rte_mbuf *
+rte_rxmbuf_alloc(struct rte_mempool *mp)
+{
+ struct rte_mbuf *m;
+
+ m = __rte_mbuf_raw_alloc(mp);
+ __rte_mbuf_sanity_check_raw(m, 0);
+ return m;
+}
+
+
static inline int enic_is_sriov_vf(struct enic *enic)
{
return enic->pdev->id.device_id == PCI_DEVICE_ID_CISCO_VIC_ENET_VF;
@@ -90,6 +101,25 @@ enic_rxmbuf_alloc(struct rte_mempool *mp)
return m;
}
+static void
+enic_rxmbuf_queue_release(struct enic *enic, struct vnic_rq *rq)
+{
+ uint16_t i;
+
+ if (!rq || !rq->mbuf_ring) {
+ dev_debug(enic, "Pointer to rq or mbuf_ring is NULL");
+ return;
+ }
+
+ for (i = 0; i < enic->config.rq_desc_count; i++) {
+ if (rq->mbuf_ring[i]) {
+ rte_pktmbuf_free_seg(rq->mbuf_ring[i]);
+ rq->mbuf_ring[i] = NULL;
+ }
+ }
+}
+
+
void enic_set_hdr_split_size(struct enic *enic, u16 split_hdr_size)
{
vnic_set_hdr_split_size(enic->vdev, split_hdr_size);
@@ -262,13 +292,13 @@ void enic_set_mac_address(struct enic *enic, uint8_t *mac_addr)
}
static void
-enic_free_rq_buf(__rte_unused struct vnic_rq *rq, struct vnic_rq_buf *buf)
+enic_free_rq_buf(struct rte_mbuf **mbuf)
{
- if (!buf->os_buf)
+ if (*mbuf == NULL)
return;
- rte_pktmbuf_free((struct rte_mbuf *)buf->os_buf);
- buf->os_buf = NULL;
+ rte_pktmbuf_free(*mbuf);
+ mbuf = NULL;
}
void enic_init_vnic_resources(struct enic *enic)
@@ -314,221 +344,47 @@ void enic_init_vnic_resources(struct enic *enic)
}
-static int enic_rq_alloc_buf(struct vnic_rq *rq)
+static int
+enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq)
{
- struct enic *enic = vnic_dev_priv(rq->vdev);
+ struct rte_mbuf *mb;
+ struct rq_enet_desc *rqd = rq->ring.descs;
+ unsigned i;
dma_addr_t dma_addr;
- struct rq_enet_desc *desc = vnic_rq_next_desc(rq);
- uint8_t type = RQ_ENET_TYPE_ONLY_SOP;
- u16 split_hdr_size = vnic_get_hdr_split_size(enic->vdev);
- struct rte_mbuf *mbuf = enic_rxmbuf_alloc(rq->mp);
- struct rte_mbuf *hdr_mbuf = NULL;
-
- if (!mbuf) {
- dev_err(enic, "mbuf alloc in enic_rq_alloc_buf failed\n");
- return -1;
- }
-
- if (unlikely(split_hdr_size)) {
- if (vnic_rq_desc_avail(rq) < 2) {
- rte_mempool_put(mbuf->pool, mbuf);
- return -1;
- }
- hdr_mbuf = enic_rxmbuf_alloc(rq->mp);
- if (!hdr_mbuf) {
- rte_mempool_put(mbuf->pool, mbuf);
- dev_err(enic,
- "hdr_mbuf alloc in enic_rq_alloc_buf failed\n");
- return -1;
- }
-
- hdr_mbuf->data_off = RTE_PKTMBUF_HEADROOM;
-
- hdr_mbuf->nb_segs = 2;
- hdr_mbuf->port = enic->port_id;
- hdr_mbuf->next = mbuf;
-
- dma_addr = (dma_addr_t)
- (hdr_mbuf->buf_physaddr + hdr_mbuf->data_off);
-
- rq_enet_desc_enc(desc, dma_addr, type, split_hdr_size);
-
- vnic_rq_post(rq, (void *)hdr_mbuf, 0 /*os_buf_index*/, dma_addr,
- (unsigned int)split_hdr_size, 0 /*wrid*/);
-
- desc = vnic_rq_next_desc(rq);
- type = RQ_ENET_TYPE_NOT_SOP;
- } else {
- mbuf->nb_segs = 1;
- mbuf->port = enic->port_id;
- }
-
- mbuf->data_off = RTE_PKTMBUF_HEADROOM;
- mbuf->next = NULL;
-
- dma_addr = (dma_addr_t)
- (mbuf->buf_physaddr + mbuf->data_off);
-
- rq_enet_desc_enc(desc, dma_addr, type, mbuf->buf_len);
-
- vnic_rq_post(rq, (void *)mbuf, 0 /*os_buf_index*/, dma_addr,
- (unsigned int)mbuf->buf_len, 0 /*wrid*/);
-
- return 0;
-}
-
-static int enic_rq_indicate_buf(struct vnic_rq *rq,
- struct cq_desc *cq_desc, struct vnic_rq_buf *buf,
- int skipped, void *opaque)
-{
- struct enic *enic = vnic_dev_priv(rq->vdev);
- struct rte_mbuf **rx_pkt_bucket = (struct rte_mbuf **)opaque;
- struct rte_mbuf *rx_pkt = NULL;
- struct rte_mbuf *hdr_rx_pkt = NULL;
-
- u8 type, color, eop, sop, ingress_port, vlan_stripped;
- u8 fcoe, fcoe_sof, fcoe_fc_crc_ok, fcoe_enc_error, fcoe_eof;
- u8 tcp_udp_csum_ok, udp, tcp, ipv4_csum_ok;
- u8 ipv6, ipv4, ipv4_fragment, fcs_ok, rss_type, csum_not_calc;
- u8 packet_error;
- u16 q_number, completed_index, bytes_written, vlan_tci, checksum;
- u32 rss_hash;
-
- cq_enet_rq_desc_dec((struct cq_enet_rq_desc *)cq_desc,
- &type, &color, &q_number, &completed_index,
- &ingress_port, &fcoe, &eop, &sop, &rss_type,
- &csum_not_calc, &rss_hash, &bytes_written,
- &packet_error, &vlan_stripped, &vlan_tci, &checksum,
- &fcoe_sof, &fcoe_fc_crc_ok, &fcoe_enc_error,
- &fcoe_eof, &tcp_udp_csum_ok, &udp, &tcp,
- &ipv4_csum_ok, &ipv6, &ipv4, &ipv4_fragment,
- &fcs_ok);
-
- rx_pkt = (struct rte_mbuf *)buf->os_buf;
- buf->os_buf = NULL;
- if (unlikely(packet_error)) {
- dev_err(enic, "packet error\n");
- rx_pkt->data_len = 0;
- return 0;
- }
+ dev_debug(enic, "queue %u, allocating %u rx queue mbufs", rq->index,
+ rq->ring.desc_count);
- if (unlikely(skipped)) {
- rx_pkt->data_len = 0;
- return 0;
- }
-
- if (likely(!vnic_get_hdr_split_size(enic->vdev))) {
- /* No header split configured */
- *rx_pkt_bucket = rx_pkt;
- rx_pkt->pkt_len = bytes_written;
-
- if (ipv4) {
- rx_pkt->packet_type = RTE_PTYPE_L3_IPV4;
- if (!csum_not_calc) {
- if (unlikely(!ipv4_csum_ok))
- rx_pkt->ol_flags |= PKT_RX_IP_CKSUM_BAD;
-
- if ((tcp || udp) && (!tcp_udp_csum_ok))
- rx_pkt->ol_flags |= PKT_RX_L4_CKSUM_BAD;
- }
- } else if (ipv6)
- rx_pkt->packet_type = RTE_PTYPE_L3_IPV6;
- } else {
- /* Header split */
- if (sop && !eop) {
- /* This piece is header */
- *rx_pkt_bucket = rx_pkt;
- rx_pkt->pkt_len = bytes_written;
- } else {
- if (sop && eop) {
- /* The packet is smaller than split_hdr_size */
- *rx_pkt_bucket = rx_pkt;
- rx_pkt->pkt_len = bytes_written;
- if (ipv4) {
- rx_pkt->packet_type = RTE_PTYPE_L3_IPV4;
- if (!csum_not_calc) {
- if (unlikely(!ipv4_csum_ok))
- rx_pkt->ol_flags |=
- PKT_RX_IP_CKSUM_BAD;
-
- if ((tcp || udp) &&
- (!tcp_udp_csum_ok))
- rx_pkt->ol_flags |=
- PKT_RX_L4_CKSUM_BAD;
- }
- } else if (ipv6)
- rx_pkt->packet_type = RTE_PTYPE_L3_IPV6;
- } else {
- /* Payload */
- hdr_rx_pkt = *rx_pkt_bucket;
- hdr_rx_pkt->pkt_len += bytes_written;
- if (ipv4) {
- hdr_rx_pkt->packet_type =
- RTE_PTYPE_L3_IPV4;
- if (!csum_not_calc) {
- if (unlikely(!ipv4_csum_ok))
- hdr_rx_pkt->ol_flags |=
- PKT_RX_IP_CKSUM_BAD;
-
- if ((tcp || udp) &&
- (!tcp_udp_csum_ok))
- hdr_rx_pkt->ol_flags |=
- PKT_RX_L4_CKSUM_BAD;
- }
- } else if (ipv6)
- hdr_rx_pkt->packet_type =
- RTE_PTYPE_L3_IPV6;
- }
+ for (i = 0; i < rq->ring.desc_count; i++, rqd++) {
+ mb = rte_rxmbuf_alloc(rq->mp);
+ if (mb == NULL) {
+ dev_err(enic, "RX mbuf alloc failed queue_id=%u",
+ (unsigned)rq->index);
+ return -ENOMEM;
}
- }
- rx_pkt->data_len = bytes_written;
+ dma_addr = (dma_addr_t)(mb->buf_physaddr + mb->data_off);
- if (rss_hash) {
- rx_pkt->ol_flags |= PKT_RX_RSS_HASH;
- rx_pkt->hash.rss = rss_hash;
+ rq_enet_desc_enc(rqd, dma_addr, RQ_ENET_TYPE_ONLY_SOP,
+ mb->buf_len);
+ rq->mbuf_ring[i] = mb;
}
- if (vlan_tci) {
- rx_pkt->ol_flags |= PKT_RX_VLAN_PKT;
- rx_pkt->vlan_tci = vlan_tci;
- }
+ /* make sure all prior writes are complete before doing the PIO write */
+ rte_rmb();
- return eop;
-}
+ /* Post all but the last 2 cache lines' worth of descriptors */
+ rq->posted_index = rq->ring.desc_count - (2 * RTE_CACHE_LINE_SIZE
+ / sizeof(struct rq_enet_desc));
+ rq->rx_nb_hold = 0;
-static int enic_rq_service(struct vnic_dev *vdev, struct cq_desc *cq_desc,
- __rte_unused u8 type, u16 q_number, u16 completed_index, void *opaque)
-{
- struct enic *enic = vnic_dev_priv(vdev);
-
- return vnic_rq_service(&enic->rq[q_number], cq_desc,
- completed_index, VNIC_RQ_RETURN_DESC,
- enic_rq_indicate_buf, opaque);
-
-}
+ dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n",
+ enic->port_id, rq->index, rq->posted_index, rq->rx_nb_hold);
+ iowrite32(rq->posted_index, &rq->ctrl->posted_index);
+ rte_rmb();
-int enic_poll(struct vnic_rq *rq, struct rte_mbuf **rx_pkts,
- unsigned int budget, unsigned int *work_done)
-{
- struct enic *enic = vnic_dev_priv(rq->vdev);
- unsigned int cq = enic_cq_rq(enic, rq->index);
- int err = 0;
-
- *work_done = vnic_cq_service(&enic->cq[cq],
- budget, enic_rq_service, (void *)rx_pkts);
-
- if (*work_done) {
- vnic_rq_fill(rq, enic_rq_alloc_buf);
+ return 0;
- /* Need at least one buffer on ring to get going */
- if (vnic_rq_desc_used(rq) == 0) {
- dev_err(enic, "Unable to alloc receive buffers\n");
- err = -1;
- }
- }
- return err;
}
static void *
@@ -576,6 +432,7 @@ enic_intr_handler(__rte_unused struct rte_intr_handle *handle,
int enic_enable(struct enic *enic)
{
unsigned int index;
+ int err;
struct rte_eth_dev *eth_dev = enic->rte_dev;
eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev);
@@ -586,15 +443,11 @@ int enic_enable(struct enic *enic)
dev_warning(enic, "Init of hash table for clsf failed."\
"Flow director feature will not work\n");
- /* Fill RQ bufs */
for (index = 0; index < enic->rq_count; index++) {
- vnic_rq_fill(&enic->rq[index], enic_rq_alloc_buf);
-
- /* Need at least one buffer on ring to get going
- */
- if (vnic_rq_desc_used(&enic->rq[index]) == 0) {
- dev_err(enic, "Unable to alloc receive buffers\n");
- return -1;
+ err = enic_alloc_rx_queue_mbufs(enic, &enic->rq[index]);
+ if (err) {
+ dev_err(enic, "Failed to alloc RX queue mbufs\n");
+ return err;
}
}
@@ -636,6 +489,9 @@ void enic_free_rq(void *rxq)
struct vnic_rq *rq = (struct vnic_rq *)rxq;
struct enic *enic = vnic_dev_priv(rq->vdev);
+ enic_rxmbuf_queue_release(enic, rq);
+ rte_free(rq->mbuf_ring);
+ rq->mbuf_ring = NULL;
vnic_rq_free(rq);
vnic_cq_free(&enic->cq[rq->index]);
}
@@ -664,7 +520,7 @@ int enic_alloc_rq(struct enic *enic, uint16_t queue_idx,
unsigned int socket_id, struct rte_mempool *mp,
uint16_t nb_desc)
{
- int err;
+ int rc;
struct vnic_rq *rq = &enic->rq[queue_idx];
rq->socket_id = socket_id;
@@ -687,23 +543,35 @@ int enic_alloc_rq(struct enic *enic, uint16_t queue_idx,
}
/* Allocate queue resources */
- err = vnic_rq_alloc(enic->vdev, &enic->rq[queue_idx], queue_idx,
- enic->config.rq_desc_count,
- sizeof(struct rq_enet_desc));
- if (err) {
+ rc = vnic_rq_alloc(enic->vdev, rq, queue_idx,
+ enic->config.rq_desc_count, sizeof(struct rq_enet_desc));
+ if (rc) {
dev_err(enic, "error in allocation of rq\n");
- return err;
+ goto err_exit;
}
- err = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx,
+ rc = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx,
socket_id, enic->config.rq_desc_count,
sizeof(struct cq_enet_rq_desc));
- if (err) {
- vnic_rq_free(rq);
+ if (rc) {
dev_err(enic, "error in allocation of cq for rq\n");
+ goto err_free_rq_exit;
}
- return err;
+ /* Allocate the mbuf ring */
+ rq->mbuf_ring = (struct rte_mbuf **)rte_zmalloc_socket("rq->mbuf_ring",
+ sizeof(struct rte_mbuf *) * enic->config.rq_desc_count,
+ RTE_CACHE_LINE_SIZE, rq->socket_id);
+
+ if (rq->mbuf_ring != NULL)
+ return 0;
+
+ /* cleanup on error */
+ vnic_cq_free(&enic->cq[queue_idx]);
+err_free_rq_exit:
+ vnic_rq_free(rq);
+err_exit:
+ return -ENOMEM;
}
void enic_free_wq(void *txq)
@@ -790,6 +658,7 @@ int enic_disable(struct enic *enic)
for (i = 0; i < enic->wq_count; i++)
vnic_wq_clean(&enic->wq[i], enic_free_wq_buf);
+
for (i = 0; i < enic->rq_count; i++)
vnic_rq_clean(&enic->rq[i], enic_free_rq_buf);
for (i = 0; i < enic->cq_count; i++)
@@ -1074,7 +943,7 @@ int enic_probe(struct enic *enic)
/* Set ingress vlan rewrite mode before vnic initialization */
err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev,
- IG_VLAN_REWRITE_MODE_PRIORITY_TAG_DEFAULT_VLAN);
+ IG_VLAN_REWRITE_MODE_PASS_THRU);
if (err) {
dev_err(enic,
"Failed to set ingress vlan rewrite mode, aborting.\n");
diff --git a/drivers/net/enic/enic_res.h b/drivers/net/enic/enic_res.h
index 49f7e22..33f2e84 100644
--- a/drivers/net/enic/enic_res.h
+++ b/drivers/net/enic/enic_res.h
@@ -52,6 +52,7 @@
#define ENIC_UNICAST_PERFECT_FILTERS 32
#define ENIC_NON_TSO_MAX_DESC 16
+#define ENIC_DEFAULT_RX_FREE_THRESH 32
#define ENIC_SETTING(enic, f) ((enic->config.flags & VENETF_##f) ? 1 : 0)
@@ -133,21 +134,6 @@ static inline void enic_queue_wq_desc_tso(struct vnic_wq *wq,
WQ_ENET_OFFLOAD_MODE_TSO,
eop, 1 /* SOP */, eop, loopback);
}
-static inline void enic_queue_rq_desc(struct vnic_rq *rq,
- void *os_buf, unsigned int os_buf_index,
- dma_addr_t dma_addr, unsigned int len)
-{
- struct rq_enet_desc *desc = vnic_rq_next_desc(rq);
- u64 wrid = 0;
- u8 type = os_buf_index ?
- RQ_ENET_TYPE_NOT_SOP : RQ_ENET_TYPE_ONLY_SOP;
-
- rq_enet_desc_enc(desc,
- (u64)dma_addr | VNIC_PADDR_TARGET,
- type, (u16)len);
-
- vnic_rq_post(rq, os_buf, os_buf_index, dma_addr, len, wrid);
-}
struct enic;
diff --git a/drivers/net/enic/enic_rx.c b/drivers/net/enic/enic_rx.c
new file mode 100644
index 0000000..45c2752
--- /dev/null
+++ b/drivers/net/enic/enic_rx.c
@@ -0,0 +1,413 @@
+/*
+ * Copyright 2008-2014 Cisco Systems, Inc. All rights reserved.
+ * Copyright 2007 Nuova Systems, Inc. All rights reserved.
+ *
+ * Copyright (c) 2014, Cisco Systems, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include <rte_mbuf.h>
+#include <rte_ethdev.h>
+#include <rte_prefetch.h>
+
+#include "enic_compat.h"
+#include "rq_enet_desc.h"
+#include "enic.h"
+
+#define RTE_PMD_USE_PREFETCH
+
+#ifdef RTE_PMD_USE_PREFETCH
+/*
+ * Prefetch a cache line into all cache levels.
+ */
+#define rte_enic_prefetch(p) rte_prefetch0(p)
+#else
+#define rte_enic_prefetch(p) do {} while (0)
+#endif
+
+#ifdef RTE_PMD_PACKET_PREFETCH
+#define rte_packet_prefetch(p) rte_prefetch1(p)
+#else
+#define rte_packet_prefetch(p) do {} while (0)
+#endif
+
+static inline struct rte_mbuf *
+rte_rxmbuf_alloc(struct rte_mempool *mp)
+{
+ struct rte_mbuf *m;
+
+ m = __rte_mbuf_raw_alloc(mp);
+ __rte_mbuf_sanity_check_raw(m, 0);
+ return m;
+}
+
+static inline uint16_t
+enic_cq_rx_desc_ciflags(struct cq_enet_rq_desc *crd)
+{
+ return le16_to_cpu(crd->completed_index_flags) & ~CQ_DESC_COMP_NDX_MASK;
+}
+
+static inline uint16_t
+enic_cq_rx_desc_bwflags(struct cq_enet_rq_desc *crd)
+{
+ return(le16_to_cpu(crd->bytes_written_flags) &
+ ~CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK);
+}
+
+static inline uint8_t
+enic_cq_rx_desc_packet_error(uint16_t bwflags)
+{
+ return((bwflags & CQ_ENET_RQ_DESC_FLAGS_TRUNCATED) ==
+ CQ_ENET_RQ_DESC_FLAGS_TRUNCATED);
+}
+
+static inline uint8_t
+enic_cq_rx_desc_eop(uint16_t ciflags)
+{
+ return (ciflags & CQ_ENET_RQ_DESC_FLAGS_EOP)
+ == CQ_ENET_RQ_DESC_FLAGS_EOP;
+}
+
+static inline uint8_t
+enic_cq_rx_desc_csum_not_calc(struct cq_enet_rq_desc *cqrd)
+{
+ return ((le16_to_cpu(cqrd->q_number_rss_type_flags) &
+ CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC) ==
+ CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC);
+}
+
+static inline uint8_t
+enic_cq_rx_desc_ipv4_csum_ok(struct cq_enet_rq_desc *cqrd)
+{
+ return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK) ==
+ CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK);
+}
+
+static inline uint8_t
+enic_cq_rx_desc_tcp_udp_csum_ok(struct cq_enet_rq_desc *cqrd)
+{
+ return((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK) ==
+ CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK);
+}
+
+static inline uint8_t
+enic_cq_rx_desc_rss_type(struct cq_enet_rq_desc *cqrd)
+{
+ return (uint8_t)((le16_to_cpu(cqrd->q_number_rss_type_flags) >>
+ CQ_DESC_Q_NUM_BITS) & CQ_ENET_RQ_DESC_RSS_TYPE_MASK);
+}
+
+static inline uint32_t
+enic_cq_rx_desc_rss_hash(struct cq_enet_rq_desc *cqrd)
+{
+ return le32_to_cpu(cqrd->rss_hash);
+}
+
+static inline uint8_t
+enic_cq_rx_desc_ipv4(struct cq_enet_rq_desc *cqrd)
+{
+ return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4) ==
+ CQ_ENET_RQ_DESC_FLAGS_IPV4);
+}
+
+static inline uint8_t
+enic_cq_rx_desc_ipv6(struct cq_enet_rq_desc *cqrd)
+{
+ return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_IPV6) ==
+ CQ_ENET_RQ_DESC_FLAGS_IPV6);
+}
+
+static inline uint8_t
+enic_cq_rx_desc_udp(struct cq_enet_rq_desc *cqrd)
+{
+ return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_UDP)
+ == CQ_ENET_RQ_DESC_FLAGS_UDP);
+}
+
+static inline uint8_t
+enic_cq_rx_desc_tcp(struct cq_enet_rq_desc *cqrd)
+{
+ return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_TCP)
+ == CQ_ENET_RQ_DESC_FLAGS_TCP);
+}
+
+static inline uint8_t
+enic_cq_rx_desc_ipv4_frag(struct cq_enet_rq_desc *cqrd)
+{
+ return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT)
+ == CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT);
+}
+
+static inline uint8_t
+enic_cq_rx_desc_fcs_ok(struct cq_enet_rq_desc *cqrd)
+{
+ return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_FCS_OK) ==
+ CQ_ENET_RQ_DESC_FLAGS_FCS_OK);
+}
+
+static inline uint16_t
+enic_cq_rx_desc_vlan(struct cq_enet_rq_desc *cqrd)
+{
+ return le16_to_cpu(cqrd->vlan);
+}
+
+static inline uint16_t
+enic_cq_rx_desc_n_bytes(struct cq_desc *cqd)
+{
+ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
+ return le16_to_cpu(cqrd->bytes_written_flags) &
+ CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK;
+}
+
+static inline uint64_t
+enic_cq_rx_to_pkt_err_flags(struct cq_desc *cqd)
+{
+ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
+ uint16_t bwflags;
+ uint64_t pkt_err_flags = 0;
+
+ bwflags = enic_cq_rx_desc_bwflags(cqrd);
+
+ /* Check for packet error. Can't be more specific than MAC error */
+ if (enic_cq_rx_desc_packet_error(bwflags)) {
+#ifdef RTE_NEXT_ABI
+ pkt_err_flags |= PKT_RX_MAC_ERR;
+#else
+ pkt_err_flags |= PKT_EXT_RX_PKT_ERROR;
+#endif
+ }
+
+ /* Check for bad FCS. MAC error isn't quite, but no other choice */
+ if (!enic_cq_rx_desc_fcs_ok(cqrd)) {
+#ifdef RTE_NEXT_ABI
+ pkt_err_flags |= PKT_RX_MAC_ERR;
+#else
+ pkt_err_flags |= PKT_EXT_RX_BAD_FCS;
+#endif
+ }
+ return pkt_err_flags;
+}
+
+/*
+ * Lookup table to translate RX CQ flags to mbuf flags.
+ */
+static inline uint32_t
+enic_cq_rx_flags_to_pkt_type(struct cq_desc *cqd)
+{
+ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
+ uint8_t cqrd_flags = cqrd->flags;
+ static const uint32_t cq_type_table[128] __rte_cache_aligned = {
+ [32] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4,
+ [34] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4
+ | RTE_PTYPE_L4_UDP,
+ [36] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4
+ | RTE_PTYPE_L4_TCP,
+ [96] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4
+ | RTE_PTYPE_L4_FRAG,
+ [16] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6,
+ [18] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6
+ | RTE_PTYPE_L4_UDP,
+ [20] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6
+ | RTE_PTYPE_L4_TCP,
+ [80] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6
+ | RTE_PTYPE_L4_FRAG,
+ /* All others reserved */
+ };
+ cqrd_flags &= CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT
+ | CQ_ENET_RQ_DESC_FLAGS_IPV4 | CQ_ENET_RQ_DESC_FLAGS_IPV6
+ | CQ_ENET_RQ_DESC_FLAGS_TCP | CQ_ENET_RQ_DESC_FLAGS_UDP;
+ return cq_type_table[cqrd_flags];
+}
+
+static inline void
+enic_cq_rx_to_pkt_flags(struct cq_desc *cqd, struct rte_mbuf *mbuf)
+{
+ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
+ uint16_t ciflags, bwflags, pkt_flags = 0;
+ ciflags = enic_cq_rx_desc_ciflags(cqrd);
+ bwflags = enic_cq_rx_desc_bwflags(cqrd);
+
+ ASSERT(mbuf->ol_flags == 0);
+
+ /* flags are meaningless if !EOP */
+ if (unlikely(!enic_cq_rx_desc_eop(ciflags)))
+ goto mbuf_flags_done;
+
+ /* VLAN stripping */
+ if (bwflags & CQ_ENET_RQ_DESC_FLAGS_VLAN_STRIPPED) {
+ pkt_flags |= PKT_RX_VLAN_PKT;
+ mbuf->vlan_tci = enic_cq_rx_desc_vlan(cqrd);
+ } else {
+ mbuf->vlan_tci = 0;
+ }
+
+ /* RSS flag */
+ if (enic_cq_rx_desc_rss_type(cqrd)) {
+ pkt_flags |= PKT_RX_RSS_HASH;
+ mbuf->hash.rss = enic_cq_rx_desc_rss_hash(cqrd);
+ }
+
+ /* checksum flags */
+ if (!enic_cq_rx_desc_csum_not_calc(cqrd) &&
+ (mbuf->packet_type & RTE_PTYPE_L3_IPV4)) {
+ if (unlikely(!enic_cq_rx_desc_ipv4_csum_ok(cqrd)))
+ pkt_flags |= PKT_RX_IP_CKSUM_BAD;
+ if (mbuf->packet_type & (RTE_PTYPE_L4_UDP | RTE_PTYPE_L4_TCP)) {
+ if (unlikely(!enic_cq_rx_desc_tcp_udp_csum_ok(cqrd)))
+ pkt_flags |= PKT_RX_L4_CKSUM_BAD;
+ }
+ }
+
+ mbuf_flags_done:
+ mbuf->ol_flags = pkt_flags;
+}
+
+static inline uint32_t
+enic_ring_add(uint32_t n_descriptors, uint32_t i0, uint32_t i1)
+{
+ uint32_t d = i0 + i1;
+ ASSERT(i0 < n_descriptors);
+ ASSERT(i1 < n_descriptors);
+ d -= (d >= n_descriptors) ? n_descriptors : 0;
+ return d;
+}
+
+
+uint16_t
+enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts)
+{
+ struct vnic_rq *rq = rx_queue;
+ struct enic *enic = vnic_dev_priv(rq->vdev);
+ unsigned int rx_id;
+ struct rte_mbuf *nmb, *rxmb;
+ uint16_t nb_rx = 0;
+ uint16_t nb_hold;
+ struct vnic_cq *cq;
+ volatile struct cq_desc *cqd_ptr;
+ uint8_t color;
+
+ ASSERT(nb_pkts <= rq->ring.desc_count);
+
+ cq = &enic->cq[enic_cq_rq(enic, rq->index)];
+ rx_id = cq->to_clean; /* index of cqd, rqd, mbuf_table */
+ cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id;
+
+ nb_hold = rq->rx_nb_hold; /* mbufs held by software */
+
+ while (nb_rx < nb_pkts) {
+ uint16_t rx_pkt_len;
+ volatile struct rq_enet_desc *rqd_ptr;
+ dma_addr_t dma_addr;
+ struct cq_desc cqd;
+ uint64_t ol_err_flags;
+
+
+ color = (cqd_ptr->type_color >> CQ_DESC_COLOR_SHIFT)
+ & CQ_DESC_COLOR_MASK;
+ if (color == cq->last_color)
+ break;
+
+ /* get the cq descriptor and rq pointer */
+ cqd = *cqd_ptr;
+ rqd_ptr = (struct rq_enet_desc *)(rq->ring.descs) + rx_id;
+
+ /* allocate a new mbuf */
+ nmb = rte_rxmbuf_alloc(rq->mp);
+ if (nmb == NULL) {
+ dev_err(enic, "RX mbuf alloc failed port=%u qid=%u",
+ enic->port_id, (unsigned)rq->index);
+ rte_eth_devices[enic->port_id].
+ data->rx_mbuf_alloc_failed++;
+ break;
+ }
+
+ /* Check for FCS or packet errors */
+ ol_err_flags = enic_cq_rx_to_pkt_err_flags(&cqd);
+ if (ol_err_flags == 0)
+ rx_pkt_len = enic_cq_rx_desc_n_bytes(&cqd);
+ else
+ rx_pkt_len = 0;
+
+ /* Increment cqd, rqd, mbuf_table index */
+ rx_id++;
+ if (unlikely(rx_id == rq->ring.desc_count)) {
+ rx_id = 0;
+ cq->last_color = cq->last_color ? 0 : 1;
+ }
+
+ /* Prefetch next mbuf & desc while processing current one */
+ cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id;
+ rte_enic_prefetch(cqd_ptr);
+ rte_enic_prefetch(rq->mbuf_ring[rx_id]);
+ rte_enic_prefetch((struct rq_enet_desc *)(rq->ring.descs)
+ + rx_id);
+
+ /* Get the mbuf to return and replace with one just allocated */
+ rxmb = rq->mbuf_ring[rx_id];
+ rq->mbuf_ring[rx_id] = nmb;
+ dma_addr = (dma_addr_t)(nmb->buf_physaddr + nmb->data_off);
+ rqd_ptr->address = rte_cpu_to_le_64(dma_addr);
+ rqd_ptr->length_type = cpu_to_le16(nmb->buf_len);
+
+ /* Fill in the rest of the mbuf */
+ rxmb->data_off = RTE_PKTMBUF_HEADROOM;
+ rxmb->nb_segs = 1;
+ rxmb->next = NULL;
+ rxmb->pkt_len = rx_pkt_len;
+ rxmb->data_len = rx_pkt_len;
+ rxmb->port = enic->port_id;
+ rxmb->ol_flags = ol_err_flags;
+ if (!ol_err_flags)
+ enic_cq_rx_to_pkt_flags(&cqd, rxmb);
+ rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd);
+
+ /* prefetch mbuf data for caller */
+ rte_packet_prefetch(RTE_PTR_ADD(rxmb->buf_addr,
+ RTE_PKTMBUF_HEADROOM));
+
+ /* store the mbuf address into the next entry of the array */
+ rx_pkts[nb_rx++] = rxmb;
+ }
+
+ nb_hold += nb_rx;
+ cq->to_clean = rx_id;
+
+ if (nb_hold > rq->rx_free_thresh) {
+ rq->posted_index = enic_ring_add(rq->ring.desc_count,
+ rq->posted_index, nb_hold);
+ nb_hold = 0;
+ rte_mb();
+ iowrite32(rq->posted_index, &rq->ctrl->posted_index);
+ }
+
+ rq->rx_nb_hold = nb_hold;
+
+ return nb_rx;
+}
--
2.7.0
^ permalink raw reply [flat|nested] 3+ messages in thread