[dpdk-dev] [PATCH 10/10] net/dpaa2: add support for multi seg buffers

[Hemant Agrawal <hemant.agrawal@nxp.com>]

Signed-off-by: Hemant Agrawal <hemant.agrawal@nxp.com>
---
 drivers/bus/fslmc/portal/dpaa2_hw_pvt.h |  27 ++++-
 drivers/net/dpaa2/dpaa2_rxtx.c          | 189 ++++++++++++++++++++++++++++++--
 2 files changed, 206 insertions(+), 10 deletions(-)

diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h b/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h
index 429eaee..f9f4e29 100644
--- a/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h
+++ b/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h
@@ -156,6 +156,19 @@ struct qbman_fle {
 	uint32_t reserved[3]; /* Not used currently */
 };
 
+struct qbman_sge {
+	uint32_t addr_lo;
+	uint32_t addr_hi;
+	uint32_t length;
+	uint32_t fin_bpid_offset;
+};
+
+/* There are three types of frames: Single, Scatter Gather and Frame Lists */
+enum qbman_fd_format {
+	qbman_fd_single = 0,
+	qbman_fd_list,
+	qbman_fd_sg
+};
 /*Macros to define operations on FD*/
 #define DPAA2_SET_FD_ADDR(fd, addr) do {			\
 	fd->simple.addr_lo = lower_32_bits((uint64_t)(addr));	\
@@ -185,7 +198,7 @@ struct qbman_fle {
 #define DPAA2_SET_FLE_OFFSET(fle, offset) \
 	((fle)->fin_bpid_offset |= (uint32_t)(offset) << 16)
 #define DPAA2_SET_FLE_BPID(fle, bpid) ((fle)->fin_bpid_offset |= (uint64_t)bpid)
-#define DPAA2_GET_FLE_BPID(fle, bpid) (fle->fin_bpid_offset & 0x000000ff)
+#define DPAA2_GET_FLE_BPID(fle) ((fle)->fin_bpid_offset & 0x000000ff)
 #define DPAA2_SET_FLE_FIN(fle)	(fle->fin_bpid_offset |= (uint64_t)1 << 31)
 #define DPAA2_SET_FLE_IVP(fle)   (((fle)->fin_bpid_offset |= 0x00004000))
 #define DPAA2_SET_FD_COMPOUND_FMT(fd)	\
@@ -197,6 +210,7 @@ struct qbman_fle {
 #define DPAA2_GET_FD_BPID(fd)	(((fd)->simple.bpid_offset & 0x00003FFF))
 #define DPAA2_GET_FD_IVP(fd)   ((fd->simple.bpid_offset & 0x00004000) >> 14)
 #define DPAA2_GET_FD_OFFSET(fd)	(((fd)->simple.bpid_offset & 0x0FFF0000) >> 16)
+#define DPAA2_GET_FLE_OFFSET(fle) (((fle)->fin_bpid_offset & 0x0FFF0000) >> 16)
 #define DPAA2_SET_FLE_SG_EXT(fle) (fle->fin_bpid_offset |= (uint64_t)1 << 29)
 #define DPAA2_IS_SET_FLE_SG_EXT(fle)	\
 	((fle->fin_bpid_offset & ((uint64_t)1 << 29)) ? 1 : 0)
@@ -206,6 +220,17 @@ struct qbman_fle {
 
 #define DPAA2_ASAL_VAL (DPAA2_MBUF_HW_ANNOTATION / 64)
 
+#define DPAA2_FD_SET_FORMAT(fd, format)	do {				\
+		(fd)->simple.bpid_offset &= 0xCFFFFFFF;			\
+		(fd)->simple.bpid_offset |= (uint32_t)format << 28;	\
+} while (0)
+#define DPAA2_FD_GET_FORMAT(fd)	(((fd)->simple.bpid_offset >> 28) & 0x3)
+
+#define DPAA2_SG_SET_FINAL(sg, fin)	do {				\
+		(sg)->fin_bpid_offset &= 0x7FFFFFFF;			\
+		(sg)->fin_bpid_offset |= (uint32_t)fin << 31;		\
+} while (0)
+#define DPAA2_SG_IS_FINAL(sg) (!!((sg)->fin_bpid_offset >> 31))
 /* Only Enqueue Error responses will be
  * pushed on FQID_ERR of Enqueue FQ
  */
diff --git a/drivers/net/dpaa2/dpaa2_rxtx.c b/drivers/net/dpaa2/dpaa2_rxtx.c
index e4bf14a..6572f25 100644
--- a/drivers/net/dpaa2/dpaa2_rxtx.c
+++ b/drivers/net/dpaa2/dpaa2_rxtx.c
@@ -133,6 +133,66 @@ dpaa2_dev_rx_offload(uint64_t hw_annot_addr, struct rte_mbuf *mbuf)
 }
 
 static inline struct rte_mbuf *__attribute__((hot))
+eth_sg_fd_to_mbuf(const struct qbman_fd *fd)
+{
+	struct qbman_sge *sgt, *sge;
+	dma_addr_t sg_addr;
+	int i = 0;
+	uint64_t fd_addr;
+	struct rte_mbuf *first_seg, *next_seg, *cur_seg, *temp;
+
+	fd_addr = (uint64_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd));
+
+	/* Get Scatter gather table address */
+	sgt = (struct qbman_sge *)(fd_addr + DPAA2_GET_FD_OFFSET(fd));
+
+	sge = &sgt[i++];
+	sg_addr = (uint64_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FLE_ADDR(sge));
+
+	/* First Scatter gather entry */
+	first_seg = DPAA2_INLINE_MBUF_FROM_BUF(sg_addr,
+		rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size);
+	/* Prepare all the metadata for first segment */
+	first_seg->buf_addr = (uint8_t *)sg_addr;
+	first_seg->ol_flags = 0;
+	first_seg->data_off = DPAA2_GET_FLE_OFFSET(sge);
+	first_seg->data_len = sge->length  & 0x1FFFF;
+	first_seg->pkt_len = DPAA2_GET_FD_LEN(fd);
+	first_seg->nb_segs = 1;
+	first_seg->next = NULL;
+
+	first_seg->packet_type = dpaa2_dev_rx_parse(
+			 (uint64_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd))
+			 + DPAA2_FD_PTA_SIZE);
+	dpaa2_dev_rx_offload((uint64_t)DPAA2_IOVA_TO_VADDR(
+			DPAA2_GET_FD_ADDR(fd)) +
+			DPAA2_FD_PTA_SIZE, first_seg);
+	rte_mbuf_refcnt_set(first_seg, 1);
+	cur_seg = first_seg;
+	while (!DPAA2_SG_IS_FINAL(sge)) {
+		sge = &sgt[i++];
+		sg_addr = (uint64_t)DPAA2_IOVA_TO_VADDR(
+				DPAA2_GET_FLE_ADDR(sge));
+		next_seg = DPAA2_INLINE_MBUF_FROM_BUF(sg_addr,
+			rte_dpaa2_bpid_info[DPAA2_GET_FLE_BPID(sge)].meta_data_size);
+		next_seg->buf_addr  = (uint8_t *)sg_addr;
+		next_seg->data_off  = DPAA2_GET_FLE_OFFSET(sge);
+		next_seg->data_len  = sge->length  & 0x1FFFF;
+		first_seg->nb_segs += 1;
+		rte_mbuf_refcnt_set(next_seg, 1);
+		cur_seg->next = next_seg;
+		next_seg->next = NULL;
+		cur_seg = next_seg;
+	}
+	temp = DPAA2_INLINE_MBUF_FROM_BUF(fd_addr,
+		rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size);
+	rte_mbuf_refcnt_set(temp, 1);
+	rte_pktmbuf_free_seg(temp);
+
+	return (void *)first_seg;
+}
+
+static inline struct rte_mbuf *__attribute__((hot))
 eth_fd_to_mbuf(const struct qbman_fd *fd)
 {
 	struct rte_mbuf *mbuf = DPAA2_INLINE_MBUF_FROM_BUF(
@@ -171,7 +231,81 @@ eth_fd_to_mbuf(const struct qbman_fd *fd)
 	return mbuf;
 }
 
-static void __rte_noinline __attribute__((hot))
+static int __attribute__ ((noinline)) __attribute__((hot))
+eth_mbuf_to_sg_fd(struct rte_mbuf *mbuf,
+		  struct qbman_fd *fd, uint16_t bpid)
+{
+	struct rte_mbuf *cur_seg = mbuf, *prev_seg, *mi, *temp;
+	struct qbman_sge *sgt, *sge = NULL;
+	int i;
+
+	/* First Prepare FD to be transmited*/
+	/* Resetting the buffer pool id and offset field*/
+	fd->simple.bpid_offset = 0;
+
+	temp = rte_pktmbuf_alloc(mbuf->pool);
+	if (temp == NULL) {
+		PMD_TX_LOG(ERR, "No memory to allocate S/G table");
+		return -ENOMEM;
+	}
+
+	DPAA2_SET_FD_ADDR(fd, DPAA2_MBUF_VADDR_TO_IOVA(temp));
+	DPAA2_SET_FD_LEN(fd, mbuf->pkt_len);
+	DPAA2_SET_FD_OFFSET(fd, temp->data_off);
+	DPAA2_SET_FD_BPID(fd, bpid);
+	DPAA2_SET_FD_ASAL(fd, DPAA2_ASAL_VAL);
+	DPAA2_FD_SET_FORMAT(fd, qbman_fd_sg);
+	/*Set Scatter gather table and Scatter gather entries*/
+	sgt = (struct qbman_sge *)(
+			(uint64_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd))
+			+ DPAA2_GET_FD_OFFSET(fd));
+
+	for (i = 0; i < mbuf->nb_segs; i++) {
+		sge = &sgt[i];
+		/*Resetting the buffer pool id and offset field*/
+		sge->fin_bpid_offset = 0;
+		DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(cur_seg));
+		DPAA2_SET_FLE_OFFSET(sge, cur_seg->data_off);
+		sge->length = cur_seg->data_len;
+		if (RTE_MBUF_DIRECT(cur_seg)) {
+			if (rte_mbuf_refcnt_read(cur_seg) > 1) {
+				/* If refcnt > 1, invalid bpid is set to ensure
+				 * buffer is not freed by HW
+				 */
+				DPAA2_SET_FLE_IVP(sge);
+				rte_mbuf_refcnt_update(cur_seg, -1);
+			} else
+				DPAA2_SET_FLE_BPID(sge,
+						mempool_to_bpid(cur_seg->pool));
+			cur_seg = cur_seg->next;
+		} else {
+			/* Get owner MBUF from indirect buffer */
+			mi = rte_mbuf_from_indirect(cur_seg);
+			if (rte_mbuf_refcnt_read(mi) > 1) {
+				/* If refcnt > 1, invalid bpid is set to ensure
+				 * owner buffer is not freed by HW
+				 */
+				DPAA2_SET_FLE_IVP(sge);
+			} else {
+				DPAA2_SET_FLE_BPID(sge,
+						   mempool_to_bpid(mi->pool));
+				rte_mbuf_refcnt_update(mi, 1);
+			}
+			prev_seg = cur_seg;
+			cur_seg = cur_seg->next;
+			prev_seg->next = NULL;
+			rte_pktmbuf_free(prev_seg);
+		}
+	}
+	DPAA2_SG_SET_FINAL(sge, true);
+	return 0;
+}
+
+static void
+eth_mbuf_to_fd(struct rte_mbuf *mbuf,
+	       struct qbman_fd *fd, uint16_t bpid) __attribute__((unused));
+
+static void __attribute__ ((noinline)) __attribute__((hot))
 eth_mbuf_to_fd(struct rte_mbuf *mbuf,
 	       struct qbman_fd *fd, uint16_t bpid)
 {
@@ -190,8 +324,22 @@ eth_mbuf_to_fd(struct rte_mbuf *mbuf,
 		DPAA2_GET_FD_OFFSET(fd), DPAA2_GET_FD_ADDR(fd),
 		rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size,
 		DPAA2_GET_FD_BPID(fd), DPAA2_GET_FD_LEN(fd));
-}
+	if (RTE_MBUF_DIRECT(mbuf)) {
+		if (rte_mbuf_refcnt_read(mbuf) > 1) {
+			DPAA2_SET_FD_IVP(fd);
+			rte_mbuf_refcnt_update(mbuf, -1);
+		}
+	} else {
+		struct rte_mbuf *mi;
 
+		mi = rte_mbuf_from_indirect(mbuf);
+		if (rte_mbuf_refcnt_read(mi) > 1)
+			DPAA2_SET_FD_IVP(fd);
+		else
+			rte_mbuf_refcnt_update(mi, 1);
+		rte_pktmbuf_free(mbuf);
+	}
+}
 
 static inline int __attribute__((hot))
 eth_copy_mbuf_to_fd(struct rte_mbuf *mbuf,
@@ -325,10 +473,11 @@ dpaa2_dev_prefetch_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 		rte_prefetch0((void *)((uint64_t)DPAA2_GET_FD_ADDR(fd[num_rx])
 				+ DPAA2_FD_PTA_SIZE + 16));
 
-		bufs[num_rx] = eth_fd_to_mbuf(fd[num_rx]);
+		if (unlikely(DPAA2_FD_GET_FORMAT(fd[num_rx]) == qbman_fd_sg))
+			bufs[num_rx] = eth_sg_fd_to_mbuf(fd[num_rx]);
+		else
+			bufs[num_rx] = eth_fd_to_mbuf(fd[num_rx]);
 		bufs[num_rx]->port = dev->data->port_id;
-		if (dev->data->dev_conf.rxmode.hw_vlan_strip)
-			rte_vlan_strip(bufs[num_rx]);
 
 		if (dev->data->dev_conf.rxmode.hw_vlan_strip)
 			rte_vlan_strip(bufs[num_rx]);
@@ -379,6 +528,7 @@ dpaa2_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 	uint32_t loop;
 	int32_t ret;
 	struct qbman_fd fd_arr[MAX_TX_RING_SLOTS];
+	struct rte_mbuf *mi;
 	uint32_t frames_to_send;
 	struct rte_mempool *mp;
 	struct qbman_eq_desc eqdesc;
@@ -419,8 +569,17 @@ dpaa2_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 			fd_arr[loop].simple.frc = 0;
 			DPAA2_RESET_FD_CTRL((&fd_arr[loop]));
 			DPAA2_SET_FD_FLC((&fd_arr[loop]), NULL);
-			mp = (*bufs)->pool;
+			if (RTE_MBUF_DIRECT(*bufs)) {
+				mp = (*bufs)->pool;
+			} else {
+				mi = rte_mbuf_from_indirect(*bufs);
+				mp = mi->pool;
+			}
 			/* Not a hw_pkt pool allocated frame */
+			if (!mp) {
+				PMD_TX_LOG(ERR, "err: no bpool attached");
+				goto skip_tx;
+			}
 			if (mp->ops_index != priv->bp_list->dpaa2_ops_index) {
 				PMD_TX_LOG(ERR, "non hw offload bufffer ");
 				/* alloc should be from the default buffer pool
@@ -429,11 +588,16 @@ dpaa2_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 				if (priv->bp_list) {
 					bpid = priv->bp_list->buf_pool.bpid;
 				} else {
-					PMD_TX_LOG(ERR, "errr: why no bpool"
-						   " attached");
+					PMD_TX_LOG(ERR,
+						   "err: no bpool attached");
 					num_tx = 0;
 					goto skip_tx;
 				}
+				if (unlikely((*bufs)->nb_segs > 1)) {
+					PMD_TX_LOG(ERR, "S/G support not added"
+						" for non hw offload buffer");
+					goto skip_tx;
+				}
 				if (eth_copy_mbuf_to_fd(*bufs,
 							&fd_arr[loop], bpid)) {
 					bufs++;
@@ -441,7 +605,14 @@ dpaa2_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 				}
 			} else {
 				bpid = mempool_to_bpid(mp);
-				eth_mbuf_to_fd(*bufs, &fd_arr[loop], bpid);
+				if (unlikely((*bufs)->nb_segs > 1)) {
+					if (eth_mbuf_to_sg_fd(*bufs,
+							&fd_arr[loop], bpid))
+						goto skip_tx;
+				} else {
+					eth_mbuf_to_fd(*bufs,
+						       &fd_arr[loop], bpid);
+				}
 			}
 			bufs++;
 		}
-- 
2.7.4