[dpdk-stable] [PATCH v1 0/1] alloc IO memory with socketid

patches for DPDK stable branches
 help / color / mirror / Atom feed

* [dpdk-stable] [PATCH v1 0/1] alloc IO memory with socketid
@ 2020-03-14 10:02 Xiaoyun wang
  2020-03-14  8:46 ` [dpdk-stable] [PATCH v1 1/1] net/hinic: " Xiaoyun wang
  0 siblings, 1 reply; 4+ messages in thread
From: Xiaoyun wang @ 2020-03-14 10:02 UTC (permalink / raw)
  To: dev
  Cc: ferruh.yigit, ktraynor, bluca, stable, luoxianjun, luoxingyu,
	zhouguoyang, xuanziyang2, shahar.belkar, yin.yinshi,
	Xiaoyun wang

This patch optimize the memory allocation, alloc IO memory with socketid.

Xiaoyun wang (1):
  net/hinic: alloc IO memory with socketid

 drivers/net/hinic/base/hinic_compat.h      | 17 ++++++++------
 drivers/net/hinic/base/hinic_pmd_api_cmd.c |  7 +++---
 drivers/net/hinic/base/hinic_pmd_cmdq.c    |  3 +--
 drivers/net/hinic/base/hinic_pmd_eqs.c     |  2 +-
 drivers/net/hinic/base/hinic_pmd_hwdev.c   | 37 +++++++++++++++---------------
 drivers/net/hinic/base/hinic_pmd_hwdev.h   |  1 -
 drivers/net/hinic/base/hinic_pmd_mbox.c    |  6 ++---
 drivers/net/hinic/base/hinic_pmd_nicio.c   | 11 +++++----
 drivers/net/hinic/base/hinic_pmd_wq.c      | 11 +++++----
 drivers/net/hinic/base/hinic_pmd_wq.h      |  2 +-
 drivers/net/hinic/hinic_pmd_ethdev.c       |  7 ++++--
 drivers/net/hinic/hinic_pmd_rx.c           | 28 +++++++++++-----------
 drivers/net/hinic/hinic_pmd_rx.h           |  5 +++-
 drivers/net/hinic/hinic_pmd_tx.c           | 11 +++++----
 drivers/net/hinic/hinic_pmd_tx.h           |  4 +++-
 15 files changed, 81 insertions(+), 71 deletions(-)

-- 
1.8.3.1


^ permalink raw reply	[flat|nested] 4+ messages in thread

* [dpdk-stable] [PATCH v1 1/1] net/hinic: alloc IO memory with socketid
  2020-03-14 10:02 [dpdk-stable] [PATCH v1 0/1] alloc IO memory with socketid Xiaoyun wang
@ 2020-03-14  8:46 ` Xiaoyun wang
  2020-03-14 10:02   ` Xiaoyun wang
  2020-03-16 17:36   ` Ferruh Yigit
  0 siblings, 2 replies; 4+ messages in thread
From: Xiaoyun wang @ 2020-03-14  8:46 UTC (permalink / raw)
  To: zhouguoyang; +Cc: Xiaoyun wang, stable

This patch makes IO memory allocation with socketid,
the txq or rxq descriptor and IO resource can be allocated with
socketid that passed by queue setup ops, which can improve
performance for cross-numa scene.

Cc: stable@dpdk.org
Signed-off-by: Xiaoyun wang <cloud.wangxiaoyun@huawei.com>
---
 drivers/net/hinic/base/hinic_compat.h      | 17 ++++++++------
 drivers/net/hinic/base/hinic_pmd_api_cmd.c |  7 +++---
 drivers/net/hinic/base/hinic_pmd_cmdq.c    |  3 +--
 drivers/net/hinic/base/hinic_pmd_eqs.c     |  2 +-
 drivers/net/hinic/base/hinic_pmd_hwdev.c   | 37 +++++++++++++++---------------
 drivers/net/hinic/base/hinic_pmd_hwdev.h   |  1 -
 drivers/net/hinic/base/hinic_pmd_mbox.c    |  6 ++---
 drivers/net/hinic/base/hinic_pmd_nicio.c   | 11 +++++----
 drivers/net/hinic/base/hinic_pmd_wq.c      | 11 +++++----
 drivers/net/hinic/base/hinic_pmd_wq.h      |  2 +-
 drivers/net/hinic/hinic_pmd_ethdev.c       |  7 ++++--
 drivers/net/hinic/hinic_pmd_rx.c           | 28 +++++++++++-----------
 drivers/net/hinic/hinic_pmd_rx.h           |  5 +++-
 drivers/net/hinic/hinic_pmd_tx.c           | 11 +++++----
 drivers/net/hinic/hinic_pmd_tx.h           |  4 +++-
 15 files changed, 81 insertions(+), 71 deletions(-)

diff --git a/drivers/net/hinic/base/hinic_compat.h b/drivers/net/hinic/base/hinic_compat.h
index b72e8cf..921b830 100644
--- a/drivers/net/hinic/base/hinic_compat.h
+++ b/drivers/net/hinic/base/hinic_compat.h
@@ -149,22 +149,25 @@ static inline int hinic_test_and_set_bit(int nr, volatile unsigned long *addr)
 }
 
 void *dma_zalloc_coherent(void *dev, size_t size, dma_addr_t *dma_handle,
-			  gfp_t flag);
-void *dma_zalloc_coherent_aligned(void *dev, size_t size,
-				dma_addr_t *dma_handle, gfp_t flag);
-void *dma_zalloc_coherent_aligned256k(void *dev, size_t size,
-				dma_addr_t *dma_handle, gfp_t flag);
+			  unsigned int socket_id);
+
+void *dma_zalloc_coherent_aligned(void *hwdev, size_t size,
+		dma_addr_t *dma_handle, unsigned int socket_id);
+
+void *dma_zalloc_coherent_aligned256k(void *hwdev, size_t size,
+			      dma_addr_t *dma_handle, unsigned int socket_id);
+
 void dma_free_coherent(void *dev, size_t size, void *virt, dma_addr_t phys);
 
 /* dma pool alloc and free */
 #define	pci_pool dma_pool
-#define	pci_pool_alloc(pool, flags, handle) dma_pool_alloc(pool, flags, handle)
+#define	pci_pool_alloc(pool, handle) dma_pool_alloc(pool, handle)
 #define	pci_pool_free(pool, vaddr, addr) dma_pool_free(pool, vaddr, addr)
 
 struct dma_pool *dma_pool_create(const char *name, void *dev, size_t size,
 				size_t align, size_t boundary);
 void dma_pool_destroy(struct dma_pool *pool);
-void *dma_pool_alloc(struct pci_pool *pool, int flags, dma_addr_t *dma_addr);
+void *dma_pool_alloc(struct pci_pool *pool, dma_addr_t *dma_addr);
 void dma_pool_free(struct pci_pool *pool, void *vaddr, dma_addr_t dma);
 
 #define kzalloc(size, flag) rte_zmalloc(NULL, size, HINIC_MEM_ALLOC_ALIGN_MIN)
diff --git a/drivers/net/hinic/base/hinic_pmd_api_cmd.c b/drivers/net/hinic/base/hinic_pmd_api_cmd.c
index dbffc2e..b72edc0 100644
--- a/drivers/net/hinic/base/hinic_pmd_api_cmd.c
+++ b/drivers/net/hinic/base/hinic_pmd_api_cmd.c
@@ -658,7 +658,7 @@ static int alloc_cmd_buf(struct hinic_api_cmd_chain *chain,
 
 	cmd_vaddr_alloc = dma_zalloc_coherent(dev, (API_CMD_BUF_SIZE +
 					      API_PAYLOAD_ALIGN_SIZE),
-					      &cmd_paddr, GFP_KERNEL);
+					      &cmd_paddr, SOCKET_ID_ANY);
 	if (!cmd_vaddr_alloc) {
 		PMD_DRV_LOG(ERR, "Allocate API CMD dma memory failed");
 		return -ENOMEM;
@@ -712,7 +712,7 @@ static int api_cmd_create_cell(struct hinic_api_cmd_chain *chain,
 
 	node_vaddr_alloc = dma_zalloc_coherent(dev, (chain->cell_size +
 					       API_CMD_NODE_ALIGN_SIZE),
-					       &node_paddr, GFP_KERNEL);
+					       &node_paddr, SOCKET_ID_ANY);
 	if (!node_vaddr_alloc) {
 		PMD_DRV_LOG(ERR, "Allocate dma API CMD cell failed");
 		return -ENOMEM;
@@ -889,8 +889,7 @@ static int api_chain_init(struct hinic_api_cmd_chain *chain,
 
 	chain->wb_status = (struct hinic_api_cmd_status *)
 			   dma_zalloc_coherent(dev, sizeof(*chain->wb_status),
-					       &chain->wb_status_paddr,
-					       GFP_KERNEL);
+				&chain->wb_status_paddr, SOCKET_ID_ANY);
 	if (!chain->wb_status) {
 		PMD_DRV_LOG(ERR, "Allocate DMA wb status failed");
 		err = -ENOMEM;
diff --git a/drivers/net/hinic/base/hinic_pmd_cmdq.c b/drivers/net/hinic/base/hinic_pmd_cmdq.c
index eb8de24..d73e2bf 100644
--- a/drivers/net/hinic/base/hinic_pmd_cmdq.c
+++ b/drivers/net/hinic/base/hinic_pmd_cmdq.c
@@ -171,8 +171,7 @@ struct hinic_cmd_buf *hinic_alloc_cmd_buf(void *hwdev)
 		return NULL;
 	}
 
-	cmd_buf->buf = pci_pool_alloc(cmdqs->cmd_buf_pool, GFP_KERNEL,
-				      &cmd_buf->dma_addr);
+	cmd_buf->buf = pci_pool_alloc(cmdqs->cmd_buf_pool, &cmd_buf->dma_addr);
 	if (!cmd_buf->buf) {
 		PMD_DRV_LOG(ERR, "Allocate cmd from the pool failed");
 		goto alloc_pci_buf_err;
diff --git a/drivers/net/hinic/base/hinic_pmd_eqs.c b/drivers/net/hinic/base/hinic_pmd_eqs.c
index abe0dae..79e1b20 100644
--- a/drivers/net/hinic/base/hinic_pmd_eqs.c
+++ b/drivers/net/hinic/base/hinic_pmd_eqs.c
@@ -240,7 +240,7 @@ static int alloc_eq_pages(struct hinic_eq *eq)
 		eq->virt_addr[pg_num] =
 			(u8 *)dma_zalloc_coherent_aligned(eq->hwdev,
 					eq->page_size, &eq->dma_addr[pg_num],
-					GFP_KERNEL);
+					SOCKET_ID_ANY);
 		if (!eq->virt_addr[pg_num]) {
 			err = -ENOMEM;
 			goto dma_alloc_err;
diff --git a/drivers/net/hinic/base/hinic_pmd_hwdev.c b/drivers/net/hinic/base/hinic_pmd_hwdev.c
index fd0292f..cf2a970 100644
--- a/drivers/net/hinic/base/hinic_pmd_hwdev.c
+++ b/drivers/net/hinic/base/hinic_pmd_hwdev.c
@@ -112,9 +112,9 @@ void hinic_be32_to_cpu(void *data, u32 len)
 	}
 }
 
-static void *
-hinic_dma_mem_zalloc(struct hinic_hwdev *hwdev, size_t size,
-		dma_addr_t *dma_handle, unsigned int flag, unsigned int align)
+static void *hinic_dma_mem_zalloc(struct hinic_hwdev *hwdev, size_t size,
+			   dma_addr_t *dma_handle, unsigned int align,
+			   unsigned int socket_id)
 {
 	int rc, alloc_cnt;
 	const struct rte_memzone *mz;
@@ -129,8 +129,8 @@ void hinic_be32_to_cpu(void *data, u32 len)
 	snprintf(z_name, sizeof(z_name), "%s_%d",
 		 hwdev->pcidev_hdl->name, alloc_cnt);
 
-	mz = rte_memzone_reserve_aligned(z_name, size, SOCKET_ID_ANY,
-					 flag, align);
+	mz = rte_memzone_reserve_aligned(z_name, size, socket_id,
+					 RTE_MEMZONE_IOVA_CONTIG, align);
 	if (!mz) {
 		PMD_DRV_LOG(ERR, "Alloc dma able memory failed, errno: %d, ma_name: %s, size: 0x%zx",
 			    rte_errno, z_name, size);
@@ -209,25 +209,26 @@ void hinic_be32_to_cpu(void *data, u32 len)
 	(void)rte_memzone_free(mz);
 }
 
-void *dma_zalloc_coherent(void *hwdev, size_t size,
-			  dma_addr_t *dma_handle, gfp_t flag)
+void *dma_zalloc_coherent(void *hwdev, size_t size, dma_addr_t *dma_handle,
+			  unsigned int socket_id)
 {
-	return hinic_dma_mem_zalloc(hwdev, size, dma_handle, flag,
-				    RTE_CACHE_LINE_SIZE);
+	return hinic_dma_mem_zalloc(hwdev, size, dma_handle,
+				    RTE_CACHE_LINE_SIZE, socket_id);
 }
 
 void *dma_zalloc_coherent_aligned(void *hwdev, size_t size,
-				  dma_addr_t *dma_handle, gfp_t flag)
+				dma_addr_t *dma_handle, unsigned int socket_id)
 {
-	return hinic_dma_mem_zalloc(hwdev, size, dma_handle, flag,
-				    HINIC_PAGE_SIZE);
+	return hinic_dma_mem_zalloc(hwdev, size, dma_handle, HINIC_PAGE_SIZE,
+				    socket_id);
 }
 
 void *dma_zalloc_coherent_aligned256k(void *hwdev, size_t size,
-				      dma_addr_t *dma_handle, gfp_t flag)
+				      dma_addr_t *dma_handle,
+				      unsigned int socket_id)
 {
-	return hinic_dma_mem_zalloc(hwdev, size, dma_handle, flag,
-				    HINIC_PAGE_SIZE * 64);
+	return hinic_dma_mem_zalloc(hwdev, size, dma_handle,
+				    HINIC_PAGE_SIZE * 64, socket_id);
 }
 
 void dma_free_coherent(void *hwdev, size_t size, void *virt, dma_addr_t phys)
@@ -304,12 +305,12 @@ void dma_pool_destroy(struct dma_pool *pool)
 	rte_free(pool);
 }
 
-void *dma_pool_alloc(struct pci_pool *pool, int flags, dma_addr_t *dma_addr)
+void *dma_pool_alloc(struct pci_pool *pool, dma_addr_t *dma_addr)
 {
 	void *buf;
 
-	buf = hinic_dma_mem_zalloc(pool->hwdev, pool->elem_size,
-				   dma_addr, flags, (u32)pool->align);
+	buf = hinic_dma_mem_zalloc(pool->hwdev, pool->elem_size, dma_addr,
+				(u32)pool->align, SOCKET_ID_ANY);
 	if (buf)
 		rte_atomic32_inc(&pool->inuse);
 
diff --git a/drivers/net/hinic/base/hinic_pmd_hwdev.h b/drivers/net/hinic/base/hinic_pmd_hwdev.h
index ac21947..d6896b3 100644
--- a/drivers/net/hinic/base/hinic_pmd_hwdev.h
+++ b/drivers/net/hinic/base/hinic_pmd_hwdev.h
@@ -442,7 +442,6 @@ struct hinic_hwdev {
 	struct hinic_msg_pf_to_mgmt *pf_to_mgmt;
 	struct hinic_cmdqs *cmdqs;
 	struct hinic_nic_io *nic_io;
-
 };
 
 int hinic_osdep_init(struct hinic_hwdev *hwdev);
diff --git a/drivers/net/hinic/base/hinic_pmd_mbox.c b/drivers/net/hinic/base/hinic_pmd_mbox.c
index 3d3c1bc..935d60b 100644
--- a/drivers/net/hinic/base/hinic_pmd_mbox.c
+++ b/drivers/net/hinic/base/hinic_pmd_mbox.c
@@ -404,10 +404,8 @@ static int alloc_mbox_wb_status(struct hinic_mbox_func_to_func *func_to_func)
 	struct hinic_hwif *hwif = hwdev->hwif;
 	u32 addr_h, addr_l;
 
-	send_mbox->wb_vaddr = dma_zalloc_coherent(hwdev,
-				  MBOX_WB_STATUS_LEN,
-				  &send_mbox->wb_paddr,
-				  GFP_KERNEL);
+	send_mbox->wb_vaddr = dma_zalloc_coherent(hwdev, MBOX_WB_STATUS_LEN,
+					&send_mbox->wb_paddr, SOCKET_ID_ANY);
 	if (!send_mbox->wb_vaddr) {
 		PMD_DRV_LOG(ERR, "Allocating memory for mailbox wb status failed");
 		return -ENOMEM;
diff --git a/drivers/net/hinic/base/hinic_pmd_nicio.c b/drivers/net/hinic/base/hinic_pmd_nicio.c
index f6cc033..59e39ed 100644
--- a/drivers/net/hinic/base/hinic_pmd_nicio.c
+++ b/drivers/net/hinic/base/hinic_pmd_nicio.c
@@ -728,9 +728,10 @@ void hinic_update_rq_local_ci(struct hinic_hwdev *hwdev, u16 q_id, int wqe_cnt)
 
 static int hinic_alloc_nicio(struct hinic_hwdev *hwdev)
 {
-	int err;
-	u16 max_qps, num_qp;
 	struct hinic_nic_io *nic_io = hwdev->nic_io;
+	struct rte_pci_device *pdev = hwdev->pcidev_hdl;
+	u16 max_qps, num_qp;
+	int err;
 
 	max_qps = hinic_func_max_qnum(hwdev);
 	if ((max_qps & (max_qps - 1))) {
@@ -751,10 +752,10 @@ static int hinic_alloc_nicio(struct hinic_hwdev *hwdev)
 		goto alloc_qps_err;
 	}
 
-	nic_io->ci_vaddr_base =
-		dma_zalloc_coherent(hwdev,
+	nic_io->ci_vaddr_base = dma_zalloc_coherent(hwdev,
 				    CI_TABLE_SIZE(num_qp, HINIC_PAGE_SIZE),
-				    &nic_io->ci_dma_base, GFP_KERNEL);
+				    &nic_io->ci_dma_base,
+				    pdev->device.numa_node);
 	if (!nic_io->ci_vaddr_base) {
 		PMD_DRV_LOG(ERR, "Failed to allocate ci area");
 		err = -ENOMEM;
diff --git a/drivers/net/hinic/base/hinic_pmd_wq.c b/drivers/net/hinic/base/hinic_pmd_wq.c
index 04c81f9..345248c 100644
--- a/drivers/net/hinic/base/hinic_pmd_wq.c
+++ b/drivers/net/hinic/base/hinic_pmd_wq.c
@@ -15,13 +15,14 @@ static void free_wq_pages(struct hinic_hwdev *hwdev, struct hinic_wq *wq)
 	wq->queue_buf_vaddr = 0;
 }
 
-static int alloc_wq_pages(struct hinic_hwdev *hwdev, struct hinic_wq *wq)
+static int alloc_wq_pages(struct hinic_hwdev *hwdev, struct hinic_wq *wq,
+			unsigned int socket_id)
 {
 	dma_addr_t dma_addr = 0;
 
 	wq->queue_buf_vaddr = (u64)(u64 *)
 		dma_zalloc_coherent_aligned256k(hwdev, wq->wq_buf_size,
-						&dma_addr, GFP_KERNEL);
+						&dma_addr, socket_id);
 	if (!wq->queue_buf_vaddr) {
 		PMD_DRV_LOG(ERR, "Failed to allocate wq page");
 		return -ENOMEM;
@@ -40,7 +41,7 @@ static int alloc_wq_pages(struct hinic_hwdev *hwdev, struct hinic_wq *wq)
 }
 
 int hinic_wq_allocate(struct hinic_hwdev *hwdev, struct hinic_wq *wq,
-		      u32 wqebb_shift, u16 q_depth)
+		      u32 wqebb_shift, u16 q_depth, unsigned int socket_id)
 {
 	int err;
 
@@ -60,7 +61,7 @@ int hinic_wq_allocate(struct hinic_hwdev *hwdev, struct hinic_wq *wq,
 		return -EINVAL;
 	}
 
-	err = alloc_wq_pages(hwdev, wq);
+	err = alloc_wq_pages(hwdev, wq, socket_id);
 	if (err) {
 		PMD_DRV_LOG(ERR, "Failed to allocate wq pages");
 		return err;
@@ -114,7 +115,7 @@ int hinic_cmdq_alloc(struct hinic_wq *wq, struct hinic_hwdev *hwdev,
 		wq[i].wq_buf_size = wq_buf_size;
 		wq[i].q_depth = q_depth;
 
-		err = alloc_wq_pages(hwdev, &wq[i]);
+		err = alloc_wq_pages(hwdev, &wq[i], SOCKET_ID_ANY);
 		if (err) {
 			PMD_DRV_LOG(ERR, "Failed to alloc CMDQ blocks");
 			goto cmdq_block_err;
diff --git a/drivers/net/hinic/base/hinic_pmd_wq.h b/drivers/net/hinic/base/hinic_pmd_wq.h
index 53ecc22..354d033 100644
--- a/drivers/net/hinic/base/hinic_pmd_wq.h
+++ b/drivers/net/hinic/base/hinic_pmd_wq.h
@@ -122,7 +122,7 @@ void hinic_cmdq_free(struct hinic_hwdev *hwdev, struct hinic_wq *wq,
 		     int cmdq_blocks);
 
 int hinic_wq_allocate(struct hinic_hwdev *hwdev, struct hinic_wq *wq,
-		      u32 wqebb_shift, u16 q_depth);
+		      u32 wqebb_shift, u16 q_depth, unsigned int socket_id);
 
 void hinic_wq_free(struct hinic_hwdev *hwdev, struct hinic_wq *wq);
 
diff --git a/drivers/net/hinic/hinic_pmd_ethdev.c b/drivers/net/hinic/hinic_pmd_ethdev.c
index 803a39e..8493f6e 100644
--- a/drivers/net/hinic/hinic_pmd_ethdev.c
+++ b/drivers/net/hinic/hinic_pmd_ethdev.c
@@ -57,6 +57,7 @@
 #define HINIC_DEFAULT_BURST_SIZE	32
 #define HINIC_DEFAULT_NB_QUEUES		1
 #define HINIC_DEFAULT_RING_SIZE		1024
+#define HINIC_MAX_LRO_SIZE		65536
 
 /*
  * vlan_id is a 12 bit number.
@@ -439,7 +440,7 @@ static int hinic_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 	nic_dev->rxqs[queue_idx] = rxq;
 
 	/* alloc rx sq hw wqepage*/
-	rc = hinic_create_rq(hwdev, queue_idx, rq_depth);
+	rc = hinic_create_rq(hwdev, queue_idx, rq_depth, socket_id);
 	if (rc) {
 		PMD_DRV_LOG(ERR, "Create rxq[%d] failed, dev_name: %s, rq_depth: %d",
 			    queue_idx, dev->data->name, rq_depth);
@@ -466,6 +467,7 @@ static int hinic_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 	rxq->q_depth = rq_depth;
 	rxq->buf_len = (u16)buf_size;
 	rxq->rx_free_thresh = rx_free_thresh;
+	rxq->socket_id = socket_id;
 
 	/* the last point cant do mbuf rearm in bulk */
 	rxq->rxinfo_align_end = rxq->q_depth - rxq->rx_free_thresh;
@@ -593,7 +595,7 @@ static int hinic_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 	nic_dev->txqs[queue_idx] = txq;
 
 	/* alloc tx sq hw wqepage */
-	rc = hinic_create_sq(hwdev, queue_idx, sq_depth);
+	rc = hinic_create_sq(hwdev, queue_idx, sq_depth, socket_id);
 	if (rc) {
 		PMD_DRV_LOG(ERR, "Create txq[%d] failed, dev_name: %s, sq_depth: %d",
 			    queue_idx, dev->data->name, sq_depth);
@@ -612,6 +614,7 @@ static int hinic_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 	txq->sq_bot_sge_addr = HINIC_GET_WQ_TAIL(txq) -
 					sizeof(struct hinic_sq_bufdesc);
 	txq->cos = nic_dev->default_cos;
+	txq->socket_id = socket_id;
 
 	/* alloc software txinfo */
 	rc = hinic_setup_tx_resources(txq);
diff --git a/drivers/net/hinic/hinic_pmd_rx.c b/drivers/net/hinic/hinic_pmd_rx.c
index f865f2f..f1b873a 100644
--- a/drivers/net/hinic/hinic_pmd_rx.c
+++ b/drivers/net/hinic/hinic_pmd_rx.c
@@ -209,7 +209,8 @@ void hinic_get_func_rx_buf_size(struct hinic_nic_dev *nic_dev)
 	nic_dev->hwdev->nic_io->rq_buf_size = buf_size;
 }
 
-int hinic_create_rq(struct hinic_hwdev *hwdev, u16 q_id, u16 rq_depth)
+int hinic_create_rq(struct hinic_hwdev *hwdev, u16 q_id,
+			u16 rq_depth, unsigned int socket_id)
 {
 	int err;
 	struct hinic_nic_io *nic_io = hwdev->nic_io;
@@ -223,17 +224,15 @@ int hinic_create_rq(struct hinic_hwdev *hwdev, u16 q_id, u16 rq_depth)
 	nic_io->rq_depth = rq_depth;
 
 	err = hinic_wq_allocate(hwdev, &nic_io->rq_wq[q_id],
-				HINIC_RQ_WQEBB_SHIFT, nic_io->rq_depth);
+			HINIC_RQ_WQEBB_SHIFT, nic_io->rq_depth, socket_id);
 	if (err) {
 		PMD_DRV_LOG(ERR, "Failed to allocate WQ for RQ");
 		return err;
 	}
 	rq->wq = &nic_io->rq_wq[q_id];
 
-	rq->pi_virt_addr =
-		(volatile u16 *)dma_zalloc_coherent(hwdev, HINIC_PAGE_SIZE,
-						    &rq->pi_dma_addr,
-						    GFP_KERNEL);
+	rq->pi_virt_addr = (volatile u16 *)dma_zalloc_coherent(hwdev,
+			HINIC_PAGE_SIZE, &rq->pi_dma_addr, socket_id);
 	if (!rq->pi_virt_addr) {
 		PMD_DRV_LOG(ERR, "Failed to allocate rq pi virt addr");
 		err = -ENOMEM;
@@ -305,15 +304,13 @@ void hinic_rxq_stats_reset(struct hinic_rxq *rxq)
 	memset(rxq_stats, 0, sizeof(*rxq_stats));
 }
 
-static int hinic_rx_alloc_cqe(struct hinic_rxq *rxq)
+static int hinic_rx_alloc_cqe(struct hinic_rxq *rxq, unsigned int socket_id)
 {
 	size_t cqe_mem_size;
 
 	cqe_mem_size = sizeof(struct hinic_rq_cqe) * rxq->q_depth;
-	rxq->cqe_start_vaddr =
-		dma_zalloc_coherent(rxq->nic_dev->hwdev,
-				    cqe_mem_size, &rxq->cqe_start_paddr,
-				    GFP_KERNEL);
+	rxq->cqe_start_vaddr = dma_zalloc_coherent(rxq->nic_dev->hwdev,
+				cqe_mem_size, &rxq->cqe_start_paddr, socket_id);
 	if (!rxq->cqe_start_vaddr) {
 		PMD_DRV_LOG(ERR, "Allocate cqe dma memory failed");
 		return -ENOMEM;
@@ -369,11 +366,12 @@ int hinic_setup_rx_resources(struct hinic_rxq *rxq)
 	int err, pkts;
 
 	rx_info_sz = rxq->q_depth * sizeof(*rxq->rx_info);
-	rxq->rx_info = kzalloc_aligned(rx_info_sz, GFP_KERNEL);
+	rxq->rx_info = rte_zmalloc_socket("rx_info", rx_info_sz,
+				RTE_CACHE_LINE_SIZE, rxq->socket_id);
 	if (!rxq->rx_info)
 		return -ENOMEM;
 
-	err = hinic_rx_alloc_cqe(rxq);
+	err = hinic_rx_alloc_cqe(rxq, rxq->socket_id);
 	if (err) {
 		PMD_DRV_LOG(ERR, "Allocate rx cqe failed");
 		goto rx_cqe_err;
@@ -392,7 +390,7 @@ int hinic_setup_rx_resources(struct hinic_rxq *rxq)
 	hinic_rx_free_cqe(rxq);
 
 rx_cqe_err:
-	kfree(rxq->rx_info);
+	rte_free(rxq->rx_info);
 	rxq->rx_info = NULL;
 
 	return err;
@@ -404,7 +402,7 @@ void hinic_free_rx_resources(struct hinic_rxq *rxq)
 		return;
 
 	hinic_rx_free_cqe(rxq);
-	kfree(rxq->rx_info);
+	rte_free(rxq->rx_info);
 	rxq->rx_info = NULL;
 }
 
diff --git a/drivers/net/hinic/hinic_pmd_rx.h b/drivers/net/hinic/hinic_pmd_rx.h
index 1a80f95..49fa565 100644
--- a/drivers/net/hinic/hinic_pmd_rx.h
+++ b/drivers/net/hinic/hinic_pmd_rx.h
@@ -82,6 +82,8 @@ struct hinic_rxq {
 	u16 rx_free_thresh;
 	u16 rxinfo_align_end;
 
+	u32 socket_id;
+
 	unsigned long status;
 	struct hinic_rxq_stats rxq_stats;
 
@@ -121,7 +123,8 @@ struct hinic_rxq {
 
 void hinic_get_func_rx_buf_size(struct hinic_nic_dev *nic_dev);
 
-int hinic_create_rq(struct hinic_hwdev *hwdev, u16 q_id, u16 rq_depth);
+int hinic_create_rq(struct hinic_hwdev *hwdev, u16 q_id,
+			u16 rq_depth, unsigned int socket_id);
 
 void hinic_destroy_rq(struct hinic_hwdev *hwdev, u16 q_id);
 
diff --git a/drivers/net/hinic/hinic_pmd_tx.c b/drivers/net/hinic/hinic_pmd_tx.c
index 985540a..64ec2c1 100644
--- a/drivers/net/hinic/hinic_pmd_tx.c
+++ b/drivers/net/hinic/hinic_pmd_tx.c
@@ -1216,7 +1216,8 @@ int hinic_setup_tx_resources(struct hinic_txq *txq)
 	u64 tx_info_sz;
 
 	tx_info_sz = txq->q_depth * sizeof(*txq->tx_info);
-	txq->tx_info = kzalloc_aligned(tx_info_sz, GFP_KERNEL);
+	txq->tx_info = rte_zmalloc_socket("tx_info", tx_info_sz,
+			RTE_CACHE_LINE_SIZE, txq->socket_id);
 	if (!txq->tx_info)
 		return -ENOMEM;
 
@@ -1228,11 +1229,12 @@ void hinic_free_tx_resources(struct hinic_txq *txq)
 	if (txq->tx_info == NULL)
 		return;
 
-	kfree(txq->tx_info);
+	rte_free(txq->tx_info);
 	txq->tx_info = NULL;
 }
 
-int hinic_create_sq(struct hinic_hwdev *hwdev, u16 q_id, u16 sq_depth)
+int hinic_create_sq(struct hinic_hwdev *hwdev, u16 q_id,
+			u16 sq_depth, unsigned int socket_id)
 {
 	int err;
 	struct hinic_nic_io *nic_io = hwdev->nic_io;
@@ -1246,7 +1248,8 @@ int hinic_create_sq(struct hinic_hwdev *hwdev, u16 q_id, u16 sq_depth)
 
 	/* alloc wq */
 	err = hinic_wq_allocate(nic_io->hwdev, &nic_io->sq_wq[q_id],
-				HINIC_SQ_WQEBB_SHIFT, nic_io->sq_depth);
+				HINIC_SQ_WQEBB_SHIFT, nic_io->sq_depth,
+				socket_id);
 	if (err) {
 		PMD_DRV_LOG(ERR, "Failed to allocate WQ for SQ");
 		return err;
diff --git a/drivers/net/hinic/hinic_pmd_tx.h b/drivers/net/hinic/hinic_pmd_tx.h
index a1ca580..dabbc6c 100644
--- a/drivers/net/hinic/hinic_pmd_tx.h
+++ b/drivers/net/hinic/hinic_pmd_tx.h
@@ -114,6 +114,7 @@ struct hinic_txq {
 	u16 q_id;
 	u16 q_depth;
 	u32 cos;
+	u32 socket_id;
 
 	/* cacheline1 */
 	struct hinic_txq_stats txq_stats;
@@ -137,7 +138,8 @@ struct hinic_txq {
 
 void hinic_txq_stats_reset(struct hinic_txq *txq);
 
-int hinic_create_sq(struct hinic_hwdev *hwdev, u16 q_id, u16 sq_depth);
+int hinic_create_sq(struct hinic_hwdev *hwdev, u16 q_id,
+			u16 sq_depth, unsigned int socket_id);
 
 void hinic_destroy_sq(struct hinic_hwdev *hwdev, u16 q_id);
 
-- 
1.8.3.1


^ permalink raw reply	[flat|nested] 4+ messages in thread

* [dpdk-stable] [PATCH v1 1/1] net/hinic: alloc IO memory with socketid
  2020-03-14  8:46 ` [dpdk-stable] [PATCH v1 1/1] net/hinic: " Xiaoyun wang
@ 2020-03-14 10:02   ` Xiaoyun wang
  2020-03-16 17:36   ` Ferruh Yigit
  1 sibling, 0 replies; 4+ messages in thread
From: Xiaoyun wang @ 2020-03-14 10:02 UTC (permalink / raw)
  To: dev
  Cc: ferruh.yigit, ktraynor, bluca, stable, luoxianjun, luoxingyu,
	zhouguoyang, xuanziyang2, shahar.belkar, yin.yinshi,
	Xiaoyun wang

This patch makes IO memory allocation with socketid,
the txq or rxq descriptor and IO resource can be allocated with
socketid that passed by queue setup ops, which can improve
performance for cross-numa scene.

Cc: stable@dpdk.org
Signed-off-by: Xiaoyun wang <cloud.wangxiaoyun@huawei.com>
---
 drivers/net/hinic/base/hinic_compat.h      | 17 ++++++++------
 drivers/net/hinic/base/hinic_pmd_api_cmd.c |  7 +++---
 drivers/net/hinic/base/hinic_pmd_cmdq.c    |  3 +--
 drivers/net/hinic/base/hinic_pmd_eqs.c     |  2 +-
 drivers/net/hinic/base/hinic_pmd_hwdev.c   | 37 +++++++++++++++---------------
 drivers/net/hinic/base/hinic_pmd_hwdev.h   |  1 -
 drivers/net/hinic/base/hinic_pmd_mbox.c    |  6 ++---
 drivers/net/hinic/base/hinic_pmd_nicio.c   | 11 +++++----
 drivers/net/hinic/base/hinic_pmd_wq.c      | 11 +++++----
 drivers/net/hinic/base/hinic_pmd_wq.h      |  2 +-
 drivers/net/hinic/hinic_pmd_ethdev.c       |  7 ++++--
 drivers/net/hinic/hinic_pmd_rx.c           | 28 +++++++++++-----------
 drivers/net/hinic/hinic_pmd_rx.h           |  5 +++-
 drivers/net/hinic/hinic_pmd_tx.c           | 11 +++++----
 drivers/net/hinic/hinic_pmd_tx.h           |  4 +++-
 15 files changed, 81 insertions(+), 71 deletions(-)

diff --git a/drivers/net/hinic/base/hinic_compat.h b/drivers/net/hinic/base/hinic_compat.h
index b72e8cf..921b830 100644
--- a/drivers/net/hinic/base/hinic_compat.h
+++ b/drivers/net/hinic/base/hinic_compat.h
@@ -149,22 +149,25 @@ static inline int hinic_test_and_set_bit(int nr, volatile unsigned long *addr)
 }
 
 void *dma_zalloc_coherent(void *dev, size_t size, dma_addr_t *dma_handle,
-			  gfp_t flag);
-void *dma_zalloc_coherent_aligned(void *dev, size_t size,
-				dma_addr_t *dma_handle, gfp_t flag);
-void *dma_zalloc_coherent_aligned256k(void *dev, size_t size,
-				dma_addr_t *dma_handle, gfp_t flag);
+			  unsigned int socket_id);
+
+void *dma_zalloc_coherent_aligned(void *hwdev, size_t size,
+		dma_addr_t *dma_handle, unsigned int socket_id);
+
+void *dma_zalloc_coherent_aligned256k(void *hwdev, size_t size,
+			      dma_addr_t *dma_handle, unsigned int socket_id);
+
 void dma_free_coherent(void *dev, size_t size, void *virt, dma_addr_t phys);
 
 /* dma pool alloc and free */
 #define	pci_pool dma_pool
-#define	pci_pool_alloc(pool, flags, handle) dma_pool_alloc(pool, flags, handle)
+#define	pci_pool_alloc(pool, handle) dma_pool_alloc(pool, handle)
 #define	pci_pool_free(pool, vaddr, addr) dma_pool_free(pool, vaddr, addr)
 
 struct dma_pool *dma_pool_create(const char *name, void *dev, size_t size,
 				size_t align, size_t boundary);
 void dma_pool_destroy(struct dma_pool *pool);
-void *dma_pool_alloc(struct pci_pool *pool, int flags, dma_addr_t *dma_addr);
+void *dma_pool_alloc(struct pci_pool *pool, dma_addr_t *dma_addr);
 void dma_pool_free(struct pci_pool *pool, void *vaddr, dma_addr_t dma);
 
 #define kzalloc(size, flag) rte_zmalloc(NULL, size, HINIC_MEM_ALLOC_ALIGN_MIN)
diff --git a/drivers/net/hinic/base/hinic_pmd_api_cmd.c b/drivers/net/hinic/base/hinic_pmd_api_cmd.c
index dbffc2e..b72edc0 100644
--- a/drivers/net/hinic/base/hinic_pmd_api_cmd.c
+++ b/drivers/net/hinic/base/hinic_pmd_api_cmd.c
@@ -658,7 +658,7 @@ static int alloc_cmd_buf(struct hinic_api_cmd_chain *chain,
 
 	cmd_vaddr_alloc = dma_zalloc_coherent(dev, (API_CMD_BUF_SIZE +
 					      API_PAYLOAD_ALIGN_SIZE),
-					      &cmd_paddr, GFP_KERNEL);
+					      &cmd_paddr, SOCKET_ID_ANY);
 	if (!cmd_vaddr_alloc) {
 		PMD_DRV_LOG(ERR, "Allocate API CMD dma memory failed");
 		return -ENOMEM;
@@ -712,7 +712,7 @@ static int api_cmd_create_cell(struct hinic_api_cmd_chain *chain,
 
 	node_vaddr_alloc = dma_zalloc_coherent(dev, (chain->cell_size +
 					       API_CMD_NODE_ALIGN_SIZE),
-					       &node_paddr, GFP_KERNEL);
+					       &node_paddr, SOCKET_ID_ANY);
 	if (!node_vaddr_alloc) {
 		PMD_DRV_LOG(ERR, "Allocate dma API CMD cell failed");
 		return -ENOMEM;
@@ -889,8 +889,7 @@ static int api_chain_init(struct hinic_api_cmd_chain *chain,
 
 	chain->wb_status = (struct hinic_api_cmd_status *)
 			   dma_zalloc_coherent(dev, sizeof(*chain->wb_status),
-					       &chain->wb_status_paddr,
-					       GFP_KERNEL);
+				&chain->wb_status_paddr, SOCKET_ID_ANY);
 	if (!chain->wb_status) {
 		PMD_DRV_LOG(ERR, "Allocate DMA wb status failed");
 		err = -ENOMEM;
diff --git a/drivers/net/hinic/base/hinic_pmd_cmdq.c b/drivers/net/hinic/base/hinic_pmd_cmdq.c
index eb8de24..d73e2bf 100644
--- a/drivers/net/hinic/base/hinic_pmd_cmdq.c
+++ b/drivers/net/hinic/base/hinic_pmd_cmdq.c
@@ -171,8 +171,7 @@ struct hinic_cmd_buf *hinic_alloc_cmd_buf(void *hwdev)
 		return NULL;
 	}
 
-	cmd_buf->buf = pci_pool_alloc(cmdqs->cmd_buf_pool, GFP_KERNEL,
-				      &cmd_buf->dma_addr);
+	cmd_buf->buf = pci_pool_alloc(cmdqs->cmd_buf_pool, &cmd_buf->dma_addr);
 	if (!cmd_buf->buf) {
 		PMD_DRV_LOG(ERR, "Allocate cmd from the pool failed");
 		goto alloc_pci_buf_err;
diff --git a/drivers/net/hinic/base/hinic_pmd_eqs.c b/drivers/net/hinic/base/hinic_pmd_eqs.c
index abe0dae..79e1b20 100644
--- a/drivers/net/hinic/base/hinic_pmd_eqs.c
+++ b/drivers/net/hinic/base/hinic_pmd_eqs.c
@@ -240,7 +240,7 @@ static int alloc_eq_pages(struct hinic_eq *eq)
 		eq->virt_addr[pg_num] =
 			(u8 *)dma_zalloc_coherent_aligned(eq->hwdev,
 					eq->page_size, &eq->dma_addr[pg_num],
-					GFP_KERNEL);
+					SOCKET_ID_ANY);
 		if (!eq->virt_addr[pg_num]) {
 			err = -ENOMEM;
 			goto dma_alloc_err;
diff --git a/drivers/net/hinic/base/hinic_pmd_hwdev.c b/drivers/net/hinic/base/hinic_pmd_hwdev.c
index fd0292f..cf2a970 100644
--- a/drivers/net/hinic/base/hinic_pmd_hwdev.c
+++ b/drivers/net/hinic/base/hinic_pmd_hwdev.c
@@ -112,9 +112,9 @@ void hinic_be32_to_cpu(void *data, u32 len)
 	}
 }
 
-static void *
-hinic_dma_mem_zalloc(struct hinic_hwdev *hwdev, size_t size,
-		dma_addr_t *dma_handle, unsigned int flag, unsigned int align)
+static void *hinic_dma_mem_zalloc(struct hinic_hwdev *hwdev, size_t size,
+			   dma_addr_t *dma_handle, unsigned int align,
+			   unsigned int socket_id)
 {
 	int rc, alloc_cnt;
 	const struct rte_memzone *mz;
@@ -129,8 +129,8 @@ void hinic_be32_to_cpu(void *data, u32 len)
 	snprintf(z_name, sizeof(z_name), "%s_%d",
 		 hwdev->pcidev_hdl->name, alloc_cnt);
 
-	mz = rte_memzone_reserve_aligned(z_name, size, SOCKET_ID_ANY,
-					 flag, align);
+	mz = rte_memzone_reserve_aligned(z_name, size, socket_id,
+					 RTE_MEMZONE_IOVA_CONTIG, align);
 	if (!mz) {
 		PMD_DRV_LOG(ERR, "Alloc dma able memory failed, errno: %d, ma_name: %s, size: 0x%zx",
 			    rte_errno, z_name, size);
@@ -209,25 +209,26 @@ void hinic_be32_to_cpu(void *data, u32 len)
 	(void)rte_memzone_free(mz);
 }
 
-void *dma_zalloc_coherent(void *hwdev, size_t size,
-			  dma_addr_t *dma_handle, gfp_t flag)
+void *dma_zalloc_coherent(void *hwdev, size_t size, dma_addr_t *dma_handle,
+			  unsigned int socket_id)
 {
-	return hinic_dma_mem_zalloc(hwdev, size, dma_handle, flag,
-				    RTE_CACHE_LINE_SIZE);
+	return hinic_dma_mem_zalloc(hwdev, size, dma_handle,
+				    RTE_CACHE_LINE_SIZE, socket_id);
 }
 
 void *dma_zalloc_coherent_aligned(void *hwdev, size_t size,
-				  dma_addr_t *dma_handle, gfp_t flag)
+				dma_addr_t *dma_handle, unsigned int socket_id)
 {
-	return hinic_dma_mem_zalloc(hwdev, size, dma_handle, flag,
-				    HINIC_PAGE_SIZE);
+	return hinic_dma_mem_zalloc(hwdev, size, dma_handle, HINIC_PAGE_SIZE,
+				    socket_id);
 }
 
 void *dma_zalloc_coherent_aligned256k(void *hwdev, size_t size,
-				      dma_addr_t *dma_handle, gfp_t flag)
+				      dma_addr_t *dma_handle,
+				      unsigned int socket_id)
 {
-	return hinic_dma_mem_zalloc(hwdev, size, dma_handle, flag,
-				    HINIC_PAGE_SIZE * 64);
+	return hinic_dma_mem_zalloc(hwdev, size, dma_handle,
+				    HINIC_PAGE_SIZE * 64, socket_id);
 }
 
 void dma_free_coherent(void *hwdev, size_t size, void *virt, dma_addr_t phys)
@@ -304,12 +305,12 @@ void dma_pool_destroy(struct dma_pool *pool)
 	rte_free(pool);
 }
 
-void *dma_pool_alloc(struct pci_pool *pool, int flags, dma_addr_t *dma_addr)
+void *dma_pool_alloc(struct pci_pool *pool, dma_addr_t *dma_addr)
 {
 	void *buf;
 
-	buf = hinic_dma_mem_zalloc(pool->hwdev, pool->elem_size,
-				   dma_addr, flags, (u32)pool->align);
+	buf = hinic_dma_mem_zalloc(pool->hwdev, pool->elem_size, dma_addr,
+				(u32)pool->align, SOCKET_ID_ANY);
 	if (buf)
 		rte_atomic32_inc(&pool->inuse);
 
diff --git a/drivers/net/hinic/base/hinic_pmd_hwdev.h b/drivers/net/hinic/base/hinic_pmd_hwdev.h
index ac21947..d6896b3 100644
--- a/drivers/net/hinic/base/hinic_pmd_hwdev.h
+++ b/drivers/net/hinic/base/hinic_pmd_hwdev.h
@@ -442,7 +442,6 @@ struct hinic_hwdev {
 	struct hinic_msg_pf_to_mgmt *pf_to_mgmt;
 	struct hinic_cmdqs *cmdqs;
 	struct hinic_nic_io *nic_io;
-
 };
 
 int hinic_osdep_init(struct hinic_hwdev *hwdev);
diff --git a/drivers/net/hinic/base/hinic_pmd_mbox.c b/drivers/net/hinic/base/hinic_pmd_mbox.c
index 3d3c1bc..935d60b 100644
--- a/drivers/net/hinic/base/hinic_pmd_mbox.c
+++ b/drivers/net/hinic/base/hinic_pmd_mbox.c
@@ -404,10 +404,8 @@ static int alloc_mbox_wb_status(struct hinic_mbox_func_to_func *func_to_func)
 	struct hinic_hwif *hwif = hwdev->hwif;
 	u32 addr_h, addr_l;
 
-	send_mbox->wb_vaddr = dma_zalloc_coherent(hwdev,
-				  MBOX_WB_STATUS_LEN,
-				  &send_mbox->wb_paddr,
-				  GFP_KERNEL);
+	send_mbox->wb_vaddr = dma_zalloc_coherent(hwdev, MBOX_WB_STATUS_LEN,
+					&send_mbox->wb_paddr, SOCKET_ID_ANY);
 	if (!send_mbox->wb_vaddr) {
 		PMD_DRV_LOG(ERR, "Allocating memory for mailbox wb status failed");
 		return -ENOMEM;
diff --git a/drivers/net/hinic/base/hinic_pmd_nicio.c b/drivers/net/hinic/base/hinic_pmd_nicio.c
index f6cc033..59e39ed 100644
--- a/drivers/net/hinic/base/hinic_pmd_nicio.c
+++ b/drivers/net/hinic/base/hinic_pmd_nicio.c
@@ -728,9 +728,10 @@ void hinic_update_rq_local_ci(struct hinic_hwdev *hwdev, u16 q_id, int wqe_cnt)
 
 static int hinic_alloc_nicio(struct hinic_hwdev *hwdev)
 {
-	int err;
-	u16 max_qps, num_qp;
 	struct hinic_nic_io *nic_io = hwdev->nic_io;
+	struct rte_pci_device *pdev = hwdev->pcidev_hdl;
+	u16 max_qps, num_qp;
+	int err;
 
 	max_qps = hinic_func_max_qnum(hwdev);
 	if ((max_qps & (max_qps - 1))) {
@@ -751,10 +752,10 @@ static int hinic_alloc_nicio(struct hinic_hwdev *hwdev)
 		goto alloc_qps_err;
 	}
 
-	nic_io->ci_vaddr_base =
-		dma_zalloc_coherent(hwdev,
+	nic_io->ci_vaddr_base = dma_zalloc_coherent(hwdev,
 				    CI_TABLE_SIZE(num_qp, HINIC_PAGE_SIZE),
-				    &nic_io->ci_dma_base, GFP_KERNEL);
+				    &nic_io->ci_dma_base,
+				    pdev->device.numa_node);
 	if (!nic_io->ci_vaddr_base) {
 		PMD_DRV_LOG(ERR, "Failed to allocate ci area");
 		err = -ENOMEM;
diff --git a/drivers/net/hinic/base/hinic_pmd_wq.c b/drivers/net/hinic/base/hinic_pmd_wq.c
index 04c81f9..345248c 100644
--- a/drivers/net/hinic/base/hinic_pmd_wq.c
+++ b/drivers/net/hinic/base/hinic_pmd_wq.c
@@ -15,13 +15,14 @@ static void free_wq_pages(struct hinic_hwdev *hwdev, struct hinic_wq *wq)
 	wq->queue_buf_vaddr = 0;
 }
 
-static int alloc_wq_pages(struct hinic_hwdev *hwdev, struct hinic_wq *wq)
+static int alloc_wq_pages(struct hinic_hwdev *hwdev, struct hinic_wq *wq,
+			unsigned int socket_id)
 {
 	dma_addr_t dma_addr = 0;
 
 	wq->queue_buf_vaddr = (u64)(u64 *)
 		dma_zalloc_coherent_aligned256k(hwdev, wq->wq_buf_size,
-						&dma_addr, GFP_KERNEL);
+						&dma_addr, socket_id);
 	if (!wq->queue_buf_vaddr) {
 		PMD_DRV_LOG(ERR, "Failed to allocate wq page");
 		return -ENOMEM;
@@ -40,7 +41,7 @@ static int alloc_wq_pages(struct hinic_hwdev *hwdev, struct hinic_wq *wq)
 }
 
 int hinic_wq_allocate(struct hinic_hwdev *hwdev, struct hinic_wq *wq,
-		      u32 wqebb_shift, u16 q_depth)
+		      u32 wqebb_shift, u16 q_depth, unsigned int socket_id)
 {
 	int err;
 
@@ -60,7 +61,7 @@ int hinic_wq_allocate(struct hinic_hwdev *hwdev, struct hinic_wq *wq,
 		return -EINVAL;
 	}
 
-	err = alloc_wq_pages(hwdev, wq);
+	err = alloc_wq_pages(hwdev, wq, socket_id);
 	if (err) {
 		PMD_DRV_LOG(ERR, "Failed to allocate wq pages");
 		return err;
@@ -114,7 +115,7 @@ int hinic_cmdq_alloc(struct hinic_wq *wq, struct hinic_hwdev *hwdev,
 		wq[i].wq_buf_size = wq_buf_size;
 		wq[i].q_depth = q_depth;
 
-		err = alloc_wq_pages(hwdev, &wq[i]);
+		err = alloc_wq_pages(hwdev, &wq[i], SOCKET_ID_ANY);
 		if (err) {
 			PMD_DRV_LOG(ERR, "Failed to alloc CMDQ blocks");
 			goto cmdq_block_err;
diff --git a/drivers/net/hinic/base/hinic_pmd_wq.h b/drivers/net/hinic/base/hinic_pmd_wq.h
index 53ecc22..354d033 100644
--- a/drivers/net/hinic/base/hinic_pmd_wq.h
+++ b/drivers/net/hinic/base/hinic_pmd_wq.h
@@ -122,7 +122,7 @@ void hinic_cmdq_free(struct hinic_hwdev *hwdev, struct hinic_wq *wq,
 		     int cmdq_blocks);
 
 int hinic_wq_allocate(struct hinic_hwdev *hwdev, struct hinic_wq *wq,
-		      u32 wqebb_shift, u16 q_depth);
+		      u32 wqebb_shift, u16 q_depth, unsigned int socket_id);
 
 void hinic_wq_free(struct hinic_hwdev *hwdev, struct hinic_wq *wq);
 
diff --git a/drivers/net/hinic/hinic_pmd_ethdev.c b/drivers/net/hinic/hinic_pmd_ethdev.c
index 803a39e..8493f6e 100644
--- a/drivers/net/hinic/hinic_pmd_ethdev.c
+++ b/drivers/net/hinic/hinic_pmd_ethdev.c
@@ -57,6 +57,7 @@
 #define HINIC_DEFAULT_BURST_SIZE	32
 #define HINIC_DEFAULT_NB_QUEUES		1
 #define HINIC_DEFAULT_RING_SIZE		1024
+#define HINIC_MAX_LRO_SIZE		65536
 
 /*
  * vlan_id is a 12 bit number.
@@ -439,7 +440,7 @@ static int hinic_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 	nic_dev->rxqs[queue_idx] = rxq;
 
 	/* alloc rx sq hw wqepage*/
-	rc = hinic_create_rq(hwdev, queue_idx, rq_depth);
+	rc = hinic_create_rq(hwdev, queue_idx, rq_depth, socket_id);
 	if (rc) {
 		PMD_DRV_LOG(ERR, "Create rxq[%d] failed, dev_name: %s, rq_depth: %d",
 			    queue_idx, dev->data->name, rq_depth);
@@ -466,6 +467,7 @@ static int hinic_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 	rxq->q_depth = rq_depth;
 	rxq->buf_len = (u16)buf_size;
 	rxq->rx_free_thresh = rx_free_thresh;
+	rxq->socket_id = socket_id;
 
 	/* the last point cant do mbuf rearm in bulk */
 	rxq->rxinfo_align_end = rxq->q_depth - rxq->rx_free_thresh;
@@ -593,7 +595,7 @@ static int hinic_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 	nic_dev->txqs[queue_idx] = txq;
 
 	/* alloc tx sq hw wqepage */
-	rc = hinic_create_sq(hwdev, queue_idx, sq_depth);
+	rc = hinic_create_sq(hwdev, queue_idx, sq_depth, socket_id);
 	if (rc) {
 		PMD_DRV_LOG(ERR, "Create txq[%d] failed, dev_name: %s, sq_depth: %d",
 			    queue_idx, dev->data->name, sq_depth);
@@ -612,6 +614,7 @@ static int hinic_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 	txq->sq_bot_sge_addr = HINIC_GET_WQ_TAIL(txq) -
 					sizeof(struct hinic_sq_bufdesc);
 	txq->cos = nic_dev->default_cos;
+	txq->socket_id = socket_id;
 
 	/* alloc software txinfo */
 	rc = hinic_setup_tx_resources(txq);
diff --git a/drivers/net/hinic/hinic_pmd_rx.c b/drivers/net/hinic/hinic_pmd_rx.c
index f865f2f..f1b873a 100644
--- a/drivers/net/hinic/hinic_pmd_rx.c
+++ b/drivers/net/hinic/hinic_pmd_rx.c
@@ -209,7 +209,8 @@ void hinic_get_func_rx_buf_size(struct hinic_nic_dev *nic_dev)
 	nic_dev->hwdev->nic_io->rq_buf_size = buf_size;
 }
 
-int hinic_create_rq(struct hinic_hwdev *hwdev, u16 q_id, u16 rq_depth)
+int hinic_create_rq(struct hinic_hwdev *hwdev, u16 q_id,
+			u16 rq_depth, unsigned int socket_id)
 {
 	int err;
 	struct hinic_nic_io *nic_io = hwdev->nic_io;
@@ -223,17 +224,15 @@ int hinic_create_rq(struct hinic_hwdev *hwdev, u16 q_id, u16 rq_depth)
 	nic_io->rq_depth = rq_depth;
 
 	err = hinic_wq_allocate(hwdev, &nic_io->rq_wq[q_id],
-				HINIC_RQ_WQEBB_SHIFT, nic_io->rq_depth);
+			HINIC_RQ_WQEBB_SHIFT, nic_io->rq_depth, socket_id);
 	if (err) {
 		PMD_DRV_LOG(ERR, "Failed to allocate WQ for RQ");
 		return err;
 	}
 	rq->wq = &nic_io->rq_wq[q_id];
 
-	rq->pi_virt_addr =
-		(volatile u16 *)dma_zalloc_coherent(hwdev, HINIC_PAGE_SIZE,
-						    &rq->pi_dma_addr,
-						    GFP_KERNEL);
+	rq->pi_virt_addr = (volatile u16 *)dma_zalloc_coherent(hwdev,
+			HINIC_PAGE_SIZE, &rq->pi_dma_addr, socket_id);
 	if (!rq->pi_virt_addr) {
 		PMD_DRV_LOG(ERR, "Failed to allocate rq pi virt addr");
 		err = -ENOMEM;
@@ -305,15 +304,13 @@ void hinic_rxq_stats_reset(struct hinic_rxq *rxq)
 	memset(rxq_stats, 0, sizeof(*rxq_stats));
 }
 
-static int hinic_rx_alloc_cqe(struct hinic_rxq *rxq)
+static int hinic_rx_alloc_cqe(struct hinic_rxq *rxq, unsigned int socket_id)
 {
 	size_t cqe_mem_size;
 
 	cqe_mem_size = sizeof(struct hinic_rq_cqe) * rxq->q_depth;
-	rxq->cqe_start_vaddr =
-		dma_zalloc_coherent(rxq->nic_dev->hwdev,
-				    cqe_mem_size, &rxq->cqe_start_paddr,
-				    GFP_KERNEL);
+	rxq->cqe_start_vaddr = dma_zalloc_coherent(rxq->nic_dev->hwdev,
+				cqe_mem_size, &rxq->cqe_start_paddr, socket_id);
 	if (!rxq->cqe_start_vaddr) {
 		PMD_DRV_LOG(ERR, "Allocate cqe dma memory failed");
 		return -ENOMEM;
@@ -369,11 +366,12 @@ int hinic_setup_rx_resources(struct hinic_rxq *rxq)
 	int err, pkts;
 
 	rx_info_sz = rxq->q_depth * sizeof(*rxq->rx_info);
-	rxq->rx_info = kzalloc_aligned(rx_info_sz, GFP_KERNEL);
+	rxq->rx_info = rte_zmalloc_socket("rx_info", rx_info_sz,
+				RTE_CACHE_LINE_SIZE, rxq->socket_id);
 	if (!rxq->rx_info)
 		return -ENOMEM;
 
-	err = hinic_rx_alloc_cqe(rxq);
+	err = hinic_rx_alloc_cqe(rxq, rxq->socket_id);
 	if (err) {
 		PMD_DRV_LOG(ERR, "Allocate rx cqe failed");
 		goto rx_cqe_err;
@@ -392,7 +390,7 @@ int hinic_setup_rx_resources(struct hinic_rxq *rxq)
 	hinic_rx_free_cqe(rxq);
 
 rx_cqe_err:
-	kfree(rxq->rx_info);
+	rte_free(rxq->rx_info);
 	rxq->rx_info = NULL;
 
 	return err;
@@ -404,7 +402,7 @@ void hinic_free_rx_resources(struct hinic_rxq *rxq)
 		return;
 
 	hinic_rx_free_cqe(rxq);
-	kfree(rxq->rx_info);
+	rte_free(rxq->rx_info);
 	rxq->rx_info = NULL;
 }
 
diff --git a/drivers/net/hinic/hinic_pmd_rx.h b/drivers/net/hinic/hinic_pmd_rx.h
index 1a80f95..49fa565 100644
--- a/drivers/net/hinic/hinic_pmd_rx.h
+++ b/drivers/net/hinic/hinic_pmd_rx.h
@@ -82,6 +82,8 @@ struct hinic_rxq {
 	u16 rx_free_thresh;
 	u16 rxinfo_align_end;
 
+	u32 socket_id;
+
 	unsigned long status;
 	struct hinic_rxq_stats rxq_stats;
 
@@ -121,7 +123,8 @@ struct hinic_rxq {
 
 void hinic_get_func_rx_buf_size(struct hinic_nic_dev *nic_dev);
 
-int hinic_create_rq(struct hinic_hwdev *hwdev, u16 q_id, u16 rq_depth);
+int hinic_create_rq(struct hinic_hwdev *hwdev, u16 q_id,
+			u16 rq_depth, unsigned int socket_id);
 
 void hinic_destroy_rq(struct hinic_hwdev *hwdev, u16 q_id);
 
diff --git a/drivers/net/hinic/hinic_pmd_tx.c b/drivers/net/hinic/hinic_pmd_tx.c
index 985540a..64ec2c1 100644
--- a/drivers/net/hinic/hinic_pmd_tx.c
+++ b/drivers/net/hinic/hinic_pmd_tx.c
@@ -1216,7 +1216,8 @@ int hinic_setup_tx_resources(struct hinic_txq *txq)
 	u64 tx_info_sz;
 
 	tx_info_sz = txq->q_depth * sizeof(*txq->tx_info);
-	txq->tx_info = kzalloc_aligned(tx_info_sz, GFP_KERNEL);
+	txq->tx_info = rte_zmalloc_socket("tx_info", tx_info_sz,
+			RTE_CACHE_LINE_SIZE, txq->socket_id);
 	if (!txq->tx_info)
 		return -ENOMEM;
 
@@ -1228,11 +1229,12 @@ void hinic_free_tx_resources(struct hinic_txq *txq)
 	if (txq->tx_info == NULL)
 		return;
 
-	kfree(txq->tx_info);
+	rte_free(txq->tx_info);
 	txq->tx_info = NULL;
 }
 
-int hinic_create_sq(struct hinic_hwdev *hwdev, u16 q_id, u16 sq_depth)
+int hinic_create_sq(struct hinic_hwdev *hwdev, u16 q_id,
+			u16 sq_depth, unsigned int socket_id)
 {
 	int err;
 	struct hinic_nic_io *nic_io = hwdev->nic_io;
@@ -1246,7 +1248,8 @@ int hinic_create_sq(struct hinic_hwdev *hwdev, u16 q_id, u16 sq_depth)
 
 	/* alloc wq */
 	err = hinic_wq_allocate(nic_io->hwdev, &nic_io->sq_wq[q_id],
-				HINIC_SQ_WQEBB_SHIFT, nic_io->sq_depth);
+				HINIC_SQ_WQEBB_SHIFT, nic_io->sq_depth,
+				socket_id);
 	if (err) {
 		PMD_DRV_LOG(ERR, "Failed to allocate WQ for SQ");
 		return err;
diff --git a/drivers/net/hinic/hinic_pmd_tx.h b/drivers/net/hinic/hinic_pmd_tx.h
index a1ca580..dabbc6c 100644
--- a/drivers/net/hinic/hinic_pmd_tx.h
+++ b/drivers/net/hinic/hinic_pmd_tx.h
@@ -114,6 +114,7 @@ struct hinic_txq {
 	u16 q_id;
 	u16 q_depth;
 	u32 cos;
+	u32 socket_id;
 
 	/* cacheline1 */
 	struct hinic_txq_stats txq_stats;
@@ -137,7 +138,8 @@ struct hinic_txq {
 
 void hinic_txq_stats_reset(struct hinic_txq *txq);
 
-int hinic_create_sq(struct hinic_hwdev *hwdev, u16 q_id, u16 sq_depth);
+int hinic_create_sq(struct hinic_hwdev *hwdev, u16 q_id,
+			u16 sq_depth, unsigned int socket_id);
 
 void hinic_destroy_sq(struct hinic_hwdev *hwdev, u16 q_id);
 
-- 
1.8.3.1


^ permalink raw reply	[flat|nested] 4+ messages in thread

* Re: [dpdk-stable] [PATCH v1 1/1] net/hinic: alloc IO memory with socketid
  2020-03-14  8:46 ` [dpdk-stable] [PATCH v1 1/1] net/hinic: " Xiaoyun wang
  2020-03-14 10:02   ` Xiaoyun wang
@ 2020-03-16 17:36   ` Ferruh Yigit
  1 sibling, 0 replies; 4+ messages in thread
From: Ferruh Yigit @ 2020-03-16 17:36 UTC (permalink / raw)
  To: Xiaoyun wang, dev
  Cc: ktraynor, bluca, stable, luoxianjun, luoxingyu, zhouguoyang,
	xuanziyang2, shahar.belkar, yin.yinshi

On 3/14/2020 10:02 AM, Xiaoyun wang wrote:
> This patch makes IO memory allocation with socketid,
> the txq or rxq descriptor and IO resource can be allocated with
> socketid that passed by queue setup ops, which can improve
> performance for cross-numa scene.
> 
> Cc: stable@dpdk.org
> Signed-off-by: Xiaoyun wang <cloud.wangxiaoyun@huawei.com>

Applied to dpdk-next-net/master, thanks.

^ permalink raw reply	[flat|nested] 4+ messages in thread

end of thread, other threads:[~2020-03-16 17:36 UTC | newest]

Thread overview: 4+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2020-03-14 10:02 [dpdk-stable] [PATCH v1 0/1] alloc IO memory with socketid Xiaoyun wang
2020-03-14  8:46 ` [dpdk-stable] [PATCH v1 1/1] net/hinic: " Xiaoyun wang
2020-03-14 10:02   ` Xiaoyun wang
2020-03-16 17:36   ` Ferruh Yigit

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).