From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 8871DA04A4; Sat, 18 Dec 2021 03:54:10 +0100 (CET) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 76B8341C28; Sat, 18 Dec 2021 03:52:40 +0100 (CET) Received: from VLXDG1SPAM1.ramaxel.com (email.unionmem.com [221.4.138.186]) by mails.dpdk.org (Postfix) with ESMTP id AA2EA411DB for ; Sat, 18 Dec 2021 03:52:36 +0100 (CET) Received: from V12DG1MBS01.ramaxel.local (v12dg1mbs01.ramaxel.local [172.26.18.31]) by VLXDG1SPAM1.ramaxel.com with ESMTPS id 1BI2q1bZ010342 (version=TLSv1.2 cipher=AES256-GCM-SHA384 bits=256 verify=FAIL); Sat, 18 Dec 2021 10:52:01 +0800 (GMT-8) (envelope-from songyl@ramaxel.com) Received: from localhost.localdomain (10.64.9.47) by V12DG1MBS01.ramaxel.local (172.26.18.31) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256_P256) id 15.1.2308.14; Sat, 18 Dec 2021 10:52:00 +0800 From: Yanling Song To: CC: , , , Subject: [PATCH v1 12/25] net/spnic: support mbuf handling of Tx/Rx Date: Sat, 18 Dec 2021 10:51:39 +0800 Message-ID: <600043103a745391c369c703b38251ad1cd0d54f.1639636621.git.songyl@ramaxel.com> X-Mailer: git-send-email 2.17.1 In-Reply-To: References: MIME-Version: 1.0 Content-Type: text/plain X-Originating-IP: [10.64.9.47] X-ClientProxiedBy: V12DG1MBS01.ramaxel.local (172.26.18.31) To V12DG1MBS01.ramaxel.local (172.26.18.31) X-DNSRBL: X-MAIL: VLXDG1SPAM1.ramaxel.com 1BI2q1bZ010342 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org This patch defines a wqe data structure for hardware to learn the sge info and offload info of packet. Furthermore, this commit implements the interfaces to fill wqe with DPDK mbuf. Signed-off-by: Yanling Song --- drivers/net/spnic/base/spnic_nic_cfg.c | 23 ++ drivers/net/spnic/base/spnic_nic_cfg.h | 23 ++ drivers/net/spnic/meson.build | 2 + drivers/net/spnic/spnic_ethdev.c | 502 ++++++++++++++++++++++++- drivers/net/spnic/spnic_rx.c | 302 +++++++++++++++ drivers/net/spnic/spnic_rx.h | 41 ++ drivers/net/spnic/spnic_tx.c | 334 ++++++++++++++++ drivers/net/spnic/spnic_tx.h | 228 +++++++++++ 8 files changed, 1454 insertions(+), 1 deletion(-) create mode 100644 drivers/net/spnic/spnic_rx.c create mode 100644 drivers/net/spnic/spnic_tx.c diff --git a/drivers/net/spnic/base/spnic_nic_cfg.c b/drivers/net/spnic/base/spnic_nic_cfg.c index 25d98d67dd..f6914f6f6d 100644 --- a/drivers/net/spnic/base/spnic_nic_cfg.c +++ b/drivers/net/spnic/base/spnic_nic_cfg.c @@ -378,6 +378,29 @@ int spnic_set_port_enable(void *hwdev, bool enable) return 0; } +int spnic_flush_qps_res(void *hwdev) +{ + struct spnic_cmd_clear_qp_resource sq_res; + u16 out_size = sizeof(sq_res); + int err; + + if (!hwdev) + return -EINVAL; + + memset(&sq_res, 0, sizeof(sq_res)); + sq_res.func_id = spnic_global_func_id(hwdev); + + err = l2nic_msg_to_mgmt_sync(hwdev, SPNIC_CMD_CLEAR_QP_RESOURCE, &sq_res, + sizeof(sq_res), &sq_res, &out_size); + if (err || !out_size || sq_res.msg_head.status) { + PMD_DRV_LOG(ERR, "Clear sq resources failed, err: %d, status: 0x%x, out size: 0x%x", + err, sq_res.msg_head.status, out_size); + return -EIO; + } + + return 0; +} + static int spnic_set_function_table(void *hwdev, u32 cfg_bitmap, struct spnic_func_tbl_cfg *cfg) { diff --git a/drivers/net/spnic/base/spnic_nic_cfg.h b/drivers/net/spnic/base/spnic_nic_cfg.h index ce9792f8ee..746c1c342d 100644 --- a/drivers/net/spnic/base/spnic_nic_cfg.h +++ b/drivers/net/spnic/base/spnic_nic_cfg.h @@ -255,6 +255,17 @@ struct spnic_cmd_register_vf { u8 rsvd[39]; }; + +struct spnic_cmd_set_rq_flush { + union { + struct { + u16 global_rq_id; + u16 local_rq_id; + }; + u32 value; + }; +}; + int l2nic_msg_to_mgmt_sync(void *hwdev, u16 cmd, void *buf_in, u16 in_size, void *buf_out, u16 *out_size); @@ -381,6 +392,18 @@ int spnic_set_port_enable(void *hwdev, bool enable); */ int spnic_get_link_state(void *hwdev, u8 *link_state); +/** + * Flush queue pairs resource in hardware + * + * @param[in] hwdev + * Device pointer to hwdev + * + * @retval zero : Success + * @retval non-zero : Failure + */ +int spnic_flush_qps_res(void *hwdev); + + /** * Init nic hwdev * diff --git a/drivers/net/spnic/meson.build b/drivers/net/spnic/meson.build index 20d5151a8d..cd8f316366 100644 --- a/drivers/net/spnic/meson.build +++ b/drivers/net/spnic/meson.build @@ -7,6 +7,8 @@ objs = [base_objs] sources = files( 'spnic_ethdev.c', 'spnic_io.c', + 'spnic_rx.c', + 'spnic_tx.c' ) includes += include_directories('base') diff --git a/drivers/net/spnic/spnic_ethdev.c b/drivers/net/spnic/spnic_ethdev.c index 4205ab43a4..27942e5d68 100644 --- a/drivers/net/spnic/spnic_ethdev.c +++ b/drivers/net/spnic/spnic_ethdev.c @@ -139,6 +139,468 @@ static int spnic_link_update(struct rte_eth_dev *dev, int wait_to_complete) return rte_eth_linkstatus_set(dev, &link); } +static void spnic_reset_rx_queue(struct rte_eth_dev *dev) +{ + struct spnic_rxq *rxq = NULL; + struct spnic_nic_dev *nic_dev; + int q_id = 0; + + nic_dev = SPNIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev); + + for (q_id = 0; q_id < nic_dev->num_rqs; q_id++) { + rxq = nic_dev->rxqs[q_id]; + + rxq->cons_idx = 0; + rxq->prod_idx = 0; + rxq->delta = rxq->q_depth; + rxq->next_to_update = 0; + } +} + +static void spnic_reset_tx_queue(struct rte_eth_dev *dev) +{ + struct spnic_nic_dev *nic_dev; + struct spnic_txq *txq = NULL; + int q_id = 0; + + nic_dev = SPNIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev); + + for (q_id = 0; q_id < nic_dev->num_sqs; q_id++) { + txq = nic_dev->txqs[q_id]; + + txq->cons_idx = 0; + txq->prod_idx = 0; + txq->owner = 1; + + /* Clear hardware ci */ + *(u16 *)txq->ci_vaddr_base = 0; + } +} + +/** + * Create the receive queue. + * + * @param[in] dev + * Pointer to ethernet device structure. + * @param[in] qid + * Receive queue index. + * @param[in] nb_desc + * Number of descriptors for receive queue. + * @param[in] socket_id + * Socket index on which memory must be allocated. + * @param rx_conf + * Thresholds parameters (unused_). + * @param mp + * Memory pool for buffer allocations. + * + * @retval zero : Success + * @retval non-zero : Failure + */ +static int spnic_rx_queue_setup(struct rte_eth_dev *dev, uint16_t qid, + uint16_t nb_desc, unsigned int socket_id, + __rte_unused const struct rte_eth_rxconf *rx_conf, + struct rte_mempool *mp) +{ + struct spnic_nic_dev *nic_dev; + struct spnic_rxq *rxq = NULL; + const struct rte_memzone *rq_mz = NULL; + const struct rte_memzone *cqe_mz = NULL; + const struct rte_memzone *pi_mz = NULL; + u16 rq_depth, rx_free_thresh; + u32 queue_buf_size, mb_buf_size; + void *db_addr = NULL; + int wqe_count; + u32 buf_size; + int err; + + nic_dev = SPNIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev); + + /* Queue depth must be power of 2, otherwise will be aligned up */ + rq_depth = (nb_desc & (nb_desc - 1)) ? + ((u16)(1U << (ilog2(nb_desc) + 1))) : nb_desc; + + /* + * Validate number of receive descriptors. + * It must not exceed hardware maximum and minimum. + */ + if (rq_depth > SPNIC_MAX_QUEUE_DEPTH || + rq_depth < SPNIC_MIN_QUEUE_DEPTH) { + PMD_DRV_LOG(ERR, "RX queue depth is out of range from %d to %d," + "(nb_desc: %d, q_depth: %d, port: %d queue: %d)", + SPNIC_MIN_QUEUE_DEPTH, SPNIC_MAX_QUEUE_DEPTH, + (int)nb_desc, (int)rq_depth, + (int)dev->data->port_id, (int)qid); + return -EINVAL; + } + + /* + * The RX descriptor ring will be cleaned after rxq->rx_free_thresh + * descriptors are used or if the number of descriptors required + * to transmit a packet is greater than the number of free RX + * descriptors. + * The following constraints must be satisfied: + * -rx_free_thresh must be greater than 0. + * -rx_free_thresh must be less than the size of the ring minus 1. + * When set to zero use default values. + */ + rx_free_thresh = (u16)((rx_conf->rx_free_thresh) ? + rx_conf->rx_free_thresh : SPNIC_DEFAULT_RX_FREE_THRESH); + if (rx_free_thresh >= (rq_depth - 1)) { + PMD_DRV_LOG(ERR, "rx_free_thresh must be less than the number " + "of RX descriptors minus 1, rx_free_thresh: %u port: %d queue: %d)", + (unsigned int)rx_free_thresh, + (int)dev->data->port_id, (int)qid); + return -EINVAL; + } + + rxq = rte_zmalloc_socket("spnic_rq", sizeof(struct spnic_rxq), + RTE_CACHE_LINE_SIZE, socket_id); + if (!rxq) { + PMD_DRV_LOG(ERR, "Allocate rxq[%d] failed, dev_name: %s", + qid, dev->data->name); + return -ENOMEM; + } + + /* Init rq parameters */ + rxq->nic_dev = nic_dev; + nic_dev->rxqs[qid] = rxq; + rxq->mb_pool = mp; + rxq->q_id = qid; + rxq->next_to_update = 0; + rxq->q_depth = rq_depth; + rxq->q_mask = rq_depth - 1; + rxq->delta = rq_depth; + rxq->cons_idx = 0; + rxq->prod_idx = 0; + rxq->wqe_type = SPNIC_NORMAL_RQ_WQE; + rxq->wqebb_shift = SPNIC_RQ_WQEBB_SHIFT + rxq->wqe_type; + rxq->wqebb_size = (u16)BIT(rxq->wqebb_shift); + rxq->rx_free_thresh = rx_free_thresh; + rxq->rxinfo_align_end = rxq->q_depth - rxq->rx_free_thresh; + rxq->port_id = dev->data->port_id; + + /* If buf_len used for function table, need to translated */ + mb_buf_size = rte_pktmbuf_data_room_size(rxq->mb_pool) - + RTE_PKTMBUF_HEADROOM; + err = spnic_convert_rx_buf_size(mb_buf_size, &buf_size); + if (err) { + PMD_DRV_LOG(ERR, "Adjust buf size failed, dev_name: %s", + dev->data->name); + goto adjust_bufsize_fail; + } + + rxq->buf_len = buf_size; + rxq->rx_buff_shift = ilog2(rxq->buf_len); + + pi_mz = rte_eth_dma_zone_reserve(dev, "spnic_rq_pi", qid, + RTE_PGSIZE_4K, RTE_CACHE_LINE_SIZE, + socket_id); + if (!pi_mz) { + PMD_DRV_LOG(ERR, "Allocate rxq[%d] pi_mz failed, dev_name: %s", + qid, dev->data->name); + err = -ENOMEM; + goto alloc_pi_mz_fail; + } + rxq->pi_mz = pi_mz; + rxq->pi_dma_addr = pi_mz->iova; + rxq->pi_virt_addr = pi_mz->addr; + + /* Rxq doesn't use direct wqe */ + err = spnic_alloc_db_addr(nic_dev->hwdev, &db_addr, NULL); + if (err) { + PMD_DRV_LOG(ERR, "Alloc rq doorbell addr failed"); + goto alloc_db_err_fail; + } + rxq->db_addr = db_addr; + + queue_buf_size = BIT(rxq->wqebb_shift) * rq_depth; + rq_mz = rte_eth_dma_zone_reserve(dev, "spnic_rq_mz", qid, + queue_buf_size, RTE_PGSIZE_256K, + socket_id); + if (!rq_mz) { + PMD_DRV_LOG(ERR, "Allocate rxq[%d] rq_mz failed, dev_name: %s", + qid, dev->data->name); + err = -ENOMEM; + goto alloc_rq_mz_fail; + } + + memset(rq_mz->addr, 0, queue_buf_size); + rxq->rq_mz = rq_mz; + rxq->queue_buf_paddr = rq_mz->iova; + rxq->queue_buf_vaddr = rq_mz->addr; + + rxq->rx_info = rte_zmalloc_socket("rx_info", + rq_depth * sizeof(*rxq->rx_info), + RTE_CACHE_LINE_SIZE, socket_id); + if (!rxq->rx_info) { + PMD_DRV_LOG(ERR, "Allocate rx_info failed, dev_name: %s", + dev->data->name); + err = -ENOMEM; + goto alloc_rx_info_fail; + } + + cqe_mz = rte_eth_dma_zone_reserve(dev, "spnic_cqe_mz", qid, + rq_depth * sizeof(*rxq->rx_cqe), + RTE_CACHE_LINE_SIZE, socket_id); + if (!cqe_mz) { + PMD_DRV_LOG(ERR, "Allocate cqe mem zone failed, dev_name: %s", + dev->data->name); + err = -ENOMEM; + goto alloc_cqe_mz_fail; + } + memset(cqe_mz->addr, 0, rq_depth * sizeof(*rxq->rx_cqe)); + rxq->cqe_mz = cqe_mz; + rxq->cqe_start_paddr = cqe_mz->iova; + rxq->cqe_start_vaddr = cqe_mz->addr; + rxq->rx_cqe = (struct spnic_rq_cqe *)rxq->cqe_start_vaddr; + + wqe_count = spnic_rx_fill_wqe(rxq); + if (wqe_count != rq_depth) { + PMD_DRV_LOG(ERR, "Fill rx wqe failed, wqe_count: %d, dev_name: %s", + wqe_count, dev->data->name); + err = -ENOMEM; + goto fill_rx_wqe_fail; + } + + /* Record rxq pointer in rte_eth rx_queues */ + dev->data->rx_queues[qid] = rxq; + + return 0; + +fill_rx_wqe_fail: + rte_memzone_free(rxq->cqe_mz); +alloc_cqe_mz_fail: + rte_free(rxq->rx_info); + +alloc_rx_info_fail: + rte_memzone_free(rxq->rq_mz); + +alloc_rq_mz_fail: + spnic_free_db_addr(nic_dev->hwdev, rxq->db_addr, NULL); + +alloc_db_err_fail: + rte_memzone_free(rxq->pi_mz); + +alloc_pi_mz_fail: +adjust_bufsize_fail: + + rte_free(rxq); + nic_dev->rxqs[qid] = NULL; + + return err; +} + +/** + * Create the transmit queue. + * + * @param[in] dev + * Pointer to ethernet device structure. + * @param[in] queue_idx + * Transmit queue index. + * @param[in] nb_desc + * Number of descriptors for transmit queue. + * @param[in] socket_id + * Socket index on which memory must be allocated. + * @param[in] tx_conf + * Tx queue configuration parameters (unused_). + * + * @retval zero : Success + * @retval non-zero : Failure + */ +static int spnic_tx_queue_setup(struct rte_eth_dev *dev, uint16_t qid, + uint16_t nb_desc, unsigned int socket_id, + __rte_unused const struct rte_eth_txconf *tx_conf) +{ + struct spnic_nic_dev *nic_dev; + struct spnic_hwdev *hwdev; + struct spnic_txq *txq = NULL; + const struct rte_memzone *sq_mz = NULL; + const struct rte_memzone *ci_mz = NULL; + void *db_addr = NULL; + u16 sq_depth, tx_free_thresh; + u32 queue_buf_size; + int err; + + nic_dev = SPNIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev); + hwdev = nic_dev->hwdev; + + /* Queue depth must be power of 2, otherwise will be aligned up */ + sq_depth = (nb_desc & (nb_desc - 1)) ? + ((u16)(1U << (ilog2(nb_desc) + 1))) : nb_desc; + + /* + * Validate number of transmit descriptors. + * It must not exceed hardware maximum and minimum. + */ + if (sq_depth > SPNIC_MAX_QUEUE_DEPTH || + sq_depth < SPNIC_MIN_QUEUE_DEPTH) { + PMD_DRV_LOG(ERR, "TX queue depth is out of range from %d to %d," + "(nb_desc: %d, q_depth: %d, port: %d queue: %d)", + SPNIC_MIN_QUEUE_DEPTH, SPNIC_MAX_QUEUE_DEPTH, + (int)nb_desc, (int)sq_depth, + (int)dev->data->port_id, (int)qid); + return -EINVAL; + } + + /* + * The TX descriptor ring will be cleaned after txq->tx_free_thresh + * descriptors are used or if the number of descriptors required + * to transmit a packet is greater than the number of free TX + * descriptors. + * The following constraints must be satisfied: + * -tx_free_thresh must be greater than 0. + * -tx_free_thresh must be less than the size of the ring minus 1. + * When set to zero use default values. + */ + tx_free_thresh = (u16)((tx_conf->tx_free_thresh) ? + tx_conf->tx_free_thresh : SPNIC_DEFAULT_TX_FREE_THRESH); + if (tx_free_thresh >= (sq_depth - 1)) { + PMD_DRV_LOG(ERR, "tx_free_thresh must be less than the number of tx " + "descriptors minus 1, tx_free_thresh: %u port: %d queue: %d", + (unsigned int)tx_free_thresh, + (int)dev->data->port_id, (int)qid); + return -EINVAL; + } + + txq = rte_zmalloc_socket("spnic_tx_queue", sizeof(struct spnic_txq), + RTE_CACHE_LINE_SIZE, socket_id); + if (!txq) { + PMD_DRV_LOG(ERR, "Allocate txq[%d] failed, dev_name: %s", + qid, dev->data->name); + return -ENOMEM; + } + nic_dev->txqs[qid] = txq; + txq->nic_dev = nic_dev; + txq->q_id = qid; + txq->q_depth = sq_depth; + txq->q_mask = sq_depth - 1; + txq->cons_idx = 0; + txq->prod_idx = 0; + txq->wqebb_shift = SPNIC_SQ_WQEBB_SHIFT; + txq->wqebb_size = (u16)BIT(txq->wqebb_shift); + txq->tx_free_thresh = tx_free_thresh; + txq->owner = 1; + txq->cos = nic_dev->default_cos; + + ci_mz = rte_eth_dma_zone_reserve(dev, "spnic_sq_ci", qid, + SPNIC_CI_Q_ADDR_SIZE, + SPNIC_CI_Q_ADDR_SIZE, socket_id); + if (!ci_mz) { + PMD_DRV_LOG(ERR, "Allocate txq[%d] ci_mz failed, dev_name: %s", + qid, dev->data->name); + err = -ENOMEM; + goto alloc_ci_mz_fail; + } + txq->ci_mz = ci_mz; + txq->ci_dma_base = ci_mz->iova; + txq->ci_vaddr_base = ci_mz->addr; + + queue_buf_size = BIT(txq->wqebb_shift) * sq_depth; + sq_mz = rte_eth_dma_zone_reserve(dev, "spnic_sq_mz", qid, + queue_buf_size, RTE_PGSIZE_256K, + socket_id); + if (!sq_mz) { + PMD_DRV_LOG(ERR, "Allocate txq[%d] sq_mz failed, dev_name: %s", + qid, dev->data->name); + err = -ENOMEM; + goto alloc_sq_mz_fail; + } + memset(sq_mz->addr, 0, queue_buf_size); + txq->sq_mz = sq_mz; + txq->queue_buf_paddr = sq_mz->iova; + txq->queue_buf_vaddr = sq_mz->addr; + txq->sq_head_addr = (u64)txq->queue_buf_vaddr; + txq->sq_bot_sge_addr = txq->sq_head_addr + queue_buf_size; + + /* Sq doesn't use direct wqe */ + err = spnic_alloc_db_addr(hwdev, &db_addr, NULL); + if (err) { + PMD_DRV_LOG(ERR, "Alloc sq doorbell addr failed"); + goto alloc_db_err_fail; + } + txq->db_addr = db_addr; + + txq->tx_info = rte_zmalloc_socket("tx_info", + sq_depth * sizeof(*txq->tx_info), + RTE_CACHE_LINE_SIZE, socket_id); + if (!txq->tx_info) { + PMD_DRV_LOG(ERR, "Allocate tx_info failed, dev_name: %s", + dev->data->name); + err = -ENOMEM; + goto alloc_tx_info_fail; + } + + /* Record txq pointer in rte_eth tx_queues */ + dev->data->tx_queues[qid] = txq; + + return 0; + +alloc_tx_info_fail: + spnic_free_db_addr(hwdev, txq->db_addr, NULL); + +alloc_db_err_fail: + rte_memzone_free(txq->sq_mz); + +alloc_sq_mz_fail: + rte_memzone_free(txq->ci_mz); + +alloc_ci_mz_fail: + rte_free(txq); + + return err; +} + +static void spnic_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid) +{ + struct spnic_rxq *rxq = dev->data->rx_queues[qid]; + struct spnic_nic_dev *nic_dev; + + if (!rxq) { + PMD_DRV_LOG(WARNING, "Rxq is null when release"); + return; + } + nic_dev = rxq->nic_dev; + + spnic_free_rxq_mbufs(rxq); + + rte_memzone_free(rxq->cqe_mz); + + rte_free(rxq->rx_info); + + rte_memzone_free(rxq->rq_mz); + + rte_memzone_free(rxq->pi_mz); + + nic_dev->rxqs[rxq->q_id] = NULL; + rte_free(rxq); +} + +static void spnic_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid) +{ + struct spnic_txq *txq = dev->data->tx_queues[qid]; + struct spnic_nic_dev *nic_dev; + + if (!txq) { + PMD_DRV_LOG(WARNING, "Txq is null when release"); + return; + } + nic_dev = txq->nic_dev; + + spnic_free_txq_mbufs(txq); + + rte_free(txq->tx_info); + txq->tx_info = NULL; + + spnic_free_db_addr(nic_dev->hwdev, txq->db_addr, NULL); + + rte_memzone_free(txq->sq_mz); + + rte_memzone_free(txq->ci_mz); + + nic_dev->txqs[txq->q_id] = NULL; + rte_free(txq); +} + static void spnic_delete_mc_addr_list(struct spnic_nic_dev *nic_dev); /** @@ -235,6 +697,7 @@ static int spnic_dev_start(struct rte_eth_dev *eth_dev) nic_dev = SPNIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev); + spnic_get_func_rx_buf_size(nic_dev); err = spnic_init_function_table(nic_dev->hwdev, nic_dev->rx_buff_len); if (err) { PMD_DRV_LOG(ERR, "Init function table failed, dev_name: %s", @@ -253,6 +716,9 @@ static int spnic_dev_start(struct rte_eth_dev *eth_dev) goto get_feature_err; } + /* reset rx and tx queue */ + spnic_reset_rx_queue(eth_dev); + spnic_reset_tx_queue(eth_dev); /* Init txq and rxq context */ err = spnic_init_qp_ctxts(nic_dev); @@ -270,6 +736,15 @@ static int spnic_dev_start(struct rte_eth_dev *eth_dev) goto set_mtu_fail; } + err = spnic_start_all_rqs(eth_dev); + if (err) { + PMD_DRV_LOG(ERR, "Set rx config failed, dev_name: %s", + eth_dev->data->name); + goto start_rqs_fail; + } + + spnic_start_all_sqs(eth_dev); + /* Update eth_dev link status */ if (eth_dev->data->dev_conf.intr_conf.lsc != 0) (void)spnic_link_update(eth_dev, 0); @@ -278,6 +753,7 @@ static int spnic_dev_start(struct rte_eth_dev *eth_dev) return 0; +start_rqs_fail: set_mtu_fail: spnic_free_qp_ctxts(nic_dev->hwdev); @@ -313,9 +789,17 @@ static int spnic_dev_stop(struct rte_eth_dev *dev) memset(&link, 0, sizeof(link)); (void)rte_eth_linkstatus_set(dev, &link); + /* Flush pending io request */ + spnic_flush_txqs(nic_dev); + + spnic_flush_qps_res(nic_dev->hwdev); /* Clean root context */ spnic_free_qp_ctxts(nic_dev->hwdev); + /* Free all tx and rx mbufs */ + spnic_free_all_txq_mbufs(nic_dev); + spnic_free_all_rxq_mbufs(nic_dev); + return 0; } @@ -329,6 +813,7 @@ static int spnic_dev_close(struct rte_eth_dev *eth_dev) { struct spnic_nic_dev *nic_dev = SPNIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev); + int qid; if (rte_bit_relaxed_test_and_set32(SPNIC_DEV_CLOSE, &nic_dev->dev_status)) { PMD_DRV_LOG(WARNING, "Device %s already closed", @@ -338,6 +823,13 @@ static int spnic_dev_close(struct rte_eth_dev *eth_dev) spnic_dev_stop(eth_dev); + /* Release io resource */ + for (qid = 0; qid < nic_dev->num_sqs; qid++) + spnic_tx_queue_release(eth_dev, qid); + + for (qid = 0; qid < nic_dev->num_rqs; qid++) + spnic_rx_queue_release(eth_dev, qid); + spnic_deinit_sw_rxtxqs(nic_dev); spnic_deinit_mac_addr(eth_dev); rte_free(nic_dev->mc_list); @@ -581,6 +1073,10 @@ static const struct eth_dev_ops spnic_pmd_ops = { .dev_set_link_up = spnic_dev_set_link_up, .dev_set_link_down = spnic_dev_set_link_down, .link_update = spnic_link_update, + .rx_queue_setup = spnic_rx_queue_setup, + .tx_queue_setup = spnic_tx_queue_setup, + .rx_queue_release = spnic_rx_queue_release, + .tx_queue_release = spnic_tx_queue_release, .dev_start = spnic_dev_start, .dev_stop = spnic_dev_stop, .dev_close = spnic_dev_close, @@ -592,8 +1088,12 @@ static const struct eth_dev_ops spnic_pmd_ops = { }; static const struct eth_dev_ops spnic_pmd_vf_ops = { - .link_update = spnic_link_update, + .rx_queue_setup = spnic_rx_queue_setup, + .tx_queue_setup = spnic_tx_queue_setup, .dev_start = spnic_dev_start, + .link_update = spnic_link_update, + .rx_queue_release = spnic_rx_queue_release, + .tx_queue_release = spnic_tx_queue_release, .dev_stop = spnic_dev_stop, .dev_close = spnic_dev_close, .mtu_set = spnic_dev_set_mtu, diff --git a/drivers/net/spnic/spnic_rx.c b/drivers/net/spnic/spnic_rx.c new file mode 100644 index 0000000000..20cd50c0c4 --- /dev/null +++ b/drivers/net/spnic/spnic_rx.c @@ -0,0 +1,302 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Ramaxel Memory Technology, Ltd + */ + +#include +#include +#include + +#include "base/spnic_compat.h" +#include "base/spnic_cmd.h" +#include "base/spnic_hwif.h" +#include "base/spnic_hwdev.h" +#include "base/spnic_wq.h" +#include "base/spnic_mgmt.h" +#include "base/spnic_nic_cfg.h" +#include "spnic_io.h" +#include "spnic_rx.h" +#include "spnic_ethdev.h" + +/** + * Get receive queue wqe + * + * @param[in] rxq + * Receive queue + * @param[out] pi + * Return current pi + * @return + * RX wqe base address + */ +static inline void *spnic_get_rq_wqe(struct spnic_rxq *rxq, u16 *pi) +{ + *pi = MASKED_QUEUE_IDX(rxq, rxq->prod_idx); + + /* Get only one rq wqe for once */ + rxq->prod_idx++; + rxq->delta--; + + return NIC_WQE_ADDR(rxq, *pi); +} + +/** + * Put receive queue wqe + * + * @param[in] rxq + * Receive queue + * @param[in] wqe_cnt + * Wqebb counters + */ +static inline void spnic_put_rq_wqe(struct spnic_rxq *rxq, u16 wqe_cnt) +{ + rxq->delta += wqe_cnt; + rxq->prod_idx -= wqe_cnt; +} + +/** + * Get receive queue local pi + * + * @param[in] rxq + * Receive queue + * @return + * Receive queue local pi + */ +static inline u16 spnic_get_rq_local_pi(struct spnic_rxq *rxq) +{ + return MASKED_QUEUE_IDX(rxq, rxq->prod_idx); +} + +/** + * Update receive queue hardware pi + * + * @param[in] rxq + * Receive queue + * @param[in] pi + * Receive queue pi to update + */ +static inline void spnic_update_rq_hw_pi(struct spnic_rxq *rxq, u16 pi) +{ + *rxq->pi_virt_addr = cpu_to_be16((pi & rxq->q_mask) << rxq->wqe_type); +} + +int spnic_rx_fill_wqe(struct spnic_rxq *rxq) +{ + struct spnic_rq_wqe *rq_wqe = NULL; + struct spnic_nic_dev *nic_dev = rxq->nic_dev; + rte_iova_t cqe_dma; + u16 pi = 0; + int i; + + cqe_dma = rxq->cqe_start_paddr; + for (i = 0; i < rxq->q_depth; i++) { + rq_wqe = spnic_get_rq_wqe(rxq, &pi); + if (!rq_wqe) { + PMD_DRV_LOG(ERR, "Get rq wqe failed, rxq id: %d, wqe id: %d", + rxq->q_id, i); + break; + } + + if (rxq->wqe_type == SPNIC_EXTEND_RQ_WQE) { + /* Unit of cqe length is 16B */ + spnic_set_sge(&rq_wqe->extend_wqe.cqe_sect.sge, + cqe_dma, + sizeof(struct spnic_rq_cqe) >> + SPNIC_CQE_SIZE_SHIFT); + /* Use fixed len */ + rq_wqe->extend_wqe.buf_desc.sge.len = + nic_dev->rx_buff_len; + } else { + rq_wqe->normal_wqe.cqe_hi_addr = upper_32_bits(cqe_dma); + rq_wqe->normal_wqe.cqe_lo_addr = lower_32_bits(cqe_dma); + } + + cqe_dma += sizeof(struct spnic_rq_cqe); + } + + spnic_put_rq_wqe(rxq, (u16)i); + + return i; +} + +static struct rte_mbuf *spnic_rx_alloc_mbuf(struct spnic_rxq *rxq, + rte_iova_t *dma_addr) +{ + struct rte_mbuf *mbuf = NULL; + + if (unlikely(rte_pktmbuf_alloc_bulk(rxq->mb_pool, &mbuf, 1) != 0)) + return NULL; + + *dma_addr = rte_mbuf_data_iova_default(mbuf); + + return mbuf; +} + +u32 spnic_rx_fill_buffers(struct spnic_rxq *rxq) +{ + struct spnic_rq_wqe *rq_wqe = NULL; + struct spnic_rx_info *rx_info = NULL; + struct rte_mbuf *mb = NULL; + rte_iova_t dma_addr; + int i, free_wqebbs; + + free_wqebbs = rxq->delta - 1; + for (i = 0; i < free_wqebbs; i++) { + rx_info = &rxq->rx_info[rxq->next_to_update]; + + mb = spnic_rx_alloc_mbuf(rxq, &dma_addr); + if (!mb) { + PMD_DRV_LOG(ERR, "Alloc mbuf failed"); + break; + } + + rx_info->mbuf = mb; + + rq_wqe = NIC_WQE_ADDR(rxq, rxq->next_to_update); + + /* Fill buffer address only */ + if (rxq->wqe_type == SPNIC_EXTEND_RQ_WQE) { + rq_wqe->extend_wqe.buf_desc.sge.hi_addr = upper_32_bits(dma_addr); + rq_wqe->extend_wqe.buf_desc.sge.lo_addr = lower_32_bits(dma_addr); + } else { + rq_wqe->normal_wqe.buf_hi_addr = upper_32_bits(dma_addr); + rq_wqe->normal_wqe.buf_lo_addr = lower_32_bits(dma_addr); + } + + rxq->next_to_update = (rxq->next_to_update + 1) & rxq->q_mask; + } + + if (likely(i > 0)) { + spnic_write_db(rxq->db_addr, rxq->q_id, 0, RQ_CFLAG_DP, + rxq->next_to_update << rxq->wqe_type); + /* Init rq contxet used, need to optimization */ + rxq->prod_idx = rxq->next_to_update; + rxq->delta -= i; + } else { + PMD_DRV_LOG(ERR, "Alloc rx buffers failed, rxq_id: %d", + rxq->q_id); + } + + return i; +} + +void spnic_free_rxq_mbufs(struct spnic_rxq *rxq) +{ + struct spnic_rx_info *rx_info = NULL; + int free_wqebbs = spnic_get_rq_free_wqebb(rxq) + 1; + volatile struct spnic_rq_cqe *rx_cqe = NULL; + u16 ci; + + while (free_wqebbs++ < rxq->q_depth) { + ci = spnic_get_rq_local_ci(rxq); + + rx_cqe = &rxq->rx_cqe[ci]; + + /* Clear done bit */ + rx_cqe->status = 0; + + rx_info = &rxq->rx_info[ci]; + rte_pktmbuf_free(rx_info->mbuf); + rx_info->mbuf = NULL; + + spnic_update_rq_local_ci(rxq, 1); + } +} + +void spnic_free_all_rxq_mbufs(struct spnic_nic_dev *nic_dev) +{ + u16 qid; + + for (qid = 0; qid < nic_dev->num_rqs; qid++) + spnic_free_rxq_mbufs(nic_dev->rxqs[qid]); +} + +static inline u32 spnic_rx_alloc_mbuf_bulk(struct spnic_rxq *rxq, + struct rte_mbuf **mbufs, + u32 exp_mbuf_cnt) +{ + u32 avail_cnt; + int err; + + err = rte_pktmbuf_alloc_bulk(rxq->mb_pool, mbufs, exp_mbuf_cnt); + if (likely(err == 0)) { + avail_cnt = exp_mbuf_cnt; + } else { + avail_cnt = 0; + rxq->rxq_stats.rx_nombuf += exp_mbuf_cnt; + } + + return avail_cnt; +} + +static inline void spnic_rearm_rxq_mbuf(struct spnic_rxq *rxq) +{ + struct spnic_rq_wqe *rq_wqe = NULL; + struct rte_mbuf **rearm_mbufs; + u32 i, free_wqebbs, rearm_wqebbs, exp_wqebbs; + rte_iova_t dma_addr; + u16 pi; + + /* Check free wqebb cnt fo rearm */ + free_wqebbs = spnic_get_rq_free_wqebb(rxq); + if (unlikely(free_wqebbs < rxq->rx_free_thresh)) + return; + + /* Get rearm mbuf array */ + pi = spnic_get_rq_local_pi(rxq); + rearm_mbufs = (struct rte_mbuf **)(&rxq->rx_info[pi]); + + /* Check rxq free wqebbs turn around */ + exp_wqebbs = rxq->q_depth - pi; + if (free_wqebbs < exp_wqebbs) + exp_wqebbs = free_wqebbs; + + /* Alloc mbuf in bulk */ + rearm_wqebbs = spnic_rx_alloc_mbuf_bulk(rxq, rearm_mbufs, exp_wqebbs); + if (unlikely(rearm_wqebbs == 0)) + return; + + /* Rearm rx mbuf */ + rq_wqe = NIC_WQE_ADDR(rxq, pi); + for (i = 0; i < rearm_wqebbs; i++) { + dma_addr = rte_mbuf_data_iova_default(rearm_mbufs[i]); + + /* Fill buffer address only */ + if (rxq->wqe_type == SPNIC_EXTEND_RQ_WQE) { + rq_wqe->extend_wqe.buf_desc.sge.hi_addr = upper_32_bits(dma_addr); + rq_wqe->extend_wqe.buf_desc.sge.lo_addr = lower_32_bits(dma_addr); + } else { + rq_wqe->normal_wqe.buf_hi_addr = upper_32_bits(dma_addr); + rq_wqe->normal_wqe.buf_lo_addr = lower_32_bits(dma_addr); + } + + rq_wqe = (struct spnic_rq_wqe *)((u64)rq_wqe + + rxq->wqebb_size); + } + rxq->prod_idx += rearm_wqebbs; + rxq->delta -= rearm_wqebbs; + +#ifndef SPNIC_RQ_DB + spnic_write_db(rxq->db_addr, rxq->q_id, 0, RQ_CFLAG_DP, + ((pi + rearm_wqebbs) & rxq->q_mask) << rxq->wqe_type); +#else + /* Update rq hw_pi */ + rte_wmb(); + spnic_update_rq_hw_pi(rxq, pi + rearm_wqebbs); +#endif +} + +int spnic_start_all_rqs(struct rte_eth_dev *eth_dev) +{ + struct spnic_nic_dev *nic_dev = NULL; + struct spnic_rxq *rxq = NULL; + int i; + + nic_dev = SPNIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev); + + for (i = 0; i < nic_dev->num_rqs; i++) { + rxq = eth_dev->data->rx_queues[i]; + spnic_rearm_rxq_mbuf(rxq); + eth_dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; + } + + return 0; +} diff --git a/drivers/net/spnic/spnic_rx.h b/drivers/net/spnic/spnic_rx.h index b2f0052533..46f4e1276d 100644 --- a/drivers/net/spnic/spnic_rx.h +++ b/drivers/net/spnic/spnic_rx.h @@ -110,4 +110,45 @@ struct spnic_rxq { struct spnic_rxq_stats rxq_stats; } __rte_cache_aligned; +int spnic_rx_fill_wqe(struct spnic_rxq *rxq); + +u32 spnic_rx_fill_buffers(struct spnic_rxq *rxq); + +void spnic_free_rxq_mbufs(struct spnic_rxq *rxq); + +void spnic_free_all_rxq_mbufs(struct spnic_nic_dev *nic_dev); + +int spnic_start_all_rqs(struct rte_eth_dev *eth_dev); +/** + * Get receive queue local ci + * + * @param[in] rxq + * Receive queue + * @return + * Receive queue local ci + */ +static inline u16 spnic_get_rq_local_ci(struct spnic_rxq *rxq) +{ + return MASKED_QUEUE_IDX(rxq, rxq->cons_idx); +} + +static inline u16 spnic_get_rq_free_wqebb(struct spnic_rxq *rxq) +{ + return rxq->delta - 1; +} + +/** + * Update receive queue local ci + * + * @param[in] rxq + * Receive queue + * @param[in] wqe_cnt + * Wqebb counters + */ +static inline void spnic_update_rq_local_ci(struct spnic_rxq *rxq, + u16 wqe_cnt) +{ + rxq->cons_idx += wqe_cnt; + rxq->delta += wqe_cnt; +} #endif /* _SPNIC_RX_H_ */ diff --git a/drivers/net/spnic/spnic_tx.c b/drivers/net/spnic/spnic_tx.c new file mode 100644 index 0000000000..d905879412 --- /dev/null +++ b/drivers/net/spnic/spnic_tx.c @@ -0,0 +1,334 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2021 Ramaxel Memory Technology, Ltd + */ + +#include +#include +#include +#include + +#include "base/spnic_compat.h" +#include "base/spnic_cmd.h" +#include "base/spnic_wq.h" +#include "base/spnic_mgmt.h" +#include "base/spnic_hwdev.h" +#include "base/spnic_nic_cfg.h" +#include "spnic_io.h" +#include "spnic_tx.h" +#include "spnic_ethdev.h" + +#define SPNIC_TX_TASK_WRAPPED 1 +#define SPNIC_TX_BD_DESC_WRAPPED 2 + +#define TX_MSS_DEFAULT 0x3E00 +#define TX_MSS_MIN 0x50 + +#define SPNIC_MAX_TX_FREE_BULK 64 + +#define MAX_PAYLOAD_OFFSET 221 + +#define SPNIC_TX_OUTER_CHECKSUM_FLAG_SET 1 +#define SPNIC_TX_OUTER_CHECKSUM_FLAG_NO_SET 0 + +/** + * Get send queue free wqebb cnt + * + * @param[in] sq + * Send queue + * @return + * Number of free wqebb + */ +static inline u16 spnic_get_sq_free_wqebbs(struct spnic_txq *sq) +{ + return (sq->q_depth - + ((sq->prod_idx - sq->cons_idx + sq->q_depth) & sq->q_mask) - 1); +} + +/** + * Update send queue local ci + * + * @param[in] sq + * Send queue + * @param[in] wqe_cnt + * Number of wqebb + */ +static inline void spnic_update_sq_local_ci(struct spnic_txq *sq, u16 wqe_cnt) +{ + sq->cons_idx += wqe_cnt; +} + +/** + * Get send queue local ci + * + * @param[in] sq + * Send queue + * @return + * Local ci + */ +static inline u16 spnic_get_sq_local_ci(struct spnic_txq *sq) +{ + return MASKED_QUEUE_IDX(sq, sq->cons_idx); +} + +/** + * Get send queue hardware ci + * + * @param[in] sq + * Send queue + * @return + * Hardware ci + */ +static inline u16 spnic_get_sq_hw_ci(struct spnic_txq *sq) +{ + return MASKED_QUEUE_IDX(sq, *(u16 *)sq->ci_vaddr_base); +} + +/** + * Get send queue wqe + * + * @param[in] sq + * Send queue + * @param[in] wqebb_cnt + * Num of wqebb counter + * @param[out] pi + * Return current pi + * @param[out] owner + * Owner bit for hardware + * @param[out] wrapped + * Indicate whether wqe is wrapped + * @return + * Send queue wqe base address + */ +static inline void *spnic_get_sq_wqe(struct spnic_txq *sq, + struct spnic_wqe_info *wqe_info) +{ + u16 cur_pi = MASKED_QUEUE_IDX(sq, sq->prod_idx); + u32 end_pi; + + end_pi = cur_pi + wqe_info->wqebb_cnt; + sq->prod_idx += wqe_info->wqebb_cnt; + + wqe_info->owner = sq->owner; + wqe_info->pi = cur_pi; + wqe_info->wrapped = 0; + + if (unlikely(end_pi >= sq->q_depth)) { + sq->owner = !sq->owner; + + if (likely(end_pi > sq->q_depth)) + wqe_info->wrapped = sq->q_depth - cur_pi; + } + + return NIC_WQE_ADDR(sq, cur_pi); +} + +/** + * Put send queue wqe + * + * @param[in] sq + * Send queue + * @param[in] wqebb_cnt + * Num of wqebb counter + * @param[out] owner + * Owner bit for hardware + */ +static inline void spnic_put_sq_wqe(struct spnic_txq *sq, + struct spnic_wqe_info *wqe_info) +{ + if (wqe_info->owner != sq->owner) + sq->owner = wqe_info->owner; + + sq->prod_idx -= wqe_info->wqebb_cnt; +} + +static inline void spnic_set_wqe_combo(struct spnic_txq *txq, + struct spnic_sq_wqe_combo *wqe_combo, + struct spnic_sq_wqe *wqe, + struct spnic_wqe_info *wqe_info) +{ + wqe_combo->hdr = &wqe->compact_wqe.wqe_desc; + + if (wqe_info->offload) { + if (wqe_info->wrapped == SPNIC_TX_TASK_WRAPPED) { + wqe_combo->task = (struct spnic_sq_task *) + (void *)txq->sq_head_addr; + wqe_combo->bds_head = (struct spnic_sq_bufdesc *) + (void *)(txq->sq_head_addr + txq->wqebb_size); + } else if (wqe_info->wrapped == SPNIC_TX_BD_DESC_WRAPPED) { + wqe_combo->task = &wqe->extend_wqe.task; + wqe_combo->bds_head = (struct spnic_sq_bufdesc *) + (void *)(txq->sq_head_addr); + } else { + wqe_combo->task = &wqe->extend_wqe.task; + wqe_combo->bds_head = wqe->extend_wqe.buf_desc; + } + + wqe_combo->wqe_type = SQ_WQE_EXTENDED_TYPE; + wqe_combo->task_type = SQ_WQE_TASKSECT_16BYTES; + return; + } + + if (wqe_info->wrapped == SPNIC_TX_TASK_WRAPPED) { + wqe_combo->bds_head = (struct spnic_sq_bufdesc *) + (void *)(txq->sq_head_addr); + } else { + wqe_combo->bds_head = + (struct spnic_sq_bufdesc *)(&wqe->extend_wqe.task); + } + + if (wqe_info->wqebb_cnt > 1) { + wqe_combo->wqe_type = SQ_WQE_EXTENDED_TYPE; + wqe_combo->task_type = SQ_WQE_TASKSECT_46BITS; + /* This section used as vlan insert, needs to clear */ + wqe_combo->bds_head->rsvd = 0; + } else { + wqe_combo->wqe_type = SQ_WQE_COMPACT_TYPE; + } +} + +void spnic_free_txq_mbufs(struct spnic_txq *txq) +{ + struct spnic_tx_info *tx_info = NULL; + u16 free_wqebbs; + u16 ci; + + free_wqebbs = spnic_get_sq_free_wqebbs(txq) + 1; + + while (free_wqebbs < txq->q_depth) { + ci = spnic_get_sq_local_ci(txq); + + tx_info = &txq->tx_info[ci]; + + rte_pktmbuf_free(tx_info->mbuf); + spnic_update_sq_local_ci(txq, tx_info->wqebb_cnt); + + free_wqebbs += tx_info->wqebb_cnt; + tx_info->mbuf = NULL; + } +} + +void spnic_free_all_txq_mbufs(struct spnic_nic_dev *nic_dev) +{ + u16 qid; + + for (qid = 0; qid < nic_dev->num_sqs; qid++) + spnic_free_txq_mbufs(nic_dev->txqs[qid]); +} + +int spnic_start_all_sqs(struct rte_eth_dev *eth_dev) +{ + struct spnic_nic_dev *nic_dev = NULL; + int i; + + nic_dev = SPNIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev); + + for (i = 0; i < nic_dev->num_rqs; i++) + eth_dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED; + + return 0; +} +static inline int spnic_xmit_mbuf_cleanup(struct spnic_txq *txq, u32 free_cnt) +{ + struct spnic_tx_info *tx_info = NULL; + struct rte_mbuf *mbuf = NULL; + struct rte_mbuf *mbuf_temp = NULL; + struct rte_mbuf *mbuf_free[SPNIC_MAX_TX_FREE_BULK]; + int nb_free = 0; + int wqebb_cnt = 0; + u16 hw_ci, sw_ci, sq_mask; + u32 i; + + hw_ci = spnic_get_sq_hw_ci(txq); + sw_ci = spnic_get_sq_local_ci(txq); + sq_mask = txq->q_mask; + + for (i = 0; i < free_cnt; ++i) { + tx_info = &txq->tx_info[sw_ci]; + if (hw_ci == sw_ci || + (((hw_ci - sw_ci) & sq_mask) < tx_info->wqebb_cnt)) + break; + + sw_ci = (sw_ci + tx_info->wqebb_cnt) & sq_mask; + + wqebb_cnt += tx_info->wqebb_cnt; + mbuf = tx_info->mbuf; + + if (likely(mbuf->nb_segs == 1)) { + mbuf_temp = rte_pktmbuf_prefree_seg(mbuf); + tx_info->mbuf = NULL; + + if (unlikely(mbuf_temp == NULL)) + continue; + + mbuf_free[nb_free++] = mbuf_temp; + if (unlikely(mbuf_temp->pool != mbuf_free[0]->pool || + nb_free >= SPNIC_MAX_TX_FREE_BULK)) { + rte_mempool_put_bulk(mbuf_free[0]->pool, + (void **)mbuf_free, (nb_free - 1)); + nb_free = 0; + mbuf_free[nb_free++] = mbuf_temp; + } + } else { + rte_pktmbuf_free(mbuf); + tx_info->mbuf = NULL; + } + } + + if (nb_free > 0) + rte_mempool_put_bulk(mbuf_free[0]->pool, (void **)mbuf_free, + nb_free); + + spnic_update_sq_local_ci(txq, wqebb_cnt); + return i; +} + +static int spnic_tx_done_cleanup(void *txq, u32 free_cnt) +{ + struct spnic_txq *tx_queue = txq; + u32 try_free_cnt = !free_cnt ? tx_queue->q_depth : free_cnt; + + return spnic_xmit_mbuf_cleanup(tx_queue, try_free_cnt); +} +int spnic_stop_sq(struct spnic_txq *txq) +{ + struct spnic_nic_dev *nic_dev = txq->nic_dev; + unsigned long timeout; + int err = -EFAULT; + int free_wqebbs; + + timeout = msecs_to_jiffies(SPNIC_FLUSH_QUEUE_TIMEOUT) + jiffies; + do { + spnic_tx_done_cleanup(txq, 0); + free_wqebbs = spnic_get_sq_free_wqebbs(txq) + 1; + if (free_wqebbs == txq->q_depth) { + err = 0; + break; + } + + rte_delay_us(1); + } while (time_before(jiffies, timeout)); + + if (err) + PMD_DRV_LOG(WARNING, "%s Wait sq empty timeout, queue_idx: %u, sw_ci: %u, " + "hw_ci: %u, sw_pi: %u, free_wqebbs: %u, q_depth:%u\n", + nic_dev->dev_name, txq->q_id, + spnic_get_sq_local_ci(txq), + spnic_get_sq_hw_ci(txq), + MASKED_QUEUE_IDX(txq, txq->prod_idx), + free_wqebbs, txq->q_depth); + + return err; +} + +/* Should stop transmiting any packets before calling this function */ +void spnic_flush_txqs(struct spnic_nic_dev *nic_dev) +{ + u16 qid; + int err; + + for (qid = 0; qid < nic_dev->num_sqs; qid++) { + err = spnic_stop_sq(nic_dev->txqs[qid]); + if (err) + PMD_DRV_LOG(ERR, "Stop sq%d failed", qid); + } +} diff --git a/drivers/net/spnic/spnic_tx.h b/drivers/net/spnic/spnic_tx.h index 7528b27bd9..d770b15c21 100644 --- a/drivers/net/spnic/spnic_tx.h +++ b/drivers/net/spnic/spnic_tx.h @@ -4,6 +4,224 @@ #ifndef _SPNIC_TX_H_ #define _SPNIC_TX_H_ +/* Tx offload info */ +struct spnic_tx_offload_info { + u8 outer_l2_len; + u8 outer_l3_type; + u16 outer_l3_len; + + u8 inner_l2_len; + u8 inner_l3_type; + u16 inner_l3_len; + + u8 tunnel_length; + u8 tunnel_type; + u8 inner_l4_type; + u8 inner_l4_len; + + u16 payload_offset; + u8 inner_l4_tcp_udp; + u8 rsvd0; +}; + +/* tx wqe ctx */ +struct spnic_wqe_info { + u8 around; + u8 cpy_mbuf_cnt; + u16 sge_cnt; + + u8 offload; + u8 rsvd0; + u16 payload_offset; + + u8 wrapped; + u8 owner; + u16 pi; + + u16 wqebb_cnt; + u16 rsvd1; + + u32 queue_info; +}; +struct spnic_sq_wqe_desc { + u32 ctrl_len; + u32 queue_info; + u32 hi_addr; + u32 lo_addr; +}; + +/* + * Engine only pass first 12B TS field directly to uCode through metadata, + * vlan_offoad is used for hardware when vlan insert in tx + */ +struct spnic_sq_task { + u32 pkt_info0; + u32 ip_identify; + u32 pkt_info2; /* Rsvd for ipsec spi */ + u32 vlan_offload; +}; + +struct spnic_sq_bufdesc { + u32 len; /* 31-bits Length, L2NIC only use length[17:0] */ + u32 rsvd; + u32 hi_addr; + u32 lo_addr; +}; + +struct spnic_sq_compact_wqe { + struct spnic_sq_wqe_desc wqe_desc; +}; + +struct spnic_sq_extend_wqe { + struct spnic_sq_wqe_desc wqe_desc; + struct spnic_sq_task task; + struct spnic_sq_bufdesc buf_desc[0]; +}; + +struct spnic_sq_wqe { + union { + struct spnic_sq_compact_wqe compact_wqe; + struct spnic_sq_extend_wqe extend_wqe; + }; +}; + +/* Use section pointer to support non continuous wqe */ +struct spnic_sq_wqe_combo { + struct spnic_sq_wqe_desc *hdr; + struct spnic_sq_task *task; + struct spnic_sq_bufdesc *bds_head; + u32 wqe_type; + u32 task_type; +}; + +/* SQ ctrl info */ +enum sq_wqe_data_format { + SQ_NORMAL_WQE = 0, +}; + +enum sq_wqe_ec_type { + SQ_WQE_COMPACT_TYPE = 0, + SQ_WQE_EXTENDED_TYPE = 1, +}; + +#define COMPACT_WQE_MAX_CTRL_LEN 0x3FFF + +enum sq_wqe_tasksect_len_type { + SQ_WQE_TASKSECT_46BITS = 0, + SQ_WQE_TASKSECT_16BYTES = 1, +}; + +#define SQ_CTRL_BD0_LEN_SHIFT 0 +#define SQ_CTRL_RSVD_SHIFT 18 +#define SQ_CTRL_BUFDESC_NUM_SHIFT 19 +#define SQ_CTRL_TASKSECT_LEN_SHIFT 27 +#define SQ_CTRL_DATA_FORMAT_SHIFT 28 +#define SQ_CTRL_DIRECT_SHIFT 29 +#define SQ_CTRL_EXTENDED_SHIFT 30 +#define SQ_CTRL_OWNER_SHIFT 31 + +#define SQ_CTRL_BD0_LEN_MASK 0x3FFFFU +#define SQ_CTRL_RSVD_MASK 0x1U +#define SQ_CTRL_BUFDESC_NUM_MASK 0xFFU +#define SQ_CTRL_TASKSECT_LEN_MASK 0x1U +#define SQ_CTRL_DATA_FORMAT_MASK 0x1U +#define SQ_CTRL_DIRECT_MASK 0x1U +#define SQ_CTRL_EXTENDED_MASK 0x1U +#define SQ_CTRL_OWNER_MASK 0x1U + +#define SQ_CTRL_SET(val, member) (((u32)(val) & \ + SQ_CTRL_##member##_MASK) << \ + SQ_CTRL_##member##_SHIFT) + +#define SQ_CTRL_GET(val, member) (((val) >> SQ_CTRL_##member##_SHIFT) \ + & SQ_CTRL_##member##_MASK) + +#define SQ_CTRL_CLEAR(val, member) ((val) & \ + (~(SQ_CTRL_##member##_MASK << \ + SQ_CTRL_##member##_SHIFT))) + +#define SQ_CTRL_QUEUE_INFO_PKT_TYPE_SHIFT 0 +#define SQ_CTRL_QUEUE_INFO_PLDOFF_SHIFT 2 +#define SQ_CTRL_QUEUE_INFO_UFO_SHIFT 10 +#define SQ_CTRL_QUEUE_INFO_TSO_SHIFT 11 +#define SQ_CTRL_QUEUE_INFO_TCPUDP_CS_SHIFT 12 +#define SQ_CTRL_QUEUE_INFO_MSS_SHIFT 13 +#define SQ_CTRL_QUEUE_INFO_SCTP_SHIFT 27 +#define SQ_CTRL_QUEUE_INFO_UC_SHIFT 28 +#define SQ_CTRL_QUEUE_INFO_PRI_SHIFT 29 + +#define SQ_CTRL_QUEUE_INFO_PKT_TYPE_MASK 0x3U +#define SQ_CTRL_QUEUE_INFO_PLDOFF_MASK 0xFFU +#define SQ_CTRL_QUEUE_INFO_UFO_MASK 0x1U +#define SQ_CTRL_QUEUE_INFO_TSO_MASK 0x1U +#define SQ_CTRL_QUEUE_INFO_TCPUDP_CS_MASK 0x1U +#define SQ_CTRL_QUEUE_INFO_MSS_MASK 0x3FFFU +#define SQ_CTRL_QUEUE_INFO_SCTP_MASK 0x1U +#define SQ_CTRL_QUEUE_INFO_UC_MASK 0x1U +#define SQ_CTRL_QUEUE_INFO_PRI_MASK 0x7U + +#define SQ_CTRL_QUEUE_INFO_SET(val, member) \ + (((u32)(val) & SQ_CTRL_QUEUE_INFO_##member##_MASK) \ + << SQ_CTRL_QUEUE_INFO_##member##_SHIFT) + +#define SQ_CTRL_QUEUE_INFO_GET(val, member) \ + (((val) >> SQ_CTRL_QUEUE_INFO_##member##_SHIFT) \ + & SQ_CTRL_QUEUE_INFO_##member##_MASK) + +#define SQ_CTRL_QUEUE_INFO_CLEAR(val, member) \ + ((val) & (~(SQ_CTRL_QUEUE_INFO_##member##_MASK << \ + SQ_CTRL_QUEUE_INFO_##member##_SHIFT))) + +#define SQ_TASK_INFO0_TUNNEL_FLAG_SHIFT 19 +#define SQ_TASK_INFO0_ESP_NEXT_PROTO_SHIFT 22 +#define SQ_TASK_INFO0_INNER_L4_EN_SHIFT 24 +#define SQ_TASK_INFO0_INNER_L3_EN_SHIFT 25 +#define SQ_TASK_INFO0_INNER_L4_PSEUDO_SHIFT 26 +#define SQ_TASK_INFO0_OUT_L4_EN_SHIFT 27 +#define SQ_TASK_INFO0_OUT_L3_EN_SHIFT 28 +#define SQ_TASK_INFO0_OUT_L4_PSEUDO_SHIFT 29 +#define SQ_TASK_INFO0_ESP_OFFLOAD_SHIFT 30 +#define SQ_TASK_INFO0_IPSEC_PROTO_SHIFT 31 + +#define SQ_TASK_INFO0_TUNNEL_FLAG_MASK 0x1U +#define SQ_TASK_INFO0_ESP_NEXT_PROTO_MASK 0x3U +#define SQ_TASK_INFO0_INNER_L4_EN_MASK 0x1U +#define SQ_TASK_INFO0_INNER_L3_EN_MASK 0x1U +#define SQ_TASK_INFO0_INNER_L4_PSEUDO_MASK 0x1U +#define SQ_TASK_INFO0_OUT_L4_EN_MASK 0x1U +#define SQ_TASK_INFO0_OUT_L3_EN_MASK 0x1U +#define SQ_TASK_INFO0_OUT_L4_PSEUDO_MASK 0x1U +#define SQ_TASK_INFO0_ESP_OFFLOAD_MASK 0x1U +#define SQ_TASK_INFO0_IPSEC_PROTO_MASK 0x1U + +#define SQ_TASK_INFO0_SET(val, member) \ + (((u32)(val) & SQ_TASK_INFO0_##member##_MASK) << \ + SQ_TASK_INFO0_##member##_SHIFT) +#define SQ_TASK_INFO0_GET(val, member) \ + (((val) >> SQ_TASK_INFO0_##member##_SHIFT) & \ + SQ_TASK_INFO0_##member##_MASK) + +#define SQ_TASK_INFO1_SET(val, member) \ + (((val) & SQ_TASK_INFO1_##member##_MASK) << \ + SQ_TASK_INFO1_##member##_SHIFT) +#define SQ_TASK_INFO1_GET(val, member) \ + (((val) >> SQ_TASK_INFO1_##member##_SHIFT) & \ + SQ_TASK_INFO1_##member##_MASK) + +#define SQ_TASK_INFO3_VLAN_TAG_SHIFT 0 +#define SQ_TASK_INFO3_VLAN_TYPE_SHIFT 16 +#define SQ_TASK_INFO3_VLAN_TAG_VALID_SHIFT 19 + +#define SQ_TASK_INFO3_VLAN_TAG_MASK 0xFFFFU +#define SQ_TASK_INFO3_VLAN_TYPE_MASK 0x7U +#define SQ_TASK_INFO3_VLAN_TAG_VALID_MASK 0x1U + +#define SQ_TASK_INFO3_SET(val, member) \ + (((val) & SQ_TASK_INFO3_##member##_MASK) << \ + SQ_TASK_INFO3_##member##_SHIFT) +#define SQ_TASK_INFO3_GET(val, member) \ + (((val) >> SQ_TASK_INFO3_##member##_SHIFT) & \ + SQ_TASK_INFO3_##member##_MASK) /* Txq info */ struct spnic_txq_stats { @@ -59,4 +277,14 @@ struct spnic_txq { struct spnic_txq_stats txq_stats; } __rte_cache_aligned; +void spnic_flush_txqs(struct spnic_nic_dev *nic_dev); + +void spnic_free_txq_mbufs(struct spnic_txq *txq); + +void spnic_free_all_txq_mbufs(struct spnic_nic_dev *nic_dev); + +u16 spnic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, u16 nb_pkts); + +int spnic_stop_sq(struct spnic_txq *txq); +int spnic_start_all_sqs(struct rte_eth_dev *eth_dev); #endif /* _SPNIC_TX_H_ */ -- 2.27.0