Add zsda compressdev qp interface implementation. Signed-off-by: Hanxiao Li --- drivers/common/zsda/zsda_qp.c | 267 ++++++++++++++++++++++++++ drivers/common/zsda/zsda_qp.h | 84 ++++++++ drivers/common/zsda/zsda_qp_common.h | 40 ++++ drivers/compress/zsda/zsda_comp_pmd.c | 66 ++++++- 4 files changed, 455 insertions(+), 2 deletions(-) diff --git a/drivers/common/zsda/zsda_qp.c b/drivers/common/zsda/zsda_qp.c index 2ddbf51665..66b212b39e 100644 --- a/drivers/common/zsda/zsda_qp.c +++ b/drivers/common/zsda/zsda_qp.c @@ -7,6 +7,8 @@ #define MAGIC_SEND 0xab #define MAGIC_RECV 0xcd #define ADMIN_VER 1 +#define RING_DIR_TX 0 +#define RING_DIR_RX 1 static uint8_t zsda_num_used_qps; @@ -516,3 +518,268 @@ zsda_queue_init(struct zsda_pci_device *zsda_pci_dev) return ret; } + +struct zsda_qp_hw * +zsda_qps_hw_per_service(struct zsda_pci_device *zsda_pci_dev, + enum zsda_service_type type) +{ + struct zsda_qp_hw *qp_hw = NULL; + + if (type < ZSDA_SERVICE_INVALID) + qp_hw = &(zsda_pci_dev->zsda_hw_qps[type]); + + return qp_hw; +} + +static const struct rte_memzone * +zsda_queue_dma_zone_reserve(const char *queue_name, + const unsigned int queue_size, + const unsigned int socket_id) +{ + const struct rte_memzone *mz; + + mz = rte_memzone_lookup(queue_name); + if (mz != 0) { + if (((size_t)queue_size <= mz->len) && + ((socket_id == (SOCKET_ID_ANY & 0xffff)) || + (socket_id == (mz->socket_id & 0xffff)))) { + ZSDA_LOG(DEBUG, + "re-use memzone already allocated for %s", + queue_name); + return mz; + } + ZSDA_LOG(ERR, "Failed! queue_name exist"); + return NULL; + } + + mz = rte_memzone_reserve_aligned(queue_name, queue_size, + (int)(socket_id & 0xfff), + RTE_MEMZONE_IOVA_CONTIG, queue_size); + + return mz; +} + +static int +zsda_queue_create(const uint8_t dev_id, struct zsda_queue *queue, + const struct zsda_qp_config *qp_conf, const uint8_t dir) +{ + void *io_addr; + const struct rte_memzone *qp_mz; + struct qinfo qcfg = {0}; + + uint16_t desc_size = ((dir == RING_DIR_TX) ? qp_conf->hw->tx_msg_size + : qp_conf->hw->rx_msg_size); + unsigned int queue_size_bytes = qp_conf->nb_descriptors * desc_size; + + queue->hw_queue_number = + ((dir == RING_DIR_TX) ? qp_conf->hw->tx_ring_num + : qp_conf->hw->rx_ring_num); + + struct rte_pci_device *pci_dev = zsda_devs[dev_id].pci_dev; + struct zsda_pci_device *zsda_dev = + (struct zsda_pci_device *)zsda_devs[dev_id].mz->addr; + + zsda_queue_cfg_by_id_get(zsda_dev, queue->hw_queue_number, &qcfg); + + if (dir == RING_DIR_TX) + snprintf(queue->memz_name, sizeof(queue->memz_name), + "%s_%d_%s_%s_%d", pci_dev->driver->driver.name, dev_id, + qp_conf->service_str, "qptxmem", + queue->hw_queue_number); + else + snprintf(queue->memz_name, sizeof(queue->memz_name), + "%s_%d_%s_%s_%d", pci_dev->driver->driver.name, dev_id, + qp_conf->service_str, "qprxmem", + queue->hw_queue_number); + + qp_mz = zsda_queue_dma_zone_reserve(queue->memz_name, queue_size_bytes, + rte_socket_id()); + if (qp_mz == NULL) { + ZSDA_LOG(ERR, "Failed! qp_mz is NULL"); + return -ENOMEM; + } + + queue->base_addr = qp_mz->addr; + queue->base_phys_addr = qp_mz->iova; + queue->modulo_mask = MAX_NUM_OPS; + queue->msg_size = desc_size; + + queue->head = (dir == RING_DIR_TX) ? qcfg.wq_head : qcfg.cq_head; + queue->tail = (dir == RING_DIR_TX) ? qcfg.wq_tail : qcfg.cq_tail; + + if ((queue->head == 0) && (queue->tail == 0)) + qcfg.cycle += 1; + + queue->valid = qcfg.cycle & (ZSDA_MAX_CYCLE - 1); + queue->queue_size = ZSDA_MAX_DESC; + queue->cycle_size = ZSDA_MAX_CYCLE; + queue->io_addr = pci_dev->mem_resource[0].addr; + + memset(queue->base_addr, 0x0, queue_size_bytes); + io_addr = pci_dev->mem_resource[0].addr; + + if (dir == RING_DIR_TX) + ZSDA_CSR_WQ_RING_BASE(io_addr, queue->hw_queue_number, + queue->base_phys_addr); + else + ZSDA_CSR_CQ_RING_BASE(io_addr, queue->hw_queue_number, + queue->base_phys_addr); + + return 0; +} + +static int +zsda_cookie_init(const uint8_t dev_id, struct zsda_qp *qp, + const uint16_t queue_pair_id, + const struct zsda_qp_config *zsda_qp_conf) +{ + char op_cookie_pool_name[RTE_RING_NAMESIZE]; + uint16_t i; + enum zsda_service_type type = zsda_qp_conf->service_type; + + if (zsda_qp_conf->nb_descriptors != ZSDA_MAX_DESC) + ZSDA_LOG(ERR, "Can't create qp for %u descriptors", + zsda_qp_conf->nb_descriptors); + + qp->srv[type].nb_descriptors = zsda_qp_conf->nb_descriptors; + + qp->srv[type].op_cookies = rte_zmalloc_socket( + "zsda PMD op cookie pointer", + zsda_qp_conf->nb_descriptors * + sizeof(*qp->srv[type].op_cookies), + RTE_CACHE_LINE_SIZE, zsda_qp_conf->socket_id); + + if (qp->srv[type].op_cookies == NULL) { + ZSDA_LOG(ERR, "Failed! op_cookies is NULL"); + return -ENOMEM; + } + + snprintf(op_cookie_pool_name, RTE_RING_NAMESIZE, "ZSDA%d_cks_%s_qp%hu", + dev_id, zsda_qp_conf->service_str, queue_pair_id); + + qp->srv[type].op_cookie_pool = rte_mempool_lookup(op_cookie_pool_name); + if (qp->srv[type].op_cookie_pool == NULL) + qp->srv[type].op_cookie_pool = rte_mempool_create( + op_cookie_pool_name, qp->srv[type].nb_descriptors, + zsda_qp_conf->cookie_size, 64, 0, NULL, NULL, NULL, + NULL, (int)(rte_socket_id() & 0xfff), 0); + if (!qp->srv[type].op_cookie_pool) { + ZSDA_LOG(ERR, "Failed! op_cookie_pool is NULL"); + goto exit; + } + + for (i = 0; i < qp->srv[type].nb_descriptors; i++) { + if (rte_mempool_get(qp->srv[type].op_cookie_pool, + &qp->srv[type].op_cookies[i])) { + ZSDA_LOG(ERR, "ZSDA PMD Cannot get op_cookie"); + goto exit; + } + memset(qp->srv[type].op_cookies[i], 0, + zsda_qp_conf->cookie_size); + } + return ZSDA_SUCCESS; + +exit: + + rte_mempool_free(qp->srv[type].op_cookie_pool); + rte_free(qp->srv[type].op_cookies); + + return -EFAULT; +} + +static int +zsda_queue_pair_setup(const uint8_t dev_id, struct zsda_qp *qp, + const uint16_t queue_pair_id, + const struct zsda_qp_config *zsda_qp_conf) +{ + struct rte_pci_device *pci_dev = zsda_devs[dev_id].pci_dev; + int ret; + enum zsda_service_type type = zsda_qp_conf->service_type; + + if (type >= ZSDA_SERVICE_INVALID) { + ZSDA_LOG(ERR, "Failed! service type"); + return -EINVAL; + } + + if (pci_dev->mem_resource[0].addr == NULL) { + ZSDA_LOG(ERR, "Failed! mem_resource[0].addr is NULL"); + return -EINVAL; + } + + if (zsda_queue_create(dev_id, &(qp->srv[type].tx_q), zsda_qp_conf, + RING_DIR_TX) != 0) { + ZSDA_LOG(ERR, "Failed! zsda_queue_create tx"); + return -EFAULT; + } + + if (zsda_queue_create(dev_id, &(qp->srv[type].rx_q), zsda_qp_conf, + RING_DIR_RX) != 0) { + ZSDA_LOG(ERR, "Failed! zsda_queue_create rx"); + zsda_queue_delete(&(qp->srv[type].tx_q)); + return -EFAULT; + } + + ret = zsda_cookie_init(dev_id, qp, queue_pair_id, zsda_qp_conf); + if (ret) { + zsda_queue_delete(&(qp->srv[type].tx_q)); + zsda_queue_delete(&(qp->srv[type].rx_q)); + qp->srv[type].used = false; + } + qp->srv[type].used = true; + return ret; +} + +static int +zsda_common_qp_setup(uint8_t zsda_dev_id, struct zsda_qp *qp, + const uint16_t queue_pair_id, const struct zsda_qp_config *conf) +{ + uint16_t i; + int ret; + rte_iova_t cookie_phys_addr; + + ret = zsda_queue_pair_setup(zsda_dev_id, qp, queue_pair_id, conf); + if (ret) + return ret; + + for (i = 0; i < qp->srv[conf->service_type].nb_descriptors; i++) { + struct zsda_op_cookie *cookie = + qp->srv[conf->service_type].op_cookies[i]; + cookie_phys_addr = rte_mempool_virt2iova(cookie); + + cookie->comp_head_phys_addr = cookie_phys_addr + + offsetof(struct zsda_op_cookie, comp_head); + + cookie->sgl_src_phys_addr = cookie_phys_addr + + offsetof(struct zsda_op_cookie, sgl_src); + + cookie->sgl_dst_phys_addr = cookie_phys_addr + + offsetof(struct zsda_op_cookie, sgl_dst); + } + return ret; +} + +int +zsda_task_queue_setup(struct zsda_pci_device *zsda_pci_dev, + struct zsda_qp *qp, struct task_queue_info *task_q_info) +{ + enum zsda_service_type type = task_q_info->type; + struct zsda_qp_config conf; + int ret; + struct zsda_qp_hw *qp_hw; + const uint16_t qp_id = task_q_info->qp_id; + + qp_hw = zsda_qps_hw_per_service(zsda_pci_dev, type); + conf.hw = qp_hw->data + qp_id; + conf.service_type = type; + conf.cookie_size = sizeof(struct zsda_op_cookie); + conf.nb_descriptors = task_q_info->nb_des; + conf.socket_id = task_q_info->socket_id; + conf.service_str = task_q_info->service_str; + + ret = zsda_common_qp_setup(zsda_pci_dev->zsda_dev_id, qp, qp_id, &conf); + qp->srv[type].rx_cb = task_q_info->rx_cb; + qp->srv[type].tx_cb = task_q_info->tx_cb; + qp->srv[type].match = task_q_info->match; + + return ret; +} diff --git a/drivers/common/zsda/zsda_qp.h b/drivers/common/zsda/zsda_qp.h index c07b8cc653..c79bfb5d36 100644 --- a/drivers/common/zsda/zsda_qp.h +++ b/drivers/common/zsda/zsda_qp.h @@ -6,6 +6,7 @@ #define _ZSDA_QP_H_ #include "zsda_qp_common.h" + #include "zsda_device.h" #define ZSDA_ADMIN_Q_START 0x100 @@ -51,6 +52,49 @@ #define ZSDA_TIME_SLEEP_US 100 #define ZSDA_TIME_NUM 500 +#define WQ_CSR_LBASE 0x1000 +#define WQ_CSR_UBASE 0x1004 +#define CQ_CSR_LBASE 0x1400 +#define CQ_CSR_UBASE 0x1404 +#define WQ_TAIL 0x1800 +#define CQ_HEAD 0x1804 + +/* CSR write macro */ +#define ZSDA_CSR_WR(csrAddr, csrOffset, val) \ + rte_write32(val, (((uint8_t *)csrAddr) + csrOffset)) +#define ZSDA_CSR_WC_WR(csrAddr, csrOffset, val) \ + rte_write32_wc(val, (((uint8_t *)csrAddr) + csrOffset)) + +/* CSR read macro */ +#define ZSDA_CSR_RD(csrAddr, csrOffset) \ + rte_read32((((uint8_t *)csrAddr) + csrOffset)) + +#define ZSDA_CSR_WQ_RING_BASE(csr_base_addr, ring, value) \ + do { \ + uint32_t l_base = 0, u_base = 0; \ + l_base = (uint32_t)(value & 0xFFFFFFFF); \ + u_base = (uint32_t)((value & 0xFFFFFFFF00000000ULL) >> 32); \ + ZSDA_CSR_WR(csr_base_addr, (ring << 3) + WQ_CSR_LBASE, \ + l_base); \ + ZSDA_LOG(INFO, "l_basg - offset:0x%x, value:0x%x", \ + ((ring << 3) + WQ_CSR_LBASE), l_base); \ + ZSDA_CSR_WR(csr_base_addr, (ring << 3) + WQ_CSR_UBASE, \ + u_base); \ + ZSDA_LOG(INFO, "h_base - offset:0x%x, value:0x%x", \ + ((ring << 3) + WQ_CSR_UBASE), u_base); \ + } while (0) + +#define ZSDA_CSR_CQ_RING_BASE(csr_base_addr, ring, value) \ + do { \ + uint32_t l_base = 0, u_base = 0; \ + l_base = (uint32_t)(value & 0xFFFFFFFF); \ + u_base = (uint32_t)((value & 0xFFFFFFFF00000000ULL) >> 32); \ + ZSDA_CSR_WR(csr_base_addr, (ring << 3) + CQ_CSR_LBASE, \ + l_base); \ + ZSDA_CSR_WR(csr_base_addr, (ring << 3) + CQ_CSR_UBASE, \ + u_base); \ + } while (0) + extern struct zsda_num_qps zsda_nb_qps; enum zsda_admin_msg_id { @@ -88,6 +132,40 @@ struct zsda_num_qps { uint16_t hash; }; +struct zsda_qp_config { + enum zsda_service_type service_type; + const struct zsda_qp_hw_data *hw; + uint16_t nb_descriptors; + uint32_t cookie_size; + int socket_id; + const char *service_str; +}; + +struct zsda_op_cookie { + struct zsda_sgl sgl_src; + struct zsda_sgl sgl_dst; + phys_addr_t sgl_src_phys_addr; + phys_addr_t sgl_dst_phys_addr; + phys_addr_t comp_head_phys_addr; + uint8_t comp_head[COMP_REMOVE_SPACE_LEN]; + uint16_t sid; + bool used; + bool decomp_no_tail; + void *op; +}; + +struct task_queue_info { + uint16_t qp_id; + uint16_t nb_des; + int socket_id; + enum zsda_service_type type; + const char *service_str; + + rx_callback rx_cb; + tx_callback tx_cb; + srv_match match; +}; + extern struct zsda_num_qps zsda_nb_qps; int zsda_queue_start(const struct rte_pci_device *pci_dev); @@ -95,4 +173,10 @@ int zsda_queue_stop(const struct rte_pci_device *pci_dev); int zsda_queue_init(struct zsda_pci_device *zsda_pci_dev); +struct zsda_qp_hw * +zsda_qps_hw_per_service(struct zsda_pci_device *zsda_pci_dev, + const enum zsda_service_type service); + +int zsda_task_queue_setup(struct zsda_pci_device *zsda_pci_dev, + struct zsda_qp *qp, struct task_queue_info *task_q_info); #endif /* _ZSDA_QP_H_ */ diff --git a/drivers/common/zsda/zsda_qp_common.h b/drivers/common/zsda/zsda_qp_common.h index f589df986e..ba77b0f027 100644 --- a/drivers/common/zsda/zsda_qp_common.h +++ b/drivers/common/zsda/zsda_qp_common.h @@ -35,6 +35,15 @@ enum zsda_service_type { #define ZSDA_CSR_READ8(addr) rte_read8((addr)) #define ZSDA_CSR_WRITE8(addr, value) rte_write8_relaxed((value), (addr)) +#define NB_DES 512 +#define ZSDA_SGL_MAX_NUMBER 512 +#define COMP_REMOVE_SPACE_LEN 16 + +#define ZSDA_MAX_DESC 512 +#define ZSDA_MAX_CYCLE 256 +#define ZSDA_MAX_DEV RTE_PMD_ZSDA_MAX_PCI_DEVICES +#define MAX_NUM_OPS 0x1FF + struct zsda_admin_req { uint16_t msg_type; uint8_t data[26]; @@ -105,10 +114,30 @@ struct zsda_qp_stat { uint64_t dequeue_err_count; }; +struct zsda_cqe { + uint8_t valid; /* cqe_cycle */ + uint8_t op_code; + uint16_t sid; + uint8_t state; + uint8_t result; + uint16_t zsda_wq_id; + uint32_t tx_real_length; + uint16_t err0; + uint16_t err1; +} __rte_packed; + +typedef int (*rx_callback)(void *cookie_in, struct zsda_cqe *cqe); +typedef int (*tx_callback)(void *op_in, const struct zsda_queue *queue, + void **op_cookies, const uint16_t new_tail); +typedef int (*srv_match)(const void *op_in); + struct qp_srv { bool used; struct zsda_queue tx_q; struct zsda_queue rx_q; + rx_callback rx_cb; + tx_callback tx_cb; + srv_match match; struct zsda_qp_stat stats; struct rte_mempool *op_cookie_pool; void **op_cookies; @@ -119,6 +148,17 @@ struct zsda_qp { struct qp_srv srv[ZSDA_MAX_SERVICES]; }; +struct zsda_buf { + uint64_t addr; + uint32_t len; + uint8_t resrvd[3]; + uint8_t type; +} __rte_packed; + +struct __rte_cache_aligned zsda_sgl { + struct zsda_buf buffers[ZSDA_SGL_MAX_NUMBER]; +}; + void zsda_queue_delete(const struct zsda_queue *queue); int zsda_queue_pair_release(struct zsda_qp **qp_addr); void zsda_stats_get(void **queue_pairs, const uint32_t nb_queue_pairs, diff --git a/drivers/compress/zsda/zsda_comp_pmd.c b/drivers/compress/zsda/zsda_comp_pmd.c index a8d2da0477..19a4932525 100644 --- a/drivers/compress/zsda/zsda_comp_pmd.c +++ b/drivers/compress/zsda/zsda_comp_pmd.c @@ -6,6 +6,7 @@ #include "zsda_logs.h" #include "zsda_qp_common.h" +#include "zsda_qp.h" #include "zsda_comp_pmd.h" static int @@ -88,11 +89,21 @@ zsda_comp_dev_stop(struct rte_compressdev *dev) zsda_queue_stop(comp_dev->zsda_pci_dev->pci_dev); } +static int +zsda_comp_qp_release(struct rte_compressdev *dev, uint16_t queue_pair_id) +{ + return zsda_queue_pair_release( + (struct zsda_qp **)&(dev->data->queue_pairs[queue_pair_id])); +} + static int zsda_comp_dev_close(struct rte_compressdev *dev) { struct zsda_comp_dev_private *comp_dev = dev->data->dev_private; + for (int i = 0; i < dev->data->nb_queue_pairs; i++) + zsda_comp_qp_release(dev, i); + rte_mempool_free(comp_dev->xformpool); comp_dev->xformpool = NULL; @@ -199,6 +210,57 @@ zsda_comp_private_xform_free(struct rte_compressdev *dev __rte_unused, return -EINVAL; } +static int +zsda_comp_qp_setup(struct rte_compressdev *dev, uint16_t qp_id, + uint32_t max_inflight_ops, int socket_id) +{ + int ret = ZSDA_SUCCESS; + struct zsda_qp *qp_new; + + struct zsda_qp **qp_addr = + (struct zsda_qp **)&(dev->data->queue_pairs[qp_id]); + struct zsda_comp_dev_private *comp_priv = dev->data->dev_private; + struct zsda_pci_device *zsda_pci_dev = comp_priv->zsda_pci_dev; + uint16_t nb_des = max_inflight_ops & 0xffff; + struct task_queue_info task_q_info; + + nb_des = (nb_des == NB_DES) ? nb_des : NB_DES; + + if (*qp_addr != NULL) { + ret = zsda_comp_qp_release(dev, qp_id); + if (ret) + return ret; + } + + qp_new = rte_zmalloc_socket("zsda PMD qp metadata", sizeof(*qp_new), + RTE_CACHE_LINE_SIZE, socket_id); + if (qp_new == NULL) { + ZSDA_LOG(ERR, "Failed! qp_new is NULL"); + return -ENOMEM; + } + + task_q_info.nb_des = nb_des; + task_q_info.socket_id = socket_id; + task_q_info.qp_id = qp_id; + + task_q_info.type = ZSDA_SERVICE_COMPRESSION; + task_q_info.service_str = "comp"; + ret = zsda_task_queue_setup(zsda_pci_dev, qp_new, &task_q_info); + + task_q_info.type = ZSDA_SERVICE_DECOMPRESSION; + task_q_info.service_str = "decomp"; + ret |= zsda_task_queue_setup(zsda_pci_dev, qp_new, &task_q_info); + + if (ret) { + rte_free(qp_new); + return ret; + } + + *qp_addr = qp_new; + + return ret; +} + static struct rte_compressdev_ops compress_zsda_ops = { .dev_configure = zsda_comp_dev_config, @@ -209,8 +271,8 @@ static struct rte_compressdev_ops compress_zsda_ops = { .stats_get = zsda_comp_stats_get, .stats_reset = zsda_comp_stats_reset, - .queue_pair_setup = NULL, - .queue_pair_release = NULL, + .queue_pair_setup = zsda_comp_qp_setup, + .queue_pair_release = zsda_comp_qp_release, .private_xform_create = zsda_comp_private_xform_create, .private_xform_free = zsda_comp_private_xform_free, -- 2.27.0