Add zsda compressdev qp interface implementation.
Signed-off-by: Hanxiao Li <li.hanxiao@zte.com.cn>
---
drivers/common/zsda/zsda_qp.c | 267 ++++++++++++++++++++++++++
drivers/common/zsda/zsda_qp.h | 84 ++++++++
drivers/common/zsda/zsda_qp_common.h | 40 ++++
drivers/compress/zsda/zsda_comp_pmd.c | 66 ++++++-
4 files changed, 455 insertions(+), 2 deletions(-)
diff --git a/drivers/common/zsda/zsda_qp.c b/drivers/common/zsda/zsda_qp.c
index 2ddbf51665..66b212b39e 100644
--- a/drivers/common/zsda/zsda_qp.c
+++ b/drivers/common/zsda/zsda_qp.c
@@ -7,6 +7,8 @@
#define MAGIC_SEND 0xab
#define MAGIC_RECV 0xcd
#define ADMIN_VER 1
+#define RING_DIR_TX 0
+#define RING_DIR_RX 1
static uint8_t zsda_num_used_qps;
@@ -516,3 +518,268 @@ zsda_queue_init(struct zsda_pci_device *zsda_pci_dev)
return ret;
}
+
+struct zsda_qp_hw *
+zsda_qps_hw_per_service(struct zsda_pci_device *zsda_pci_dev,
+ enum zsda_service_type type)
+{
+ struct zsda_qp_hw *qp_hw = NULL;
+
+ if (type < ZSDA_SERVICE_INVALID)
+ qp_hw = &(zsda_pci_dev->zsda_hw_qps[type]);
+
+ return qp_hw;
+}
+
+static const struct rte_memzone *
+zsda_queue_dma_zone_reserve(const char *queue_name,
+ const unsigned int queue_size,
+ const unsigned int socket_id)
+{
+ const struct rte_memzone *mz;
+
+ mz = rte_memzone_lookup(queue_name);
+ if (mz != 0) {
+ if (((size_t)queue_size <= mz->len) &&
+ ((socket_id == (SOCKET_ID_ANY & 0xffff)) ||
+ (socket_id == (mz->socket_id & 0xffff)))) {
+ ZSDA_LOG(DEBUG,
+ "re-use memzone already allocated for %s",
+ queue_name);
+ return mz;
+ }
+ ZSDA_LOG(ERR, "Failed! queue_name exist");
+ return NULL;
+ }
+
+ mz = rte_memzone_reserve_aligned(queue_name, queue_size,
+ (int)(socket_id & 0xfff),
+ RTE_MEMZONE_IOVA_CONTIG, queue_size);
+
+ return mz;
+}
+
+static int
+zsda_queue_create(const uint8_t dev_id, struct zsda_queue *queue,
+ const struct zsda_qp_config *qp_conf, const uint8_t dir)
+{
+ void *io_addr;
+ const struct rte_memzone *qp_mz;
+ struct qinfo qcfg = {0};
+
+ uint16_t desc_size = ((dir == RING_DIR_TX) ? qp_conf->hw->tx_msg_size
+ : qp_conf->hw->rx_msg_size);
+ unsigned int queue_size_bytes = qp_conf->nb_descriptors * desc_size;
+
+ queue->hw_queue_number =
+ ((dir == RING_DIR_TX) ? qp_conf->hw->tx_ring_num
+ : qp_conf->hw->rx_ring_num);
+
+ struct rte_pci_device *pci_dev = zsda_devs[dev_id].pci_dev;
+ struct zsda_pci_device *zsda_dev =
+ (struct zsda_pci_device *)zsda_devs[dev_id].mz->addr;
+
+ zsda_queue_cfg_by_id_get(zsda_dev, queue->hw_queue_number, &qcfg);
+
+ if (dir == RING_DIR_TX)
+ snprintf(queue->memz_name, sizeof(queue->memz_name),
+ "%s_%d_%s_%s_%d", pci_dev->driver->driver.name, dev_id,
+ qp_conf->service_str, "qptxmem",
+ queue->hw_queue_number);
+ else
+ snprintf(queue->memz_name, sizeof(queue->memz_name),
+ "%s_%d_%s_%s_%d", pci_dev->driver->driver.name, dev_id,
+ qp_conf->service_str, "qprxmem",
+ queue->hw_queue_number);
+
+ qp_mz = zsda_queue_dma_zone_reserve(queue->memz_name, queue_size_bytes,
+ rte_socket_id());
+ if (qp_mz == NULL) {
+ ZSDA_LOG(ERR, "Failed! qp_mz is NULL");
+ return -ENOMEM;
+ }
+
+ queue->base_addr = qp_mz->addr;
+ queue->base_phys_addr = qp_mz->iova;
+ queue->modulo_mask = MAX_NUM_OPS;
+ queue->msg_size = desc_size;
+
+ queue->head = (dir == RING_DIR_TX) ? qcfg.wq_head : qcfg.cq_head;
+ queue->tail = (dir == RING_DIR_TX) ? qcfg.wq_tail : qcfg.cq_tail;
+
+ if ((queue->head == 0) && (queue->tail == 0))
+ qcfg.cycle += 1;
+
+ queue->valid = qcfg.cycle & (ZSDA_MAX_CYCLE - 1);
+ queue->queue_size = ZSDA_MAX_DESC;
+ queue->cycle_size = ZSDA_MAX_CYCLE;
+ queue->io_addr = pci_dev->mem_resource[0].addr;
+
+ memset(queue->base_addr, 0x0, queue_size_bytes);
+ io_addr = pci_dev->mem_resource[0].addr;
+
+ if (dir == RING_DIR_TX)
+ ZSDA_CSR_WQ_RING_BASE(io_addr, queue->hw_queue_number,
+ queue->base_phys_addr);
+ else
+ ZSDA_CSR_CQ_RING_BASE(io_addr, queue->hw_queue_number,
+ queue->base_phys_addr);
+
+ return 0;
+}
+
+static int
+zsda_cookie_init(const uint8_t dev_id, struct zsda_qp *qp,
+ const uint16_t queue_pair_id,
+ const struct zsda_qp_config *zsda_qp_conf)
+{
+ char op_cookie_pool_name[RTE_RING_NAMESIZE];
+ uint16_t i;
+ enum zsda_service_type type = zsda_qp_conf->service_type;
+
+ if (zsda_qp_conf->nb_descriptors != ZSDA_MAX_DESC)
+ ZSDA_LOG(ERR, "Can't create qp for %u descriptors",
+ zsda_qp_conf->nb_descriptors);
+
+ qp->srv[type].nb_descriptors = zsda_qp_conf->nb_descriptors;
+
+ qp->srv[type].op_cookies = rte_zmalloc_socket(
+ "zsda PMD op cookie pointer",
+ zsda_qp_conf->nb_descriptors *
+ sizeof(*qp->srv[type].op_cookies),
+ RTE_CACHE_LINE_SIZE, zsda_qp_conf->socket_id);
+
+ if (qp->srv[type].op_cookies == NULL) {
+ ZSDA_LOG(ERR, "Failed! op_cookies is NULL");
+ return -ENOMEM;
+ }
+
+ snprintf(op_cookie_pool_name, RTE_RING_NAMESIZE, "ZSDA%d_cks_%s_qp%hu",
+ dev_id, zsda_qp_conf->service_str, queue_pair_id);
+
+ qp->srv[type].op_cookie_pool = rte_mempool_lookup(op_cookie_pool_name);
+ if (qp->srv[type].op_cookie_pool == NULL)
+ qp->srv[type].op_cookie_pool = rte_mempool_create(
+ op_cookie_pool_name, qp->srv[type].nb_descriptors,
+ zsda_qp_conf->cookie_size, 64, 0, NULL, NULL, NULL,
+ NULL, (int)(rte_socket_id() & 0xfff), 0);
+ if (!qp->srv[type].op_cookie_pool) {
+ ZSDA_LOG(ERR, "Failed! op_cookie_pool is NULL");
+ goto exit;
+ }
+
+ for (i = 0; i < qp->srv[type].nb_descriptors; i++) {
+ if (rte_mempool_get(qp->srv[type].op_cookie_pool,
+ &qp->srv[type].op_cookies[i])) {
+ ZSDA_LOG(ERR, "ZSDA PMD Cannot get op_cookie");
+ goto exit;
+ }
+ memset(qp->srv[type].op_cookies[i], 0,
+ zsda_qp_conf->cookie_size);
+ }
+ return ZSDA_SUCCESS;
+
+exit:
+
+ rte_mempool_free(qp->srv[type].op_cookie_pool);
+ rte_free(qp->srv[type].op_cookies);
+
+ return -EFAULT;
+}
+
+static int
+zsda_queue_pair_setup(const uint8_t dev_id, struct zsda_qp *qp,
+ const uint16_t queue_pair_id,
+ const struct zsda_qp_config *zsda_qp_conf)
+{
+ struct rte_pci_device *pci_dev = zsda_devs[dev_id].pci_dev;
+ int ret;
+ enum zsda_service_type type = zsda_qp_conf->service_type;
+
+ if (type >= ZSDA_SERVICE_INVALID) {
+ ZSDA_LOG(ERR, "Failed! service type");
+ return -EINVAL;
+ }
+
+ if (pci_dev->mem_resource[0].addr == NULL) {
+ ZSDA_LOG(ERR, "Failed! mem_resource[0].addr is NULL");
+ return -EINVAL;
+ }
+
+ if (zsda_queue_create(dev_id, &(qp->srv[type].tx_q), zsda_qp_conf,
+ RING_DIR_TX) != 0) {
+ ZSDA_LOG(ERR, "Failed! zsda_queue_create tx");
+ return -EFAULT;
+ }
+
+ if (zsda_queue_create(dev_id, &(qp->srv[type].rx_q), zsda_qp_conf,
+ RING_DIR_RX) != 0) {
+ ZSDA_LOG(ERR, "Failed! zsda_queue_create rx");
+ zsda_queue_delete(&(qp->srv[type].tx_q));
+ return -EFAULT;
+ }
+
+ ret = zsda_cookie_init(dev_id, qp, queue_pair_id, zsda_qp_conf);
+ if (ret) {
+ zsda_queue_delete(&(qp->srv[type].tx_q));
+ zsda_queue_delete(&(qp->srv[type].rx_q));
+ qp->srv[type].used = false;
+ }
+ qp->srv[type].used = true;
+ return ret;
+}
+
+static int
+zsda_common_qp_setup(uint8_t zsda_dev_id, struct zsda_qp *qp,
+ const uint16_t queue_pair_id, const struct zsda_qp_config *conf)
+{
+ uint16_t i;
+ int ret;
+ rte_iova_t cookie_phys_addr;
+
+ ret = zsda_queue_pair_setup(zsda_dev_id, qp, queue_pair_id, conf);
+ if (ret)
+ return ret;
+
+ for (i = 0; i < qp->srv[conf->service_type].nb_descriptors; i++) {
+ struct zsda_op_cookie *cookie =
+ qp->srv[conf->service_type].op_cookies[i];
+ cookie_phys_addr = rte_mempool_virt2iova(cookie);
+
+ cookie->comp_head_phys_addr = cookie_phys_addr +
+ offsetof(struct zsda_op_cookie, comp_head);
+
+ cookie->sgl_src_phys_addr = cookie_phys_addr +
+ offsetof(struct zsda_op_cookie, sgl_src);
+
+ cookie->sgl_dst_phys_addr = cookie_phys_addr +
+ offsetof(struct zsda_op_cookie, sgl_dst);
+ }
+ return ret;
+}
+
+int
+zsda_task_queue_setup(struct zsda_pci_device *zsda_pci_dev,
+ struct zsda_qp *qp, struct task_queue_info *task_q_info)
+{
+ enum zsda_service_type type = task_q_info->type;
+ struct zsda_qp_config conf;
+ int ret;
+ struct zsda_qp_hw *qp_hw;
+ const uint16_t qp_id = task_q_info->qp_id;
+
+ qp_hw = zsda_qps_hw_per_service(zsda_pci_dev, type);
+ conf.hw = qp_hw->data + qp_id;
+ conf.service_type = type;
+ conf.cookie_size = sizeof(struct zsda_op_cookie);
+ conf.nb_descriptors = task_q_info->nb_des;
+ conf.socket_id = task_q_info->socket_id;
+ conf.service_str = task_q_info->service_str;
+
+ ret = zsda_common_qp_setup(zsda_pci_dev->zsda_dev_id, qp, qp_id, &conf);
+ qp->srv[type].rx_cb = task_q_info->rx_cb;
+ qp->srv[type].tx_cb = task_q_info->tx_cb;
+ qp->srv[type].match = task_q_info->match;
+
+ return ret;
+}
diff --git a/drivers/common/zsda/zsda_qp.h b/drivers/common/zsda/zsda_qp.h
index c07b8cc653..c79bfb5d36 100644
--- a/drivers/common/zsda/zsda_qp.h
+++ b/drivers/common/zsda/zsda_qp.h
@@ -6,6 +6,7 @@
#define _ZSDA_QP_H_
#include "zsda_qp_common.h"
+
#include "zsda_device.h"
#define ZSDA_ADMIN_Q_START 0x100
@@ -51,6 +52,49 @@
#define ZSDA_TIME_SLEEP_US 100
#define ZSDA_TIME_NUM 500
+#define WQ_CSR_LBASE 0x1000
+#define WQ_CSR_UBASE 0x1004
+#define CQ_CSR_LBASE 0x1400
+#define CQ_CSR_UBASE 0x1404
+#define WQ_TAIL 0x1800
+#define CQ_HEAD 0x1804
+
+/* CSR write macro */
+#define ZSDA_CSR_WR(csrAddr, csrOffset, val) \
+ rte_write32(val, (((uint8_t *)csrAddr) + csrOffset))
+#define ZSDA_CSR_WC_WR(csrAddr, csrOffset, val) \
+ rte_write32_wc(val, (((uint8_t *)csrAddr) + csrOffset))
+
+/* CSR read macro */
+#define ZSDA_CSR_RD(csrAddr, csrOffset) \
+ rte_read32((((uint8_t *)csrAddr) + csrOffset))
+
+#define ZSDA_CSR_WQ_RING_BASE(csr_base_addr, ring, value) \
+ do { \
+ uint32_t l_base = 0, u_base = 0; \
+ l_base = (uint32_t)(value & 0xFFFFFFFF); \
+ u_base = (uint32_t)((value & 0xFFFFFFFF00000000ULL) >> 32); \
+ ZSDA_CSR_WR(csr_base_addr, (ring << 3) + WQ_CSR_LBASE, \
+ l_base); \
+ ZSDA_LOG(INFO, "l_basg - offset:0x%x, value:0x%x", \
+ ((ring << 3) + WQ_CSR_LBASE), l_base); \
+ ZSDA_CSR_WR(csr_base_addr, (ring << 3) + WQ_CSR_UBASE, \
+ u_base); \
+ ZSDA_LOG(INFO, "h_base - offset:0x%x, value:0x%x", \
+ ((ring << 3) + WQ_CSR_UBASE), u_base); \
+ } while (0)
+
+#define ZSDA_CSR_CQ_RING_BASE(csr_base_addr, ring, value) \
+ do { \
+ uint32_t l_base = 0, u_base = 0; \
+ l_base = (uint32_t)(value & 0xFFFFFFFF); \
+ u_base = (uint32_t)((value & 0xFFFFFFFF00000000ULL) >> 32); \
+ ZSDA_CSR_WR(csr_base_addr, (ring << 3) + CQ_CSR_LBASE, \
+ l_base); \
+ ZSDA_CSR_WR(csr_base_addr, (ring << 3) + CQ_CSR_UBASE, \
+ u_base); \
+ } while (0)
+
extern struct zsda_num_qps zsda_nb_qps;
enum zsda_admin_msg_id {
@@ -88,6 +132,40 @@ struct zsda_num_qps {
uint16_t hash;
};
+struct zsda_qp_config {
+ enum zsda_service_type service_type;
+ const struct zsda_qp_hw_data *hw;
+ uint16_t nb_descriptors;
+ uint32_t cookie_size;
+ int socket_id;
+ const char *service_str;
+};
+
+struct zsda_op_cookie {
+ struct zsda_sgl sgl_src;
+ struct zsda_sgl sgl_dst;
+ phys_addr_t sgl_src_phys_addr;
+ phys_addr_t sgl_dst_phys_addr;
+ phys_addr_t comp_head_phys_addr;
+ uint8_t comp_head[COMP_REMOVE_SPACE_LEN];
+ uint16_t sid;
+ bool used;
+ bool decomp_no_tail;
+ void *op;
+};
+
+struct task_queue_info {
+ uint16_t qp_id;
+ uint16_t nb_des;
+ int socket_id;
+ enum zsda_service_type type;
+ const char *service_str;
+
+ rx_callback rx_cb;
+ tx_callback tx_cb;
+ srv_match match;
+};
+
extern struct zsda_num_qps zsda_nb_qps;
int zsda_queue_start(const struct rte_pci_device *pci_dev);
@@ -95,4 +173,10 @@ int zsda_queue_stop(const struct rte_pci_device *pci_dev);
int zsda_queue_init(struct zsda_pci_device *zsda_pci_dev);
+struct zsda_qp_hw *
+zsda_qps_hw_per_service(struct zsda_pci_device *zsda_pci_dev,
+ const enum zsda_service_type service);
+
+int zsda_task_queue_setup(struct zsda_pci_device *zsda_pci_dev,
+ struct zsda_qp *qp, struct task_queue_info *task_q_info);
#endif /* _ZSDA_QP_H_ */
diff --git a/drivers/common/zsda/zsda_qp_common.h b/drivers/common/zsda/zsda_qp_common.h
index f589df986e..ba77b0f027 100644
--- a/drivers/common/zsda/zsda_qp_common.h
+++ b/drivers/common/zsda/zsda_qp_common.h
@@ -35,6 +35,15 @@ enum zsda_service_type {
#define ZSDA_CSR_READ8(addr) rte_read8((addr))
#define ZSDA_CSR_WRITE8(addr, value) rte_write8_relaxed((value), (addr))
+#define NB_DES 512
+#define ZSDA_SGL_MAX_NUMBER 512
+#define COMP_REMOVE_SPACE_LEN 16
+
+#define ZSDA_MAX_DESC 512
+#define ZSDA_MAX_CYCLE 256
+#define ZSDA_MAX_DEV RTE_PMD_ZSDA_MAX_PCI_DEVICES
+#define MAX_NUM_OPS 0x1FF
+
struct zsda_admin_req {
uint16_t msg_type;
uint8_t data[26];
@@ -105,10 +114,30 @@ struct zsda_qp_stat {
uint64_t dequeue_err_count;
};
+struct zsda_cqe {
+ uint8_t valid; /* cqe_cycle */
+ uint8_t op_code;
+ uint16_t sid;
+ uint8_t state;
+ uint8_t result;
+ uint16_t zsda_wq_id;
+ uint32_t tx_real_length;
+ uint16_t err0;
+ uint16_t err1;
+} __rte_packed;
+
+typedef int (*rx_callback)(void *cookie_in, struct zsda_cqe *cqe);
+typedef int (*tx_callback)(void *op_in, const struct zsda_queue *queue,
+ void **op_cookies, const uint16_t new_tail);
+typedef int (*srv_match)(const void *op_in);
+
struct qp_srv {
bool used;
struct zsda_queue tx_q;
struct zsda_queue rx_q;
+ rx_callback rx_cb;
+ tx_callback tx_cb;
+ srv_match match;
struct zsda_qp_stat stats;
struct rte_mempool *op_cookie_pool;
void **op_cookies;
@@ -119,6 +148,17 @@ struct zsda_qp {
struct qp_srv srv[ZSDA_MAX_SERVICES];
};
+struct zsda_buf {
+ uint64_t addr;
+ uint32_t len;
+ uint8_t resrvd[3];
+ uint8_t type;
+} __rte_packed;
+
+struct __rte_cache_aligned zsda_sgl {
+ struct zsda_buf buffers[ZSDA_SGL_MAX_NUMBER];
+};
+
void zsda_queue_delete(const struct zsda_queue *queue);
int zsda_queue_pair_release(struct zsda_qp **qp_addr);
void zsda_stats_get(void **queue_pairs, const uint32_t nb_queue_pairs,
diff --git a/drivers/compress/zsda/zsda_comp_pmd.c b/drivers/compress/zsda/zsda_comp_pmd.c
index a8d2da0477..19a4932525 100644
--- a/drivers/compress/zsda/zsda_comp_pmd.c
+++ b/drivers/compress/zsda/zsda_comp_pmd.c
@@ -6,6 +6,7 @@
#include "zsda_logs.h"
#include "zsda_qp_common.h"
+#include "zsda_qp.h"
#include "zsda_comp_pmd.h"
static int
@@ -88,11 +89,21 @@ zsda_comp_dev_stop(struct rte_compressdev *dev)
zsda_queue_stop(comp_dev->zsda_pci_dev->pci_dev);
}
+static int
+zsda_comp_qp_release(struct rte_compressdev *dev, uint16_t queue_pair_id)
+{
+ return zsda_queue_pair_release(
+ (struct zsda_qp **)&(dev->data->queue_pairs[queue_pair_id]));
+}
+
static int
zsda_comp_dev_close(struct rte_compressdev *dev)
{
struct zsda_comp_dev_private *comp_dev = dev->data->dev_private;
+ for (int i = 0; i < dev->data->nb_queue_pairs; i++)
+ zsda_comp_qp_release(dev, i);
+
rte_mempool_free(comp_dev->xformpool);
comp_dev->xformpool = NULL;
@@ -199,6 +210,57 @@ zsda_comp_private_xform_free(struct rte_compressdev *dev __rte_unused,
return -EINVAL;
}
+static int
+zsda_comp_qp_setup(struct rte_compressdev *dev, uint16_t qp_id,
+ uint32_t max_inflight_ops, int socket_id)
+{
+ int ret = ZSDA_SUCCESS;
+ struct zsda_qp *qp_new;
+
+ struct zsda_qp **qp_addr =
+ (struct zsda_qp **)&(dev->data->queue_pairs[qp_id]);
+ struct zsda_comp_dev_private *comp_priv = dev->data->dev_private;
+ struct zsda_pci_device *zsda_pci_dev = comp_priv->zsda_pci_dev;
+ uint16_t nb_des = max_inflight_ops & 0xffff;
+ struct task_queue_info task_q_info;
+
+ nb_des = (nb_des == NB_DES) ? nb_des : NB_DES;
+
+ if (*qp_addr != NULL) {
+ ret = zsda_comp_qp_release(dev, qp_id);
+ if (ret)
+ return ret;
+ }
+
+ qp_new = rte_zmalloc_socket("zsda PMD qp metadata", sizeof(*qp_new),
+ RTE_CACHE_LINE_SIZE, socket_id);
+ if (qp_new == NULL) {
+ ZSDA_LOG(ERR, "Failed! qp_new is NULL");
+ return -ENOMEM;
+ }
+
+ task_q_info.nb_des = nb_des;
+ task_q_info.socket_id = socket_id;
+ task_q_info.qp_id = qp_id;
+
+ task_q_info.type = ZSDA_SERVICE_COMPRESSION;
+ task_q_info.service_str = "comp";
+ ret = zsda_task_queue_setup(zsda_pci_dev, qp_new, &task_q_info);
+
+ task_q_info.type = ZSDA_SERVICE_DECOMPRESSION;
+ task_q_info.service_str = "decomp";
+ ret |= zsda_task_queue_setup(zsda_pci_dev, qp_new, &task_q_info);
+
+ if (ret) {
+ rte_free(qp_new);
+ return ret;
+ }
+
+ *qp_addr = qp_new;
+
+ return ret;
+}
+
static struct rte_compressdev_ops compress_zsda_ops = {
.dev_configure = zsda_comp_dev_config,
@@ -209,8 +271,8 @@ static struct rte_compressdev_ops compress_zsda_ops = {
.stats_get = zsda_comp_stats_get,
.stats_reset = zsda_comp_stats_reset,
- .queue_pair_setup = NULL,
- .queue_pair_release = NULL,
+ .queue_pair_setup = zsda_comp_qp_setup,
+ .queue_pair_release = zsda_comp_qp_release,
.private_xform_create = zsda_comp_private_xform_create,
.private_xform_free = zsda_comp_private_xform_free,
--
2.27.0