[dpdk-dev] [PATCH v4 3/6] raw/octeontx2_ep: add device uninitialization

[Mahipal Challa <mchalla@marvell.com>]

Add rawdev close/uninitialize operation for SDP
VF devices.

Signed-off-by: Mahipal Challa <mchalla@marvell.com>
---
 drivers/raw/octeontx2_ep/otx2_ep_enqdeq.c | 111 ++++++++++++++++++++++++++++++
 drivers/raw/octeontx2_ep/otx2_ep_rawdev.c |  78 +++++++++++++++++++++
 drivers/raw/octeontx2_ep/otx2_ep_rawdev.h |   8 +++
 drivers/raw/octeontx2_ep/otx2_ep_vf.c     |  44 ++++++++++++
 4 files changed, 241 insertions(+)

diff --git a/drivers/raw/octeontx2_ep/otx2_ep_enqdeq.c b/drivers/raw/octeontx2_ep/otx2_ep_enqdeq.c
index 8857004..584b818 100644
--- a/drivers/raw/octeontx2_ep/otx2_ep_enqdeq.c
+++ b/drivers/raw/octeontx2_ep/otx2_ep_enqdeq.c
@@ -21,6 +21,59 @@
 #include "otx2_common.h"
 #include "otx2_ep_enqdeq.h"
 
+static void
+sdp_dmazone_free(const struct rte_memzone *mz)
+{
+	const struct rte_memzone *mz_tmp;
+	int ret = 0;
+
+	if (mz == NULL) {
+		otx2_err("Memzone %s : NULL", mz->name);
+		return;
+	}
+
+	mz_tmp = rte_memzone_lookup(mz->name);
+	if (mz_tmp == NULL) {
+		otx2_err("Memzone %s Not Found", mz->name);
+		return;
+	}
+
+	ret = rte_memzone_free(mz);
+	if (ret)
+		otx2_err("Memzone free failed : ret = %d", ret);
+
+}
+
+/* Free IQ resources */
+int
+sdp_delete_iqs(struct sdp_device *sdpvf, uint32_t iq_no)
+{
+	struct sdp_instr_queue *iq;
+
+	iq = sdpvf->instr_queue[iq_no];
+	if (iq == NULL) {
+		otx2_err("Invalid IQ[%d]\n", iq_no);
+		return -ENOMEM;
+	}
+
+	rte_free(iq->req_list);
+	iq->req_list = NULL;
+
+	if (iq->iq_mz) {
+		sdp_dmazone_free(iq->iq_mz);
+		iq->iq_mz = NULL;
+	}
+
+	rte_free(sdpvf->instr_queue[iq_no]);
+	sdpvf->instr_queue[iq_no] = NULL;
+
+	sdpvf->num_iqs--;
+
+	otx2_info("IQ[%d] is deleted", iq_no);
+
+	return 0;
+}
+
 /* IQ initialization */
 static int
 sdp_init_instr_queue(struct sdp_device *sdpvf, int iq_no)
@@ -126,6 +179,7 @@
 	return 0;
 
 delete_IQ:
+	sdp_delete_iqs(sdpvf, iq_no);
 	return -ENOMEM;
 }
 
@@ -139,6 +193,61 @@
 	rte_atomic64_set(&droq->pkts_pending, 0);
 }
 
+static void
+sdp_droq_destroy_ring_buffers(struct sdp_device *sdpvf,
+				struct sdp_droq *droq)
+{
+	uint32_t idx;
+
+	for (idx = 0; idx < droq->nb_desc; idx++) {
+		if (droq->recv_buf_list[idx].buffer) {
+			rte_mempool_put(sdpvf->enqdeq_mpool,
+				droq->recv_buf_list[idx].buffer);
+
+			droq->recv_buf_list[idx].buffer = NULL;
+		}
+	}
+
+	sdp_droq_reset_indices(droq);
+}
+
+/* Free OQs resources */
+int
+sdp_delete_oqs(struct sdp_device *sdpvf, uint32_t oq_no)
+{
+	struct sdp_droq *droq;
+
+	droq = sdpvf->droq[oq_no];
+	if (droq == NULL) {
+		otx2_err("Invalid droq[%d]", oq_no);
+		return -ENOMEM;
+	}
+
+	sdp_droq_destroy_ring_buffers(sdpvf, droq);
+	rte_free(droq->recv_buf_list);
+	droq->recv_buf_list = NULL;
+
+	if (droq->info_mz) {
+		sdp_dmazone_free(droq->info_mz);
+		droq->info_mz = NULL;
+	}
+
+	if (droq->desc_ring_mz) {
+		sdp_dmazone_free(droq->desc_ring_mz);
+		droq->desc_ring_mz = NULL;
+	}
+
+	memset(droq, 0, SDP_DROQ_SIZE);
+
+	rte_free(sdpvf->droq[oq_no]);
+	sdpvf->droq[oq_no] = NULL;
+
+	sdpvf->num_oqs--;
+
+	otx2_info("OQ[%d] is deleted", oq_no);
+	return 0;
+}
+
 static int
 sdp_droq_setup_ring_buffers(struct sdp_device *sdpvf,
 		struct sdp_droq *droq)
@@ -290,5 +399,7 @@
 	return 0;
 
 delete_OQ:
+	sdp_delete_oqs(sdpvf, oq_no);
 	return -ENOMEM;
 }
+
diff --git a/drivers/raw/octeontx2_ep/otx2_ep_rawdev.c b/drivers/raw/octeontx2_ep/otx2_ep_rawdev.c
index 0c56609..3db5a74 100644
--- a/drivers/raw/octeontx2_ep/otx2_ep_rawdev.c
+++ b/drivers/raw/octeontx2_ep/otx2_ep_rawdev.c
@@ -63,6 +63,45 @@
 }
 
 static int
+sdp_vfdev_exit(struct rte_rawdev *rawdev)
+{
+	struct sdp_device *sdpvf;
+	uint32_t rawdev_queues, q;
+
+	otx2_info("%s:", __func__);
+
+	sdpvf = (struct sdp_device *)rawdev->dev_private;
+
+	sdpvf->fn_list.disable_io_queues(sdpvf);
+
+	rawdev_queues = sdpvf->num_oqs;
+	for (q = 0; q < rawdev_queues; q++) {
+		if (sdp_delete_oqs(sdpvf, q)) {
+			otx2_err("Failed to delete OQ:%d", q);
+			return -ENOMEM;
+		}
+	}
+	otx2_info("Num OQs:%d freed", sdpvf->num_oqs);
+
+	/* Free the oqbuf_pool */
+	rte_mempool_free(sdpvf->enqdeq_mpool);
+	sdpvf->enqdeq_mpool = NULL;
+
+	otx2_info("Enqdeq_mpool free done");
+
+	rawdev_queues = sdpvf->num_iqs;
+	for (q = 0; q < rawdev_queues; q++) {
+		if (sdp_delete_iqs(sdpvf, q)) {
+			otx2_err("Failed to delete IQ:%d", q);
+			return -ENOMEM;
+		}
+	}
+	otx2_sdp_dbg("Num IQs:%d freed", sdpvf->num_iqs);
+
+	return 0;
+}
+
+static int
 sdp_chip_specific_setup(struct sdp_device *sdpvf)
 {
 	struct rte_pci_device *pdev = sdpvf->pci_dev;
@@ -142,13 +181,49 @@
 
 	return 0;
 
+/* Error handling  */
 oq_fail:
+	/* Free the allocated OQs */
+	for (q = 0; q < sdpvf->num_oqs; q++)
+		sdp_delete_oqs(sdpvf, q);
+
 iq_fail:
+	/* Free the allocated IQs */
+	for (q = 0; q < sdpvf->num_iqs; q++)
+		sdp_delete_iqs(sdpvf, q);
+
 setup_fail:
 	return -ENOMEM;
 }
 
 static int
+sdp_rawdev_start(struct rte_rawdev *dev)
+{
+	dev->started = 1;
+
+	return 0;
+}
+
+static void
+sdp_rawdev_stop(struct rte_rawdev *dev)
+{
+	dev->started = 0;
+}
+
+static int
+sdp_rawdev_close(struct rte_rawdev *dev)
+{
+	int ret;
+	ret = sdp_vfdev_exit(dev);
+	if (ret) {
+		otx2_err(" SDP_EP rawdev exit error");
+		return ret;
+	}
+
+	return 0;
+}
+
+static int
 sdp_rawdev_configure(const struct rte_rawdev *dev, rte_rawdev_obj_t config)
 {
 	struct sdp_rawdev_info *app_info = (struct sdp_rawdev_info *)config;
@@ -173,6 +248,9 @@
 /* SDP VF endpoint rawdev ops */
 static const struct rte_rawdev_ops sdp_rawdev_ops = {
 	.dev_configure  = sdp_rawdev_configure,
+	.dev_start      = sdp_rawdev_start,
+	.dev_stop       = sdp_rawdev_stop,
+	.dev_close      = sdp_rawdev_close,
 };
 
 static int
diff --git a/drivers/raw/octeontx2_ep/otx2_ep_rawdev.h b/drivers/raw/octeontx2_ep/otx2_ep_rawdev.h
index bb36b6a..a01f48d 100644
--- a/drivers/raw/octeontx2_ep/otx2_ep_rawdev.h
+++ b/drivers/raw/octeontx2_ep/otx2_ep_rawdev.h
@@ -378,10 +378,16 @@ struct sdp_config {
 struct sdp_fn_list {
 	void (*setup_iq_regs)(struct sdp_device *sdpvf, uint32_t q_no);
 	void (*setup_oq_regs)(struct sdp_device *sdpvf, uint32_t q_no);
+
 	int (*setup_device_regs)(struct sdp_device *sdpvf);
 	void (*enable_io_queues)(struct sdp_device *sdpvf);
+	void (*disable_io_queues)(struct sdp_device *sdpvf);
+
 	void (*enable_iq)(struct sdp_device *sdpvf, uint32_t q_no);
+	void (*disable_iq)(struct sdp_device *sdpvf, uint32_t q_no);
+
 	void (*enable_oq)(struct sdp_device *sdpvf, uint32_t q_no);
+	void (*disable_oq)(struct sdp_device *sdpvf, uint32_t q_no);
 };
 
 /* SRIOV information */
@@ -447,7 +453,9 @@ struct sdp_device {
 
 const struct sdp_config *sdp_get_defconf(struct sdp_device *sdp_dev);
 int sdp_setup_iqs(struct sdp_device *sdpvf, uint32_t iq_no);
+int sdp_delete_iqs(struct sdp_device *sdpvf, uint32_t iq_no);
 
 int sdp_setup_oqs(struct sdp_device *sdpvf, uint32_t oq_no);
+int sdp_delete_oqs(struct sdp_device *sdpvf, uint32_t oq_no);
 
 #endif /* _OTX2_EP_RAWDEV_H_ */
diff --git a/drivers/raw/octeontx2_ep/otx2_ep_vf.c b/drivers/raw/octeontx2_ep/otx2_ep_vf.c
index b6120eb..8e79fe8 100644
--- a/drivers/raw/octeontx2_ep/otx2_ep_vf.c
+++ b/drivers/raw/octeontx2_ep/otx2_ep_vf.c
@@ -371,6 +371,44 @@
 		sdp_vf_enable_oq(sdpvf, q_no);
 }
 
+static void
+sdp_vf_disable_iq(struct sdp_device *sdpvf, uint32_t q_no)
+{
+	volatile uint64_t reg_val = 0ull;
+
+	/* Reset the doorbell register for this Input Queue. */
+	reg_val = otx2_read64(sdpvf->hw_addr + SDP_VF_R_IN_ENABLE(q_no));
+	reg_val &= ~0x1ull;
+
+	otx2_write64(reg_val, sdpvf->hw_addr + SDP_VF_R_IN_ENABLE(q_no));
+}
+
+static void
+sdp_vf_disable_oq(struct sdp_device *sdpvf, uint32_t q_no)
+{
+	volatile uint64_t reg_val = 0ull;
+
+	reg_val = otx2_read64(sdpvf->hw_addr + SDP_VF_R_OUT_ENABLE(q_no));
+	reg_val &= ~0x1ull;
+
+	otx2_write64(reg_val, sdpvf->hw_addr + SDP_VF_R_OUT_ENABLE(q_no));
+
+}
+
+static void
+sdp_vf_disable_io_queues(struct sdp_device *sdpvf)
+{
+	uint32_t q_no = 0;
+
+	/* Disable Input Queues. */
+	for (q_no = 0; q_no < sdpvf->num_iqs; q_no++)
+		sdp_vf_disable_iq(sdpvf, q_no);
+
+	/* Disable Output Queues. */
+	for (q_no = 0; q_no < sdpvf->num_oqs; q_no++)
+		sdp_vf_disable_oq(sdpvf, q_no);
+}
+
 int
 sdp_vf_setup_device(struct sdp_device *sdpvf)
 {
@@ -396,10 +434,16 @@
 
 	sdpvf->fn_list.setup_iq_regs       = sdp_vf_setup_iq_regs;
 	sdpvf->fn_list.setup_oq_regs       = sdp_vf_setup_oq_regs;
+
 	sdpvf->fn_list.setup_device_regs   = sdp_vf_setup_device_regs;
 	sdpvf->fn_list.enable_io_queues    = sdp_vf_enable_io_queues;
+	sdpvf->fn_list.disable_io_queues   = sdp_vf_disable_io_queues;
+
 	sdpvf->fn_list.enable_iq           = sdp_vf_enable_iq;
+	sdpvf->fn_list.disable_iq          = sdp_vf_disable_iq;
+
 	sdpvf->fn_list.enable_oq           = sdp_vf_enable_oq;
+	sdpvf->fn_list.disable_oq          = sdp_vf_disable_oq;
 
 
 	return 0;
-- 
1.8.3.1