[dpdk-dev] [PATCH v3 11/32] crypto/octeontx: add mailbox routines

[Anoob Joseph <anoob.joseph@caviumnetworks.com>]

From: Murthy NSSR <nidadavolu.murthy@caviumnetworks.com>

Adding mailbox routines to interact with the pf driver

Signed-off-by: Ankur Dwivedi <ankur.dwivedi@caviumnetworks.com>
Signed-off-by: Anoob Joseph <anoob.joseph@caviumnetworks.com>
Signed-off-by: Murthy NSSR <nidadavolu.murthy@caviumnetworks.com>
Signed-off-by: Nithin Dabilpuram <nithin.dabilpuram@caviumnetworks.com>
Signed-off-by: Ragothaman Jayaraman <rjayaraman@caviumnetworks.com>
Signed-off-by: Srisivasubramanian S <ssrinivasan@caviumnetworks.com>
Signed-off-by: Tejasree Kondoj <kondoj.tejasree@caviumnetworks.com>
---
 drivers/crypto/octeontx/Makefile                  |   1 +
 drivers/crypto/octeontx/meson.build               |   1 +
 drivers/crypto/octeontx/otx_cryptodev_hw_access.c |  13 ++
 drivers/crypto/octeontx/otx_cryptodev_mbox.c      | 178 ++++++++++++++++++++++
 drivers/crypto/octeontx/otx_cryptodev_mbox.h      |  92 +++++++++++
 5 files changed, 285 insertions(+)
 create mode 100644 drivers/crypto/octeontx/otx_cryptodev_mbox.c
 create mode 100644 drivers/crypto/octeontx/otx_cryptodev_mbox.h

diff --git a/drivers/crypto/octeontx/Makefile b/drivers/crypto/octeontx/Makefile
index 1808244..2e78e69 100644
--- a/drivers/crypto/octeontx/Makefile
+++ b/drivers/crypto/octeontx/Makefile
@@ -27,6 +27,7 @@ CFLAGS += -I$(RTE_SDK)/drivers/common/cpt
 SRCS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_CRYPTO) += otx_cryptodev.c
 SRCS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_CRYPTO) += otx_cryptodev_capabilities.c
 SRCS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_CRYPTO) += otx_cryptodev_hw_access.c
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_CRYPTO) += otx_cryptodev_mbox.c
 SRCS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_CRYPTO) += otx_cryptodev_ops.c
 
 # export include files
diff --git a/drivers/crypto/octeontx/meson.build b/drivers/crypto/octeontx/meson.build
index 7f65476..6511b40 100644
--- a/drivers/crypto/octeontx/meson.build
+++ b/drivers/crypto/octeontx/meson.build
@@ -11,6 +11,7 @@ name = 'octeontx_crypto'
 sources = files('otx_cryptodev.c',
 		'otx_cryptodev_capabilities.c',
 		'otx_cryptodev_hw_access.c',
+		'otx_cryptodev_mbox.c',
 		'otx_cryptodev_ops.c')
 
 includes += include_directories('../../common/cpt')
diff --git a/drivers/crypto/octeontx/otx_cryptodev_hw_access.c b/drivers/crypto/octeontx/otx_cryptodev_hw_access.c
index e8a2b0b..eb9fbcf 100644
--- a/drivers/crypto/octeontx/otx_cryptodev_hw_access.c
+++ b/drivers/crypto/octeontx/otx_cryptodev_hw_access.c
@@ -7,6 +7,7 @@
 #include <rte_common.h>
 
 #include "otx_cryptodev_hw_access.h"
+#include "otx_cryptodev_mbox.h"
 
 #include "cpt_pmd_logs.h"
 #include "cpt_hw_types.h"
@@ -22,8 +23,19 @@ otx_cpt_vf_init(struct cpt_vf *cptvf)
 {
 	int ret = 0;
 
+	/* Check ready with PF */
+	/* Gets chip ID / device Id from PF if ready */
+	ret = otx_cpt_check_pf_ready(cptvf);
+	if (ret) {
+		CPT_LOG_ERR("%s: PF not responding to READY msg",
+				cptvf->dev_name);
+		ret = -EBUSY;
+		goto exit;
+	}
+
 	CPT_LOG_DP_DEBUG("%s: %s done", cptvf->dev_name, __func__);
 
+exit:
 	return ret;
 }
 
@@ -178,6 +190,7 @@ otx_cpt_poll_misc(struct cpt_vf *cptvf)
 	if (likely(intr & CPT_VF_INTR_MBOX_MASK)) {
 		CPT_LOG_DP_DEBUG("%s: Mailbox interrupt 0x%lx on CPT VF %d",
 			cptvf->dev_name, (unsigned int long)intr, cptvf->vfid);
+		otx_cpt_handle_mbox_intr(cptvf);
 		otx_cpt_clear_mbox_intr(cptvf);
 	} else if (unlikely(intr & CPT_VF_INTR_IRDE_MASK)) {
 		otx_cpt_clear_irde_intr(cptvf);
diff --git a/drivers/crypto/octeontx/otx_cryptodev_mbox.c b/drivers/crypto/octeontx/otx_cryptodev_mbox.c
new file mode 100644
index 0000000..a8e51a8
--- /dev/null
+++ b/drivers/crypto/octeontx/otx_cryptodev_mbox.c
@@ -0,0 +1,178 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium, Inc
+ */
+
+#include <unistd.h>
+
+#include "otx_cryptodev_hw_access.h"
+#include "otx_cryptodev_mbox.h"
+
+void
+otx_cpt_handle_mbox_intr(struct cpt_vf *cptvf)
+{
+	struct cpt_mbox mbx = {0, 0};
+
+	/*
+	 * MBOX[0] contains msg
+	 * MBOX[1] contains data
+	 */
+	mbx.msg  = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf),
+				CPTX_VFX_PF_MBOXX(0, 0, 0));
+	mbx.data = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf),
+				CPTX_VFX_PF_MBOXX(0, 0, 1));
+
+	CPT_LOG_DP_DEBUG("%s: Mailbox msg 0x%lx from PF",
+		    cptvf->dev_name, (unsigned int long)mbx.msg);
+	switch (mbx.msg) {
+	case OTX_CPT_MSG_READY:
+		{
+			otx_cpt_chipid_vfid_t cid;
+
+			cid.u64 = mbx.data;
+			cptvf->pf_acked = true;
+			cptvf->vfid = cid.s.vfid;
+			CPT_LOG_DP_DEBUG("%s: Received VFID %d chip_id %d",
+					 cptvf->dev_name,
+					 cptvf->vfid, cid.s.chip_id);
+		}
+		break;
+	case OTX_CPT_MSG_QBIND_GRP:
+		cptvf->pf_acked = true;
+		cptvf->vftype = mbx.data;
+		CPT_LOG_DP_DEBUG("%s: VF %d type %s group %d",
+				 cptvf->dev_name, cptvf->vfid,
+				 ((mbx.data == SE_TYPE) ? "SE" : "AE"),
+				 cptvf->vfgrp);
+		break;
+	case OTX_CPT_MBOX_MSG_TYPE_ACK:
+		cptvf->pf_acked = true;
+		break;
+	case OTX_CPT_MBOX_MSG_TYPE_NACK:
+		cptvf->pf_nacked = true;
+		break;
+	default:
+		CPT_LOG_DP_DEBUG("%s: Invalid msg from PF, msg 0x%lx",
+				 cptvf->dev_name, (unsigned int long)mbx.msg);
+		break;
+	}
+}
+
+/* Send a mailbox message to PF
+ * @vf: vf from which this message to be sent
+ * @mbx: Message to be sent
+ */
+static void
+otx_cpt_send_msg_to_pf(struct cpt_vf *cptvf, struct cpt_mbox *mbx)
+{
+	/* Writing mbox(1) causes interrupt */
+	CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf),
+		      CPTX_VFX_PF_MBOXX(0, 0, 0), mbx->msg);
+	CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf),
+		      CPTX_VFX_PF_MBOXX(0, 0, 1), mbx->data);
+}
+
+static int32_t
+otx_cpt_send_msg_to_pf_timeout(struct cpt_vf *cptvf, struct cpt_mbox *mbx)
+{
+	int timeout = OTX_CPT_MBOX_MSG_TIMEOUT;
+	int sleep_ms = 10;
+
+	cptvf->pf_acked = false;
+	cptvf->pf_nacked = false;
+
+	otx_cpt_send_msg_to_pf(cptvf, mbx);
+
+	/* Wait for previous message to be acked, timeout 2sec */
+	while (!cptvf->pf_acked) {
+		if (cptvf->pf_nacked)
+			return -EINVAL;
+		usleep(sleep_ms * 1000);
+		otx_cpt_poll_misc(cptvf);
+		if (cptvf->pf_acked)
+			break;
+		timeout -= sleep_ms;
+		if (!timeout) {
+			CPT_LOG_ERR("%s: PF didn't ack mbox msg %lx(vfid %u)",
+				    cptvf->dev_name,
+				    (unsigned int long)(mbx->msg & 0xFF),
+				    cptvf->vfid);
+			return -EBUSY;
+		}
+	}
+	return 0;
+}
+
+int
+otx_cpt_check_pf_ready(struct cpt_vf *cptvf)
+{
+	struct cpt_mbox mbx = {0, 0};
+
+	mbx.msg = OTX_CPT_MSG_READY;
+	if (otx_cpt_send_msg_to_pf_timeout(cptvf, &mbx)) {
+		CPT_LOG_ERR("%s: PF didn't respond to READY msg",
+			    cptvf->dev_name);
+		return 1;
+	}
+	return 0;
+}
+
+int
+otx_cpt_send_vq_size_msg(struct cpt_vf *cptvf)
+{
+	struct cpt_mbox mbx = {0, 0};
+
+	mbx.msg = OTX_CPT_MSG_QLEN;
+
+	mbx.data = cptvf->qsize;
+	if (otx_cpt_send_msg_to_pf_timeout(cptvf, &mbx)) {
+		CPT_LOG_ERR("%s: PF didn't respond to vq_size msg",
+			    cptvf->dev_name);
+		return 1;
+	}
+	return 0;
+}
+
+int
+otx_cpt_send_vf_grp_msg(struct cpt_vf *cptvf, uint32_t group)
+{
+	struct cpt_mbox mbx = {0, 0};
+
+	mbx.msg = OTX_CPT_MSG_QBIND_GRP;
+
+	/* Convey group of the VF */
+	mbx.data = group;
+	if (otx_cpt_send_msg_to_pf_timeout(cptvf, &mbx)) {
+		CPT_LOG_ERR("%s: PF didn't respond to vf_type msg",
+			    cptvf->dev_name);
+		return 1;
+	}
+	return 0;
+}
+
+int
+otx_cpt_send_vf_up(struct cpt_vf *cptvf)
+{
+	struct cpt_mbox mbx = {0, 0};
+
+	mbx.msg = OTX_CPT_MSG_VF_UP;
+	if (otx_cpt_send_msg_to_pf_timeout(cptvf, &mbx)) {
+		CPT_LOG_ERR("%s: PF didn't respond to UP msg",
+			    cptvf->dev_name);
+		return 1;
+	}
+	return 0;
+}
+
+int
+otx_cpt_send_vf_down(struct cpt_vf *cptvf)
+{
+	struct cpt_mbox mbx = {0, 0};
+
+	mbx.msg = OTX_CPT_MSG_VF_DOWN;
+	if (otx_cpt_send_msg_to_pf_timeout(cptvf, &mbx)) {
+		CPT_LOG_ERR("%s: PF didn't respond to DOWN msg",
+			    cptvf->dev_name);
+		return 1;
+	}
+	return 0;
+}
diff --git a/drivers/crypto/octeontx/otx_cryptodev_mbox.h b/drivers/crypto/octeontx/otx_cryptodev_mbox.h
new file mode 100644
index 0000000..b05d1c5
--- /dev/null
+++ b/drivers/crypto/octeontx/otx_cryptodev_mbox.h
@@ -0,0 +1,92 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Cavium, Inc
+ */
+
+#ifndef _OTX_CRYPTODEV_MBOX_H_
+#define _OTX_CRYPTODEV_MBOX_H_
+
+#include <rte_byteorder.h>
+#include <rte_common.h>
+
+#include "cpt_common.h"
+#include "cpt_pmd_logs.h"
+
+#include "otx_cryptodev_hw_access.h"
+
+#define OTX_CPT_MBOX_MSG_TIMEOUT    2000 /* In Milli Seconds */
+
+#define OTX_CPT_MBOX_MSG_TYPE_REQ	0
+#define OTX_CPT_MBOX_MSG_TYPE_ACK	1
+#define OTX_CPT_MBOX_MSG_TYPE_NACK	2
+#define OTX_CPT_MBOX_MSG_TYPE_NOP	3
+
+/* CPT mailbox structure */
+struct cpt_mbox {
+	/** Message type MBOX[0] */
+	uint64_t msg;
+	/** Data         MBOX[1] */
+	uint64_t data;
+};
+
+typedef enum {
+	OTX_CPT_MSG_VF_UP = 1,
+	OTX_CPT_MSG_VF_DOWN,
+	OTX_CPT_MSG_READY,
+	OTX_CPT_MSG_QLEN,
+	OTX_CPT_MSG_QBIND_GRP,
+	OTX_CPT_MSG_VQ_PRIORITY,
+	OTX_CPT_MSG_PF_TYPE,
+} otx_cpt_mbox_opcode_t;
+
+typedef union {
+	uint64_t u64;
+	struct {
+#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
+		uint32_t chip_id;
+		uint8_t vfid;
+		uint8_t reserved[3];
+#else
+		uint8_t reserved[3];
+		uint8_t vfid;
+		uint32_t chip_id;
+#endif
+	} s;
+} otx_cpt_chipid_vfid_t;
+
+/* Poll handler to handle mailbox messages from VFs */
+void
+otx_cpt_handle_mbox_intr(struct cpt_vf *cptvf);
+
+/*
+ * Checks if VF is able to comminicate with PF
+ * and also gets the CPT number this VF is associated to.
+ */
+int
+otx_cpt_check_pf_ready(struct cpt_vf *cptvf);
+
+/*
+ * Communicate VQs size to PF to program CPT(0)_PF_Q(0-15)_CTL of the VF.
+ * Must be ACKed.
+ */
+int
+otx_cpt_send_vq_size_msg(struct cpt_vf *cptvf);
+
+/*
+ * Communicate VF group required to PF and get the VQ binded to that group
+ */
+int
+otx_cpt_send_vf_grp_msg(struct cpt_vf *cptvf, uint32_t group);
+
+/*
+ * Communicate to PF that VF is UP and running
+ */
+int
+otx_cpt_send_vf_up(struct cpt_vf *cptvf);
+
+/*
+ * Communicate to PF that VF is DOWN and running
+ */
+int
+otx_cpt_send_vf_down(struct cpt_vf *cptvf);
+
+#endif /* _OTX_CRYPTODEV_MBOX_H_ */
-- 
2.7.4