[dpdk-dev] [PATCH 2/7] net/octeontx: add application domain validation

[<pbhagavatula@marvell.com>]

From: Pavan Nikhilesh <pbhagavatula@marvell.com>

Add domain validation for PKI and PKO vfs

Signed-off-by: Pavan Nikhilesh <pbhagavatula@marvell.com>
---
 drivers/common/octeontx/octeontx_mbox.c       | 15 ++++-
 drivers/common/octeontx/octeontx_mbox.h       |  6 +-
 .../octeontx/rte_common_octeontx_version.map  |  1 +
 drivers/event/octeontx/ssovf_probe.c          |  5 +-
 drivers/net/octeontx/base/octeontx_pkivf.c    | 66 +++++++++++++++++--
 drivers/net/octeontx/base/octeontx_pkivf.h    |  8 +--
 drivers/net/octeontx/base/octeontx_pkovf.c    | 37 +++++++++--
 drivers/net/octeontx/base/octeontx_pkovf.h    |  3 +
 drivers/net/octeontx/octeontx_ethdev.c        | 13 +++-
 drivers/net/octeontx/octeontx_ethdev.h        |  1 +
 10 files changed, 135 insertions(+), 20 deletions(-)

diff --git a/drivers/common/octeontx/octeontx_mbox.c b/drivers/common/octeontx/octeontx_mbox.c
index 68cb0351f..2fd253107 100644
--- a/drivers/common/octeontx/octeontx_mbox.c
+++ b/drivers/common/octeontx/octeontx_mbox.c
@@ -35,6 +35,7 @@ struct mbox {
 	uint8_t *ram_mbox_base; /* Base address of mbox message stored in ram */
 	uint8_t *reg; /* Store to this register triggers PF mbox interrupt */
 	uint16_t tag_own; /* Last tag which was written to own channel */
+	uint16_t domain; /* Domain */
 	rte_spinlock_t lock;
 };
 
@@ -198,7 +199,7 @@ mbox_send(struct mbox *m, struct octeontx_mbox_hdr *hdr, const void *txmsg,
 }
 
 int
-octeontx_mbox_set_ram_mbox_base(uint8_t *ram_mbox_base)
+octeontx_mbox_set_ram_mbox_base(uint8_t *ram_mbox_base, uint16_t domain)
 {
 	struct mbox *m = &octeontx_mbox;
 
@@ -215,13 +216,14 @@ octeontx_mbox_set_ram_mbox_base(uint8_t *ram_mbox_base)
 	if (m->reg != NULL) {
 		rte_spinlock_init(&m->lock);
 		m->init_once = 1;
+		m->domain = domain;
 	}
 
 	return 0;
 }
 
 int
-octeontx_mbox_set_reg(uint8_t *reg)
+octeontx_mbox_set_reg(uint8_t *reg, uint16_t domain)
 {
 	struct mbox *m = &octeontx_mbox;
 
@@ -238,6 +240,7 @@ octeontx_mbox_set_reg(uint8_t *reg)
 	if (m->ram_mbox_base != NULL) {
 		rte_spinlock_init(&m->lock);
 		m->init_once = 1;
+		m->domain = domain;
 	}
 
 	return 0;
@@ -344,3 +347,11 @@ octeontx_mbox_init(void)
 
 	return 0;
 }
+
+uint16_t
+octeontx_get_global_domain(void)
+{
+	struct mbox *m = &octeontx_mbox;
+
+	return m->domain;
+}
diff --git a/drivers/common/octeontx/octeontx_mbox.h b/drivers/common/octeontx/octeontx_mbox.h
index 1f794c7f7..e56719cb8 100644
--- a/drivers/common/octeontx/octeontx_mbox.h
+++ b/drivers/common/octeontx/octeontx_mbox.h
@@ -36,8 +36,10 @@ struct octeontx_mbox_hdr {
 };
 
 int octeontx_mbox_init(void);
-int octeontx_mbox_set_ram_mbox_base(uint8_t *ram_mbox_base);
-int octeontx_mbox_set_reg(uint8_t *reg);
+void octeontx_set_global_domain(uint16_t global_domain);
+uint16_t octeontx_get_global_domain(void);
+int octeontx_mbox_set_ram_mbox_base(uint8_t *ram_mbox_base, uint16_t domain);
+int octeontx_mbox_set_reg(uint8_t *reg, uint16_t domain);
 int octeontx_mbox_send(struct octeontx_mbox_hdr *hdr,
 		void *txdata, uint16_t txlen, void *rxdata, uint16_t rxlen);
 
diff --git a/drivers/common/octeontx/rte_common_octeontx_version.map b/drivers/common/octeontx/rte_common_octeontx_version.map
index fdc036a62..2bba7cc93 100644
--- a/drivers/common/octeontx/rte_common_octeontx_version.map
+++ b/drivers/common/octeontx/rte_common_octeontx_version.map
@@ -10,4 +10,5 @@ DPDK_19.08 {
 	global:
 
 	octeontx_mbox_init;
+	octeontx_get_global_domain;
 };
diff --git a/drivers/event/octeontx/ssovf_probe.c b/drivers/event/octeontx/ssovf_probe.c
index 9252998c1..4da7d1ae4 100644
--- a/drivers/event/octeontx/ssovf_probe.c
+++ b/drivers/event/octeontx/ssovf_probe.c
@@ -181,7 +181,8 @@ ssowvf_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
 	sdev.total_ssowvfs++;
 	if (vfid == 0) {
 		ram_mbox_base = ssovf_bar(OCTEONTX_SSO_HWS, 0, 4);
-		if (octeontx_mbox_set_ram_mbox_base(ram_mbox_base)) {
+		if (octeontx_mbox_set_ram_mbox_base(ram_mbox_base,
+						    res->domain)) {
 			mbox_log_err("Invalid Failed to set ram mbox base");
 			return -EINVAL;
 		}
@@ -257,7 +258,7 @@ ssovf_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
 	if (vfid == 0) {
 		reg = ssovf_bar(OCTEONTX_SSO_GROUP, 0, 0);
 		reg += SSO_VHGRP_PF_MBOX(1);
-		if (octeontx_mbox_set_reg(reg)) {
+		if (octeontx_mbox_set_reg(reg, res->domain)) {
 			mbox_log_err("Invalid Failed to set mbox_reg");
 			return -EINVAL;
 		}
diff --git a/drivers/net/octeontx/base/octeontx_pkivf.c b/drivers/net/octeontx/base/octeontx_pkivf.c
index 1babea0e8..783b2a2e5 100644
--- a/drivers/net/octeontx/base/octeontx_pkivf.c
+++ b/drivers/net/octeontx/base/octeontx_pkivf.c
@@ -7,19 +7,50 @@
 #include <rte_eal.h>
 #include <rte_bus_pci.h>
 
+#include "../octeontx_logs.h"
+#include "octeontx_io.h"
 #include "octeontx_pkivf.h"
 
+
+struct octeontx_pkivf {
+	uint8_t		*bar0;
+	uint8_t		status;
+	uint16_t	domain;
+	uint16_t	vfid;
+};
+
+struct octeontx_pki_vf_ctl_s {
+	struct octeontx_pkivf pki[PKI_VF_MAX];
+};
+
+static struct octeontx_pki_vf_ctl_s pki_vf_ctl;
+
 int
 octeontx_pki_port_open(int port)
 {
+	uint16_t global_domain = octeontx_get_global_domain();
 	struct octeontx_mbox_hdr hdr;
-	int res;
+	mbox_pki_port_t port_type = {
+		.port_type = OCTTX_PORT_TYPE_NET,
+	};
+	int i, res;
+
+	/* Check if atleast one PKI vf is in application domain. */
+	for (i = 0; i < PKI_VF_MAX; i++) {
+		if (pki_vf_ctl.pki[i].domain != global_domain)
+			continue;
+		break;
+	}
+
+	if (i == PKI_VF_MAX)
+		return -ENODEV;
 
 	hdr.coproc = OCTEONTX_PKI_COPROC;
 	hdr.msg = MBOX_PKI_PORT_OPEN;
 	hdr.vfid = port;
 
-	res = octeontx_mbox_send(&hdr, NULL, 0, NULL, 0);
+	res = octeontx_mbox_send(&hdr, &port_type, sizeof(mbox_pki_port_t),
+				 NULL, 0);
 	if (res < 0)
 		return -EACCES;
 	return res;
@@ -113,13 +144,40 @@ octeontx_pki_port_errchk_config(int port, pki_errchk_cfg_t *cfg)
 static int
 pkivf_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
 {
-	RTE_SET_USED(pci_drv);
-	RTE_SET_USED(pci_dev);
+	struct octeontx_pkivf *res;
+	static uint8_t vf_cnt;
+	uint16_t domain;
+	uint16_t vfid;
+	uint8_t *bar0;
+	uint64_t val;
 
+	RTE_SET_USED(pci_drv);
 	/* For secondary processes, the primary has done all the work */
 	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 		return 0;
 
+	if (pci_dev->mem_resource[0].addr == NULL) {
+		octeontx_log_err("PKI Empty bar[0] %p",
+				 pci_dev->mem_resource[0].addr);
+		return -ENODEV;
+	}
+
+	bar0 = pci_dev->mem_resource[0].addr;
+	val = octeontx_read64(bar0);
+	domain = val & 0xffff;
+	vfid = (val >> 16) & 0xffff;
+
+	if (unlikely(vfid >= PKI_VF_MAX)) {
+		octeontx_log_err("pki: Invalid vfid %d", vfid);
+		return -EINVAL;
+	}
+
+	res = &pki_vf_ctl.pki[vf_cnt++];
+	res->vfid = vfid;
+	res->domain = domain;
+	res->bar0 = bar0;
+
+	octeontx_log_dbg("PKI Domain=%d vfid=%d", res->domain, res->vfid);
 	return 0;
 }
 
diff --git a/drivers/net/octeontx/base/octeontx_pkivf.h b/drivers/net/octeontx/base/octeontx_pkivf.h
index 7f19a4bb8..c2a944404 100644
--- a/drivers/net/octeontx/base/octeontx_pkivf.h
+++ b/drivers/net/octeontx/base/octeontx_pkivf.h
@@ -48,6 +48,10 @@ enum  {
 	MBOX_PKI_PARSE_NOTHING = 0x7f
 };
 
+/* PKI maximum constants */
+#define PKI_VF_MAX			(32)
+#define PKI_MAX_PKTLEN			(32768)
+
 /* Interface types: */
 enum {
 	OCTTX_PORT_TYPE_NET, /* Network interface ports */
@@ -231,10 +235,6 @@ typedef struct mbox_pki_port_delete_qos_entry {
 	uint16_t index;
 } mbox_pki_del_qos_t;
 
-/* PKI maximum constants */
-#define PKI_VF_MAX			(1)
-#define PKI_MAX_PKTLEN			(32768)
-
 /* pki pkind parse mode */
 enum  {
 	PKI_PARSE_LA_TO_LG = 0,
diff --git a/drivers/net/octeontx/base/octeontx_pkovf.c b/drivers/net/octeontx/base/octeontx_pkovf.c
index 0a6d64b8e..dacbdd0b4 100644
--- a/drivers/net/octeontx/base/octeontx_pkovf.c
+++ b/drivers/net/octeontx/base/octeontx_pkovf.c
@@ -24,6 +24,8 @@ struct octeontx_pko_iomem {
 };
 
 #define PKO_IOMEM_NULL (struct octeontx_pko_iomem){0, 0, 0}
+#define PKO_VALID	0x1
+#define PKO_INUSE	0x2
 
 struct octeontx_pko_fc_ctl_s {
 	int64_t buf_cnt;
@@ -33,13 +35,14 @@ struct octeontx_pko_fc_ctl_s {
 struct octeontx_pkovf {
 	uint8_t		*bar0;
 	uint8_t		*bar2;
+	uint8_t		status;
 	uint16_t	domain;
 	uint16_t	vfid;
 };
 
 struct octeontx_pko_vf_ctl_s {
 	rte_spinlock_t lock;
-
+	uint16_t global_domain;
 	struct octeontx_pko_iomem fc_iomem;
 	struct octeontx_pko_fc_ctl_s *fc_ctl;
 	struct octeontx_pkovf pko[PKO_VF_MAX];
@@ -403,7 +406,7 @@ octeontx_pko_channel_query(struct octeontx_pko_vf_ctl_s *ctl, uint64_t chanid,
 	curr.lmtline_va = ctl->pko[dq_vf].bar2;
 	curr.ioreg_va = (void *)((uintptr_t)ctl->pko[dq_vf].bar0
 		+ PKO_VF_DQ_OP_SEND((dq), 0));
-	curr.fc_status_va = ctl->fc_ctl + dq;
+	curr.fc_status_va = ctl->fc_ctl + dq_num;
 
 	octeontx_log_dbg("lmtline=%p ioreg_va=%p fc_status_va=%p",
 			 curr.lmtline_va, curr.ioreg_va,
@@ -431,8 +434,10 @@ octeontx_pko_channel_query_dqs(int chanid, void *out, size_t out_elem_size,
 int
 octeontx_pko_vf_count(void)
 {
+	uint16_t global_domain = octeontx_get_global_domain();
 	int vf_cnt;
 
+	pko_vf_ctl.global_domain = global_domain;
 	vf_cnt = 0;
 	while (pko_vf_ctl.pko[vf_cnt].bar0)
 		vf_cnt++;
@@ -440,6 +445,26 @@ octeontx_pko_vf_count(void)
 	return vf_cnt;
 }
 
+size_t
+octeontx_pko_get_vfid(void)
+{
+	size_t vf_cnt = octeontx_pko_vf_count();
+	size_t vf_idx;
+
+
+	for (vf_idx = 0; vf_idx < vf_cnt; vf_idx++) {
+		if (!(pko_vf_ctl.pko[vf_idx].status & PKO_VALID))
+			continue;
+		if (pko_vf_ctl.pko[vf_idx].status & PKO_INUSE)
+			continue;
+
+		pko_vf_ctl.pko[vf_idx].status |= PKO_INUSE;
+		return pko_vf_ctl.pko[vf_idx].vfid;
+	}
+
+	return SIZE_MAX;
+}
+
 int
 octeontx_pko_init_fc(const size_t pko_vf_count)
 {
@@ -467,8 +492,10 @@ octeontx_pko_init_fc(const size_t pko_vf_count)
 
 	/* Configure Flow-Control feature for all DQs of open VFs */
 	for (vf_idx = 0; vf_idx < pko_vf_count; vf_idx++) {
-		dq_ix = vf_idx * PKO_VF_NUM_DQ;
+		if (pko_vf_ctl.pko[vf_idx].domain != pko_vf_ctl.global_domain)
+			continue;
 
+		dq_ix = pko_vf_ctl.pko[vf_idx].vfid * PKO_VF_NUM_DQ;
 		vf_bar0 = pko_vf_ctl.pko[vf_idx].bar0;
 
 		reg = (pko_vf_ctl.fc_iomem.iova +
@@ -479,6 +506,7 @@ octeontx_pko_init_fc(const size_t pko_vf_count)
 		    (0x1 << 0);		/* ENABLE */
 
 		octeontx_write64(reg, vf_bar0 + PKO_VF_DQ_FC_CONFIG);
+		pko_vf_ctl.pko[vf_idx].status = PKO_VALID;
 
 		octeontx_log_dbg("PKO: bar0 %p VF_idx %d DQ_FC_CFG=%" PRIx64 "",
 				 vf_bar0, (int)vf_idx, reg);
@@ -528,6 +556,7 @@ pkovf_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
 	uint16_t domain;
 	uint8_t *bar0;
 	uint8_t *bar2;
+	static uint8_t vf_cnt;
 	struct octeontx_pkovf *res;
 
 	RTE_SET_USED(pci_drv);
@@ -558,7 +587,7 @@ pkovf_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
 		return -EINVAL;
 	}
 
-	res = &pko_vf_ctl.pko[vfid];
+	res = &pko_vf_ctl.pko[vf_cnt++];
 	res->vfid = vfid;
 	res->domain = domain;
 	res->bar0 = bar0;
diff --git a/drivers/net/octeontx/base/octeontx_pkovf.h b/drivers/net/octeontx/base/octeontx_pkovf.h
index cbd28249f..4208ef880 100644
--- a/drivers/net/octeontx/base/octeontx_pkovf.h
+++ b/drivers/net/octeontx/base/octeontx_pkovf.h
@@ -5,6 +5,8 @@
 #ifndef	__OCTEONTX_PKO_H__
 #define	__OCTEONTX_PKO_H__
 
+#include <octeontx_mbox.h>
+
 /* PKO maximum constants */
 #define	PKO_VF_MAX			(32)
 #define	PKO_VF_NUM_DQ			(8)
@@ -63,6 +65,7 @@ int octeontx_pko_channel_close(int chanid);
 int octeontx_pko_channel_start(int chanid);
 int octeontx_pko_channel_stop(int chanid);
 int octeontx_pko_vf_count(void);
+size_t octeontx_pko_get_vfid(void);
 int octeontx_pko_init_fc(const size_t pko_vf_count);
 void octeontx_pko_fc_free(void);
 
diff --git a/drivers/net/octeontx/octeontx_ethdev.c b/drivers/net/octeontx/octeontx_ethdev.c
index 00686ea26..c2258d136 100644
--- a/drivers/net/octeontx/octeontx_ethdev.c
+++ b/drivers/net/octeontx/octeontx_ethdev.c
@@ -308,7 +308,7 @@ octeontx_dev_configure(struct rte_eth_dev *dev)
 
 	nic->num_tx_queues = dev->data->nb_tx_queues;
 
-	ret = octeontx_pko_channel_open(nic->port_id * PKO_VF_NUM_DQ,
+	ret = octeontx_pko_channel_open(nic->pko_vfid * PKO_VF_NUM_DQ,
 					nic->num_tx_queues,
 					nic->base_ochan);
 	if (ret) {
@@ -719,7 +719,7 @@ octeontx_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx,
 	RTE_SET_USED(nb_desc);
 	RTE_SET_USED(socket_id);
 
-	dq_num = (nic->port_id * PKO_VF_NUM_DQ) + qidx;
+	dq_num = (nic->pko_vfid * PKO_VF_NUM_DQ) + qidx;
 
 	/* Socket id check */
 	if (socket_id != (unsigned int)SOCKET_ID_ANY &&
@@ -1001,6 +1001,7 @@ octeontx_create(struct rte_vdev_device *dev, int port, uint8_t evdev,
 			int socket_id)
 {
 	int res;
+	size_t pko_vfid;
 	char octtx_name[OCTEONTX_MAX_NAME_LEN];
 	struct octeontx_nic *nic = NULL;
 	struct rte_eth_dev *eth_dev = NULL;
@@ -1039,7 +1040,15 @@ octeontx_create(struct rte_vdev_device *dev, int port, uint8_t evdev,
 		goto err;
 	}
 	data->dev_private = nic;
+	pko_vfid = octeontx_pko_get_vfid();
 
+	if (pko_vfid == SIZE_MAX) {
+		octeontx_log_err("failed to get pko vfid");
+		res = -ENODEV;
+		goto err;
+	}
+
+	nic->pko_vfid = pko_vfid;
 	nic->port_id = port;
 	nic->evdev = evdev;
 
diff --git a/drivers/net/octeontx/octeontx_ethdev.h b/drivers/net/octeontx/octeontx_ethdev.h
index fd2e99edf..50fae35d9 100644
--- a/drivers/net/octeontx/octeontx_ethdev.h
+++ b/drivers/net/octeontx/octeontx_ethdev.h
@@ -58,6 +58,7 @@ struct octeontx_nic {
 	uint8_t mcast_mode;
 	uint16_t num_tx_queues;
 	uint64_t hwcap;
+	uint8_t pko_vfid;
 	uint8_t link_up;
 	uint8_t	duplex;
 	uint8_t speed;
-- 
2.24.0