[RFC PATCH 21/29] net/qdma: add mailbox communication library

[Aman Kumar <aman.kumar@vvdntech.in>]

this patch implements mailbox communication
mechanism to enable communication between
virtual functions and physical function.

Signed-off-by: Aman Kumar <aman.kumar@vvdntech.in>
---
 drivers/net/qdma/meson.build |   1 +
 drivers/net/qdma/qdma.h      |   2 +
 drivers/net/qdma/qdma_mbox.c | 400 +++++++++++++++++++++++++++++++++++
 drivers/net/qdma/qdma_mbox.h |  47 ++++
 4 files changed, 450 insertions(+)
 create mode 100644 drivers/net/qdma/qdma_mbox.c
 create mode 100644 drivers/net/qdma/qdma_mbox.h

diff --git a/drivers/net/qdma/meson.build b/drivers/net/qdma/meson.build
index 8c86412b83..dd2478be6c 100644
--- a/drivers/net/qdma/meson.build
+++ b/drivers/net/qdma/meson.build
@@ -25,6 +25,7 @@ sources = files(
         'qdma_common.c',
         'qdma_devops.c',
         'qdma_ethdev.c',
+        'qdma_mbox.c',
         'qdma_user.c',
         'qdma_rxtx.c',
         'qdma_access/eqdma_soft_access/eqdma_soft_access.c',
diff --git a/drivers/net/qdma/qdma.h b/drivers/net/qdma/qdma.h
index 8515ebe60e..8fb64c21b0 100644
--- a/drivers/net/qdma/qdma.h
+++ b/drivers/net/qdma/qdma.h
@@ -19,6 +19,7 @@
 #include "qdma_user.h"
 #include "qdma_resource_mgmt.h"
 #include "qdma_access_common.h"
+#include "qdma_mbox.h"
 #include "rte_pmd_qdma.h"
 #include "qdma_log.h"
 
@@ -278,6 +279,7 @@ struct qdma_pci_dev {
 	uint32_t ip_type:4;
 
 	struct queue_info *q_info;
+	struct qdma_dev_mbox mbox;
 	uint8_t init_q_range;
 
 	uint32_t g_ring_sz[QDMA_NUM_RING_SIZES];
diff --git a/drivers/net/qdma/qdma_mbox.c b/drivers/net/qdma/qdma_mbox.c
new file mode 100644
index 0000000000..eec2ecaf6a
--- /dev/null
+++ b/drivers/net/qdma/qdma_mbox.c
@@ -0,0 +1,400 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017-2022 Xilinx, Inc. All rights reserved.
+ */
+
+#include <ethdev_pci.h>
+#include <rte_malloc.h>
+#include <rte_spinlock.h>
+#include <rte_alarm.h>
+#include <rte_eal.h>
+
+#include "qdma.h"
+#include "qdma_mbox.h"
+
+/*
+ * Get index from VF info array of PF device for a given VF function id.
+ */
+static int qdma_get_internal_vf_index(struct rte_eth_dev *dev, uint8_t devfn)
+{
+	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+	struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+	uint16_t  i;
+
+	for (i = 0; i < pci_dev->max_vfs; i++) {
+		if (qdma_dev->vfinfo[i].func_id == devfn)
+			return i;
+	}
+
+	return QDMA_FUNC_ID_INVALID;
+}
+
+static void qdma_mbox_process_msg_from_vf(void *arg)
+{
+	struct qdma_mbox_msg *mbox_msg_rsp = qdma_mbox_msg_alloc();
+	struct rte_eth_dev *dev = (struct rte_eth_dev *)arg;
+	struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+	uint16_t vf_func_id;
+	uint16_t vf_index;
+	int i, rv;
+
+	if (mbox_msg_rsp == NULL)
+		return;
+
+	if (!qdma_dev)
+		return;
+
+	rv = qdma_mbox_pf_rcv_msg_handler(dev,
+					  qdma_dev->dma_device_index,
+					  qdma_dev->func_id,
+					  qdma_dev->mbox.rx_data,
+					  mbox_msg_rsp->raw_data);
+	if (rv != QDMA_MBOX_VF_OFFLINE &&
+			rv != QDMA_MBOX_VF_RESET &&
+			rv != QDMA_MBOX_PF_RESET_DONE &&
+			rv != QDMA_MBOX_PF_BYE)
+		qdma_mbox_msg_send(dev, mbox_msg_rsp, 0);
+	else
+		qdma_mbox_msg_free(mbox_msg_rsp);
+
+	if (rv == QDMA_MBOX_VF_ONLINE) {
+		vf_func_id = qdma_mbox_vf_func_id_get(qdma_dev->mbox.rx_data,
+			qdma_dev->is_vf);
+		/* Mapping internal VF function id to a valid VF function id */
+		for (i = 0; i < pci_dev->max_vfs; i++) {
+			if (qdma_dev->vfinfo[i].func_id ==
+					QDMA_FUNC_ID_INVALID) {
+				qdma_dev->vfinfo[i].func_id =
+					vf_func_id;
+				break;
+			}
+		}
+
+		if (i == pci_dev->max_vfs) {
+			PMD_DRV_LOG(INFO,
+			"PF-%d failed to create function id mapping VF func_id%d",
+				    qdma_dev->func_id, vf_func_id);
+			return;
+		}
+
+		qdma_dev->vf_online_count++;
+	} else if (rv == QDMA_MBOX_VF_OFFLINE) {
+		if (!qdma_dev->reset_in_progress) {
+			vf_func_id =
+				qdma_mbox_vf_func_id_get(qdma_dev->mbox.rx_data,
+					qdma_dev->is_vf);
+			vf_index = qdma_get_internal_vf_index(dev, vf_func_id);
+			if (vf_index != QDMA_FUNC_ID_INVALID)
+				qdma_dev->vfinfo[vf_index].func_id =
+					QDMA_FUNC_ID_INVALID;
+		}
+		qdma_dev->vf_online_count--;
+	}
+}
+
+static void *qdma_reset_task(void *arg)
+{
+	struct rte_eth_dev *dev = (struct rte_eth_dev *)arg;
+	struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+
+	if (!qdma_dev)
+		return NULL;
+
+	rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET,
+					      NULL);
+
+	return NULL;
+}
+
+static void *qdma_remove_task(void *arg)
+{
+	struct rte_eth_dev *dev = (struct rte_eth_dev *)arg;
+	struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+
+	if (!qdma_dev)
+		return NULL;
+
+	rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RMV,
+					      NULL);
+
+	return NULL;
+}
+
+static void qdma_mbox_process_msg_from_pf(void *arg)
+{
+	struct qdma_mbox_msg *mbox_msg_rsp = NULL;
+	struct rte_eth_dev *dev = (struct rte_eth_dev *)arg;
+	struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+	pthread_t thread;
+	pthread_attr_t tattr;
+	int rv;
+
+	if (!qdma_dev)
+		return;
+
+	mbox_msg_rsp = qdma_mbox_msg_alloc();
+	if (!mbox_msg_rsp)
+		return;
+
+	rv = qdma_mbox_vf_rcv_msg_handler(qdma_dev->mbox.rx_data,
+					  mbox_msg_rsp->raw_data);
+	if (rv) {
+		qdma_mbox_msg_send(dev, mbox_msg_rsp, 0);
+	} else {
+		qdma_mbox_msg_free(mbox_msg_rsp);
+		return;
+	}
+
+	if (rv == QDMA_MBOX_VF_RESET) {
+		qdma_dev->reset_state = RESET_STATE_RECV_PF_RESET_REQ;
+
+		rv = pthread_attr_init(&tattr);
+		if (rv)
+			PMD_DRV_LOG(ERR,
+				"Failed pthread_attr_init for PF reset\n");
+
+		rv = pthread_attr_setdetachstate(&tattr,
+					PTHREAD_CREATE_DETACHED);
+		if (rv)
+			PMD_DRV_LOG(ERR,
+				"Failed pthread_attr_setdetachstate for PF reset\n");
+
+		if (pthread_create(&thread, NULL,
+				qdma_reset_task, (void *)dev)) {
+			PMD_DRV_LOG(ERR, "Could not create qdma reset"
+					" starter thread\n");
+		}
+	} else if (rv == QDMA_MBOX_PF_RESET_DONE) {
+		qdma_dev->reset_state = RESET_STATE_RECV_PF_RESET_DONE;
+	} else if (rv == QDMA_MBOX_PF_BYE) {
+		rv = pthread_attr_init(&tattr);
+		if (rv)
+			PMD_DRV_LOG(ERR,
+				"Failed pthread_attr_init for PF shutdown\n");
+
+		rv = pthread_attr_setdetachstate(&tattr,
+					PTHREAD_CREATE_DETACHED);
+		if (rv)
+			PMD_DRV_LOG(ERR,
+				"Failed pthread_attr_setdetachstate for PF shutdown\n");
+
+		if (pthread_create(&thread, NULL,
+				qdma_remove_task, (void *)dev)) {
+			PMD_DRV_LOG(ERR,
+				"Could not create qdma remove"
+				" starter thread\n");
+		}
+	}
+}
+
+static void qdma_mbox_process_rsp_from_pf(void *arg)
+{
+	struct rte_eth_dev *dev = (struct rte_eth_dev *)arg;
+	struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+	struct qdma_list_head *entry = NULL;
+	struct qdma_list_head *tmp = NULL;
+
+	if (!qdma_dev)
+		return;
+
+	if (qdma_dev->is_vf && qdma_dev->func_id == 0) {
+		qdma_dev->func_id =
+			qdma_mbox_vf_func_id_get(qdma_dev->mbox.rx_data,
+					qdma_dev->is_vf);
+		PMD_DRV_LOG(INFO, "VF function ID: %d", qdma_dev->func_id);
+	}
+
+	rte_spinlock_lock(&qdma_dev->mbox.list_lock);
+	qdma_list_for_each_safe(entry, tmp,
+				&qdma_dev->mbox.rx_pend_list) {
+		struct qdma_mbox_msg *msg = QDMA_LIST_GET_DATA(entry);
+
+		if (qdma_mbox_is_msg_response(msg->raw_data,
+					      qdma_dev->mbox.rx_data)) {
+			/* copy response message back to tx buffer */
+			memcpy(msg->raw_data, qdma_dev->mbox.rx_data,
+			       MBOX_MSG_REG_MAX * sizeof(uint32_t));
+			msg->rsp_rcvd = 1;
+			qdma_list_del(entry);
+			break;
+		}
+	}
+	rte_spinlock_unlock(&qdma_dev->mbox.list_lock);
+}
+
+static void qdma_mbox_rcv_task(void *arg)
+{
+	struct rte_eth_dev *dev = (struct rte_eth_dev *)arg;
+	struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+	int rv;
+
+	if (!qdma_dev)
+		return;
+
+	do {
+		memset(qdma_dev->mbox.rx_data, 0,
+		       MBOX_MSG_REG_MAX * sizeof(uint32_t));
+		rv = qdma_mbox_rcv(dev, qdma_dev->is_vf,
+				   qdma_dev->mbox.rx_data);
+		if (rv < 0)
+			break;
+		if (qdma_dev->is_vf) {
+			qdma_mbox_process_msg_from_pf(arg);
+			qdma_mbox_process_rsp_from_pf(arg);
+		} else {
+			qdma_mbox_process_msg_from_vf(arg);
+		}
+
+	} while (1);
+
+	if (!qdma_dev->dev_cap.mailbox_intr)
+		rte_eal_alarm_set(MBOX_POLL_FRQ, qdma_mbox_rcv_task, arg);
+}
+
+static void qdma_mbox_send_task(void *arg)
+{
+	struct rte_eth_dev *dev = (struct rte_eth_dev *)arg;
+	struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+	struct qdma_list_head *entry = NULL;
+	struct qdma_list_head *tmp = NULL;
+	int rv;
+
+	rte_spinlock_lock(&qdma_dev->mbox.list_lock);
+	qdma_list_for_each_safe(entry, tmp, &qdma_dev->mbox.tx_todo_list) {
+		struct qdma_mbox_msg *msg = QDMA_LIST_GET_DATA(entry);
+
+		rv = qdma_mbox_send(dev, qdma_dev->is_vf, msg->raw_data);
+		if (rv < 0) {
+			msg->retry_cnt--;
+			if (!msg->retry_cnt) {
+				qdma_list_del(entry);
+				if (msg->rsp_wait == QDMA_MBOX_RSP_NO_WAIT)
+					qdma_mbox_msg_free(msg);
+			}
+		} else {
+			qdma_list_del(entry);
+			if (msg->rsp_wait == QDMA_MBOX_RSP_WAIT)
+				qdma_list_add_tail(entry,
+					   &qdma_dev->mbox.rx_pend_list);
+			else
+				qdma_mbox_msg_free(msg);
+		}
+	}
+	if (!qdma_list_is_empty(&qdma_dev->mbox.tx_todo_list))
+		rte_eal_alarm_set(MBOX_POLL_FRQ, qdma_mbox_send_task, arg);
+	rte_spinlock_unlock(&qdma_dev->mbox.list_lock);
+}
+
+int qdma_mbox_msg_send(struct rte_eth_dev *dev, struct qdma_mbox_msg *msg,
+		       unsigned int timeout_ms)
+{
+	struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+
+	if (!msg)
+		return -EINVAL;
+
+	msg->retry_cnt = timeout_ms ? ((timeout_ms / MBOX_POLL_FRQ) + 1) :
+			MBOX_SEND_RETRY_COUNT;
+	QDMA_LIST_SET_DATA(&msg->node, msg);
+
+	rte_spinlock_lock(&qdma_dev->mbox.list_lock);
+	qdma_list_add_tail(&msg->node, &qdma_dev->mbox.tx_todo_list);
+	rte_spinlock_unlock(&qdma_dev->mbox.list_lock);
+
+	msg->rsp_wait = (!timeout_ms) ? QDMA_MBOX_RSP_NO_WAIT :
+			QDMA_MBOX_RSP_WAIT;
+	rte_eal_alarm_set(MBOX_POLL_FRQ, qdma_mbox_send_task, dev);
+
+	if (!timeout_ms)
+		return 0;
+
+	/* if code reached here, caller should free the buffer */
+	while (msg->retry_cnt && !msg->rsp_rcvd)
+		rte_delay_ms(1);
+
+	if (!msg->rsp_rcvd)
+		return  -EPIPE;
+
+	return 0;
+}
+
+void *qdma_mbox_msg_alloc(void)
+{
+	return rte_zmalloc(NULL, sizeof(struct qdma_mbox_msg), 0);
+}
+
+void qdma_mbox_msg_free(void *buffer)
+{
+	rte_free(buffer);
+}
+
+int qdma_mbox_init(struct rte_eth_dev *dev)
+{
+	struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+	uint32_t raw_data[MBOX_MSG_REG_MAX] = {0};
+	struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
+
+	if (!qdma_dev->is_vf) {
+		int i;
+
+		for (i = 0; i < pci_dev->max_vfs; i++)
+			qdma_mbox_rcv(dev, 0, raw_data);
+	} else {
+		qdma_mbox_rcv(dev, 1, raw_data);
+	}
+
+	qdma_mbox_hw_init(dev, qdma_dev->is_vf);
+	qdma_list_init_head(&qdma_dev->mbox.tx_todo_list);
+	qdma_list_init_head(&qdma_dev->mbox.rx_pend_list);
+	rte_spinlock_init(&qdma_dev->mbox.list_lock);
+
+	if (qdma_dev->dev_cap.mailbox_intr) {
+		/* Register interrupt call back handler */
+		rte_intr_callback_register(intr_handle,
+					qdma_mbox_rcv_task, dev);
+
+		/* enable uio/vfio intr/eventfd mapping */
+		rte_intr_enable(intr_handle);
+
+		/* enable qdma mailbox interrupt */
+		qdma_mbox_enable_interrupts((void *)dev, qdma_dev->is_vf);
+	} else {
+		rte_eal_alarm_set(MBOX_POLL_FRQ, qdma_mbox_rcv_task,
+				  (void *)dev);
+	}
+
+	return 0;
+}
+
+void qdma_mbox_uninit(struct rte_eth_dev *dev)
+{
+	struct qdma_pci_dev *qdma_dev = dev->data->dev_private;
+	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+	struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
+
+	do {
+		rte_spinlock_lock(&qdma_dev->mbox.list_lock);
+		if (!qdma_list_is_empty(&qdma_dev->mbox.tx_todo_list)) {
+			rte_spinlock_unlock(&qdma_dev->mbox.list_lock);
+			rte_delay_ms(100);
+			continue;
+		}
+		rte_spinlock_unlock(&qdma_dev->mbox.list_lock);
+		break;
+	} while (1);
+
+	rte_eal_alarm_cancel(qdma_mbox_send_task, (void *)dev);
+	if (qdma_dev->dev_cap.mailbox_intr) {
+		/* Disable the mailbox interrupt */
+		qdma_mbox_disable_interrupts((void *)dev, qdma_dev->is_vf);
+
+		/* Disable uio intr before callback unregister */
+		rte_intr_disable(intr_handle);
+
+		rte_intr_callback_unregister(intr_handle,
+				qdma_mbox_rcv_task, dev);
+	} else {
+		rte_eal_alarm_cancel(qdma_mbox_rcv_task, (void *)dev);
+	}
+}
diff --git a/drivers/net/qdma/qdma_mbox.h b/drivers/net/qdma/qdma_mbox.h
new file mode 100644
index 0000000000..99d6149cec
--- /dev/null
+++ b/drivers/net/qdma/qdma_mbox.h
@@ -0,0 +1,47 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017-2022 Xilinx, Inc. All rights reserved.
+ */
+
+#ifndef QDMA_DPDK_MBOX_H_
+#define QDMA_DPDK_MBOX_H_
+
+#include "qdma_list.h"
+#include "qdma_mbox_protocol.h"
+#include <rte_ethdev.h>
+
+#define MBOX_POLL_FRQ 1000
+#define MBOX_OP_RSP_TIMEOUT (10000 * MBOX_POLL_FRQ) /* 10 sec */
+#define MBOX_SEND_RETRY_COUNT (MBOX_OP_RSP_TIMEOUT / MBOX_POLL_FRQ)
+
+enum qdma_mbox_rsp_state {
+	QDMA_MBOX_RSP_NO_WAIT,
+	QDMA_MBOX_RSP_WAIT
+};
+
+struct qdma_dev_mbox {
+	struct qdma_list_head tx_todo_list;
+	struct qdma_list_head rx_pend_list;
+	rte_spinlock_t list_lock;
+	uint32_t rx_data[MBOX_MSG_REG_MAX];
+};
+
+struct qdma_mbox_msg {
+	uint8_t rsp_rcvd;
+	uint32_t retry_cnt;
+	enum qdma_mbox_rsp_state rsp_wait;
+	uint32_t raw_data[MBOX_MSG_REG_MAX];
+	struct qdma_list_head node;
+};
+
+int qdma_mbox_init(struct rte_eth_dev *dev);
+void qdma_mbox_uninit(struct rte_eth_dev *dev);
+void *qdma_mbox_msg_alloc(void);
+void qdma_mbox_msg_free(void *buffer);
+int qdma_mbox_msg_send(struct rte_eth_dev *dev, struct qdma_mbox_msg *msg,
+		       unsigned int timeout_ms);
+int qdma_dev_notify_qadd(struct rte_eth_dev *dev, uint32_t qidx_hw,
+						enum qdma_dev_q_type q_type);
+int qdma_dev_notify_qdel(struct rte_eth_dev *dev, uint32_t qidx_hw,
+						enum qdma_dev_q_type q_type);
+
+#endif /* QDMA_DPDK_MBOX_H_ */
-- 
2.36.1