[dpdk-stable] [PATCH v1 01/15] net/hinic/base: add mbox command channel for SRIOV

[Ziyang Xuan <xuanziyang2@huawei.com>]

From: Xiaoyun Wang <cloud.wangxiaoyun@huawei.com>

Add mbox command channel for SRIOV, which is used to
communicate between VF and VF, VF and PF. This patch
introduces data structures, initialization, interfaces
and commands of mbox channel.

Signed-off-by: Xiaoyun Wang <cloud.wangxiaoyun@huawei.com>
---
 drivers/net/hinic/Makefile              |   1 +
 drivers/net/hinic/base/hinic_pmd_mbox.c | 854 ++++++++++++++++++++++++++++++++
 drivers/net/hinic/base/hinic_pmd_mbox.h |  92 ++++
 drivers/net/hinic/base/meson.build      |   1 +
 4 files changed, 948 insertions(+)
 create mode 100644 drivers/net/hinic/base/hinic_pmd_mbox.c
 create mode 100644 drivers/net/hinic/base/hinic_pmd_mbox.h

diff --git a/drivers/net/hinic/Makefile b/drivers/net/hinic/Makefile
index 42b4a78..20a338e 100644
--- a/drivers/net/hinic/Makefile
+++ b/drivers/net/hinic/Makefile
@@ -59,6 +59,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_HINIC_PMD) += hinic_pmd_mgmt.c
 SRCS-$(CONFIG_RTE_LIBRTE_HINIC_PMD) += hinic_pmd_niccfg.c
 SRCS-$(CONFIG_RTE_LIBRTE_HINIC_PMD) += hinic_pmd_nicio.c
 SRCS-$(CONFIG_RTE_LIBRTE_HINIC_PMD) += hinic_pmd_wq.c
+SRCS-$(CONFIG_RTE_LIBRTE_HINIC_PMD) += hinic_pmd_mbox.c
 
 SRCS-$(CONFIG_RTE_LIBRTE_HINIC_PMD) += hinic_pmd_ethdev.c
 SRCS-$(CONFIG_RTE_LIBRTE_HINIC_PMD) += hinic_pmd_rx.c
diff --git a/drivers/net/hinic/base/hinic_pmd_mbox.c b/drivers/net/hinic/base/hinic_pmd_mbox.c
new file mode 100644
index 0000000..4b1a2bf
--- /dev/null
+++ b/drivers/net/hinic/base/hinic_pmd_mbox.c
@@ -0,0 +1,854 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017 Huawei Technologies Co., Ltd
+ */
+
+#include "hinic_compat.h"
+#include "hinic_csr.h"
+#include "hinic_pmd_hwdev.h"
+#include "hinic_pmd_hwif.h"
+#include "hinic_pmd_eqs.h"
+#include "hinic_pmd_mgmt.h"
+#include "hinic_pmd_mbox.h"
+
+#define HINIC_MBOX_INT_DST_FUNC_SHIFT				0
+#define HINIC_MBOX_INT_DST_AEQN_SHIFT				10
+#define HINIC_MBOX_INT_SRC_RESP_AEQN_SHIFT			12
+#define HINIC_MBOX_INT_STAT_DMA_SHIFT				14
+/* The size of data to be send (unit of 4 bytes) */
+#define HINIC_MBOX_INT_TX_SIZE_SHIFT				20
+/* SO_RO(strong order, relax order)  */
+#define HINIC_MBOX_INT_STAT_DMA_SO_RO_SHIFT			25
+#define HINIC_MBOX_INT_WB_EN_SHIFT				28
+
+
+#define HINIC_MBOX_INT_DST_FUNC_MASK				0x3FF
+#define HINIC_MBOX_INT_DST_AEQN_MASK				0x3
+#define HINIC_MBOX_INT_SRC_RESP_AEQN_MASK			0x3
+#define HINIC_MBOX_INT_STAT_DMA_MASK				0x3F
+#define HINIC_MBOX_INT_TX_SIZE_MASK				0x1F
+#define HINIC_MBOX_INT_STAT_DMA_SO_RO_MASK			0x3
+#define HINIC_MBOX_INT_WB_EN_MASK				0x1
+
+#define HINIC_MBOX_INT_SET(val, field)	\
+			(((val) & HINIC_MBOX_INT_##field##_MASK) << \
+			HINIC_MBOX_INT_##field##_SHIFT)
+
+enum hinic_mbox_tx_status {
+	TX_DONE = 0,
+	TX_NOT_DONE,
+};
+
+#define HINIC_MBOX_CTRL_TRIGGER_AEQE_SHIFT			0
+/* specifies the issue request for the message data.
+ * 0 - Tx request is done;
+ * 1 - Tx request is in process.
+ */
+#define HINIC_MBOX_CTRL_TX_STATUS_SHIFT				1
+
+#define HINIC_MBOX_CTRL_TRIGGER_AEQE_MASK			0x1
+#define HINIC_MBOX_CTRL_TX_STATUS_MASK				0x1
+
+#define HINIC_MBOX_CTRL_SET(val, field)	\
+			(((val) & HINIC_MBOX_CTRL_##field##_MASK) << \
+			HINIC_MBOX_CTRL_##field##_SHIFT)
+
+#define HINIC_MBOX_HEADER_MSG_LEN_SHIFT				0
+#define HINIC_MBOX_HEADER_MODULE_SHIFT				11
+#define HINIC_MBOX_HEADER_SEG_LEN_SHIFT				16
+#define HINIC_MBOX_HEADER_NO_ACK_SHIFT				22
+#define HINIC_MBOX_HEADER_SEQID_SHIFT				24
+#define HINIC_MBOX_HEADER_LAST_SHIFT				30
+
+/* specifies the mailbox message direction
+ * 0 - send
+ * 1 - receive
+ */
+#define HINIC_MBOX_HEADER_DIRECTION_SHIFT			31
+#define HINIC_MBOX_HEADER_CMD_SHIFT				32
+#define HINIC_MBOX_HEADER_MSG_ID_SHIFT				40
+#define HINIC_MBOX_HEADER_STATUS_SHIFT				48
+#define HINIC_MBOX_HEADER_SRC_GLB_FUNC_IDX_SHIFT		54
+
+#define HINIC_MBOX_HEADER_MSG_LEN_MASK				0x7FF
+#define HINIC_MBOX_HEADER_MODULE_MASK				0x1F
+#define HINIC_MBOX_HEADER_SEG_LEN_MASK				0x3F
+#define HINIC_MBOX_HEADER_NO_ACK_MASK				0x1
+#define HINIC_MBOX_HEADER_SEQID_MASK				0x3F
+#define HINIC_MBOX_HEADER_LAST_MASK				0x1
+#define HINIC_MBOX_HEADER_DIRECTION_MASK			0x1
+#define HINIC_MBOX_HEADER_CMD_MASK				0xFF
+#define HINIC_MBOX_HEADER_MSG_ID_MASK				0xFF
+#define HINIC_MBOX_HEADER_STATUS_MASK				0x3F
+#define HINIC_MBOX_HEADER_SRC_GLB_FUNC_IDX_MASK			0x3FF
+
+#define HINIC_MBOX_HEADER_GET(val, field)	\
+			(((val) >> HINIC_MBOX_HEADER_##field##_SHIFT) & \
+			HINIC_MBOX_HEADER_##field##_MASK)
+#define HINIC_MBOX_HEADER_SET(val, field)	\
+			((u64)((val) & HINIC_MBOX_HEADER_##field##_MASK) << \
+			HINIC_MBOX_HEADER_##field##_SHIFT)
+
+#define HINIC_MBOX_COMP_TIME_MS			8000U
+#define MBOX_MSG_POLLING_TIMEOUT_MS		5000
+
+/* The size unite is Bytes */
+#define HINIC_MBOX_DATA_SIZE			2040
+#define MBOX_MAX_BUF_SZ				2048UL
+#define MBOX_HEADER_SZ				8
+
+/* MBOX size is 64B, 8B for mbox_header, 4B reserved */
+#define MBOX_SEG_LEN				48
+#define MBOX_SEG_LEN_ALIGN			4
+#define MBOX_WB_STATUS_LEN			16UL
+#define MBOX_SIZE				64
+
+/* mbox write back status is 16B, only first 4B is used */
+#define MBOX_WB_STATUS_ERRCODE_MASK		0xFFFF
+#define MBOX_WB_STATUS_MASK			0xFF
+#define MBOX_WB_ERROR_CODE_MASK			0xFF00
+#define MBOX_WB_STATUS_FINISHED_SUCCESS		0xFF
+#define MBOX_WB_STATUS_FINISHED_WITH_ERR	0xFE
+#define MBOX_WB_STATUS_NOT_FINISHED		0x00
+
+#define MBOX_STATUS_FINISHED(wb)	\
+	(((wb) & MBOX_WB_STATUS_MASK) != MBOX_WB_STATUS_NOT_FINISHED)
+#define MBOX_STATUS_SUCCESS(wb)		\
+	(((wb) & MBOX_WB_STATUS_MASK) == MBOX_WB_STATUS_FINISHED_SUCCESS)
+#define MBOX_STATUS_ERRCODE(wb)		\
+	((wb) & MBOX_WB_ERROR_CODE_MASK)
+
+#define SEQ_ID_START_VAL			0
+
+#define DST_AEQ_IDX_DEFAULT_VAL			0
+#define SRC_AEQ_IDX_DEFAULT_VAL			0
+#define NO_DMA_ATTRIBUTE_VAL			0
+
+#define MBOX_MSG_NO_DATA_LEN			1
+
+#define FUNC_ID_OFF_SET_8B		8
+#define FUNC_ID_OFF_SET_10B		10
+
+#define MBOX_BODY_FROM_HDR(header)	((u8 *)(header) + MBOX_HEADER_SZ)
+#define MBOX_AREA(hwif)			\
+		((hwif)->cfg_regs_base + HINIC_FUNC_CSR_MAILBOX_DATA_OFF)
+
+#define MBOX_RESPONSE_ERROR		0x1
+#define MBOX_MSG_ID_MASK		0xFF
+#define MBOX_MSG_ID(func_to_func)	((func_to_func)->send_msg_id)
+#define MBOX_MSG_ID_INC(func_to_func)	(MBOX_MSG_ID(func_to_func) =	\
+			(MBOX_MSG_ID(func_to_func) + 1) & MBOX_MSG_ID_MASK)
+
+enum hinic_hwif_direction_type {
+	HINIC_HWIF_DIRECT_SEND	= 0,
+	HINIC_HWIF_RESPONSE,
+};
+
+enum mbox_send_mod {
+	MBOX_SEND_MSG_POLL = 1
+};
+
+enum mbox_seg_type {
+	NOT_LAST_SEG,
+	LAST_SEG,
+};
+
+enum mbox_ordering_type {
+	STRONG_ORDER,
+	RELAX_ORDER,
+};
+
+enum mbox_write_back_type {
+	NOT_WRITE_BACK = 0,
+	WRITE_BACK,
+};
+
+enum mbox_aeq_trig_type {
+	NOT_TRIGGER,
+	TRIGGER,
+};
+
+static int send_mbox_to_func(struct hinic_mbox_func_to_func *func_to_func,
+				enum hinic_mod_type mod, u16 cmd, void *msg,
+				u16 msg_len, u16 dst_func,
+				enum hinic_hwif_direction_type direction,
+				enum hinic_mbox_ack_type ack_type,
+				struct mbox_msg_info *msg_info);
+
+static int recv_vf_mbox_handler(struct hinic_mbox_func_to_func *func_to_func,
+				struct hinic_recv_mbox *recv_mbox,
+				void *buf_out, u16 *out_size, void *param)
+{
+	int rc = 0;
+	*out_size = 0;
+
+	switch (recv_mbox->mod) {
+	case HINIC_MOD_COMM:
+		hinic_comm_async_event_handle(func_to_func->hwdev,
+						recv_mbox->cmd, recv_mbox->mbox,
+						recv_mbox->mbox_len,
+						buf_out, out_size);
+		break;
+	case HINIC_MOD_L2NIC:
+		hinic_l2nic_async_event_handle(func_to_func->hwdev, param,
+						recv_mbox->cmd, recv_mbox->mbox,
+						recv_mbox->mbox_len,
+						buf_out, out_size);
+		break;
+	default:
+		PMD_DRV_LOG(ERR, "No handler, mod = %d",
+				recv_mbox->mod);
+		rc = HINIC_MBOX_VF_CMD_ERROR;
+		break;
+	}
+
+	return rc;
+}
+
+static void set_mbx_msg_status(struct mbox_msg_info *msg_info, int status)
+{
+	if (status == HINIC_DEV_BUSY_ACTIVE_FW)
+		msg_info->status = HINIC_MBOX_PF_BUSY_ACTIVE_FW;
+	else if (status == HINIC_MBOX_VF_CMD_ERROR)
+		msg_info->status = HINIC_MBOX_VF_CMD_ERROR;
+	else if (status)
+		msg_info->status = HINIC_MBOX_PF_SEND_ERR;
+}
+
+static void recv_func_mbox_handler(struct hinic_mbox_func_to_func *func_to_func,
+				struct hinic_recv_mbox *recv_mbox,
+				u16 src_func_idx, void *param)
+{
+	struct hinic_hwdev *dev = func_to_func->hwdev;
+	struct mbox_msg_info msg_info = { 0 };
+	u16 out_size = MBOX_MAX_BUF_SZ;
+	void *buf_out = recv_mbox->buf_out;
+	int err = 0;
+
+	if (HINIC_IS_VF(dev)) {
+		err = recv_vf_mbox_handler(func_to_func, recv_mbox, buf_out,
+						&out_size, param);
+	} else {
+		err = -EINVAL;
+		PMD_DRV_LOG(ERR, "PMD doesn't support non-VF handle mailbox message");
+	}
+
+	if (!out_size || err)
+		out_size = MBOX_MSG_NO_DATA_LEN;
+
+	if (recv_mbox->ack_type == MBOX_ACK) {
+		msg_info.msg_id = recv_mbox->msg_info.msg_id;
+		set_mbx_msg_status(&msg_info, err);
+		send_mbox_to_func(func_to_func, recv_mbox->mod, recv_mbox->cmd,
+				buf_out, out_size, src_func_idx,
+				HINIC_HWIF_RESPONSE, MBOX_ACK, &msg_info);
+	}
+}
+
+static bool check_mbox_seq_id_and_seg_len(struct hinic_recv_mbox *recv_mbox,
+					  u8 seq_id, u8 seg_len)
+{
+	if (seq_id > HINIC_SEQ_ID_MAX_VAL || seg_len > HINIC_MSG_SEG_LEN)
+		return false;
+
+	if (seq_id == 0) {
+		recv_mbox->sed_id = seq_id;
+	} else {
+		if (seq_id != recv_mbox->sed_id + 1) {
+			recv_mbox->sed_id = 0;
+			return false;
+		}
+
+		recv_mbox->sed_id = seq_id;
+	}
+
+	return true;
+}
+
+static void clear_mbox_status(struct hinic_send_mbox *mbox)
+{
+	/* clear mailbox write back status */
+	*mbox->wb_status = 0;
+	rte_wmb();
+}
+
+static void mbox_copy_header(struct hinic_send_mbox *mbox, u64 *header)
+{
+	u32 *data = (u32 *)header;
+	u32 i, idx_max = MBOX_HEADER_SZ / sizeof(u32);
+
+	for (i = 0; i < idx_max; i++)
+		__raw_writel(*(data + i), mbox->data + i * sizeof(u32));
+}
+
+static void
+mbox_copy_send_data(struct hinic_send_mbox *mbox, void *seg, u16 seg_len)
+{
+	u32 *data = (u32 *)seg;
+	u32 data_len, chk_sz = sizeof(u32);
+	u32 i, idx_max;
+	u8 mbox_max_buf[MBOX_SEG_LEN] = {0};
+
+	/* The mbox message should be aligned in 4 bytes. */
+	if (seg_len % chk_sz) {
+		memcpy(mbox_max_buf, seg, seg_len);
+		data = (u32 *)mbox_max_buf;
+	}
+
+	data_len = seg_len;
+	idx_max = ALIGN(data_len, chk_sz) / chk_sz;
+
+	for (i = 0; i < idx_max; i++)
+		__raw_writel(*(data + i),
+				mbox->data + MBOX_HEADER_SZ + i * sizeof(u32));
+}
+
+static void write_mbox_msg_attr(struct hinic_mbox_func_to_func *func_to_func,
+			u16 dst_func, u16 dst_aeqn,
+			__rte_unused u16 seg_len, int poll)
+{
+	u32 mbox_int, mbox_ctrl;
+
+	mbox_int = HINIC_MBOX_INT_SET(dst_func, DST_FUNC) |
+		HINIC_MBOX_INT_SET(dst_aeqn, DST_AEQN) |
+		HINIC_MBOX_INT_SET(HINIC_MBOX_RSP_AEQN, SRC_RESP_AEQN) |
+		HINIC_MBOX_INT_SET(NO_DMA_ATTRIBUTE_VAL, STAT_DMA) |
+		HINIC_MBOX_INT_SET(ALIGN(MBOX_SIZE, MBOX_SEG_LEN_ALIGN) >> 2,
+					TX_SIZE) |
+		HINIC_MBOX_INT_SET(STRONG_ORDER, STAT_DMA_SO_RO) |
+		HINIC_MBOX_INT_SET(WRITE_BACK, WB_EN);
+
+	hinic_hwif_write_reg(func_to_func->hwdev->hwif,
+			HINIC_FUNC_CSR_MAILBOX_INT_OFFSET_OFF, mbox_int);
+
+	rte_wmb();
+	mbox_ctrl = HINIC_MBOX_CTRL_SET(TX_NOT_DONE, TX_STATUS);
+
+	if (poll)
+		mbox_ctrl |= HINIC_MBOX_CTRL_SET(NOT_TRIGGER, TRIGGER_AEQE);
+	else
+		mbox_ctrl |= HINIC_MBOX_CTRL_SET(TRIGGER, TRIGGER_AEQE);
+
+	hinic_hwif_write_reg(func_to_func->hwdev->hwif,
+				HINIC_FUNC_CSR_MAILBOX_CONTROL_OFF, mbox_ctrl);
+}
+
+static int init_mbox_info(struct hinic_recv_mbox *mbox_info)
+{
+	int err;
+
+	mbox_info->mbox = kzalloc(MBOX_MAX_BUF_SZ, GFP_KERNEL);
+	if (!mbox_info->mbox) {
+		PMD_DRV_LOG(ERR, "Alloc mbox buf_in mem failed\n");
+		return -ENOMEM;
+	}
+
+	mbox_info->buf_out = kzalloc(MBOX_MAX_BUF_SZ, GFP_KERNEL);
+	if (!mbox_info->buf_out) {
+		PMD_DRV_LOG(ERR, "Alloc mbox buf_out mem failed\n");
+		err = -ENOMEM;
+		goto alloc_buf_out_err;
+	}
+
+	return 0;
+
+alloc_buf_out_err:
+	kfree(mbox_info->mbox);
+
+	return err;
+}
+
+static void clean_mbox_info(struct hinic_recv_mbox *mbox_info)
+{
+	kfree(mbox_info->buf_out);
+	kfree(mbox_info->mbox);
+}
+
+static int alloc_mbox_info(struct hinic_recv_mbox *mbox_info)
+{
+	u16 func_idx, i;
+	int err;
+
+	for (func_idx = 0; func_idx < HINIC_MAX_FUNCTIONS; func_idx++) {
+		err = init_mbox_info(&mbox_info[func_idx]);
+		if (err) {
+			PMD_DRV_LOG(ERR, "Initialize function[%d] mailbox information failed, err: %d",
+				    func_idx, err);
+			goto init_mbox_info_err;
+		}
+	}
+
+	return 0;
+
+init_mbox_info_err:
+	for (i = 0; i < func_idx; i++)
+		clean_mbox_info(&mbox_info[i]);
+
+	return err;
+}
+
+static void free_mbox_info(struct hinic_recv_mbox *mbox_info)
+{
+	u16 func_idx;
+
+	for (func_idx = 0; func_idx < HINIC_MAX_FUNCTIONS; func_idx++)
+		clean_mbox_info(&mbox_info[func_idx]);
+}
+
+static void prepare_send_mbox(struct hinic_mbox_func_to_func *func_to_func)
+{
+	struct hinic_send_mbox *send_mbox = &func_to_func->send_mbox;
+
+	send_mbox->data = MBOX_AREA(func_to_func->hwdev->hwif);
+}
+
+static int alloc_mbox_wb_status(struct hinic_mbox_func_to_func *func_to_func)
+{
+	struct hinic_send_mbox *send_mbox = &func_to_func->send_mbox;
+	struct hinic_hwdev *hwdev = func_to_func->hwdev;
+	struct hinic_hwif *hwif = hwdev->hwif;
+	u32 addr_h, addr_l;
+
+	send_mbox->wb_vaddr = dma_zalloc_coherent(hwdev,
+				  MBOX_WB_STATUS_LEN,
+				  &send_mbox->wb_paddr,
+				  GFP_KERNEL);
+	if (!send_mbox->wb_vaddr) {
+		PMD_DRV_LOG(ERR, "Allocating memory for mailbox wb status failed");
+		return -ENOMEM;
+	}
+	send_mbox->wb_status = (volatile u64 *)send_mbox->wb_vaddr;
+
+	addr_h = upper_32_bits(send_mbox->wb_paddr);
+	addr_l = lower_32_bits(send_mbox->wb_paddr);
+	hinic_hwif_write_reg(hwif, HINIC_FUNC_CSR_MAILBOX_RESULT_H_OFF, addr_h);
+	hinic_hwif_write_reg(hwif, HINIC_FUNC_CSR_MAILBOX_RESULT_L_OFF, addr_l);
+
+	return 0;
+}
+
+static void free_mbox_wb_status(struct hinic_mbox_func_to_func *func_to_func)
+{
+	struct hinic_send_mbox *send_mbox = &func_to_func->send_mbox;
+	struct hinic_hwdev *hwdev = func_to_func->hwdev;
+	struct hinic_hwif *hwif = hwdev->hwif;
+
+	hinic_hwif_write_reg(hwif, HINIC_FUNC_CSR_MAILBOX_RESULT_H_OFF, 0);
+	hinic_hwif_write_reg(hwif, HINIC_FUNC_CSR_MAILBOX_RESULT_L_OFF, 0);
+
+	dma_free_coherent(hwdev, MBOX_WB_STATUS_LEN,
+				send_mbox->wb_vaddr, send_mbox->wb_paddr);
+}
+
+static int recv_mbox_handler(struct hinic_mbox_func_to_func *func_to_func,
+		void *header, struct hinic_recv_mbox *recv_mbox, void *param)
+{
+	u64 mbox_header = *((u64 *)header);
+	void *mbox_body = MBOX_BODY_FROM_HDR(header);
+	u16 src_func_idx;
+	enum hinic_hwif_direction_type direction;
+	u8 seq_id, seg_len;
+
+	seq_id = HINIC_MBOX_HEADER_GET(mbox_header, SEQID);
+	seg_len = HINIC_MBOX_HEADER_GET(mbox_header, SEG_LEN);
+	direction = HINIC_MBOX_HEADER_GET(mbox_header, DIRECTION);
+	src_func_idx = HINIC_MBOX_HEADER_GET(mbox_header, SRC_GLB_FUNC_IDX);
+
+	if (!check_mbox_seq_id_and_seg_len(recv_mbox, seq_id, seg_len)) {
+		PMD_DRV_LOG(ERR,
+			"Mailbox sequence and segment check fail, src func id: 0x%x, front id: 0x%x, current id: 0x%x, seg len: 0x%x\n",
+			src_func_idx, recv_mbox->sed_id, seq_id, seg_len);
+		return HINIC_RECV_NEXT_AEQE;
+	}
+
+	memcpy((u8 *)recv_mbox->mbox + seq_id * HINIC_MSG_SEG_LEN,
+		mbox_body, seg_len);
+
+	if (!HINIC_MBOX_HEADER_GET(mbox_header, LAST))
+		return HINIC_RECV_NEXT_AEQE;
+
+	recv_mbox->sed_id = 0;
+	recv_mbox->cmd = HINIC_MBOX_HEADER_GET(mbox_header, CMD);
+	recv_mbox->mod = HINIC_MBOX_HEADER_GET(mbox_header, MODULE);
+	recv_mbox->mbox_len = HINIC_MBOX_HEADER_GET(mbox_header, MSG_LEN);
+	recv_mbox->ack_type = HINIC_MBOX_HEADER_GET(mbox_header, NO_ACK);
+	recv_mbox->msg_info.msg_id = HINIC_MBOX_HEADER_GET(mbox_header, MSG_ID);
+	recv_mbox->msg_info.status = HINIC_MBOX_HEADER_GET(mbox_header, STATUS);
+
+	if (direction == HINIC_HWIF_RESPONSE) {
+		if (recv_mbox->msg_info.msg_id == func_to_func->send_msg_id &&
+			func_to_func->event_flag == EVENT_START) {
+			return HINIC_RECV_DONE;
+		}
+
+		PMD_DRV_LOG(ERR, "Mbox response timeout, current send msg id(0x%x), recv msg id(0x%x), status(0x%x)",
+			func_to_func->send_msg_id, recv_mbox->msg_info.msg_id,
+			recv_mbox->msg_info.status);
+		return HINIC_RECV_NEXT_AEQE;
+	}
+
+	recv_func_mbox_handler(func_to_func, recv_mbox, src_func_idx, param);
+
+	return HINIC_RECV_NEXT_AEQE;
+}
+
+int hinic_mbox_func_aeqe_handler(void *handle, u8 *header,
+					__rte_unused u8 size, void *param)
+{
+	struct hinic_mbox_func_to_func *func_to_func =
+				((struct hinic_hwdev *)handle)->func_to_func;
+	struct hinic_recv_mbox *recv_mbox;
+	u64 mbox_header = *((u64 *)header);
+	u16 src = HINIC_MBOX_HEADER_GET(mbox_header, SRC_GLB_FUNC_IDX);
+
+	if (src >= HINIC_MAX_FUNCTIONS) {
+		PMD_DRV_LOG(ERR, "Mailbox source function id: %d is invalid",
+				src);
+		return HINIC_RECV_NEXT_AEQE;
+	}
+
+	recv_mbox = (HINIC_MBOX_HEADER_GET(mbox_header, DIRECTION) ==
+			HINIC_HWIF_DIRECT_SEND) ?
+			&func_to_func->mbox_send[src] :
+			&func_to_func->mbox_resp[src];
+
+	return recv_mbox_handler(func_to_func, (u64 *)header, recv_mbox, param);
+}
+
+static u16 get_mbox_status(struct hinic_send_mbox *mbox)
+{
+	/* write back is 16B, but only use first 4B */
+	u64 wb_val = be64_to_cpu(*mbox->wb_status);
+
+	rte_rmb(); /* verify reading before check */
+
+	return (u16)(wb_val & MBOX_WB_STATUS_ERRCODE_MASK);
+}
+
+static void dump_mox_reg(struct hinic_hwdev *hwdev)
+{
+	u32 val;
+
+	val = hinic_hwif_read_reg(hwdev->hwif,
+					HINIC_FUNC_CSR_MAILBOX_CONTROL_OFF);
+	PMD_DRV_LOG(WARNING, "Mailbox control reg: 0x%x", val);
+	val = hinic_hwif_read_reg(hwdev->hwif,
+					HINIC_FUNC_CSR_MAILBOX_INT_OFFSET_OFF);
+	PMD_DRV_LOG(WARNING, "Mailbox interrupt offset: 0x%x", val);
+}
+
+static int send_mbox_seg(struct hinic_mbox_func_to_func *func_to_func,
+			 u64 header, u16 dst_func, void *seg, u16 seg_len)
+{
+	struct hinic_send_mbox *send_mbox = &func_to_func->send_mbox;
+	struct hinic_hwdev *hwdev = func_to_func->hwdev;
+	u16 seq_dir = HINIC_MBOX_HEADER_GET(header, DIRECTION);
+	u16 dst_aeqn = (seq_dir == HINIC_HWIF_DIRECT_SEND) ?
+				HINIC_MBOX_RECV_AEQN : HINIC_MBOX_RSP_AEQN;
+	u16 errcode, wb_status = 0;
+	u32 cnt = 0;
+
+	clear_mbox_status(send_mbox);
+
+	mbox_copy_header(send_mbox, &header);
+
+	mbox_copy_send_data(send_mbox, seg, seg_len);
+
+	write_mbox_msg_attr(func_to_func, dst_func, dst_aeqn, seg_len,
+			    MBOX_SEND_MSG_POLL);
+
+	rte_wmb();
+
+	while (cnt < MBOX_MSG_POLLING_TIMEOUT_MS) {
+		wb_status = get_mbox_status(send_mbox);
+		if (MBOX_STATUS_FINISHED(wb_status))
+			break;
+
+		rte_delay_ms(1); /* loop every ms */
+		cnt++;
+	}
+
+	if (cnt == MBOX_MSG_POLLING_TIMEOUT_MS) {
+		PMD_DRV_LOG(ERR, "Send mailbox segment timeout, wb status: 0x%x",
+				wb_status);
+		dump_mox_reg(hwdev);
+		return -ETIMEDOUT;
+	}
+
+	if (!MBOX_STATUS_SUCCESS(wb_status)) {
+		PMD_DRV_LOG(ERR, "Send mailbox segment to function %d error, wb status: 0x%x",
+				dst_func, wb_status);
+		errcode = MBOX_STATUS_ERRCODE(wb_status);
+		return errcode ? errcode : -EFAULT;
+	}
+
+	return 0;
+}
+
+static void set_mbox_to_func_event(struct hinic_mbox_func_to_func *func_to_func,
+				   enum mbox_event_state event_flag)
+{
+	spin_lock(&func_to_func->mbox_lock);
+	func_to_func->event_flag = event_flag;
+	spin_unlock(&func_to_func->mbox_lock);
+}
+
+static int send_mbox_to_func(struct hinic_mbox_func_to_func *func_to_func,
+				enum hinic_mod_type mod, u16 cmd, void *msg,
+				u16 msg_len, u16 dst_func,
+				enum hinic_hwif_direction_type direction,
+				enum hinic_mbox_ack_type ack_type,
+				struct mbox_msg_info *msg_info)
+{
+	struct hinic_hwdev *hwdev = func_to_func->hwdev;
+	int err = 0;
+	u32 seq_id = 0;
+	u16 seg_len = HINIC_MSG_SEG_LEN;
+	u16 left = msg_len;
+	u8 *msg_seg = (u8 *)msg;
+	u64 header = 0;
+
+	spin_lock(&func_to_func->msg_send_lock);
+
+	header = HINIC_MBOX_HEADER_SET(msg_len, MSG_LEN) |
+		HINIC_MBOX_HEADER_SET(mod, MODULE) |
+		HINIC_MBOX_HEADER_SET(seg_len, SEG_LEN) |
+		HINIC_MBOX_HEADER_SET(ack_type, NO_ACK) |
+		HINIC_MBOX_HEADER_SET(SEQ_ID_START_VAL, SEQID) |
+		HINIC_MBOX_HEADER_SET(NOT_LAST_SEG, LAST) |
+		HINIC_MBOX_HEADER_SET(direction, DIRECTION) |
+		HINIC_MBOX_HEADER_SET(cmd, CMD) |
+		HINIC_MBOX_HEADER_SET(msg_info->msg_id, MSG_ID) |
+		HINIC_MBOX_HEADER_SET(msg_info->status, STATUS) |
+		HINIC_MBOX_HEADER_SET(hinic_global_func_id(hwdev),
+					SRC_GLB_FUNC_IDX);
+
+	while (!(HINIC_MBOX_HEADER_GET(header, LAST))) {
+		if (left <= HINIC_MSG_SEG_LEN) {
+			header &=
+			~(HINIC_MBOX_HEADER_SET(HINIC_MBOX_HEADER_SEG_LEN_MASK,
+						SEG_LEN));
+			header |= HINIC_MBOX_HEADER_SET(left, SEG_LEN);
+			header |= HINIC_MBOX_HEADER_SET(LAST_SEG, LAST);
+
+			seg_len = left;
+		}
+
+		err = send_mbox_seg(func_to_func, header, dst_func, msg_seg,
+				    seg_len);
+		if (err) {
+			PMD_DRV_LOG(ERR, "Fail to send mbox seg, seq_id: 0x%lx, err: %d",
+				HINIC_MBOX_HEADER_GET(header, SEQID), err);
+			goto send_err;
+		}
+
+		left -= HINIC_MSG_SEG_LEN;
+		msg_seg += HINIC_MSG_SEG_LEN;
+
+		seq_id++;
+		header &= ~(HINIC_MBOX_HEADER_SET(HINIC_MBOX_HEADER_SEQID_MASK,
+							SEQID));
+		header |= HINIC_MBOX_HEADER_SET(seq_id, SEQID);
+	}
+
+send_err:
+	spin_unlock(&func_to_func->msg_send_lock);
+
+	return err;
+}
+
+static int hinic_mbox_to_func(struct hinic_mbox_func_to_func *func_to_func,
+			enum hinic_mod_type mod, u16 cmd, u16 dst_func,
+			void *buf_in, u16 in_size, void *buf_out, u16 *out_size,
+			u32 timeout)
+{
+	struct hinic_recv_mbox *mbox_for_resp =
+					&func_to_func->mbox_resp[dst_func];
+	struct mbox_msg_info msg_info = {0};
+	u32 time;
+	int err;
+
+	spin_lock(&func_to_func->mbox_send_lock);
+
+	msg_info.msg_id = MBOX_MSG_ID_INC(func_to_func);
+
+	set_mbox_to_func_event(func_to_func, EVENT_START);
+
+	err = send_mbox_to_func(func_to_func, mod, cmd, buf_in, in_size,
+				dst_func, HINIC_HWIF_DIRECT_SEND,
+				MBOX_ACK, &msg_info);
+	if (err)
+		goto send_err;
+
+	time = msecs_to_jiffies(timeout ? timeout : HINIC_MBOX_COMP_TIME_MS);
+	err = hinic_aeq_poll_msg(func_to_func->rsp_aeq, time, NULL);
+	if (err) {
+		set_mbox_to_func_event(func_to_func, EVENT_TIMEOUT);
+		PMD_DRV_LOG(ERR, "Send mailbox message time out");
+		err = -ETIMEDOUT;
+		goto send_err;
+	}
+
+	set_mbox_to_func_event(func_to_func, EVENT_END);
+
+	if (mbox_for_resp->msg_info.status) {
+		err = mbox_for_resp->msg_info.status;
+		if (err != HINIC_MBOX_PF_BUSY_ACTIVE_FW)
+			PMD_DRV_LOG(ERR, "Mailbox response error: 0x%x",
+					mbox_for_resp->msg_info.status);
+		else
+			PMD_DRV_LOG(ERR, "Chip is in active, PF can't process VF message");
+		goto send_err;
+	}
+
+	rte_rmb();
+
+	if (mbox_for_resp->mbox_len && buf_out && out_size) {
+		if (mbox_for_resp->mbox_len <= *out_size) {
+			memcpy(buf_out, mbox_for_resp->mbox,
+				mbox_for_resp->mbox_len);
+			*out_size = mbox_for_resp->mbox_len;
+		} else {
+			PMD_DRV_LOG(ERR, "Mailbox response message len[%u] overflow",
+					mbox_for_resp->mbox_len);
+			err = -ERANGE;
+		}
+	}
+
+send_err:
+	if (err && out_size)
+		*out_size = 0;
+	spin_unlock(&func_to_func->mbox_send_lock);
+
+	return err;
+}
+
+static int
+mbox_func_params_valid(__rte_unused struct hinic_mbox_func_to_func *mbox_obj,
+			void *buf_in, u16 in_size)
+{
+	if (!buf_in || !in_size)
+		return -EINVAL;
+
+	if (in_size > HINIC_MBOX_DATA_SIZE) {
+		PMD_DRV_LOG(ERR, "Mailbox message len(%d) exceed limit(%d)",
+				in_size, HINIC_MBOX_DATA_SIZE);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+int hinic_mbox_to_pf(struct hinic_hwdev *hwdev,
+		      enum hinic_mod_type mod, u8 cmd, void *buf_in,
+		      u16 in_size, void *buf_out, u16 *out_size, u32 timeout)
+{
+	struct hinic_mbox_func_to_func *func_to_func = hwdev->func_to_func;
+	int err;
+
+	err = mbox_func_params_valid(func_to_func, buf_in, in_size);
+	if (err) {
+		PMD_DRV_LOG(ERR, "Mailbox parameters check failed: %d", err);
+		return err;
+	}
+
+	if (!HINIC_IS_VF(hwdev)) {
+		PMD_DRV_LOG(ERR, "Input function type error, func_type: %d",
+				hinic_func_type(hwdev));
+		return -EINVAL;
+	}
+
+	return hinic_mbox_to_func(func_to_func, mod, cmd,
+				   hinic_pf_id_of_vf(hwdev), buf_in, in_size,
+				   buf_out, out_size, timeout);
+}
+
+int hinic_mbox_to_pf_no_ack(struct hinic_hwdev *hwdev, enum hinic_mod_type mod,
+			u8 cmd, void *buf_in, u16 in_size)
+{
+	int err;
+	struct mbox_msg_info msg_info = {0};
+
+	spin_lock(&hwdev->func_to_func->mbox_send_lock); /*pmd use spinlock*/
+
+	err = send_mbox_to_func(hwdev->func_to_func, mod, cmd, buf_in, in_size,
+			hinic_pf_id_of_vf(hwdev), HINIC_HWIF_DIRECT_SEND,
+			MBOX_NO_ACK, &msg_info);
+	if (err)
+		PMD_DRV_LOG(ERR, "Send mailbox no ack failed, err: %d", err);
+
+	spin_unlock(&hwdev->func_to_func->mbox_send_lock);
+
+	return err;
+}
+
+static int hinic_func_to_func_init(struct hinic_hwdev *hwdev)
+{
+	struct hinic_mbox_func_to_func *func_to_func;
+	int err;
+
+	func_to_func = kzalloc(sizeof(*func_to_func), GFP_KERNEL);
+	if (!func_to_func) {
+		PMD_DRV_LOG(ERR, "Allocating memory for func_to_func object failed");
+		return -ENOMEM;
+	}
+	hwdev->func_to_func = func_to_func;
+	func_to_func->hwdev = hwdev;
+	spin_lock_init(&func_to_func->mbox_send_lock);
+	spin_lock_init(&func_to_func->msg_send_lock);
+
+	err = alloc_mbox_info(func_to_func->mbox_send);
+	if (err) {
+		PMD_DRV_LOG(ERR, "Allocating memory for mailbox sending failed");
+		goto alloc_mbox_for_send_err;
+	}
+
+	err = alloc_mbox_info(func_to_func->mbox_resp);
+	if (err) {
+		PMD_DRV_LOG(ERR, "Allocating memory for mailbox responsing failed");
+		goto alloc_mbox_for_resp_err;
+	}
+
+	err = alloc_mbox_wb_status(func_to_func);
+	if (err)
+		goto alloc_wb_status_err;
+
+	prepare_send_mbox(func_to_func);
+
+	return 0;
+
+alloc_wb_status_err:
+	free_mbox_info(func_to_func->mbox_resp);
+
+alloc_mbox_for_resp_err:
+	free_mbox_info(func_to_func->mbox_send);
+
+alloc_mbox_for_send_err:
+	kfree(func_to_func);
+
+	return err;
+}
+
+void hinic_comm_func_to_func_free(struct hinic_hwdev *hwdev)
+{
+	struct hinic_mbox_func_to_func *func_to_func = hwdev->func_to_func;
+
+	free_mbox_wb_status(func_to_func);
+	free_mbox_info(func_to_func->mbox_resp);
+	free_mbox_info(func_to_func->mbox_send);
+	kfree(func_to_func);
+}
+
+int hinic_comm_func_to_func_init(struct hinic_hwdev *hwdev)
+{
+	int rc;
+
+	rc = hinic_func_to_func_init(hwdev);
+	if (rc)
+		return rc;
+
+	hwdev->func_to_func->rsp_aeq = &hwdev->aeqs->aeq[HINIC_MBOX_RSP_AEQN];
+	hwdev->func_to_func->recv_aeq = &hwdev->aeqs->aeq[HINIC_MBOX_RECV_AEQN];
+
+	return 0;
+}
+
diff --git a/drivers/net/hinic/base/hinic_pmd_mbox.h b/drivers/net/hinic/base/hinic_pmd_mbox.h
new file mode 100644
index 0000000..ac1b3d7
--- /dev/null
+++ b/drivers/net/hinic/base/hinic_pmd_mbox.h
@@ -0,0 +1,92 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017 Huawei Technologies Co., Ltd
+ */
+
+#ifndef _HINIC_PMD_MBOX_H_
+#define _HINIC_PMD_MBOX_H_
+
+#define HINIC_MBOX_RECV_AEQN		(0)
+#define HINIC_MBOX_RSP_AEQN		(2)
+
+#define HINIC_MBOX_PF_SEND_ERR		0x1
+#define HINIC_MBOX_PF_BUSY_ACTIVE_FW	0x2
+#define HINIC_MBOX_VF_CMD_ERROR		0x3
+
+/* PFs does not support enable SR-IOV cap when PFs use PMD, VFs just receive
+ * mailbox message from PFs. The max number of PFs is 16, so the max number
+ * of mailbox buffer for functions is also 16.
+ */
+#define HINIC_MAX_FUNCTIONS			16
+#define HINIC_MAX_PF_FUNCS			16
+
+#define HINIC_MGMT_CMD_UNSUPPORTED	0xFF
+
+#define HINIC_SEQ_ID_MAX_VAL		42
+#define HINIC_MSG_SEG_LEN			48
+
+enum hinic_mbox_ack_type {
+	MBOX_ACK,
+	MBOX_NO_ACK,
+};
+
+struct mbox_msg_info {
+	u8 msg_id;
+	u8 status; /*can only use 6 bit*/
+};
+
+struct hinic_recv_mbox {
+	void *mbox;
+	u8 cmd;
+	enum hinic_mod_type mod;
+	u16 mbox_len;
+	void *buf_out;
+	enum hinic_mbox_ack_type ack_type;
+	struct mbox_msg_info msg_info;
+	u8 sed_id;
+};
+
+struct hinic_send_mbox {
+	u8 *data;
+	volatile u64 *wb_status; /* write back status */
+	void *wb_vaddr;
+	dma_addr_t wb_paddr;
+};
+
+enum mbox_event_state {
+	EVENT_START = 0,
+	EVENT_TIMEOUT,
+	EVENT_END,
+};
+
+struct hinic_mbox_func_to_func {
+	struct hinic_hwdev *hwdev;
+
+	spinlock_t mbox_send_lock; /* lock for mbox cmd sending */
+	spinlock_t msg_send_lock; /* lock for mbox segments sending */
+	struct hinic_send_mbox send_mbox;
+
+	struct hinic_recv_mbox mbox_resp[HINIC_MAX_FUNCTIONS];
+	struct hinic_recv_mbox mbox_send[HINIC_MAX_FUNCTIONS];
+
+	struct hinic_eq *rsp_aeq;
+	struct hinic_eq *recv_aeq;
+
+	u8 send_msg_id;
+	enum mbox_event_state event_flag;
+	spinlock_t mbox_lock; /* lock for mbox event flag */
+};
+
+/*
+ * mbox function prototypes
+ */
+int hinic_comm_func_to_func_init(struct hinic_hwdev *hwdev);
+void hinic_comm_func_to_func_free(struct hinic_hwdev *hwdev);
+int hinic_mbox_func_aeqe_handler(void *handle, u8 *header,
+					u8 size, void *param);
+int hinic_mbox_to_pf(struct hinic_hwdev *hwdev, enum hinic_mod_type mod, u8 cmd,
+			void *buf_in, u16 in_size,
+			void *buf_out, u16 *out_size, u32 timeout);
+int hinic_mbox_to_pf_no_ack(struct hinic_hwdev *hwdev, enum hinic_mod_type mod,
+				u8 cmd, void *buf_in, u16 in_size);
+
+#endif /* _HINIC_PMD_MBOX_H_ */
diff --git a/drivers/net/hinic/base/meson.build b/drivers/net/hinic/base/meson.build
index cde394a..6cf947f 100644
--- a/drivers/net/hinic/base/meson.build
+++ b/drivers/net/hinic/base/meson.build
@@ -12,6 +12,7 @@ sources = [
 	'hinic_pmd_niccfg.c',
 	'hinic_pmd_nicio.c',
 	'hinic_pmd_wq.c',
+ 	'hinic_pmd_mbox.c',
 ]
 
 extra_flags = []
-- 
1.8.3.1