[PATCH v3 09/25] net/spnic: support MAC and link event handling

[Yanling Song <songyl@ramaxel.com>]

This commit adds interfaces to add/remove MAC addresses
and registers related ops to struct eth_dev_ops. Furthermore,
this commit adds callback to handle link events.

Signed-off-by: Yanling Song <songyl@ramaxel.com>
---
 drivers/net/spnic/base/meson.build       |   3 +-
 drivers/net/spnic/base/spnic_hw_cfg.c    |  12 +
 drivers/net/spnic/base/spnic_hw_cfg.h    |   2 +
 drivers/net/spnic/base/spnic_nic_cfg.c   | 291 +++++++++++++++++++
 drivers/net/spnic/base/spnic_nic_cfg.h   | 180 ++++++++++++
 drivers/net/spnic/base/spnic_nic_event.c |  27 +-
 drivers/net/spnic/base/spnic_nic_event.h |   9 +-
 drivers/net/spnic/spnic_ethdev.c         | 348 ++++++++++++++++++++++-
 8 files changed, 861 insertions(+), 11 deletions(-)
 create mode 100644 drivers/net/spnic/base/spnic_nic_cfg.c
 create mode 100644 drivers/net/spnic/base/spnic_nic_cfg.h

diff --git a/drivers/net/spnic/base/meson.build b/drivers/net/spnic/base/meson.build
index 77a56ca41e..f4bb4469ae 100644
--- a/drivers/net/spnic/base/meson.build
+++ b/drivers/net/spnic/base/meson.build
@@ -11,7 +11,8 @@ sources = [
 	'spnic_cmdq.c',
 	'spnic_hw_comm.c',
 	'spnic_wq.c',
-	'spnic_hw_cfg.c'
+	'spnic_hw_cfg.c',
+	'spnic_nic_cfg.c'
 ]
 
 extra_flags = []
diff --git a/drivers/net/spnic/base/spnic_hw_cfg.c b/drivers/net/spnic/base/spnic_hw_cfg.c
index e8856ce9fe..6505f48273 100644
--- a/drivers/net/spnic/base/spnic_hw_cfg.c
+++ b/drivers/net/spnic/base/spnic_hw_cfg.c
@@ -155,3 +155,15 @@ void spnic_free_capability(void *dev)
 {
 	rte_free(((struct spnic_hwdev *)dev)->cfg_mgmt);
 }
+
+u8 spnic_physical_port_id(void *hwdev)
+{
+	struct spnic_hwdev *dev = hwdev;
+
+	if (!dev) {
+		PMD_DRV_LOG(INFO, "Hwdev is NULL for getting physical port id");
+		return 0;
+	}
+
+	return dev->cfg_mgmt->svc_cap.port_id;
+}
diff --git a/drivers/net/spnic/base/spnic_hw_cfg.h b/drivers/net/spnic/base/spnic_hw_cfg.h
index 1b1b598726..9ab51f2875 100644
--- a/drivers/net/spnic/base/spnic_hw_cfg.h
+++ b/drivers/net/spnic/base/spnic_hw_cfg.h
@@ -112,6 +112,8 @@ struct spnic_cfg_cmd_dev_cap {
 int spnic_init_capability(void *dev);
 void spnic_free_capability(void *dev);
 
+u8 spnic_physical_port_id(void *hwdev);
+
 int cfg_mbx_vf_proc_msg(void *hwdev, void *pri_handle, u16 cmd, void *buf_in,
 			u16 in_size, void *buf_out, u16 *out_size);
 #endif /* _SPNIC_HW_CFG_H_ */
diff --git a/drivers/net/spnic/base/spnic_nic_cfg.c b/drivers/net/spnic/base/spnic_nic_cfg.c
new file mode 100644
index 0000000000..c47bc330a3
--- /dev/null
+++ b/drivers/net/spnic/base/spnic_nic_cfg.c
@@ -0,0 +1,291 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 Ramaxel Memory Technology, Ltd
+ */
+
+#include <rte_ether.h>
+#include "spnic_compat.h"
+#include "spnic_cmd.h"
+#include "spnic_mgmt.h"
+#include "spnic_hwif.h"
+#include "spnic_mbox.h"
+#include "spnic_hwdev.h"
+#include "spnic_wq.h"
+#include "spnic_cmdq.h"
+#include "spnic_nic_cfg.h"
+#include "spnic_hw_cfg.h"
+
+struct vf_msg_handler {
+	u16 cmd;
+};
+
+const struct vf_msg_handler vf_cmd_handler[] = {
+	{
+		.cmd = SPNIC_CMD_VF_REGISTER,
+	},
+
+	{
+		.cmd = SPNIC_CMD_GET_MAC,
+	},
+
+	{
+		.cmd = SPNIC_CMD_SET_MAC,
+	},
+
+	{
+		.cmd = SPNIC_CMD_DEL_MAC,
+	},
+
+	{
+		.cmd = SPNIC_CMD_UPDATE_MAC,
+	},
+
+	{
+		.cmd = SPNIC_CMD_VF_COS,
+	},
+};
+
+static const struct vf_msg_handler vf_mag_cmd_handler[] = {
+	{
+		.cmd = MAG_CMD_GET_LINK_STATUS,
+	},
+};
+
+static int mag_msg_to_mgmt_sync(void *hwdev, u16 cmd, void *buf_in, u16 in_size,
+				void *buf_out, u16 *out_size);
+
+int l2nic_msg_to_mgmt_sync(void *hwdev, u16 cmd, void *buf_in, u16 in_size,
+			   void *buf_out, u16 *out_size)
+{
+	u32 i, cmd_cnt = ARRAY_LEN(vf_cmd_handler);
+	bool cmd_to_pf = false;
+
+	if (spnic_func_type(hwdev) == TYPE_VF) {
+		for (i = 0; i < cmd_cnt; i++) {
+			if (cmd == vf_cmd_handler[i].cmd)
+				cmd_to_pf = true;
+		}
+	}
+
+	if (cmd_to_pf) {
+		return spnic_mbox_to_pf(hwdev, SPNIC_MOD_L2NIC, cmd, buf_in,
+					in_size, buf_out, out_size, 0);
+	}
+
+	return spnic_msg_to_mgmt_sync(hwdev, SPNIC_MOD_L2NIC, cmd, buf_in,
+				      in_size, buf_out, out_size, 0);
+}
+
+static int spnic_check_mac_info(u8 status, u16 vlan_id)
+{
+	if ((status && status != SPNIC_MGMT_STATUS_EXIST &&
+	     status != SPNIC_PF_SET_VF_ALREADY) ||
+	    (vlan_id & CHECK_IPSU_15BIT &&
+	     status == SPNIC_MGMT_STATUS_EXIST))
+		return -EINVAL;
+
+	return 0;
+}
+
+#define VLAN_N_VID		4096
+
+int spnic_set_mac(void *hwdev, const u8 *mac_addr, u16 vlan_id, u16 func_id)
+{
+	struct spnic_port_mac_set mac_info;
+	u16 out_size = sizeof(mac_info);
+	int err;
+
+	if (!hwdev || !mac_addr)
+		return -EINVAL;
+
+	memset(&mac_info, 0, sizeof(mac_info));
+
+	if (vlan_id >= VLAN_N_VID) {
+		PMD_DRV_LOG(ERR, "Invalid VLAN number: %d", vlan_id);
+		return -EINVAL;
+	}
+
+	mac_info.func_id = func_id;
+	mac_info.vlan_id = vlan_id;
+	memmove(mac_info.mac, mac_addr, ETH_ALEN);
+
+	err = l2nic_msg_to_mgmt_sync(hwdev, SPNIC_CMD_SET_MAC, &mac_info,
+				     sizeof(mac_info), &mac_info, &out_size);
+	if (err || !out_size ||
+	    spnic_check_mac_info(mac_info.msg_head.status, mac_info.vlan_id)) {
+		PMD_DRV_LOG(ERR, "Update MAC failed, err: %d, status: 0x%x, out size: 0x%x",
+			    err, mac_info.msg_head.status, out_size);
+		return -EINVAL;
+	}
+
+	if (mac_info.msg_head.status == SPNIC_PF_SET_VF_ALREADY) {
+		PMD_DRV_LOG(WARNING, "PF has already set VF mac, Ignore set operation");
+		return SPNIC_PF_SET_VF_ALREADY;
+	}
+
+	if (mac_info.msg_head.status == SPNIC_MGMT_STATUS_EXIST) {
+		PMD_DRV_LOG(WARNING, "MAC is repeated. Ignore update operation");
+		return 0;
+	}
+
+	return 0;
+}
+
+int spnic_del_mac(void *hwdev, const u8 *mac_addr, u16 vlan_id, u16 func_id)
+{
+	struct spnic_port_mac_set mac_info;
+	u16 out_size = sizeof(mac_info);
+	int err;
+
+	if (!hwdev || !mac_addr)
+		return -EINVAL;
+
+	if (vlan_id >= VLAN_N_VID) {
+		PMD_DRV_LOG(ERR, "Invalid VLAN number: %d", vlan_id);
+		return -EINVAL;
+	}
+
+	memset(&mac_info, 0, sizeof(mac_info));
+	mac_info.func_id = func_id;
+	mac_info.vlan_id = vlan_id;
+	memmove(mac_info.mac, mac_addr, ETH_ALEN);
+
+	err = l2nic_msg_to_mgmt_sync(hwdev, SPNIC_CMD_DEL_MAC, &mac_info,
+				     sizeof(mac_info), &mac_info, &out_size);
+	if (err || !out_size || (mac_info.msg_head.status &&
+	    mac_info.msg_head.status != SPNIC_PF_SET_VF_ALREADY)) {
+		PMD_DRV_LOG(ERR, "Delete MAC failed, err: %d, status: 0x%x, out size: 0x%x",
+			    err, mac_info.msg_head.status, out_size);
+		return -EINVAL;
+	}
+
+	if (mac_info.msg_head.status == SPNIC_PF_SET_VF_ALREADY) {
+		PMD_DRV_LOG(WARNING, "PF has already set VF mac, Ignore delete operation");
+		return SPNIC_PF_SET_VF_ALREADY;
+	}
+
+	return 0;
+}
+
+int spnic_update_mac(void *hwdev, u8 *old_mac, u8 *new_mac, u16 vlan_id,
+		     u16 func_id)
+{
+	struct spnic_port_mac_update mac_info;
+	u16 out_size = sizeof(mac_info);
+	int err;
+
+	if (!hwdev || !old_mac || !new_mac)
+		return -EINVAL;
+
+	if (vlan_id >= VLAN_N_VID) {
+		PMD_DRV_LOG(ERR, "Invalid VLAN number: %d", vlan_id);
+		return -EINVAL;
+	}
+
+	memset(&mac_info, 0, sizeof(mac_info));
+	mac_info.func_id = func_id;
+	mac_info.vlan_id = vlan_id;
+	memcpy(mac_info.old_mac, old_mac, ETH_ALEN);
+	memcpy(mac_info.new_mac, new_mac, ETH_ALEN);
+
+	err = l2nic_msg_to_mgmt_sync(hwdev, SPNIC_CMD_UPDATE_MAC, &mac_info,
+				     sizeof(mac_info), &mac_info, &out_size);
+	if (err || !out_size ||
+	    spnic_check_mac_info(mac_info.msg_head.status, mac_info.vlan_id)) {
+		PMD_DRV_LOG(ERR, "Update MAC failed, err: %d, status: 0x%x, out size: 0x%x",
+			    err, mac_info.msg_head.status, out_size);
+		return -EINVAL;
+	}
+
+	if (mac_info.msg_head.status == SPNIC_PF_SET_VF_ALREADY) {
+		PMD_DRV_LOG(WARNING, "PF has already set VF MAC. Ignore update operation");
+		return SPNIC_PF_SET_VF_ALREADY;
+	}
+
+	if (mac_info.msg_head.status == SPNIC_MGMT_STATUS_EXIST) {
+		PMD_DRV_LOG(INFO, "MAC is repeated. Ignore update operation");
+		return 0;
+	}
+
+	return 0;
+}
+
+int spnic_get_default_mac(void *hwdev, u8 *mac_addr, int ether_len)
+{
+	struct spnic_port_mac_set mac_info;
+	u16 out_size = sizeof(mac_info);
+	int err;
+
+	if (!hwdev || !mac_addr)
+		return -EINVAL;
+
+	memset(&mac_info, 0, sizeof(mac_info));
+	mac_info.func_id = spnic_global_func_id(hwdev);
+
+	err = l2nic_msg_to_mgmt_sync(hwdev, SPNIC_CMD_GET_MAC,
+				     &mac_info, sizeof(mac_info),
+		&mac_info, &out_size);
+	if (err || !out_size || mac_info.msg_head.status) {
+		PMD_DRV_LOG(ERR, "Get MAC failed, err: %d, status: 0x%x, out size: 0x%x",
+			    err, mac_info.msg_head.status, out_size);
+		return -EINVAL;
+	}
+
+	memmove(mac_addr, mac_info.mac, ether_len);
+
+	return 0;
+}
+
+int spnic_get_port_info(void *hwdev, struct nic_port_info *port_info)
+{
+	struct spnic_cmd_port_info port_msg;
+	u16 out_size = sizeof(port_msg);
+	int err;
+
+	if (!hwdev || !port_info)
+		return -EINVAL;
+
+	memset(&port_msg, 0, sizeof(port_msg));
+	port_msg.port_id = spnic_physical_port_id(hwdev);
+
+	err = mag_msg_to_mgmt_sync(hwdev, MAG_CMD_GET_PORT_INFO, &port_msg,
+				   sizeof(port_msg), &port_msg, &out_size);
+	if (err || !out_size || port_msg.msg_head.status) {
+		PMD_DRV_LOG(ERR, "Get port info failed, err: %d, status: 0x%x, out size: 0x%x",
+			    err, port_msg.msg_head.status, out_size);
+		return -EINVAL;
+	}
+
+	port_info->autoneg_cap = port_msg.autoneg_cap;
+	port_info->autoneg_state = port_msg.autoneg_state;
+	port_info->duplex = port_msg.duplex;
+	port_info->port_type = port_msg.port_type;
+	port_info->speed = port_msg.speed;
+	port_info->fec = port_msg.fec;
+
+	return 0;
+}
+
+static int _mag_msg_to_mgmt_sync(void *hwdev, u16 cmd, void *buf_in,
+				 u16 in_size, void *buf_out, u16 *out_size)
+{
+	u32 i, cmd_cnt = ARRAY_LEN(vf_mag_cmd_handler);
+
+	if (spnic_func_type(hwdev) == TYPE_VF) {
+		for (i = 0; i < cmd_cnt; i++) {
+			if (cmd == vf_mag_cmd_handler[i].cmd)
+				return spnic_mbox_to_pf(hwdev, SPNIC_MOD_HILINK,
+							cmd, buf_in, in_size,
+							buf_out, out_size, 0);
+		}
+	}
+
+	return spnic_msg_to_mgmt_sync(hwdev, SPNIC_MOD_HILINK, cmd, buf_in,
+				      in_size, buf_out, out_size, 0);
+}
+
+static int mag_msg_to_mgmt_sync(void *hwdev, u16 cmd, void *buf_in, u16 in_size,
+				void *buf_out, u16 *out_size)
+{
+	return _mag_msg_to_mgmt_sync(hwdev, cmd, buf_in, in_size, buf_out,
+				     out_size);
+}
diff --git a/drivers/net/spnic/base/spnic_nic_cfg.h b/drivers/net/spnic/base/spnic_nic_cfg.h
new file mode 100644
index 0000000000..a10a310530
--- /dev/null
+++ b/drivers/net/spnic/base/spnic_nic_cfg.h
@@ -0,0 +1,180 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 Ramaxel Memory Technology, Ltd
+ */
+
+#ifndef _SPNIC_NIC_CFG_H_
+#define _SPNIC_NIC_CFG_H_
+
+#ifndef ETH_ALEN
+#define ETH_ALEN			6
+#endif
+
+#define OS_VF_ID_TO_HW(os_vf_id) ((os_vf_id) + 1)
+#define HW_VF_ID_TO_OS(hw_vf_id) ((hw_vf_id) - 1)
+
+#define SPNIC_PF_SET_VF_ALREADY		0x4
+#define SPNIC_MGMT_STATUS_EXIST		0x6
+#define CHECK_IPSU_15BIT		0x8000
+
+/* Structures for port info */
+struct nic_port_info {
+	u8 port_type;
+	u8 autoneg_cap;
+	u8 autoneg_state;
+	u8 duplex;
+	u8 speed;
+	u8 fec;
+};
+
+enum spnic_link_status {
+	SPNIC_LINK_DOWN = 0,
+	SPNIC_LINK_UP
+};
+
+enum nic_media_type {
+	MEDIA_UNKNOWN = -1,
+	MEDIA_FIBRE = 0,
+	MEDIA_COPPER,
+	MEDIA_BACKPLANE
+};
+
+enum nic_speed_level {
+	LINK_SPEED_10MB = 0,
+	LINK_SPEED_100MB,
+	LINK_SPEED_1GB,
+	LINK_SPEED_10GB,
+	LINK_SPEED_25GB,
+	LINK_SPEED_40GB,
+	LINK_SPEED_100GB,
+	LINK_SPEED_LEVELS,
+};
+
+struct spnic_port_mac_set {
+	struct mgmt_msg_head msg_head;
+
+	u16 func_id;
+	u16 vlan_id;
+	u16 rsvd1;
+	u8 mac[ETH_ALEN];
+};
+
+struct spnic_port_mac_update {
+	struct mgmt_msg_head msg_head;
+
+	u16 func_id;
+	u16 vlan_id;
+	u16 rsvd1;
+	u8 old_mac[ETH_ALEN];
+	u16 rsvd2;
+	u8 new_mac[ETH_ALEN];
+};
+
+struct spnic_cmd_port_info {
+	struct mgmt_msg_head msg_head;
+
+	u8 port_id;
+	u8 rsvd1[3];
+	u8 port_type;
+	u8 autoneg_cap;
+	u8 autoneg_state;
+	u8 duplex;
+	u8 speed;
+	u8 fec;
+	u16 rsvd2;
+	u32 rsvd3[4];
+};
+
+struct spnic_cmd_link_state {
+	struct mgmt_msg_head msg_head;
+
+	u8 port_id;
+	u8 state;
+	u16 rsvd1;
+};
+
+int l2nic_msg_to_mgmt_sync(void *hwdev, u16 cmd, void *buf_in, u16 in_size,
+			   void *buf_out, u16 *out_size);
+
+/**
+ * Update MAC address to hardware
+ *
+ * @param[in] hwdev
+ *   Device pointer to hwdev
+ * @param[in] old_mac
+ *   Old MAC addr to delete
+ * @param[in] new_mac
+ *   New MAC addr to update
+ * @param[in] vlan_id
+ *   Vlan id
+ * @param func_id
+ *   Function index
+ *
+ * @retval zero : Success
+ * @retval non-zero : Failure
+ */
+int spnic_update_mac(void *hwdev, u8 *old_mac, u8 *new_mac, u16 vlan_id,
+		     u16 func_id);
+
+/**
+ * Get the default mac address
+ *
+ * @param[in] hwdev
+ *   Device pointer to hwdev
+ * @param[in] mac_addr
+ *   Mac address from hardware
+ * @param[in] ether_len
+ *   The length of mac address
+ *
+ * @retval zero : Success
+ * @retval non-zero : Failure
+ */
+int spnic_get_default_mac(void *hwdev, u8 *mac_addr, int ether_len);
+
+/**
+ * Set mac address
+ *
+ * @param[in] hwdev
+ *   Device pointer to hwdev
+ * @param[in] mac_addr
+ *   Mac address from hardware
+ * @param[in] vlan_id
+ *   Vlan id
+ * @param[in] func_id
+ *   Function index
+ *
+ * @retval zero : Success
+ * @retval non-zero : Failure
+ */
+int spnic_set_mac(void *hwdev, const u8 *mac_addr, u16 vlan_id, u16 func_id);
+
+/**
+ * Delete MAC address
+ *
+ * @param[in] hwdev
+ *   Device pointer to hwdev
+ * @param[in] mac_addr
+ *   MAC address from hardware
+ * @param[in] vlan_id
+ *   Vlan id
+ * @param[in] func_id
+ *   Function index
+ *
+ * @retval zero : Success
+ * @retval non-zero : Failure
+ */
+int spnic_del_mac(void *hwdev, const u8 *mac_addr, u16 vlan_id, u16 func_id);
+
+/**
+ * Get port info
+ *
+ * @param[in] hwdev
+ *   Device pointer to hwdev
+ * @param[out] port_info
+ *   Port info, including autoneg, port type, duplex, speed and fec mode
+ *
+ * @retval zero : Success
+ * @retval non-zero : Failure
+ */
+int spnic_get_port_info(void *hwdev, struct nic_port_info *port_info);
+
+#endif /* _SPNIC_NIC_CFG_H_ */
diff --git a/drivers/net/spnic/base/spnic_nic_event.c b/drivers/net/spnic/base/spnic_nic_event.c
index 5a1e109bf8..959d0fbaf3 100644
--- a/drivers/net/spnic/base/spnic_nic_event.c
+++ b/drivers/net/spnic/base/spnic_nic_event.c
@@ -9,16 +9,39 @@
 #include "spnic_hwif.h"
 #include "spnic_hwdev.h"
 #include "spnic_mgmt.h"
+#include "spnic_nic_cfg.h"
 #include "spnic_hwdev.h"
 #include "spnic_nic_event.h"
 
-static void get_port_info(u8 link_state, struct rte_eth_link *link)
+void get_port_info(struct spnic_hwdev *hwdev, u8 link_state,
+		   struct rte_eth_link *link)
 {
+	uint32_t port_speed[LINK_SPEED_LEVELS] = {ETH_SPEED_NUM_10M,
+					ETH_SPEED_NUM_100M, ETH_SPEED_NUM_1G,
+					ETH_SPEED_NUM_10G, ETH_SPEED_NUM_25G,
+					ETH_SPEED_NUM_40G, ETH_SPEED_NUM_100G};
+	struct nic_port_info port_info = {0};
+	int err;
+
 	if (!link_state) {
 		link->link_status = ETH_LINK_DOWN;
 		link->link_speed = ETH_SPEED_NUM_NONE;
 		link->link_duplex = ETH_LINK_HALF_DUPLEX;
 		link->link_autoneg = ETH_LINK_FIXED;
+	} else {
+		link->link_status = ETH_LINK_UP;
+
+		err = spnic_get_port_info(hwdev, &port_info);
+		if (err) {
+			link->link_speed = ETH_SPEED_NUM_NONE;
+			link->link_duplex = ETH_LINK_FULL_DUPLEX;
+			link->link_autoneg = ETH_LINK_FIXED;
+		} else {
+			link->link_speed = port_speed[port_info.speed %
+						LINK_SPEED_LEVELS];
+			link->link_duplex = port_info.duplex;
+			link->link_autoneg = port_info.autoneg_state;
+		}
 	}
 }
 
@@ -51,7 +74,7 @@ static void link_status_event_handler(void *hwdev, void *buf_in,
 	spnic_link_event_stats(hwdev, link_status->state);
 
 	/* Link event reported only after set vport enable */
-	get_port_info(link_status->state, &link);
+	get_port_info(dev, link_status->state, &link);
 	err = rte_eth_linkstatus_set((struct rte_eth_dev *)(dev->eth_dev),
 				     &link);
 	if (!err)
diff --git a/drivers/net/spnic/base/spnic_nic_event.h b/drivers/net/spnic/base/spnic_nic_event.h
index bf1c15941e..3a86c31538 100644
--- a/drivers/net/spnic/base/spnic_nic_event.h
+++ b/drivers/net/spnic/base/spnic_nic_event.h
@@ -5,13 +5,8 @@
 #ifndef _SPNIC_NIC_EVENT_H_
 #define _SPNIC_NIC_EVENT_H_
 
-struct spnic_cmd_link_state {
-	struct mgmt_msg_head msg_head;
-
-	u8 port_id;
-	u8 state;
-	u16 rsvd1;
-};
+void get_port_info(struct spnic_hwdev *hwdev, u8 link_state,
+		   struct rte_eth_link *link);
 
 void spnic_pf_event_handler(void *hwdev, __rte_unused void *pri_handle,
 			    u16 cmd, void *buf_in, u16 in_size,
diff --git a/drivers/net/spnic/spnic_ethdev.c b/drivers/net/spnic/spnic_ethdev.c
index 228ed0c936..8f71280fa7 100644
--- a/drivers/net/spnic/spnic_ethdev.c
+++ b/drivers/net/spnic/spnic_ethdev.c
@@ -5,14 +5,23 @@
 #include <rte_pci.h>
 #include <rte_bus_pci.h>
 #include <ethdev_pci.h>
+#include <rte_malloc.h>
 #include <rte_errno.h>
 #include <rte_ether.h>
 
 #include "base/spnic_compat.h"
+#include "base/spnic_cmd.h"
 #include "base/spnic_csr.h"
+#include "base/spnic_wq.h"
+#include "base/spnic_eqs.h"
+#include "base/spnic_mgmt.h"
+#include "base/spnic_cmdq.h"
 #include "base/spnic_hwdev.h"
 #include "base/spnic_hwif.h"
-
+#include "base/spnic_hw_cfg.h"
+#include "base/spnic_hw_comm.h"
+#include "base/spnic_nic_cfg.h"
+#include "base/spnic_nic_event.h"
 #include "spnic_ethdev.h"
 
 /* Driver-specific log messages type */
@@ -21,6 +30,58 @@ int spnic_logtype;
 #define SPNIC_MAX_UC_MAC_ADDRS		128
 #define SPNIC_MAX_MC_MAC_ADDRS		128
 
+static void spnic_delete_mc_addr_list(struct spnic_nic_dev *nic_dev)
+{
+	u16 func_id;
+	u32 i;
+
+	func_id = spnic_global_func_id(nic_dev->hwdev);
+
+	for (i = 0; i < SPNIC_MAX_MC_MAC_ADDRS; i++) {
+		if (rte_is_zero_ether_addr(&nic_dev->mc_list[i]))
+			break;
+
+		spnic_del_mac(nic_dev->hwdev, nic_dev->mc_list[i].addr_bytes,
+			      0, func_id);
+		memset(&nic_dev->mc_list[i], 0, sizeof(struct rte_ether_addr));
+	}
+}
+
+/**
+ * Deinit mac_vlan table in hardware.
+ *
+ * @param[in] eth_dev
+ *   Pointer to ethernet device structure.
+ */
+static void spnic_deinit_mac_addr(struct rte_eth_dev *eth_dev)
+{
+	struct spnic_nic_dev *nic_dev =
+				SPNIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
+	u16 func_id = 0;
+	int err;
+	int i;
+
+	func_id = spnic_global_func_id(nic_dev->hwdev);
+
+	for (i = 0; i < SPNIC_MAX_UC_MAC_ADDRS; i++) {
+		if (rte_is_zero_ether_addr(&eth_dev->data->mac_addrs[i]))
+			continue;
+
+		err = spnic_del_mac(nic_dev->hwdev,
+				    eth_dev->data->mac_addrs[i].addr_bytes,
+				    0, func_id);
+		if (err && err != SPNIC_PF_SET_VF_ALREADY)
+			PMD_DRV_LOG(ERR, "Delete mac table failed, dev_name: %s",
+				    eth_dev->data->name);
+
+		memset(&eth_dev->data->mac_addrs[i], 0,
+		       sizeof(struct rte_ether_addr));
+	}
+
+	/* Delete multicast mac addrs */
+	spnic_delete_mc_addr_list(nic_dev);
+}
+
 /**
  * Close the device.
  *
@@ -38,13 +99,247 @@ static int spnic_dev_close(struct rte_eth_dev *eth_dev)
 		return 0;
 	}
 
+	spnic_deinit_mac_addr(eth_dev);
+	rte_free(nic_dev->mc_list);
+
+	rte_bit_relaxed_clear32(SPNIC_DEV_INTR_EN, &nic_dev->dev_status);
+
 	spnic_free_hwdev(nic_dev->hwdev);
 
+	eth_dev->dev_ops = NULL;
+
 	rte_free(nic_dev->hwdev);
 	nic_dev->hwdev = NULL;
 
 	return 0;
 }
+/**
+ * Update MAC address
+ *
+ * @param[in] dev
+ *   Pointer to ethernet device structure.
+ * @param[in] addr
+ *   Pointer to MAC address
+ *
+ * @retval zero: Success
+ * @retval non-zero: Failure
+ */
+static int spnic_set_mac_addr(struct rte_eth_dev *dev,
+			      struct rte_ether_addr *addr)
+{
+	struct spnic_nic_dev *nic_dev = SPNIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
+	char mac_addr[RTE_ETHER_ADDR_FMT_SIZE];
+	u16 func_id;
+	int err;
+
+	if (!rte_is_valid_assigned_ether_addr(addr)) {
+		rte_ether_format_addr(mac_addr, RTE_ETHER_ADDR_FMT_SIZE, addr);
+		PMD_DRV_LOG(ERR, "Set invalid MAC address %s", mac_addr);
+		return -EINVAL;
+	}
+
+	func_id = spnic_global_func_id(nic_dev->hwdev);
+	err = spnic_update_mac(nic_dev->hwdev,
+				nic_dev->default_addr.addr_bytes,
+				addr->addr_bytes, 0, func_id);
+	if (err)
+		return err;
+
+	rte_ether_addr_copy(addr, &nic_dev->default_addr);
+	rte_ether_format_addr(mac_addr, RTE_ETHER_ADDR_FMT_SIZE,
+			      &nic_dev->default_addr);
+
+	PMD_DRV_LOG(INFO, "Set new MAC address %s", mac_addr);
+
+	return 0;
+}
+
+/**
+ * Remove a MAC address.
+ *
+ * @param[in] dev
+ *   Pointer to ethernet device structure.
+ * @param[in] index
+ *   MAC address index.
+ */
+static void spnic_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
+{
+	struct spnic_nic_dev *nic_dev = SPNIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
+	u16 func_id;
+	int err;
+
+	if (index >= SPNIC_MAX_UC_MAC_ADDRS) {
+		PMD_DRV_LOG(INFO, "Remove MAC index(%u) is out of range",
+			    index);
+		return;
+	}
+
+	func_id = spnic_global_func_id(nic_dev->hwdev);
+	err = spnic_del_mac(nic_dev->hwdev,
+			     dev->data->mac_addrs[index].addr_bytes,
+			     0, func_id);
+	if (err)
+		PMD_DRV_LOG(ERR, "Remove MAC index(%u) failed", index);
+}
+
+/**
+ * Add a MAC address.
+ *
+ * @param[in] dev
+ *   Pointer to ethernet device structure.
+ * @param[in] mac_addr
+ *   MAC address to register.
+ * @param[in] index
+ *   MAC address index.
+ * @param[in] vmdq
+ *   VMDq pool index to associate address with (unused_).
+ *
+ * @retval zero: Success
+ * @retval non-zero: Failure
+ */
+static int spnic_mac_addr_add(struct rte_eth_dev *dev,
+			      struct rte_ether_addr *mac_addr, uint32_t index,
+			      __rte_unused uint32_t vmdq)
+{
+	struct spnic_nic_dev *nic_dev = SPNIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
+	unsigned int i;
+	u16 func_id;
+	int err;
+
+	if (!rte_is_valid_assigned_ether_addr(mac_addr)) {
+		PMD_DRV_LOG(ERR, "Add invalid MAC address");
+		return -EINVAL;
+	}
+
+	if (index >= SPNIC_MAX_UC_MAC_ADDRS) {
+		PMD_DRV_LOG(ERR, "Add MAC index(%u) is out of range", index);
+		return -EINVAL;
+	}
+
+	/* Make sure this address doesn't already be configured */
+	for (i = 0; i < SPNIC_MAX_UC_MAC_ADDRS; i++) {
+		if (rte_is_same_ether_addr(mac_addr,
+			&dev->data->mac_addrs[i])) {
+			PMD_DRV_LOG(ERR, "MAC address is already configured");
+			return -EADDRINUSE;
+		}
+	}
+
+	func_id = spnic_global_func_id(nic_dev->hwdev);
+	err = spnic_set_mac(nic_dev->hwdev, mac_addr->addr_bytes, 0, func_id);
+	if (err)
+		return err;
+
+	return 0;
+}
+
+/**
+ * Set multicast MAC address
+ *
+ * @param[in] dev
+ *   Pointer to ethernet device structure.
+ * @param[in] mc_addr_set
+ *   Pointer to multicast MAC address
+ * @param[in] nb_mc_addr
+ *   The number of multicast MAC address to set
+ *
+ * @retval zero: Success
+ * @retval non-zero: Failure
+ */
+static int spnic_set_mc_addr_list(struct rte_eth_dev *dev,
+				  struct rte_ether_addr *mc_addr_set,
+				  uint32_t nb_mc_addr)
+{
+	struct spnic_nic_dev *nic_dev = SPNIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
+	char mac_addr[RTE_ETHER_ADDR_FMT_SIZE];
+	u16 func_id;
+	int err;
+	u32 i;
+
+	func_id = spnic_global_func_id(nic_dev->hwdev);
+
+	/* Delete old multi_cast addrs firstly */
+	spnic_delete_mc_addr_list(nic_dev);
+
+	if (nb_mc_addr > SPNIC_MAX_MC_MAC_ADDRS)
+		return -EINVAL;
+
+	for (i = 0; i < nb_mc_addr; i++) {
+		if (!rte_is_multicast_ether_addr(&mc_addr_set[i])) {
+			rte_ether_format_addr(mac_addr, RTE_ETHER_ADDR_FMT_SIZE,
+					      &mc_addr_set[i]);
+			PMD_DRV_LOG(ERR, "Set mc MAC addr failed, addr(%s) invalid",
+				    mac_addr);
+			return -EINVAL;
+		}
+	}
+
+	for (i = 0; i < nb_mc_addr; i++) {
+		err = spnic_set_mac(nic_dev->hwdev, mc_addr_set[i].addr_bytes,
+				    0, func_id);
+		if (err) {
+			spnic_delete_mc_addr_list(nic_dev);
+			return err;
+		}
+
+		rte_ether_addr_copy(&mc_addr_set[i], &nic_dev->mc_list[i]);
+	}
+
+	return 0;
+}
+static const struct eth_dev_ops spnic_pmd_ops = {
+	.mac_addr_set                  = spnic_set_mac_addr,
+	.mac_addr_remove               = spnic_mac_addr_remove,
+	.mac_addr_add                  = spnic_mac_addr_add,
+	.set_mc_addr_list              = spnic_set_mc_addr_list,
+};
+
+static const struct eth_dev_ops spnic_pmd_vf_ops = {
+	.mac_addr_set                  = spnic_set_mac_addr,
+	.mac_addr_remove               = spnic_mac_addr_remove,
+	.mac_addr_add                  = spnic_mac_addr_add,
+	.set_mc_addr_list              = spnic_set_mc_addr_list,
+};
+
+/**
+ * Init mac_vlan table in hardwares.
+ *
+ * @param[in] eth_dev
+ *   Pointer to ethernet device structure.
+ *
+ * @retval zero: Success
+ * @retval non-zero: Failure
+ */
+static int spnic_init_mac_table(struct rte_eth_dev *eth_dev)
+{
+	struct spnic_nic_dev *nic_dev =
+		SPNIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
+	u8 addr_bytes[RTE_ETHER_ADDR_LEN];
+	u16 func_id = 0;
+	int err = 0;
+
+	err = spnic_get_default_mac(nic_dev->hwdev, addr_bytes,
+				     RTE_ETHER_ADDR_LEN);
+	if (err)
+		return err;
+
+	rte_ether_addr_copy((struct rte_ether_addr *)addr_bytes,
+			    &eth_dev->data->mac_addrs[0]);
+	if (rte_is_zero_ether_addr(&eth_dev->data->mac_addrs[0]))
+		rte_eth_random_addr(eth_dev->data->mac_addrs[0].addr_bytes);
+
+	func_id = spnic_global_func_id(nic_dev->hwdev);
+	err = spnic_set_mac(nic_dev->hwdev,
+			    eth_dev->data->mac_addrs[0].addr_bytes,
+			    0, func_id);
+	if (err && err != SPNIC_PF_SET_VF_ALREADY)
+		return err;
+
+	rte_ether_addr_copy(&eth_dev->data->mac_addrs[0],
+			    &nic_dev->default_addr);
+
+	return 0;
+}
 
 static int spnic_func_init(struct rte_eth_dev *eth_dev)
 {
@@ -63,11 +358,37 @@ static int spnic_func_init(struct rte_eth_dev *eth_dev)
 	}
 
 	nic_dev = SPNIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
+	memset(nic_dev, 0, sizeof(*nic_dev));
 	snprintf(nic_dev->dev_name, sizeof(nic_dev->dev_name),
 		 "spnic-%.4x:%.2x:%.2x.%x",
 		 pci_dev->addr.domain, pci_dev->addr.bus,
 		 pci_dev->addr.devid, pci_dev->addr.function);
 
+	/* Alloc mac_addrs */
+	eth_dev->data->mac_addrs = rte_zmalloc("spnic_mac",
+		SPNIC_MAX_UC_MAC_ADDRS * sizeof(struct rte_ether_addr), 0);
+	if (!eth_dev->data->mac_addrs) {
+		PMD_DRV_LOG(ERR, "Allocate %zx bytes to store MAC addresses "
+			    "failed, dev_name: %s",
+			    SPNIC_MAX_UC_MAC_ADDRS *
+			    sizeof(struct rte_ether_addr),
+			    eth_dev->data->name);
+		err = -ENOMEM;
+		goto alloc_eth_addr_fail;
+	}
+
+	nic_dev->mc_list = rte_zmalloc("spnic_mc",
+		SPNIC_MAX_MC_MAC_ADDRS * sizeof(struct rte_ether_addr), 0);
+	if (!nic_dev->mc_list) {
+		PMD_DRV_LOG(ERR, "Allocate %zx bytes to store multicast "
+			    "addresses failed, dev_name: %s",
+			    SPNIC_MAX_MC_MAC_ADDRS *
+			    sizeof(struct rte_ether_addr),
+			    eth_dev->data->name);
+		err = -ENOMEM;
+		goto alloc_mc_list_fail;
+	}
+
 	eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
 	/* Create hardware device */
 	nic_dev->hwdev = rte_zmalloc("spnic_hwdev", sizeof(*nic_dev->hwdev),
@@ -90,17 +411,42 @@ static int spnic_func_init(struct rte_eth_dev *eth_dev)
 		goto init_hwdev_fail;
 	}
 
+	if (SPNIC_FUNC_TYPE(nic_dev->hwdev) == TYPE_VF)
+		eth_dev->dev_ops = &spnic_pmd_vf_ops;
+	else
+		eth_dev->dev_ops = &spnic_pmd_ops;
+	err = spnic_init_mac_table(eth_dev);
+	if (err) {
+		PMD_DRV_LOG(ERR, "Init mac table failed, dev_name: %s",
+			    eth_dev->data->name);
+		goto init_mac_table_fail;
+	}
+
+	rte_bit_relaxed_set32(SPNIC_DEV_INTR_EN, &nic_dev->dev_status);
+
 	rte_bit_relaxed_set32(SPNIC_DEV_INIT, &nic_dev->dev_status);
 	PMD_DRV_LOG(INFO, "Initialize %s in primary succeed",
 		    eth_dev->data->name);
 
 	return 0;
 
+init_mac_table_fail:
+	spnic_free_hwdev(nic_dev->hwdev);
+	eth_dev->dev_ops = NULL;
+
 init_hwdev_fail:
 	rte_free(nic_dev->hwdev);
 	nic_dev->hwdev = NULL;
 
 alloc_hwdev_mem_fail:
+	rte_free(nic_dev->mc_list);
+	nic_dev->mc_list = NULL;
+
+alloc_mc_list_fail:
+	rte_free(eth_dev->data->mac_addrs);
+	eth_dev->data->mac_addrs = NULL;
+
+alloc_eth_addr_fail:
 	PMD_DRV_LOG(ERR, "Initialize %s in primary failed",
 		    eth_dev->data->name);
 	return err;
-- 
2.32.0